From f167864f2c591faef7928a4bff4416a27c32c78b Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Thu, 5 Jan 2023 11:56:10 +0100 Subject: [PATCH] armbian-next: split of `config-prepare` into multiple functions, allowing for logging of all non-interactive sections - introduce `do_with_conditional_logging()` which only starts logging sections if `do_logging=no` - with this we should get complete logs (ofc except for the interactive sections) --- lib/functions/cli/cli-build.sh | 15 +-- lib/functions/cli/cli-configdump.sh | 2 +- .../configuration/compilation-config.sh | 29 ++++++ lib/functions/logging/logging.sh | 15 ++- lib/functions/main/config-interactive.sh | 27 ++++++ lib/functions/main/config-prepare.sh | 97 +++++++------------ lib/library-functions.sh | 18 ++++ 7 files changed, 129 insertions(+), 74 deletions(-) create mode 100644 lib/functions/configuration/compilation-config.sh create mode 100644 lib/functions/main/config-interactive.sh diff --git a/lib/functions/cli/cli-build.sh b/lib/functions/cli/cli-build.sh index 8e0d154361..8c36672afe 100644 --- a/lib/functions/cli/cli-build.sh +++ b/lib/functions/cli/cli-build.sh @@ -1,21 +1,14 @@ function cli_standard_build_pre_run() { declare -g ARMBIAN_COMMAND_REQUIRE_BASIC_DEPS="yes" # Require prepare_host_basic to run before the command. - # "gimme root on a Linux machine" + # "gimme root on a Linux machine" cli_standard_relaunch_docker_or_sudo } function cli_standard_build_run() { - # @TODO: then many other interesting possibilities like a REPL, which we lost somewhere along the way. docker-shell? - - # configuration etc - it initializes the extension manager + # configuration etc - it initializes the extension manager; handles its own logging sections prepare_and_config_main_build_single - # Allow for custom user-invoked functions, or do the default build. - if [[ -z $1 ]]; then - main_default_build_single - else - # @TODO: rpardini: check this with extensions usage? - eval "$@" - fi + # main_default_build_single() handles its own logging sections... + main_default_build_single } diff --git a/lib/functions/cli/cli-configdump.sh b/lib/functions/cli/cli-configdump.sh index a311a9fa98..d99f256f1a 100644 --- a/lib/functions/cli/cli-configdump.sh +++ b/lib/functions/cli/cli-configdump.sh @@ -9,7 +9,7 @@ function cli_config_dump_run() { } function config_and_remove_useless() { - prepare_and_config_main_build_single + do_logging=no prepare_and_config_main_build_single # avoid logging during configdump; it's useless unset FINALDEST unset FINAL_HOST_DEPS unset HOOK_ORDER HOOK_POINT HOOK_POINT_TOTAL_FUNCS diff --git a/lib/functions/configuration/compilation-config.sh b/lib/functions/configuration/compilation-config.sh new file mode 100644 index 0000000000..a160d287b4 --- /dev/null +++ b/lib/functions/configuration/compilation-config.sh @@ -0,0 +1,29 @@ +function prepare_compilation_vars() { + # moved from config: rpardini: ccache belongs in compilation, not config. I think. + if [[ $USE_CCACHE != no ]]; then + CCACHE=ccache + export PATH="/usr/lib/ccache:$PATH" + # private ccache directory to avoid permission issues when using build script with "sudo" + # see https://ccache.samba.org/manual.html#_sharing_a_cache for alternative solution + [[ $PRIVATE_CCACHE == yes ]] && export CCACHE_DIR=$SRC/cache/ccache + else + CCACHE="" + fi + + # moved from config: this does not belong in configuration. it's a compilation thing. + # optimize build time with 100% CPU usage + CPUS=$(grep -c 'processor' /proc/cpuinfo) + if [[ $USEALLCORES != no ]]; then + CTHREADS="-j$((CPUS + CPUS / 2))" + else + CTHREADS="-j1" + fi + + call_extension_method "post_determine_cthreads" "config_post_determine_cthreads" <<- 'POST_DETERMINE_CTHREADS' + *give config a chance modify CTHREADS programatically. A build server may work better with hyperthreads-1 for example.* + Called early, before any compilation work starts. + POST_DETERMINE_CTHREADS + + return 0 +} + diff --git a/lib/functions/logging/logging.sh b/lib/functions/logging/logging.sh index ff0d891f97..eb8f9d77cf 100644 --- a/lib/functions/logging/logging.sh +++ b/lib/functions/logging/logging.sh @@ -120,10 +120,21 @@ function do_with_logging() { function do_with_logging_unless_user_terminal() { # Is user on a terminal? If so, don't log, just show on screen. if [[ -t 1 ]]; then - display_alert "User is on a terminal, not logging output" "terminal" "debug" + display_alert "User is on a terminal, not logging output" "do_with_logging_unless_user_terminal" "debug" "$@" else - display_alert "User is not on a terminal, logging output" "terminal" "debug" + display_alert "User is not on a terminal, logging output" "do_with_logging_unless_user_terminal" "debug" + do_with_logging "$@" + fi +} + +function do_with_conditional_logging() { + # if "do_logging=no", just run the command, otherwise, log it. + if [[ "${do_logging:-"yes"}" == "no" ]]; then + display_alert "do_logging=no, not starting logging section" "do_with_conditional_logging" "debug" + "$@" + else + display_alert "normally logging output" "do_with_conditional_logging" "debug" do_with_logging "$@" fi } diff --git a/lib/functions/main/config-interactive.sh b/lib/functions/main/config-interactive.sh new file mode 100644 index 0000000000..1dbad84fb5 --- /dev/null +++ b/lib/functions/main/config-interactive.sh @@ -0,0 +1,27 @@ +function config_possibly_interactive_kernel_board() { + # if KERNEL_ONLY, KERNEL_CONFIGURE, BOARD, BRANCH or RELEASE are not set, display selection menu + + interactive_config_ask_kernel + [[ -z $KERNEL_ONLY ]] && exit_with_error "No option selected: KERNEL_ONLY" + [[ -z $KERNEL_CONFIGURE ]] && exit_with_error "No option selected: KERNEL_CONFIGURE" + + interactive_config_ask_board_list # this uses get_list_of_all_buildable_boards too + [[ -z $BOARD ]] && exit_with_error "No board selected: BOARD" + + return 0 # shortcircuit above +} + +function config_possibly_interactive_branch_release_desktop_minimal() { + interactive_config_ask_branch + [[ -z $BRANCH ]] && exit_with_error "No kernel branch selected: BRANCH" + [[ ${KERNEL_TARGET} != *${BRANCH}* && ${BRANCH} != "ddk" ]] && exit_with_error "Kernel branch not defined for this board: '${BRANCH}' for '${BOARD}'" + + interactive_config_ask_release + [[ -z $RELEASE && ${KERNEL_ONLY} != yes ]] && exit_with_error "No release selected: RELEASE" + + interactive_config_ask_desktop_build + interactive_config_ask_standard_or_minimal + + return 0 # protect against eventual shortcircuit above +} + diff --git a/lib/functions/main/config-prepare.sh b/lib/functions/main/config-prepare.sh index 9a26b5d180..83263ab649 100644 --- a/lib/functions/main/config-prepare.sh +++ b/lib/functions/main/config-prepare.sh @@ -1,58 +1,24 @@ #!/usr/bin/env bash -function prepare_compilation_vars() { - # moved from config: rpardini: ccache belongs in compilation, not config. I think. - if [[ $USE_CCACHE != no ]]; then - CCACHE=ccache - export PATH="/usr/lib/ccache:$PATH" - # private ccache directory to avoid permission issues when using build script with "sudo" - # see https://ccache.samba.org/manual.html#_sharing_a_cache for alternative solution - [[ $PRIVATE_CCACHE == yes ]] && export CCACHE_DIR=$SRC/cache/ccache - else - CCACHE="" - fi +function prepare_and_config_main_build_single() { + LOG_SECTION="config_early_init" do_with_conditional_logging config_early_init - # moved from config: this does not belong in configuration. it's a compilation thing. - # optimize build time with 100% CPU usage - CPUS=$(grep -c 'processor' /proc/cpuinfo) - if [[ $USEALLCORES != no ]]; then - CTHREADS="-j$((CPUS + CPUS / 2))" - else - CTHREADS="-j1" - fi + # those are possibly interactive. interactive (dialog) and logging don't mix, for obvious reasons. + interactive_config_prepare_terminal # init vars used for interactive + config_possibly_interactive_kernel_board - call_extension_method "post_determine_cthreads" "config_post_determine_cthreads" <<- 'POST_DETERMINE_CTHREADS' - *give config a chance modify CTHREADS programatically. A build server may work better with hyperthreads-1 for example.* - Called early, before any compilation work starts. - POST_DETERMINE_CTHREADS + LOG_SECTION="config_source_board_file" do_with_conditional_logging config_source_board_file - return 0 + config_possibly_interactive_branch_release_desktop_minimal + interactive_finish # cleans up vars used for interactive + + LOG_SECTION="config_pre_main" do_with_conditional_logging config_pre_main + LOG_SECTION="do_main_configuration" do_with_conditional_logging do_main_configuration # This initializes the extension manager among a lot of other things, and call extension_prepare_config() hook + LOG_SECTION="config_post_main" do_with_conditional_logging config_post_main + display_alert "Done with prepare_and_config_main_build_single" "${BOARD}.${BOARD_TYPE}" "info" } -function prepare_and_config_main_build_single() { - # default umask for root is 022 so parent directories won't be group writeable without this - # this is used instead of making the chmod in prepare_host() recursive - umask 002 - - interactive_config_prepare_terminal - - # Warnings mitigation - [[ -z $LANGUAGE ]] && export LANGUAGE="en_US:en" # set to english if not set - [[ -z $CONSOLE_CHAR ]] && export CONSOLE_CHAR="UTF-8" # set console to UTF-8 if not set - - declare -g SHOW_WARNING=yes # If you try something that requires EXPERT=yes. - - display_alert "Starting single build process" "${BOARD}" "info" - - # if KERNEL_ONLY, KERNEL_CONFIGURE, BOARD, BRANCH or RELEASE are not set, display selection menu - - interactive_config_ask_kernel - [[ -z $KERNEL_ONLY ]] && exit_with_error "No option selected: KERNEL_ONLY" - [[ -z $KERNEL_CONFIGURE ]] && exit_with_error "No option selected: KERNEL_CONFIGURE" - - interactive_config_ask_board_list # this uses get_list_of_all_buildable_boards too - [[ -z $BOARD ]] && exit_with_error "No board selected: BOARD" - +function config_source_board_file() { declare -a arr_all_board_names=() # arrays declare -A dict_all_board_types=() dict_all_board_source_files=() # dictionaries get_list_of_all_buildable_boards arr_all_board_names "" dict_all_board_types dict_all_board_source_files "" # invoke @@ -82,19 +48,27 @@ function prepare_and_config_main_build_single() { [[ -z $KERNEL_TARGET ]] && exit_with_error "Board ('${BOARD}') configuration does not define valid kernel config" - interactive_config_ask_branch - [[ -z $BRANCH ]] && exit_with_error "No kernel branch selected: BRANCH" - [[ ${KERNEL_TARGET} != *${BRANCH}* && ${BRANCH} != "ddk" ]] && exit_with_error "Kernel branch not defined for this board: '${BRANCH}' for '${BOARD}'" + return 0 # shortcircuit above +} - interactive_config_ask_release - [[ -z $RELEASE && ${KERNEL_ONLY} != yes ]] && exit_with_error "No release selected: RELEASE" +function config_early_init() { - interactive_config_ask_desktop_build + # default umask for root is 022 so parent directories won't be group writeable without this + # this is used instead of making the chmod in prepare_host() recursive + umask 002 - interactive_config_ask_standard_or_minimal + # Warnings mitigation + [[ -z $LANGUAGE ]] && export LANGUAGE="en_US:en" # set to english if not set + [[ -z $CONSOLE_CHAR ]] && export CONSOLE_CHAR="UTF-8" # set console to UTF-8 if not set - interactive_finish # cleans up vars + declare -g SHOW_WARNING=yes # If you try something that requires EXPERT=yes. + display_alert "Starting single build process" "${BOARD:-"no BOARD set"}" "info" + + return 0 # protect against eventual shortcircuit above +} + +function config_pre_main() { #prevent conflicting setup if [[ $BUILD_DESKTOP == "yes" ]]; then BUILD_MINIMAL=no @@ -111,8 +85,10 @@ function prepare_and_config_main_build_single() { [[ ${KERNEL_CONFIGURE} == prebuilt ]] && [[ -z ${REPOSITORY_INSTALL} ]] && REPOSITORY_INSTALL="u-boot,kernel,bsp,armbian-zsh,armbian-config,armbian-bsp-cli,armbian-firmware${BUILD_DESKTOP:+,armbian-desktop,armbian-bsp-desktop}" - do_main_configuration # This initializes the extension manager among a lot of other things, and call extension_prepare_config() hook + return 0 # shortcircuit above +} +function config_post_main() { if [[ "$BETA" == "yes" ]]; then IMAGE_TYPE=nightly elif [ "$BETA" == "no" ] || [ "$RC" == "yes" ]; then @@ -197,7 +173,7 @@ function prepare_and_config_main_build_single() { Don't change anything not coming from other variables or meant to be configured by the user. EXTENSION_FINISH_CONFIG - display_alert "Done with prepare_and_config_main_build_single" "${BOARD}.${BOARD_TYPE}" "info" + return 0 # protect against eventual shortcircuit above } # cli-bsp also uses this @@ -211,10 +187,11 @@ function set_distribution_status() { [[ "${DISTRIBUTION_STATUS}" != "supported" ]] && [[ "${EXPERT}" != "yes" ]] && exit_with_error "Armbian ${RELEASE} is unsupported and, therefore, only available to experts (EXPERT=yes)" - return 0 # due to last stmt above being a shortcut conditional + return 0 # due to last stmt above being a shortcircuit conditional } # Some utility functions -branch2dir() { +function branch2dir() { [[ "${1}" == "head" ]] && echo "HEAD" || echo "${1##*:}" + return 0 } diff --git a/lib/library-functions.sh b/lib/library-functions.sh index 0162d50a7e..28446c6fcd 100644 --- a/lib/library-functions.sh +++ b/lib/library-functions.sh @@ -316,6 +316,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/configuration/aggregation.sh source "${SRC}"/lib/functions/configuration/aggregation.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/configuration/compilation-config.sh +# shellcheck source=lib/functions/configuration/compilation-config.sh +source "${SRC}"/lib/functions/configuration/compilation-config.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled @@ -622,6 +631,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/logging/traps.sh source "${SRC}"/lib/functions/logging/traps.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/main/config-interactive.sh +# shellcheck source=lib/functions/main/config-interactive.sh +source "${SRC}"/lib/functions/main/config-interactive.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled