From d244c20dece3770f8511cdb85ebb6471f8fa8250 Mon Sep 17 00:00:00 2001 From: dongheng Date: Tue, 18 Jun 2019 13:08:26 +0800 Subject: [PATCH] tools(ci): add script to build all examples Using new cmake project declare. --- .gitlab-ci.yml | 79 +++++++ .../bootloader/subproject/CMakeLists.txt | 4 +- .../bootloader/subproject/main/CMakeLists.txt | 4 + tools/ci/build_examples.sh | 210 +++++++++++++++++ tools/ci/build_examples_cmake.sh | 218 ++++++++++++++++++ tools/ci/envsubst.py | 27 +++ 6 files changed, 539 insertions(+), 3 deletions(-) create mode 100644 components/bootloader/subproject/main/CMakeLists.txt create mode 100755 tools/ci/build_examples.sh create mode 100755 tools/ci/build_examples_cmake.sh create mode 100755 tools/ci/envsubst.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a61b56fb..4c887a49 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -101,3 +101,82 @@ build_docs: - make gh-linkcheck - make html - ../check_doc_warnings.sh + + +.build_examples_make_template: &build_examples_make_template + <<: *build_template + # This is a workaround for a rarely encountered issue with building examples in CI. + # Probably related to building of Kconfig in 'make clean' stage + retry: 1 + artifacts: + when: always + paths: + - $LOG_PATH + expire_in: 2 days + variables: + LOG_PATH: "$CI_PROJECT_DIR/log_examples_make" + only: + variables: + - $BOT_TRIGGER_WITH_LABEL == null + - $BOT_LABEL_BUILD + - $BOT_LABEL_EXAMPLE_TEST + - $BOT_LABEL_REGULAR_TEST + script: + # it's not possible to build 100% out-of-tree and have the "artifacts" + # mechanism work, but this is the next best thing + - rm -rf build_examples + - mkdir build_examples + - cd build_examples + # build some of examples + - mkdir -p ${LOG_PATH} + - ${IDF_PATH}/tools/ci/build_examples.sh "${CI_JOB_NAME}" + +# same as above, but for CMake +.build_examples_cmake_template: &build_examples_cmake_template + <<: *build_template + artifacts: + when: always + paths: + - $LOG_PATH + expire_in: 2 days + variables: + LOG_PATH: "$CI_PROJECT_DIR/log_examples_cmake" + only: + variables: + - $BOT_TRIGGER_WITH_LABEL == null + - $BOT_LABEL_BUILD + - $BOT_LABEL_EXAMPLE_TEST + - $BOT_LABEL_REGULAR_TEST + script: + # it's not possible to build 100% out-of-tree and have the "artifacts" + # mechanism work, but this is the next best thing + - rm -rf build_examples_cmake + - mkdir build_examples_cmake + - cd build_examples_cmake + # build some of examples + - mkdir -p ${LOG_PATH} + - ${IDF_PATH}/tools/ci/build_examples_cmake.sh "${CI_JOB_NAME}" + +build_examples_make_00: + <<: *build_examples_make_template + +build_examples_make_01: + <<: *build_examples_make_template + +build_examples_make_02: + <<: *build_examples_make_template + +build_examples_make_03: + <<: *build_examples_make_template + +build_examples_cmake_00: + <<: *build_examples_cmake_template + +build_examples_cmake_01: + <<: *build_examples_cmake_template + +build_examples_cmake_02: + <<: *build_examples_cmake_template + +build_examples_cmake_03: + <<: *build_examples_cmake_template diff --git a/components/bootloader/subproject/CMakeLists.txt b/components/bootloader/subproject/CMakeLists.txt index 50121e24..d2e14d2c 100644 --- a/components/bootloader/subproject/CMakeLists.txt +++ b/components/bootloader/subproject/CMakeLists.txt @@ -10,14 +10,12 @@ if(NOT IDF_PATH) "in by the parent build process.") endif() -set(COMPONENTS esptool_py bootloader bootloader_support spi_flash log esp8266 util partition_table) +set(COMPONENTS esptool_py bootloader bootloader_support spi_flash log esp8266 util partition_table main) set(BOOTLOADER_BUILD 1) add_definitions(-DBOOTLOADER_BUILD=1) set(COMPONENT_REQUIRES_COMMON log esp8266 spi_flash) -set(MAIN_SRCS main/bootloader_start.c) - include("${IDF_PATH}/tools/cmake/project.cmake") project(bootloader) diff --git a/components/bootloader/subproject/main/CMakeLists.txt b/components/bootloader/subproject/main/CMakeLists.txt new file mode 100644 index 00000000..6398604d --- /dev/null +++ b/components/bootloader/subproject/main/CMakeLists.txt @@ -0,0 +1,4 @@ +set(COMPONENT_SRCS "bootloader_start.c") +set(COMPONENT_ADD_INCLUDEDIRS "") +set(COMPONENT_REQUIRES "bootloader_support") +register_component() diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh new file mode 100755 index 00000000..032e336c --- /dev/null +++ b/tools/ci/build_examples.sh @@ -0,0 +1,210 @@ +#!/bin/bash +# +# Build all examples from the examples directory, out of tree to +# ensure they can run when copied to a new directory. +# +# Runs as part of CI process. +# +# Assumes PWD is an out-of-tree build directory, and will copy examples +# to individual subdirectories, one by one. +# +# +# Without arguments it just builds all examples +# +# With one argument it builds part of the examples. This is a useful for +# parallel execution in CI. +# must look like this: +# _ +# It scans .gitlab-ci.yaml to count number of jobs which have name "_" +# It scans the filesystem to count all examples +# Based on this, it decides to run qa set of examples. +# + +# ----------------------------------------------------------------------------- +# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d). + +if [[ ! -z ${DEBUG_SHELL} ]] +then + set -x # Activate the expand mode if DEBUG is anything but empty. +fi + +set -o errexit # Exit if command failed. +set -o pipefail # Exit if pipe failed. +set -o nounset # Exit if variable not set. + +# Remove the initial space and instead use '\n'. +IFS=$'\n\t' + +# ----------------------------------------------------------------------------- + +die() { + echo "${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +[ -z ${IDF_PATH} ] && die "IDF_PATH is not set" +[ -z ${LOG_PATH} ] && die "LOG_PATH is not set" +[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH} + +echo "build_examples running in ${PWD}" + +# only 0 or 1 arguments +[ $# -le 1 ] || die "Have to run as $(basename $0) []" + +export BATCH_BUILD=1 +export V=0 # only build verbose if there's an error + +export IDF_CI_BUILD=1 + +export EXAMPLE_MQTT_BROKER_CERTIFICATE="https://www.espressif.com/" +export EXAMPLE_MQTT_BROKER_WS="https://www.espressif.com/" +export EXAMPLE_MQTT_BROKER_WSS="https://www.espressif.com/" +export EXAMPLE_MQTT_BROKER_SSL="https://www.espressif.com/" + +shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024) + +RESULT=0 +FAILED_EXAMPLES="" +RESULT_ISSUES=22 # magic number result code for issues found +LOG_SUSPECTED=${LOG_PATH}/common_log.txt +touch ${LOG_SUSPECTED} +SDKCONFIG_DEFAULTS_CI=sdkconfig.ci + +if [ $# -eq 0 ] +then + START_NUM=0 + END_NUM=999 +else + JOB_NAME=$1 + + # parse text prefix at the beginning of string 'some_your_text_NUM' + # (will be 'some_your_text' without last '_') + JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' ) + [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad" + + # parse number 'NUM' at the end of string 'some_your_text_NUM' + # NOTE: Getting rid of the leading zero to get the decimal + JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' ) + [ -z ${JOB_NUM} ] && die "JOB_NUM is bad" + + # count number of the jobs + NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${IDF_PATH}/.gitlab-ci.yml" ) + [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad" + + # count number of examples + NUM_OF_EXAMPLES=$( find ${IDF_PATH}/examples/ -type f -name Makefile | wc -l ) + [ -z ${NUM_OF_EXAMPLES} ] && die "NUM_OF_EXAMPLES is bad" + + # separate intervals + #57 / 5 == 12 + NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} )) + [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad" + + # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60) + START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} )) + [ -z ${START_NUM} ] && die "START_NUM is bad" + + END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} )) + [ -z ${END_NUM} ] && die "END_NUM is bad" +fi + +build_example () { + local ID=$1 + shift + local MAKE_FILE=$1 + shift + + local EXAMPLE_DIR=$(dirname "${MAKE_FILE}") + local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}") + + if [[ -f "example_builds/${ID}/${EXAMPLE_NAME}/build/ci_build_success" ]]; then + echo "Project ${EXAMPLE_NAME} has been built and skip building ..." + else + echo "Building ${EXAMPLE_NAME} as ${ID}..." + mkdir -p "example_builds/${ID}" + cp -r "${EXAMPLE_DIR}" "example_builds/${ID}" + pushd "example_builds/${ID}/${EXAMPLE_NAME}" + # be stricter in the CI build than the default IDF settings + export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations" + export EXTRA_CXXFLAGS=${EXTRA_CFLAGS} + + # sdkconfig files are normally not checked into git, but may be present when + # a developer runs this script locally + rm -f sdkconfig + + # If sdkconfig.ci file is present, append it to sdkconfig.defaults, + # replacing environment variables + if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then + cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults + fi + + # build non-verbose first + local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt + touch ${BUILDLOG} + + local FLASH_ARGS=build/download.config + + make clean >>${BUILDLOG} 2>&1 && + make defconfig >>${BUILDLOG} 2>&1 && + make all -j4 >>${BUILDLOG} 2>&1 && + make ota >>${BUILDLOG} 2>&1 && + make print_flash_cmd >${FLASH_ARGS}.full 2>>${BUILDLOG} && + touch build/ci_build_success || + { + RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ; + } + + tail -n 1 ${FLASH_ARGS}.full > ${FLASH_ARGS} || : + test -s ${FLASH_ARGS} || die "Error: ${FLASH_ARGS} file is empty" + + cat ${BUILDLOG} + popd + + grep -i "error\|warning" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || : + fi +} + +EXAMPLE_NUM=0 + +EXAMPLE_PATHS=$( find ${IDF_PATH}/examples/ -type f -name Makefile | grep -v "/build_system/cmake/" | sort ) +for FN in ${EXAMPLE_PATHS} +do + if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]] + then + EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) + continue + fi + echo ">>> example [ ${EXAMPLE_NUM} ] - $FN" + + build_example "${EXAMPLE_NUM}" "${FN}" + + EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) +done + +# show warnings +echo -e "\nFound issues:" + +# Ignore the next messages: +# "error.o" or "-Werror" in compiler's command line +# "reassigning to symbol" or "changes choice state" in sdkconfig +# 'Compiler and toochain versions is not supported' from make/project.mk +IGNORE_WARNS="\ +library/error\.o\ +\|\ -Werror\ +\|error\.d\ +\|reassigning to symbol\ +\|changes choice state\ +\|Compiler version is not supported\ +\|Toolchain version is not supported\ +" + +sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \ + && RESULT=$RESULT_ISSUES \ + || echo -e "\tNone" + +[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES" +[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!" + +echo -e "\nReturn code = $RESULT" + +exit $RESULT diff --git a/tools/ci/build_examples_cmake.sh b/tools/ci/build_examples_cmake.sh new file mode 100755 index 00000000..90bfa36f --- /dev/null +++ b/tools/ci/build_examples_cmake.sh @@ -0,0 +1,218 @@ +#!/bin/bash +# +# Build all examples from the examples directory, out of tree to +# ensure they can run when copied to a new directory. +# +# Runs as part of CI process. +# +# Assumes PWD is an out-of-tree build directory, and will copy examples +# to individual subdirectories, one by one. +# +# +# Without arguments it just builds all examples +# +# With one argument it builds part of the examples. This is a useful for +# parallel execution in CI. +# must look like this: +# _ +# It scans .gitlab-ci.yaml to count number of jobs which have name "_" +# It scans the filesystem to count all examples +# Based on this, it decides to run qa set of examples. +# + +# ----------------------------------------------------------------------------- +# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d). + +if [[ ! -z ${DEBUG_SHELL} ]] +then + set -x # Activate the expand mode if DEBUG is anything but empty. +fi + +set -o errexit # Exit if command failed. +set -o pipefail # Exit if pipe failed. + +# Remove the initial space and instead use '\n'. +IFS=$'\n\t' + +export PATH="$IDF_PATH/tools:$PATH" # for idf.py + +# ----------------------------------------------------------------------------- + +die() { + echo "${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +[ -z ${IDF_PATH} ] && die "IDF_PATH is not set" +[ -z ${LOG_PATH} ] && die "LOG_PATH is not set" +[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH} + +set -o nounset # Exit if variable not set. + +echo "build_examples running in ${PWD}" + +# only 0 or 1 arguments +[ $# -le 1 ] || die "Have to run as $(basename $0) []" + +export BATCH_BUILD=1 +export V=0 # only build verbose if there's an error + +export IDF_CI_BUILD=1 + +export EXAMPLE_MQTT_BROKER_CERTIFICATE="https://www.espressif.com/" +export EXAMPLE_MQTT_BROKER_WS="https://www.espressif.com/" +export EXAMPLE_MQTT_BROKER_WSS="https://www.espressif.com/" +export EXAMPLE_MQTT_BROKER_SSL="https://www.espressif.com/" + +shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024) + +RESULT=0 +FAILED_EXAMPLES="" +RESULT_ISSUES=22 # magic number result code for issues found +LOG_SUSPECTED=${LOG_PATH}/common_log.txt +touch ${LOG_SUSPECTED} +SDKCONFIG_DEFAULTS_CI=sdkconfig.ci + +EXAMPLE_PATHS=$( find ${IDF_PATH}/examples/ -type f -name CMakeLists.txt | grep -v "/components/" | grep -v "/main/" | sort ) + +if [ $# -eq 0 ] +then + START_NUM=0 + END_NUM=999 +else + JOB_NAME=$1 + + # parse text prefix at the beginning of string 'some_your_text_NUM' + # (will be 'some_your_text' without last '_') + JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' ) + [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad" + + # parse number 'NUM' at the end of string 'some_your_text_NUM' + # NOTE: Getting rid of the leading zero to get the decimal + JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' ) + [ -z ${JOB_NUM} ] && die "JOB_NUM is bad" + + # count number of the jobs + NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${IDF_PATH}/.gitlab-ci.yml" ) + [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad" + + # count number of examples + NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l ) + [ ${NUM_OF_EXAMPLES} -lt 50 ] && die "NUM_OF_EXAMPLES is bad" + + # separate intervals + #57 / 5 == 12 + NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} )) + [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad" + + # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60) + START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} )) + [ -z ${START_NUM} ] && die "START_NUM is bad" + + END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} )) + [ -z ${END_NUM} ] && die "END_NUM is bad" +fi + +prepare_build() { + if [[ $1 == "subscribe_publish" ]]; then + echo "Dummy certificate data for continuous integration" > main/certs/certificate.pem.crt + echo "Dummy certificate data for continuous integration" > main/certs/private.pem.key + elif [[ $1 == "thing_shadow" ]]; then + echo "Dummy certificate data for continuous integration" > main/certs/certificate.pem.crt + echo "Dummy certificate data for continuous integration" > main/certs/private.pem.key + fi +} + +build_example () { + local ID=$1 + shift + local CMAKELISTS=$1 + shift + + local EXAMPLE_DIR=$(dirname "${CMAKELISTS}") + local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}") + + if [[ -f "example_builds/${ID}/${EXAMPLE_NAME}/build/ci_build_success" ]]; then + echo "Project ${EXAMPLE_NAME} has been built and skip building ..." + else + echo "Building ${EXAMPLE_NAME} as ${ID}..." + mkdir -p "example_builds/${ID}" + cp -r "${EXAMPLE_DIR}" "example_builds/${ID}" + pushd "example_builds/${ID}/${EXAMPLE_NAME}" + # be stricter in the CI build than the default IDF settings + export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations" + export EXTRA_CXXFLAGS=${EXTRA_CFLAGS} + + prepare_build ${EXAMPLE_NAME} + + # sdkconfig files are normally not checked into git, but may be present when + # a developer runs this script locally + rm -f sdkconfig + + # If sdkconfig.ci file is present, append it to sdkconfig.defaults, + # replacing environment variables + if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then + cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults + fi + + # build non-verbose first + local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt + touch ${BUILDLOG} + + idf.py build >>${BUILDLOG} 2>&1 && + cp build/flash_project_args build/download.config && # backwards compatible download.config filename + touch build/ci_build_success || + { + RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ; + } + + cat ${BUILDLOG} + popd + + grep -i "error\|warning" "${BUILDLOG}" 2>&1 | grep -v "error.c.obj" >> "${LOG_SUSPECTED}" || : + fi +} + +EXAMPLE_NUM=0 + +for EXAMPLE_PATH in ${EXAMPLE_PATHS} +do + if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]] + then + EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) + continue + fi + echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH" + + build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}" + + EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) +done + +# show warnings +echo -e "\nFound issues:" + +# Ignore the next messages: +# "error.o" or "-Werror" in compiler's command line +# "reassigning to symbol" or "changes choice state" in sdkconfig +# 'Compiler and toochain versions is not supported' from crosstool_version_check.cmake +IGNORE_WARNS="\ +library/error\.o\ +\|\ -Werror\ +\|error\.d\ +\|reassigning to symbol\ +\|changes choice state\ +\|crosstool_version_check\.cmake\ +\| -Wno-dev \ +" + +sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \ + && RESULT=$RESULT_ISSUES \ + || echo -e "\tNone" + +[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES" +[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!" + +echo -e "\nReturn code = $RESULT" + +exit $RESULT diff --git a/tools/ci/envsubst.py b/tools/ci/envsubst.py new file mode 100755 index 00000000..dbfbb1c1 --- /dev/null +++ b/tools/ci/envsubst.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# +# A script similar to GNU envsubst, but filters out +# some CI related variables. + +import os +import sys + + +def main(): + # Sanitize environment variables + vars_to_remove = [] + for var_name in os.environ.keys(): + if var_name.startswith('CI_'): + vars_to_remove.append(var_name) + for var_name in vars_to_remove: + del os.environ[var_name] + + for line in sys.stdin: + if not line: + break + sys.stdout.write(os.path.expandvars(line)) + sys.stdout.flush() + + +if __name__ == '__main__': + main()