tools(ci): add script to build all examples

Using new cmake project declare.

.gitlab-ci.yml

@@ -101,3 +101,82 @@ build_docs:
     - make gh-linkcheck
     - make html
     - ../check_doc_warnings.sh
+
+    
+.build_examples_make_template: &build_examples_make_template
+  <<: *build_template
+  # This is a workaround for a rarely encountered issue with building examples in CI.
+  # Probably related to building of Kconfig in 'make clean' stage
+  retry: 1
+  artifacts:
+    when: always
+    paths:
+      - $LOG_PATH
+    expire_in: 2 days
+  variables:
+    LOG_PATH: "$CI_PROJECT_DIR/log_examples_make"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_EXAMPLE_TEST
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    # it's not possible to build 100% out-of-tree and have the "artifacts"
+    # mechanism work, but this is the next best thing
+    - rm -rf build_examples
+    - mkdir build_examples
+    - cd build_examples
+    # build some of examples
+    - mkdir -p ${LOG_PATH}
+    - ${IDF_PATH}/tools/ci/build_examples.sh "${CI_JOB_NAME}"
+
+# same as above, but for CMake
+.build_examples_cmake_template: &build_examples_cmake_template
+  <<: *build_template
+  artifacts:
+    when: always
+    paths:
+      - $LOG_PATH
+    expire_in: 2 days
+  variables:
+    LOG_PATH: "$CI_PROJECT_DIR/log_examples_cmake"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_EXAMPLE_TEST
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    # it's not possible to build 100% out-of-tree and have the "artifacts"
+    # mechanism work, but this is the next best thing
+    - rm -rf build_examples_cmake
+    - mkdir build_examples_cmake
+    - cd build_examples_cmake
+    # build some of examples
+    - mkdir -p ${LOG_PATH}
+    - ${IDF_PATH}/tools/ci/build_examples_cmake.sh "${CI_JOB_NAME}"
+
+build_examples_make_00:
+  <<: *build_examples_make_template
+
+build_examples_make_01:
+  <<: *build_examples_make_template
+
+build_examples_make_02:
+  <<: *build_examples_make_template
+
+build_examples_make_03:
+  <<: *build_examples_make_template
+
+build_examples_cmake_00:
+  <<: *build_examples_cmake_template
+
+build_examples_cmake_01:
+  <<: *build_examples_cmake_template
+
+build_examples_cmake_02:
+  <<: *build_examples_cmake_template
+
+build_examples_cmake_03:
+  <<: *build_examples_cmake_template

components/bootloader/subproject/CMakeLists.txt

@@ -10,14 +10,12 @@ if(NOT IDF_PATH)
         "in by the parent build process.")
 endif()
 
-set(COMPONENTS esptool_py bootloader bootloader_support spi_flash log esp8266 util partition_table)
+set(COMPONENTS esptool_py bootloader bootloader_support spi_flash log esp8266 util partition_table main)
 set(BOOTLOADER_BUILD 1)
 add_definitions(-DBOOTLOADER_BUILD=1)
 
 set(COMPONENT_REQUIRES_COMMON log esp8266 spi_flash)
 
-set(MAIN_SRCS main/bootloader_start.c)
-
 include("${IDF_PATH}/tools/cmake/project.cmake")
 project(bootloader)
 

components/bootloader/subproject/main/CMakeLists.txt

@@ -0,0 +1,4 @@
+set(COMPONENT_SRCS "bootloader_start.c")
+set(COMPONENT_ADD_INCLUDEDIRS "")
+set(COMPONENT_REQUIRES "bootloader_support")
+register_component()

tools/ci/build_examples.sh

@@ -0,0 +1,210 @@
+#!/bin/bash
+#
+# Build all examples from the examples directory, out of tree to
+# ensure they can run when copied to a new directory.
+#
+# Runs as part of CI process.
+#
+# Assumes PWD is an out-of-tree build directory, and will copy examples
+# to individual subdirectories, one by one.
+#
+#
+# Without arguments it just builds all examples
+#
+# With one argument <JOB_NAME> it builds part of the examples. This is a useful for
+#   parallel execution in CI.
+#   <JOB_NAME> must look like this:
+#               <some_text_label>_<num>
+#   It scans .gitlab-ci.yaml to count number of jobs which have name "<some_text_label>_<num>"
+#   It scans the filesystem to count all examples
+#   Based on this, it decides to run qa set of examples.
+#
+
+# -----------------------------------------------------------------------------
+# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
+
+if [[ ! -z ${DEBUG_SHELL} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+set -o nounset # Exit if variable not set.
+
+# Remove the initial space and instead use '\n'.
+IFS=$'\n\t'
+
+# -----------------------------------------------------------------------------
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
+[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
+[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
+
+echo "build_examples running in ${PWD}"
+
+# only 0 or 1 arguments
+[ $# -le 1 ] || die "Have to run as $(basename $0) [<JOB_NAME>]"
+
+export BATCH_BUILD=1
+export V=0 # only build verbose if there's an error
+
+export IDF_CI_BUILD=1
+
+export EXAMPLE_MQTT_BROKER_CERTIFICATE="https://www.espressif.com/"
+export EXAMPLE_MQTT_BROKER_WS="https://www.espressif.com/"
+export EXAMPLE_MQTT_BROKER_WSS="https://www.espressif.com/"
+export EXAMPLE_MQTT_BROKER_SSL="https://www.espressif.com/"
+
+shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024)
+
+RESULT=0
+FAILED_EXAMPLES=""
+RESULT_ISSUES=22  # magic number result code for issues found
+LOG_SUSPECTED=${LOG_PATH}/common_log.txt
+touch ${LOG_SUSPECTED}
+SDKCONFIG_DEFAULTS_CI=sdkconfig.ci
+
+if [ $# -eq 0 ]
+then
+    START_NUM=0
+    END_NUM=999
+else
+    JOB_NAME=$1
+
+    # parse text prefix at the beginning of string 'some_your_text_NUM'
+    # (will be 'some_your_text' without last '_')
+    JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' )
+    [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad"
+
+    # parse number 'NUM' at the end of string 'some_your_text_NUM'
+    # NOTE: Getting rid of the leading zero to get the decimal
+    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' )
+    [ -z ${JOB_NUM} ] && die "JOB_NUM is bad"
+
+    # count number of the jobs
+    NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${IDF_PATH}/.gitlab-ci.yml" )
+    [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad"
+
+    # count number of examples
+    NUM_OF_EXAMPLES=$( find ${IDF_PATH}/examples/ -type f -name Makefile | wc -l )
+    [ -z ${NUM_OF_EXAMPLES} ] && die "NUM_OF_EXAMPLES is bad"
+
+    # separate intervals
+    #57 / 5 == 12
+    NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} ))
+    [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
+
+    # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
+    START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
+    [ -z ${START_NUM} ] && die "START_NUM is bad"
+
+    END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} ))
+    [ -z ${END_NUM} ] && die "END_NUM is bad"
+fi
+
+build_example () {
+    local ID=$1
+    shift
+    local MAKE_FILE=$1
+    shift
+
+    local EXAMPLE_DIR=$(dirname "${MAKE_FILE}")
+    local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}")
+
+    if [[ -f "example_builds/${ID}/${EXAMPLE_NAME}/build/ci_build_success" ]]; then
+        echo "Project ${EXAMPLE_NAME} has been built and skip building ..."
+    else
+        echo "Building ${EXAMPLE_NAME} as ${ID}..."
+        mkdir -p "example_builds/${ID}"
+        cp -r "${EXAMPLE_DIR}" "example_builds/${ID}"
+        pushd "example_builds/${ID}/${EXAMPLE_NAME}"
+            # be stricter in the CI build than the default IDF settings
+            export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations"
+            export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
+
+            # sdkconfig files are normally not checked into git, but may be present when
+            # a developer runs this script locally
+            rm -f sdkconfig
+
+            # If sdkconfig.ci file is present, append it to sdkconfig.defaults,
+            # replacing environment variables
+            if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then
+                cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults
+            fi
+
+            # build non-verbose first
+            local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt
+            touch ${BUILDLOG}
+
+            local FLASH_ARGS=build/download.config
+
+            make clean >>${BUILDLOG} 2>&1 &&
+            make defconfig >>${BUILDLOG} 2>&1 &&
+            make all -j4 >>${BUILDLOG} 2>&1 &&
+            make ota >>${BUILDLOG} 2>&1 &&
+            make print_flash_cmd >${FLASH_ARGS}.full 2>>${BUILDLOG} &&
+            touch build/ci_build_success ||
+            {
+                RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ;
+            }
+
+            tail -n 1 ${FLASH_ARGS}.full > ${FLASH_ARGS} || :
+            test -s ${FLASH_ARGS} || die "Error: ${FLASH_ARGS} file is empty"
+
+            cat ${BUILDLOG}
+        popd
+
+        grep -i "error\|warning" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || :
+    fi
+}
+
+EXAMPLE_NUM=0
+
+EXAMPLE_PATHS=$( find ${IDF_PATH}/examples/ -type f -name Makefile | grep -v "/build_system/cmake/" | sort )
+for FN in ${EXAMPLE_PATHS}
+do
+    if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]
+    then
+        EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
+        continue
+    fi
+    echo ">>> example [ ${EXAMPLE_NUM} ] - $FN"
+
+    build_example "${EXAMPLE_NUM}" "${FN}"
+
+    EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
+done
+
+# show warnings
+echo -e "\nFound issues:"
+
+#       Ignore the next messages:
+# "error.o" or "-Werror" in compiler's command line
+# "reassigning to symbol" or "changes choice state" in sdkconfig
+# 'Compiler and toochain versions is not supported' from make/project.mk
+IGNORE_WARNS="\
+library/error\.o\
+\|\ -Werror\
+\|error\.d\
+\|reassigning to symbol\
+\|changes choice state\
+\|Compiler version is not supported\
+\|Toolchain version is not supported\
+"
+
+sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \
+    && RESULT=$RESULT_ISSUES \
+    || echo -e "\tNone"
+
+[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES"
+[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!"
+
+echo -e "\nReturn code = $RESULT"
+
+exit $RESULT

tools/ci/build_examples_cmake.sh

@@ -0,0 +1,218 @@
+#!/bin/bash
+#
+# Build all examples from the examples directory, out of tree to
+# ensure they can run when copied to a new directory.
+#
+# Runs as part of CI process.
+#
+# Assumes PWD is an out-of-tree build directory, and will copy examples
+# to individual subdirectories, one by one.
+#
+#
+# Without arguments it just builds all examples
+#
+# With one argument <JOB_NAME> it builds part of the examples. This is a useful for
+#   parallel execution in CI.
+#   <JOB_NAME> must look like this:
+#               <some_text_label>_<num>
+#   It scans .gitlab-ci.yaml to count number of jobs which have name "<some_text_label>_<num>"
+#   It scans the filesystem to count all examples
+#   Based on this, it decides to run qa set of examples.
+#
+
+# -----------------------------------------------------------------------------
+# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
+
+if [[ ! -z ${DEBUG_SHELL} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+
+# Remove the initial space and instead use '\n'.
+IFS=$'\n\t'
+
+export PATH="$IDF_PATH/tools:$PATH"  # for idf.py
+
+# -----------------------------------------------------------------------------
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
+[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
+[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
+
+set -o nounset # Exit if variable not set.
+
+echo "build_examples running in ${PWD}"
+
+# only 0 or 1 arguments
+[ $# -le 1 ] || die "Have to run as $(basename $0) [<JOB_NAME>]"
+
+export BATCH_BUILD=1
+export V=0 # only build verbose if there's an error
+
+export IDF_CI_BUILD=1
+
+export EXAMPLE_MQTT_BROKER_CERTIFICATE="https://www.espressif.com/"
+export EXAMPLE_MQTT_BROKER_WS="https://www.espressif.com/"
+export EXAMPLE_MQTT_BROKER_WSS="https://www.espressif.com/"
+export EXAMPLE_MQTT_BROKER_SSL="https://www.espressif.com/"
+
+shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024)
+
+RESULT=0
+FAILED_EXAMPLES=""
+RESULT_ISSUES=22  # magic number result code for issues found
+LOG_SUSPECTED=${LOG_PATH}/common_log.txt
+touch ${LOG_SUSPECTED}
+SDKCONFIG_DEFAULTS_CI=sdkconfig.ci
+
+EXAMPLE_PATHS=$( find ${IDF_PATH}/examples/ -type f -name CMakeLists.txt | grep -v "/components/" | grep -v "/main/" | sort )
+
+if [ $# -eq 0 ]
+then
+    START_NUM=0
+    END_NUM=999
+else
+    JOB_NAME=$1
+
+    # parse text prefix at the beginning of string 'some_your_text_NUM'
+    # (will be 'some_your_text' without last '_')
+    JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' )
+    [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad"
+
+    # parse number 'NUM' at the end of string 'some_your_text_NUM'
+    # NOTE: Getting rid of the leading zero to get the decimal
+    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' )
+    [ -z ${JOB_NUM} ] && die "JOB_NUM is bad"
+
+    # count number of the jobs
+    NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${IDF_PATH}/.gitlab-ci.yml" )
+    [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad"
+
+    # count number of examples
+    NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l )
+    [ ${NUM_OF_EXAMPLES} -lt 50 ] && die "NUM_OF_EXAMPLES is bad"
+
+    # separate intervals
+    #57 / 5 == 12
+    NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} ))
+    [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
+
+    # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
+    START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
+    [ -z ${START_NUM} ] && die "START_NUM is bad"
+
+    END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} ))
+    [ -z ${END_NUM} ] && die "END_NUM is bad"
+fi
+
+prepare_build() {
+    if [[ $1 == "subscribe_publish" ]]; then
+        echo "Dummy certificate data for continuous integration" > main/certs/certificate.pem.crt
+        echo "Dummy certificate data for continuous integration" > main/certs/private.pem.key
+    elif [[ $1 == "thing_shadow" ]]; then
+        echo "Dummy certificate data for continuous integration" > main/certs/certificate.pem.crt
+        echo "Dummy certificate data for continuous integration" > main/certs/private.pem.key
+    fi
+}
+
+build_example () {
+    local ID=$1
+    shift
+    local CMAKELISTS=$1
+    shift
+
+    local EXAMPLE_DIR=$(dirname "${CMAKELISTS}")
+    local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}")
+
+    if [[ -f "example_builds/${ID}/${EXAMPLE_NAME}/build/ci_build_success" ]]; then
+        echo "Project ${EXAMPLE_NAME} has been built and skip building ..."
+    else
+        echo "Building ${EXAMPLE_NAME} as ${ID}..."
+        mkdir -p "example_builds/${ID}"
+        cp -r "${EXAMPLE_DIR}" "example_builds/${ID}"
+        pushd "example_builds/${ID}/${EXAMPLE_NAME}"
+            # be stricter in the CI build than the default IDF settings
+            export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations"
+            export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
+
+            prepare_build ${EXAMPLE_NAME}
+
+            # sdkconfig files are normally not checked into git, but may be present when
+            # a developer runs this script locally
+            rm -f sdkconfig
+
+            # If sdkconfig.ci file is present, append it to sdkconfig.defaults,
+            # replacing environment variables
+            if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then
+                cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults
+            fi
+
+            # build non-verbose first
+            local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt
+            touch ${BUILDLOG}
+
+            idf.py build >>${BUILDLOG} 2>&1 &&
+            cp build/flash_project_args build/download.config && # backwards compatible download.config filename
+            touch build/ci_build_success ||
+            {
+                RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ;
+            }
+
+            cat ${BUILDLOG}
+        popd
+
+        grep -i "error\|warning" "${BUILDLOG}" 2>&1 | grep -v "error.c.obj" >> "${LOG_SUSPECTED}" || :
+    fi
+}
+
+EXAMPLE_NUM=0
+
+for EXAMPLE_PATH in ${EXAMPLE_PATHS}
+do
+    if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]
+    then
+        EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
+        continue
+    fi
+    echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH"
+
+    build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}"
+
+    EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
+done
+
+# show warnings
+echo -e "\nFound issues:"
+
+#       Ignore the next messages:
+# "error.o" or "-Werror" in compiler's command line
+# "reassigning to symbol" or "changes choice state" in sdkconfig
+# 'Compiler and toochain versions is not supported' from crosstool_version_check.cmake
+IGNORE_WARNS="\
+library/error\.o\
+\|\ -Werror\
+\|error\.d\
+\|reassigning to symbol\
+\|changes choice state\
+\|crosstool_version_check\.cmake\
+\| -Wno-dev \
+"
+
+sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \
+    && RESULT=$RESULT_ISSUES \
+    || echo -e "\tNone"
+
+[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES"
+[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!"
+
+echo -e "\nReturn code = $RESULT"
+
+exit $RESULT

tools/ci/envsubst.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+#
+# A script similar to GNU envsubst, but filters out
+# some CI related variables.
+
+import os
+import sys
+
+
+def main():
+    # Sanitize environment variables
+    vars_to_remove = []
+    for var_name in os.environ.keys():
+        if var_name.startswith('CI_'):
+            vars_to_remove.append(var_name)
+    for var_name in vars_to_remove:
+        del os.environ[var_name]
+
+    for line in sys.stdin:
+        if not line:
+            break
+        sys.stdout.write(os.path.expandvars(line))
+    sys.stdout.flush()
+
+
+if __name__ == '__main__':
+    main()