feat(make): Sync code from esp-idf and modify for ESP8266

Commit ID: f6bfe13e

components/app_update/CMakeLists.txt

@@ -0,0 +1,15 @@
+set(COMPONENT_SRCDIRS ".")
+set(COMPONENT_ADD_INCLUDEDIRS "include")
+
+set(COMPONENT_REQUIRES "spi_flash")
+set(COMPONENT_PRIV_REQUIRES "bootloader_support" "util")
+
+register_component()
+
+if(CONFIG_APP_UPDATE_CHECK_APP_SUM)
+target_compile_definitions(${COMPONENT_NAME} PUBLIC -D CONFIG_ENABLE_BOOT_CHECK_SUM=1)
+endif()
+
+if(CONFIG_APP_UPDATE_CHECK_APP_HASH)
+target_compile_definitions(${COMPONENT_NAME} PUBLIC -D CONFIG_ENABLE_BOOT_CHECK_SHA256=1)
+endif()

components/aws_iot/CMakeLists.txt

@@ -0,0 +1,11 @@
+if(CONFIG_AWS_IOT_SDK)
+    set(COMPONENT_ADD_INCLUDEDIRS "include aws-iot-device-sdk-embedded-C/include")
+    set(COMPONENT_SRCDIRS "aws-iot-device-sdk-embedded-C/src port")
+else()
+    message(STATUS "Building empty aws_iot component due to configuration")
+endif()
+
+set(COMPONENT_REQUIRES ssl)
+set(COMPONENT_PRIV_REQUIRES jsmn esp-tls)
+
+register_component()

components/bootloader/CMakeLists.txt

@@ -0,0 +1,7 @@
+# bootloader component logic is all in project_include.cmake,
+# and subproject/CMakeLists.txt.
+#
+# This file is only included so the build system finds the
+# component
+
+

components/bootloader/project_include.cmake

@@ -0,0 +1,33 @@
+if(BOOTLOADER_BUILD)
+    return()  # don't keep recursing!
+endif()
+
+# Glue to build the bootloader subproject binary as an external
+# cmake project under this one
+#
+#
+set(bootloader_build_dir "${CMAKE_BINARY_DIR}/bootloader")
+set(bootloader_binary_files
+    "${bootloader_build_dir}/bootloader.elf"
+    "${bootloader_build_dir}/bootloader.bin"
+    "${bootloader_build_dir}/bootloader.map"
+    )
+
+externalproject_add(bootloader
+    # TODO: support overriding the bootloader in COMPONENT_PATHS
+    SOURCE_DIR "${IDF_PATH}/components/bootloader/subproject"
+    BINARY_DIR "${bootloader_build_dir}"
+    CMAKE_ARGS -DSDKCONFIG=${SDKCONFIG} -DIDF_PATH=${IDF_PATH}
+    INSTALL_COMMAND ""
+    BUILD_ALWAYS 1  # no easy way around this...
+    BUILD_BYPRODUCTS ${bootloader_binary_files}
+    )
+
+# this is a hack due to an (annoying) shortcoming in cmake, it can't
+# extend the 'clean' target to the external project
+# see thread: https://cmake.org/pipermail/cmake/2016-December/064660.html
+#
+# So for now we just have the top-level build remove the final build products...
+set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY
+    ADDITIONAL_MAKE_CLEAN_FILES
+    ${bootloader_binary_files})

components/bootloader/subproject/CMakeLists.txt

@@ -0,0 +1,29 @@
+cmake_minimum_required(VERSION 3.5)
+
+if(NOT SDKCONFIG)
+    message(FATAL_ERROR "Bootloader subproject expects the SDKCONFIG variable to be passed "
+        "in by the parent build process.")
+endif()
+
+if(NOT IDF_PATH)
+    message(FATAL_ERROR "Bootloader subproject expects the IDF_PATH variable to be passed "
+        "in by the parent build process.")
+endif()
+
+set(COMPONENTS esptool_py bootloader bootloader_support spi_flash log esp8266 util partition_table)
+set(BOOTLOADER_BUILD 1)
+add_definitions(-DBOOTLOADER_BUILD=1)
+
+set(COMPONENT_REQUIRES_COMMON log esp8266 spi_flash)
+
+set(MAIN_SRCS main/bootloader_start.c)
+
+include("${IDF_PATH}/tools/cmake/project.cmake")
+project(bootloader)
+
+target_linker_script(bootloader.elf
+    "main/esp8266.bootloader.ld"
+    "main/esp8266.bootloader.rom.ld")
+# Imported from esp8266 component
+target_linker_script(bootloader.elf ${ESP8266_BOOTLOADER_LINKER_SCRIPTS})
+target_link_libraries(bootloader.elf ${ESP8266_BOOTLOADER_LIBS})

components/bootloader_support/CMakeLists.txt

@@ -0,0 +1,22 @@
+set(COMPONENT_SRCDIRS "src")
+
+if(${BOOTLOADER_BUILD})
+    set(COMPONENT_ADD_INCLUDEDIRS "include include_priv")
+    set(COMPONENT_REQUIRES)
+    set(COMPONENT_PRIV_REQUIRES spi_flash util)
+else()
+    set(COMPONENT_ADD_INCLUDEDIRS "include")
+    set(COMPONENT_PRIV_INCLUDEDIRS "include_priv")
+    set(COMPONENT_REQUIRES)
+    set(COMPONENT_PRIV_REQUIRES spi_flash util ssl)
+endif()
+
+register_component()
+
+if(CONFIG_APP_UPDATE_CHECK_APP_SUM)
+target_compile_definitions(${COMPONENT_NAME} PUBLIC -D CONFIG_ENABLE_BOOT_CHECK_SUM=1)
+endif()
+
+if(CONFIG_APP_UPDATE_CHECK_APP_HASH)
+target_compile_definitions(${COMPONENT_NAME} PUBLIC -D CONFIG_ENABLE_BOOT_CHECK_SHA256=1)
+endif()

components/cjson/CMakeLists.txt

@@ -1,6 +1,10 @@
 set(COMPONENT_SRCDIRS cJSON)
 set(COMPONENT_ADD_INCLUDEDIRS cJSON)
 
-set(COMPONENT_REQUIRES "")
+set(COMPONENT_REQUIRES newlib)
 
 register_component()
+
+if(CONFIG_NEWLIB_LIBRARY_LEVEL_NORMAL)
+target_compile_definitions(${COMPONENT_NAME} PRIVATE -D CJSON_SPRINTF_FLOAT=1)
+endif()

components/coap/CMakeLists.txt

@@ -40,3 +40,4 @@ set_source_files_properties(
     PROPERTIES COMPILE_FLAGS
     -Wno-implicit-fallthrough)
 
+target_compile_definitions(${COMPONENT_NAME} PUBLIC -D WITH_POSIX)

components/esp-tls/CMakeLists.txt

@@ -1,7 +1,7 @@
 set(COMPONENT_SRCS "esp_tls.c")
 set(COMPONENT_ADD_INCLUDEDIRS ".")
 
-set(COMPONENT_REQUIRES mbedtls)
+set(COMPONENT_REQUIRES ssl)
-set(COMPONENT_PRIV_REQUIRES lwip nghttp)
+set(COMPONENT_PRIV_REQUIRES lwip)
 
 register_component()

components/esp8266/CMakeLists.txt

@@ -1,57 +1,96 @@
-set(COMPONENT_SRCDIRS source driver)
+if(BOOTLOADER_BUILD)
-
+    # For bootloader, all we need from esp8266 is headers
     set(COMPONENT_ADD_INCLUDEDIRS include)
+    # set(COMPONENT_REQUIRES ${COMPONENTS})
+    set(COMPONENT_SRCS source/ets_printf.c)
+    register_component(esp8266)
 
-set(COMPONENT_PRIV_INCLUDEDIRS include/driver)
+    # as cmake won't attach linker args to a header-only library, attach
+    # linker args directly to the bootloader.elf
+    set(ESP8266_BOOTLOADER_LINKER_SCRIPTS
+        "${CMAKE_CURRENT_SOURCE_DIR}/ld/esp8266.rom.ld"
+        PARENT_SCOPE
+        )
 
-set(COMPONENT_REQUIRES lwip)
+    set(ESP8266_BOOTLOADER_LIBS
-set(COMPONENT_PRIV_REQUIRES freertos)
+        "-L ${CMAKE_CURRENT_SOURCE_DIR}/lib"
+        "core"
+        PARENT_SCOPE
+        )
+
+else()
+    # Regular app build
+
+    set(COMPONENT_SRCDIRS "driver source")
+    set(COMPONENT_ADD_INCLUDEDIRS "include")
+    set(COMPONENT_PRIV_INCLUDEDIRS "include/driver")
+
+    set(COMPONENT_REQUIRES newlib)
+    # driver is a public requirement because esp_sleep.h uses gpio_num_t & touch_pad_t
+    # tcpip_adapter is a public requirement because esp_event.h uses tcpip_adapter types
+    set(COMPONENT_PRIV_REQUIRES "log" "nvs_flash" "spi_flash" "tcpip_adapter" "bootloader_support" "util" "esp_ringbuf")
 
     register_component()
 
-target_link_libraries(${COMPONENT_NAME} "-L ${CMAKE_CURRENT_SOURCE_DIR}/lib")
+    target_link_libraries(esp8266 "-L ${CMAKE_CURRENT_SOURCE_DIR}/lib")
     if(NOT CONFIG_NO_BLOBS)
-    target_link_libraries(${COMPONENT_NAME} gcc hal core
+        target_link_libraries(esp8266 gcc hal core net80211 phy pp smartconfig ssc wpa espnow wps)
-                                            net80211 phy pp smartconfig ssc wpa espnow wps)
+    endif()
+    target_linker_script(esp8266 "${CMAKE_CURRENT_BINARY_DIR}/esp8266_out.ld" "${CMAKE_CURRENT_BINARY_DIR}/esp8266_common_out.ld")
+
+    target_linker_script(esp8266
+        "ld/esp8266.rom.ld"
+        "ld/esp8266.peripherals.ld"
+        )
+
+    target_link_libraries(esp8266 "-u call_user_start")
+
+    # Preprocess esp8266.ld linker script to include configuration, becomes esp8266_out.ld
+    set(LD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ld)
+    add_custom_command(
+        OUTPUT esp8266_out.ld
+        COMMAND "${CMAKE_C_COMPILER}" -C -P -x c -E -o esp8266_out.ld ${CFLAGS} -I ${CONFIG_DIR} ${LD_DIR}/esp8266.ld
+        MAIN_DEPENDENCY ${LD_DIR}/esp8266.ld ${SDKCONFIG_H}
+        COMMENT "Generating memory map linker script..."
+        VERBATIM)
+    add_custom_command(
+        OUTPUT esp8266_common_out.ld
+        COMMAND "${CMAKE_C_COMPILER}" -C -P -x c -E -o esp8266_common_out.ld -I ${CONFIG_DIR} ${LD_DIR}/esp8266.common.ld
+        MAIN_DEPENDENCY ${LD_DIR}/esp8266.common.ld ${SDKCONFIG_H}
+        COMMENT "Generating section linker script..."
+        VERBATIM)
+    add_custom_target(esp8266_linker_script DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/esp8266_out.ld" "${CMAKE_CURRENT_BINARY_DIR}/esp8266_common_out.ld")
+    add_dependencies(esp8266 esp8266_linker_script)
+
+    if(CONFIG_ESP8266_PHY_INIT_DATA_IN_PARTITION)
+        set(PHY_INIT_DATA_BIN phy_init_data.bin)
+
+        # To get the phy_init_data.bin file, compile phy_init_data.h as a C file and then objcopy
+        # the object file to a raw binary
+        add_custom_command(
+            OUTPUT ${PHY_INIT_DATA_BIN}
+            DEPENDS ${CMAKE_CURRENT_LIST_DIR}/phy_init_data.h
+            COMMAND ${CMAKE_C_COMPILER} -x c -c
+            -I ${CMAKE_CURRENT_LIST_DIR} -I ${CMAKE_CURRENT_LIST_DIR}/include -I ${CMAKE_BINARY_DIR}
+            -o phy_init_data.obj
+            ${CMAKE_CURRENT_LIST_DIR}/phy_init_data.h
+            COMMAND ${CMAKE_OBJCOPY} -O binary phy_init_data.obj ${PHY_INIT_DATA_BIN}
+            )
+        add_custom_target(phy_init_data ALL DEPENDS ${PHY_INIT_DATA_BIN})
+        add_dependencies(flash phy_init_data)
+
     endif()
 
-target_link_libraries(${COMPONENT_NAME} "-u call_user_start")
+    if(CONFIG_ESP_FILENAME_MACRO_NO_PATH)
-
+        target_compile_definitions(${COMPONENT_NAME} PUBLIC -D __ESP_FILE__=__FILE__)
-set(ESPTOOLPY_FLASHSIZE ${CONFIG_ESPTOOLPY_FLASHSIZE})
-
-if(ESPTOOLPY_FLASHSIZE STREQUAL "512KB")
-set(ESP8266_LINKER_SCRIPTS eagle.app.v6.new.512.${CONFIG_ESPTOOLPY_APP_NUM}.ld)
-endif()
-if(ESPTOOLPY_FLASHSIZE STREQUAL "1MB")
-set(ESP8266_LINKER_SCRIPTS eagle.app.v6.new.1024.${CONFIG_ESPTOOLPY_APP_NUM}.ld)
-endif()
-if(ESPTOOLPY_FLASHSIZE STREQUAL "2MB")
-set(ESP8266_LINKER_SCRIPTS eagle.app.v6.new.1024.${CONFIG_ESPTOOLPY_APP_NUM}.ld)
-endif()
-if(ESPTOOLPY_FLASHSIZE STREQUAL "2MB-c1")
-set(ESP8266_LINKER_SCRIPTS eagle.app.v6.new.2048.ld)
-endif()
-if(ESPTOOLPY_FLASHSIZE STREQUAL "4MB")
-set(ESP8266_LINKER_SCRIPTS eagle.app.v6.new.1024.${CONFIG_ESPTOOLPY_APP_NUM}.ld)
-endif()
-if(ESPTOOLPY_FLASHSIZE STREQUAL "4MB-c1")
-set(ESP8266_LINKER_SCRIPTS eagle.app.v6.new.2048.ld)
-endif()
-if(ESPTOOLPY_FLASHSIZE STREQUAL "8MB")
-set(ESP8266_LINKER_SCRIPTS eagle.app.v6.new.2048.ld)
-endif()
-if(ESPTOOLPY_FLASHSIZE STREQUAL "16MB")
-set(ESP8266_LINKER_SCRIPTS eagle.app.v6.new.2048.ld)
     endif()
 
-target_linker_script(${COMPONENT_NAME}
+    if(CONFIG_ESP_FILENAME_MACRO_RAW)
-                     ld/${ESP8266_LINKER_SCRIPTS}
+        target_compile_definitions(${COMPONENT_NAME} PUBLIC -D __ESP_FILE__=__FILE__)
-                     ld/eagle.app.v6.common.ld
+    endif()
-                     ld/eagle.rom.addr.v6.ld)
 
-target_compile_options(${COMPONENT_NAME} PUBLIC -Wno-error=char-subscripts -Wno-error=unknown-pragmas -Wno-error=implicit-function-declaration
+    if(CONFIG_ESP_FILENAME_MACRO_NULL)
-                                                -Wno-error=pointer-sign -Wno-error=switch -Wno-error=maybe-uninitialized -Wno-error=format=
+        target_compile_definitions(${COMPONENT_NAME} PUBLIC -D __ESP_FILE__="null")
-                                                -Wno-error=unused-value -Wno-error=address -Wno-error=return-type -Wno-error=format-extra-args
+    endif()
-                                                -Wno-error=format-zero-length -Wno-error=unused-label -Wno-error=sizeof-pointer-memaccess)
 
-target_compile_options(${COMPONENT_NAME} PUBLIC -DICACHE_FLASH)
+endif()

components/esp8266/ld/esp8266.common.ld

@@ -105,7 +105,7 @@ SECTIONS
     *(.init.literal)
     *(.init)
     *(.iram1 .iram1.*)
-    *libspi_flash.a:spi_flash_raw.o(.literal .text .literal.* .text.*)
+    *libspi_flash.a:spi_flash_raw.*(.literal .text .literal.* .text.*)
 #ifdef CONFIG_ESP8266_WIFI_DEBUG_LOG_ENABLE
     *libpp_dbg.a:(.literal .text .literal.* .text.*)
 #else
@@ -134,13 +134,13 @@ SECTIONS
 #endif
 
 #ifdef CONFIG_FREERTOS_GLOBAL_DATA_LINK_IRAM
-    *libfreertos.a:tasks.o(.bss .data .bss.* .data.* COMMON)
+    *libfreertos.a:tasks.*(.bss .data .bss.* .data.* COMMON)
-    *libfreertos.a:timers.o(.bss .data .bss.* .data.* COMMON)
+    *libfreertos.a:timers.*(.bss .data .bss.* .data.* COMMON)
-    *libfreertos.a:freertos_hooks.o(.bss .data .bss.* .data.* COMMON)
+    *libfreertos.a:freertos_hooks.*(.bss .data .bss.* .data.* COMMON)
 #endif
 
 #ifdef CONFIG_LINK_ETS_PRINTF_TO_IRAM
-    *libesp8266.a:ets_printf.o(.literal .text .literal.* .text.* .rodata.* .rodata)
+    *libesp8266.a:ets_printf.*(.literal .text .literal.* .text.* .rodata.* .rodata)
 #endif
 
     _text_end = ABSOLUTE(.);
@@ -182,13 +182,13 @@ SECTIONS
     . = (. + 3) & ~ 3;
     /*  C++ constructor and destructor tables, properly ordered:  */
     __init_array_start = ABSOLUTE(.);
-    KEEP (*crtbegin.o(.ctors))
+    KEEP (*crtbegin.*(.ctors))
-    KEEP (*(EXCLUDE_FILE (*crtend.o) .ctors))
+    KEEP (*(EXCLUDE_FILE (*crtend.*) .ctors))
     KEEP (*(SORT(.ctors.*)))
     KEEP (*(.ctors))
     __init_array_end = ABSOLUTE(.);
-    KEEP (*crtbegin.o(.dtors))
+    KEEP (*crtbegin.*(.dtors))
-    KEEP (*(EXCLUDE_FILE (*crtend.o) .dtors))
+    KEEP (*(EXCLUDE_FILE (*crtend.*) .dtors))
     KEEP (*(SORT(.dtors.*)))
     KEEP (*(.dtors))
     /*  C++ exception handlers table:  */

components/esp8266/project_include.cmake

@@ -1,162 +0,0 @@
-set(BOOTLOADER_FIRMWARE_DIR ${CMAKE_CURRENT_LIST_DIR}/firmware)
-
-#configurate downloading parameters
-set(ESPTOOLPY_FLASHSIZE ${CONFIG_ESPTOOLPY_FLASHSIZE})
-set(ESPTOOLPY_FLASHMODE ${CONFIG_ESPTOOLPY_FLASHMODE})
-set(ESPTOOLPY_FLASHFREQ ${CONFIG_ESPTOOLPY_FLASHFREQ})
-
-if(${ESPTOOLPY_FLASHSIZE} STREQUAL "512KB")
-set(BLANK_BIN_OFFSET1 0x7B000)
-set(BLANK_BIN_OFFSET2 0x7E000)
-set(ESP_INIT_DATA_DEFAULT_BIN_OFFSET 0x7C000)
-set(ESP8266_SIZEMAP 0)
-endif()
-if(${ESPTOOLPY_FLASHSIZE} STREQUAL "1MB")
-set(BLANK_BIN_OFFSET1 0xFB000)
-set(BLANK_BIN_OFFSET2 0xFE000)
-set(ESP_INIT_DATA_DEFAULT_BIN_OFFSET 0xFC000)
-set(ESP8266_SIZEMAP 2)
-endif()
-if(${ESPTOOLPY_FLASHSIZE} STREQUAL "2MB")
-set(BLANK_BIN_OFFSET1 0x1FB000)
-set(BLANK_BIN_OFFSET2 0x1FE000)
-set(ESP_INIT_DATA_DEFAULT_BIN_OFFSET 0x1FC000)
-set(ESP8266_SIZEMAP 3)
-endif()
-if(${ESPTOOLPY_FLASHSIZE} STREQUAL "2MB-c1")
-set(BLANK_BIN_OFFSET1 0x1FB000)
-set(BLANK_BIN_OFFSET2 0x1FE000)
-set(ESP_INIT_DATA_DEFAULT_BIN_OFFSET 0x1FC000)
-set(ESP8266_SIZEMAP 5)
-endif()
-if(${ESPTOOLPY_FLASHSIZE} STREQUAL "4MB")
-set(BLANK_BIN_OFFSET1 0x3FB000)
-set(BLANK_BIN_OFFSET2 0x3FE000)
-set(ESP_INIT_DATA_DEFAULT_BIN_OFFSET 0x3FC000)
-set(ESP8266_SIZEMAP 4)
-endif()
-if(${ESPTOOLPY_FLASHSIZE} STREQUAL "4MB-c1")
-set(BLANK_BIN_OFFSET1 0x3FB000)
-set(BLANK_BIN_OFFSET2 0x3FE000)
-set(ESP_INIT_DATA_DEFAULT_BIN_OFFSET 0x3FC000)
-set(ESP8266_SIZEMAP 6)
-endif()
-if(${ESPTOOLPY_FLASHSIZE} STREQUAL "8MB")
-set(BLANK_BIN_OFFSET1 0x7FB000)
-set(BLANK_BIN_OFFSET2 0x7FE000)
-set(ESP_INIT_DATA_DEFAULT_BIN_OFFSET 0x7FC000)
-set(ESP8266_SIZEMAP 8)
-endif()
-if(${ESPTOOLPY_FLASHSIZE} STREQUAL "16MB")
-set(BLANK_BIN_OFFSET1 0xFFB000)
-set(BLANK_BIN_OFFSET2 0xFFE000)
-set(ESP_INIT_DATA_DEFAULT_BIN_OFFSET 0xFFC000)
-set(ESP8266_SIZEMAP 9)
-endif()
-
-set(BOOTLOADER_BIN_OFFSET 0)
-set(APP_OFFSET 0x1000)
-
-set(ESP8266_BOOTMODE 2) # always be 2
-
-if(${ESPTOOLPY_FLASHMODE} STREQUAL "qio")
-set(ESP8266_FLASHMODE 0)
-endif()
-if(${ESPTOOLPY_FLASHMODE} STREQUAL "qout")
-set(ESP8266_FLASHMODE 1)
-endif()
-if(${ESPTOOLPY_FLASHMODE} STREQUAL "dio")
-set(ESP8266_FLASHMODE 2)
-endif()
-if(${ESPTOOLPY_FLASHMODE} STREQUAL "dout")
-set(ESP8266_FLASHMODE 3)
-endif()
-
-if(${ESPTOOLPY_FLASHFREQ} STREQUAL "20m")
-set(ESP8266_FREQDIV 2)
-endif()
-if(${ESPTOOLPY_FLASHFREQ} STREQUAL "26m")
-set(ESP8266_FREQDIV 1)
-endif()
-if(${ESPTOOLPY_FLASHFREQ} STREQUAL "40m")
-set(ESP8266_FREQDIV 0)
-endif()
-if(${ESPTOOLPY_FLASHFREQ} STREQUAL "80m")
-set(ESP8266_FREQDIV 15)
-endif()
-
-set(ESP8266_BINSCRIPT ${PYTHON} $(IDF_PATH)/tools/gen_appbin.py)
-
-#
-# Add 'app.bin' target - generates with elf2image
-#
-add_custom_command(OUTPUT ${PROJECT_NAME}.bin
-    COMMAND ${CMAKE_OBJCOPY_COMPILER} --only-section .text -O binary ${PROJECT_NAME}.elf eagle.app.v6.text.bin
-    COMMAND ${CMAKE_OBJCOPY_COMPILER} --only-section .data -O binary ${PROJECT_NAME}.elf eagle.app.v6.data.bin
-    COMMAND ${CMAKE_OBJCOPY_COMPILER} --only-section .rodata -O binary ${PROJECT_NAME}.elf eagle.app.v6.rodata.bin
-	COMMAND ${CMAKE_OBJCOPY_COMPILER} --only-section .irom0.text -O binary ${PROJECT_NAME}.elf eagle.app.v6.irom0text.bin
-	COMMAND ${ESP8266_BINSCRIPT} ${PROJECT_NAME}.elf ${ESP8266_BOOTMODE} ${ESP8266_FLASHMODE} ${ESP8266_FREQDIV} ${ESP8266_SIZEMAP}
-	COMMAND mv eagle.app.flash.bin ${PROJECT_NAME}.bin
-	COMMAND rm eagle.app.v6.text.bin eagle.app.v6.data.bin eagle.app.v6.rodata.bin eagle.app.v6.irom0text.bin
-    DEPENDS ${PROJECT_NAME}.elf
-    VERBATIM
-    )
-add_custom_target(app ALL DEPENDS ${PROJECT_NAME}.bin)
-
-set(BLANK_BIN ${BOOTLOADER_FIRMWARE_DIR}/blank.bin)
-set(ESP_INIT_DATA_DEFAULT_BIN ${BOOTLOADER_FIRMWARE_DIR}/esp_init_data_default.bin)
-set(BOOTLOADER_BIN ${BOOTLOADER_FIRMWARE_DIR}/boot_v1.7.bin)
-
-set(PYTHON ${CONFIG_PYTHON})
-set(ESPTOOLPY_SRC $(IDF_PATH)/components/esptool_py/esptool/esptool.py)
-
-set(CHIP esp8266)
-set(ESPPORT ${CONFIG_ESPTOOLPY_PORT})
-set(ESPBAUD ${CONFIG_ESPTOOLPY_BAUD})
-set(ESPFLASHMODE ${CONFIG_ESPTOOLPY_FLASHMODE})
-set(ESPFLASHFREQ ${CONFIG_ESPTOOLPY_FLASHFREQ})
-set(ESPFLASHSIZE ${CONFIG_ESPTOOLPY_FLASHSIZE})
-set(ESPTOOLPY ${PYTHON} ${ESPTOOLPY_SRC} --chip ${CHIP})
-
-set(ESPTOOL_WRITE_FLASH_OPTIONS --flash_mode ${ESPFLASHMODE} --flash_freq ${ESPFLASHFREQ} --flash_size ${ESPFLASHSIZE})
-
-set(ESPTOOLPY_SERIAL ${ESPTOOLPY} --port ${ESPPORT} --baud ${ESPBAUD} --before ${CONFIG_ESPTOOLPY_BEFORE} --after ${CONFIG_ESPTOOLPY_AFTER})
-
-set(ESPTOOLPY_WRITE_FLASH ${ESPTOOLPY_SERIAL} write_flash -z ${ESPTOOL_WRITE_FLASH_OPTIONS})
-
-set(APP_BIN ${PROJECT_NAME}.bin)
-
-set(ESPTOOL_ALL_FLASH_ARGS ${BOOTLOADER_BIN_OFFSET} ${BOOTLOADER_BIN}
-                           ${APP_OFFSET} ${APP_BIN}
-                           ${ESP_INIT_DATA_DEFAULT_BIN_OFFSET} ${ESP_INIT_DATA_DEFAULT_BIN}
-                           ${BLANK_BIN_OFFSET1} ${BLANK_BIN}
-                           ${BLANK_BIN_OFFSET2} ${BLANK_BIN})
-
-add_custom_target(flash DEPENDS ${PROJECT_NAME}.bin
-    COMMAND echo "Flashing binaries to serial port ${ESPPORT} app at offset ${APP_OFFSET}..."
-    COMMAND echo ${ESPTOOL_ALL_FLASH_ARGS} 
-    COMMAND ${ESPTOOLPY_WRITE_FLASH} ${ESPTOOL_ALL_FLASH_ARGS}
-    COMMAND echo "success"
-    )
-
-add_custom_target(erase_flash DEPENDS ""
-    COMMAND echo "Erasing entire flash..."
-    COMMAND ${ESPTOOLPY_SERIAL} erase_flash
-    COMMAND echo "success"
-    )
-
-set(MONITOR_PYTHON ${PYTHON})
-set(MONITORBAUD ${CONFIG_MONITOR_BAUD})
-set(APP_ELF ${PROJECT_NAME}.elf)
-set(MONITOR_OPTS --baud ${MONITORBAUD} --port ${ESPPORT} --toolchain-prefix ${CONFIG_TOOLPREFIX} ${APP_ELF})
-
-function(esp_monitor func dependencies)
-    add_custom_target(${func} DEPENDS ${dependencies}
-        COMMAND echo "start monitor ... "
-        COMMAND echo $(MONITOR_PYTHON) ${IDF_PATH}/tools/idf_monitor.py ${MONITOR_OPTS} 
-        COMMAND $(MONITOR_PYTHON) ${IDF_PATH}/tools/idf_monitor.py ${MONITOR_OPTS} 
-        COMMAND echo "idf monitor exit"
-        )
-endfunction()
-
-esp_monitor(monitor "")

components/esp_http_client/CMakeLists.txt

@@ -5,7 +5,7 @@ set(COMPONENT_SRCS "esp_http_client.c"
 set(COMPONENT_ADD_INCLUDEDIRS "include")
 set(COMPONENT_PRIV_INCLUDEDIRS "lib/include")
 
-set(COMPONENT_REQUIRES "nghttp")
+set(COMPONENT_REQUIRES "http_parser")
-set(COMPONENT_PRIV_REQUIRES "mbedtls" "lwip" "esp-tls" "tcp_transport")
+set(COMPONENT_PRIV_REQUIRES "ssl" "lwip" "esp-tls" "tcp_transport" "tcpip_adapter")
 
 register_component()

components/esp_http_server/CMakeLists.txt

@@ -7,7 +7,6 @@ set(COMPONENT_SRCS "src/httpd_main.c"
                    "src/httpd_uri.c"
                    "src/util/ctrl_sock.c")
 
-set(COMPONENT_REQUIRES nghttp)  # for http_parser.h
+set(COMPONENT_PRIV_REQUIRES lwip http_parser)
-set(COMPONENT_PRIV_REQUIRES lwip)
 
 register_component()

components/esptool_py/CMakeLists.txt

@@ -1 +1,14 @@
 register_config_only_component()
+
+# Generate pre-canned flasher args files suitable for passing to esptool.py
+foreach(part project app bootloader partition_table)
+    configure_file(
+        "${CMAKE_CURRENT_LIST_DIR}/flash_${part}_args.in"
+        "${CMAKE_BINARY_DIR}/flash_${part}_args"
+        )
+endforeach()
+
+configure_file(
+    "${CMAKE_CURRENT_LIST_DIR}/flasher_args.json.in"
+    "${CMAKE_BINARY_DIR}/flasher_args.json"
+    )

components/esptool_py/Makefile.projbuild

@@ -70,9 +70,6 @@ APP_BIN_UNSIGNED ?= $(APP_BIN)
 
 $(APP_BIN_UNSIGNED): $(APP_ELF) $(ESPTOOLPY_SRC)
 	$(ESPTOOLPY) elf2image $(ESPTOOL_FLASH_OPTIONS) $(ESPTOOL_ELF2IMAGE_OPTIONS) -o $@ $<
-ifdef IS_BOOTLOADER_BUILD
-	@mv $@0x00000.bin $@
-endif
 
 flash: all_binaries $(ESPTOOLPY_SRC) $(call prereq_if_explicit,erase_flash)
 	@echo "Flashing binaries to serial port $(ESPPORT) (app at offset $(APP_OFFSET))..."

components/esptool_py/esptool/esptool.py

@@ -1373,7 +1373,7 @@ class ESP8266ROMFirmwareImage(BaseFirmwareImage):
 
         # everything but IROM goes at 0x00000 in an image file
         normal_segments = self.get_non_irom_segments()
-        with open("%s0x00000.bin" % basename, 'wb') as f:
+        with open("%s" % basename, 'wb') as f:
             self.write_common_header(f, normal_segments)
             checksum = ESPLoader.ESP_CHECKSUM_MAGIC
             for segment in normal_segments:

components/esptool_py/flash_app_args.in

@@ -0,0 +1 @@
+${APP_PARTITION_OFFSET} ${PROJECT_NAME}.bin

components/esptool_py/flash_bootloader_args.in

@@ -0,0 +1,4 @@
+--flash_mode ${ESPFLASHMODE}
+--flash_size ${ESPFLASHSIZE}
+--flash_freq ${ESPFLASHFREQ}
+0x0000 bootloader/bootloader.bin

components/esptool_py/flash_partition_table_args.in

@@ -0,0 +1 @@
+${PARTITION_TABLE_OFFSET} partition_table/partition-table.bin

components/esptool_py/flash_project_args.in

@@ -0,0 +1,7 @@
+--flash_mode ${ESPFLASHMODE}
+--flash_size ${ESPFLASHSIZE}
+--flash_freq ${ESPFLASHFREQ}
+0x0000 bootloader/bootloader.bin
+${PARTITION_TABLE_OFFSET} partition_table/partition-table.bin
+${APP_PARTITION_OFFSET} ${PROJECT_NAME}.bin
+${PHY_PARTITION_OFFSET} ${PHY_PARTITION_BIN_FILE}

components/esptool_py/flasher_args.json.in

@@ -0,0 +1,17 @@
+{
+    "write_flash_args" : [ "--flash_mode", "${ESPFLASHMODE}",
+                           "--flash_size", "${ESPFLASHSIZE}",
+                           "--flash_freq", "${ESPFLASHFREQ}" ],
+    "flash_files" : {
+        "0x0000" : "bootloader/bootloader.bin",
+        "${PARTITION_TABLE_OFFSET}" : "partition_table/partition-table.bin",
+        "${APP_PARTITION_OFFSET}" : "${PROJECT_NAME}.bin",
+        "${PHY_PARTITION_OFFSET}" : "${PHY_PARTITION_BIN_FILE}"
+    },
+    "bootloader" :      { "offset" : "0x1000",
+                          "file" : "bootloader/bootloader.bin" },
+    "app" :             { "offset" : "${APP_PARTITION_OFFSET}",
+                          "file" : "${PROJECT_NAME}.bin" },
+    "partition_table" : { "offset" : "${PARTITION_TABLE_OFFSET}",
+                          "file" : "partition_table/partition-table.bin" }
+}

components/esptool_py/project_include.cmake

@@ -0,0 +1,61 @@
+# Set some global esptool.py variables
+#
+# Many of these are read when generating flash_app_args & flash_project_args
+set(ESPCHIP "esp8266")
+set(ESPTOOLPY "${PYTHON}" "${CMAKE_CURRENT_LIST_DIR}/esptool/esptool.py" --chip "${ESPCHIP}")
+set(ESPSECUREPY "${PYTHON}" "${CMAKE_CURRENT_LIST_DIR}/esptool/espsecure.py")
+
+set(ESPFLASHMODE ${CONFIG_ESPTOOLPY_FLASHMODE})
+set(ESPFLASHFREQ ${CONFIG_ESPTOOLPY_FLASHFREQ})
+set(ESPFLASHSIZE ${CONFIG_ESPTOOLPY_FLASHSIZE})
+
+set(ESPTOOLPY_SERIAL "${ESPTOOLPY}" --port "${ESPPORT}" --baud ${ESPBAUD})
+
+set(ESPTOOLPY_ELF2IMAGE_FLASH_OPTIONS
+    --flash_mode ${ESPFLASHMODE}
+    --flash_freq ${ESPFLASHFREQ}
+    --flash_size ${ESPFLASHSIZE}
+    )
+
+if(CONFIG_ESPTOOLPY_FLASHSIZE_DETECT)
+    # Set ESPFLASHSIZE to 'detect' *after* elf2image options are generated,
+    # as elf2image can't have 'detect' as an option...
+    set(ESPFLASHSIZE detect)
+endif()
+
+# Set variables if the PHY data partition is in the flash
+if(CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION)
+    set(PHY_PARTITION_OFFSET   ${CONFIG_PHY_DATA_OFFSET})
+    set(PHY_PARTITION_BIN_FILE "esp32/phy_init_data.bin")
+endif()
+
+if(BOOTLOADER_BUILD)
+set(ESPTOOL_ELF2IMAGE_OPTIONS "")
+else()
+set(ESPTOOL_ELF2IMAGE_OPTIONS "--version=3")
+endif()
+
+#
+# Add 'app.bin' target - generates with elf2image
+#
+add_custom_command(OUTPUT "${PROJECT_NAME}.bin"
+    COMMAND ${ESPTOOLPY} elf2image ${ESPTOOLPY_ELF2IMAGE_FLASH_OPTIONS} ${ESPTOOL_ELF2IMAGE_OPTIONS} -o "${PROJECT_NAME}.bin" "${PROJECT_NAME}.elf"
+    DEPENDS ${PROJECT_NAME}.elf
+    VERBATIM
+    )
+add_custom_target(app ALL DEPENDS "${PROJECT_NAME}.bin")
+
+#
+# Add 'flash' target - not all build systems can run this directly
+#
+function(esptool_py_custom_target target_name flasher_filename dependencies)
+    add_custom_target(${target_name} DEPENDS ${dependencies}
+        COMMAND ${ESPTOOLPY} -p ${CONFIG_ESPTOOLPY_PORT} -b ${CONFIG_ESPTOOLPY_BAUD}
+        write_flash @flash_${flasher_filename}_args
+        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+        )
+endfunction()
+
+esptool_py_custom_target(flash project "app;partition_table;bootloader")
+esptool_py_custom_target(app-flash app "app")
+esptool_py_custom_target(bootloader-flash bootloader "bootloader")

components/freertos/CMakeLists.txt

@@ -1,9 +1,17 @@
-set(COMPONENT_ADD_INCLUDEDIRS include 
+set(COMPONENT_ADD_INCLUDEDIRS include include/freertos include/freertos/private port/esp8266/include port/esp8266/include/freertos)
-                              include/freertos
+set(COMPONENT_SRCDIRS "freertos" "port/esp8266")
-                              include/freertos/private
+set(COMPONENT_REQUIRES "esp8266")
-                              port/esp8266/include
-                              port/esp8266/include/freertos)
-set(COMPONENT_SRCDIRS freertos port/esp8266)
-set(COMPONENT_REQUIRES esp8266)
 
 register_component()
+
+target_link_libraries(freertos "-Wl,--undefined=uxTopUsedPriority")
+
+set_source_files_properties(
+    tasks.c
+    event_groups.c
+    timers.c
+    queue.c
+    ringbuf.c
+    PROPERTIES COMPILE_DEFINITIONS
+    _ESP_FREERTOS_INTERNAL
+    )

components/heap/CMakeLists.txt

@@ -0,0 +1,25 @@
+set(COMPONENT_SRCDIRS "src" "port/esp8266")
+
+set(COMPONENT_ADD_INCLUDEDIRS "include" "port/esp8266/include")
+
+set(COMPONENT_REQUIRES "log" "newlib")
+
+register_component()
+
+if(CONFIG_HEAP_TRACING)
+    set(WRAP_FUNCTIONS
+        calloc
+        malloc
+        free
+        realloc
+        heap_caps_malloc
+        heap_caps_free
+        heap_caps_realloc
+        heap_caps_malloc_default
+        heap_caps_realloc_default)
+
+    foreach(wrap ${WRAP_FUNCTIONS})
+        target_link_libraries(heap "-Wl,--wrap=${wrap}")
+    endforeach()
+
+endif()

components/http_parser/CMakeLists.txt

@@ -0,0 +1,4 @@
+set(COMPONENT_ADD_INCLUDEDIRS "include")
+set(COMPONENT_SRCDIRS "src")
+
+register_component()

components/jsmn/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(COMPONENT_SRCDIRS "src")
+set(COMPONENT_ADD_INCLUDEDIRS "include")
+
+set(COMPONENT_REQUIRES "")
+
+register_component()

components/libsodium/CMakeLists.txt

@@ -1,134 +1,78 @@
 set(SRC libsodium/src/libsodium)
 
-set(COMPONENT_REQUIRES "mbedtls")
+set(COMPONENT_REQUIRES "ssl")
 
+set(COMPONENT_SRCDIRS
+    port
     # Derived from libsodium/src/libsodium/Makefile.am
     # (ignoring the !MINIMAL set)
-set(COMPONENT_SRCS "${SRC}/crypto_aead/chacha20poly1305/sodium/aead_chacha20poly1305.c"
+    ${SRC}/crypto_aead/chacha20poly1305/sodium
-                   "${SRC}/crypto_aead/xchacha20poly1305/sodium/aead_xchacha20poly1305.c"
+    ${SRC}/crypto_aead/xchacha20poly1305/sodium
-                   "${SRC}/crypto_auth/crypto_auth.c"
+    ${SRC}/crypto_auth
-                   "${SRC}/crypto_auth/hmacsha256/auth_hmacsha256.c"
+    ${SRC}/crypto_auth/hmacsha256
-                   "${SRC}/crypto_auth/hmacsha512/auth_hmacsha512.c"
+    ${SRC}/crypto_auth/hmacsha512
-                   "${SRC}/crypto_auth/hmacsha512256/auth_hmacsha512256.c"
+    ${SRC}/crypto_auth/hmacsha512256
-                   "${SRC}/crypto_box/crypto_box.c"
+    ${SRC}/crypto_box
-                   "${SRC}/crypto_box/crypto_box_easy.c"
+    ${SRC}/crypto_box/curve25519xsalsa20poly1305
-                   "${SRC}/crypto_box/crypto_box_seal.c"
+    ${SRC}/crypto_core/curve25519/ref10
-                   "${SRC}/crypto_box/curve25519xchacha20poly1305/box_curve25519xchacha20poly1305.c"
+    ${SRC}/crypto_core/hchacha20
-                   "${SRC}/crypto_box/curve25519xsalsa20poly1305/box_curve25519xsalsa20poly1305.c"
+    ${SRC}/crypto_core/hsalsa20/ref2
-                   "${SRC}/crypto_core/curve25519/ref10/curve25519_ref10.c"
+    ${SRC}/crypto_core/hsalsa20
-                   "${SRC}/crypto_core/hchacha20/core_hchacha20.c"
+    ${SRC}/crypto_core/salsa/ref
-                   "${SRC}/crypto_core/hsalsa20/core_hsalsa20.c"
+    ${SRC}/crypto_generichash
-                   "${SRC}/crypto_core/hsalsa20/ref2/core_hsalsa20_ref2.c"
+    ${SRC}/crypto_generichash/blake2b
-                   "${SRC}/crypto_core/salsa/ref/core_salsa_ref.c"
+    ${SRC}/crypto_generichash/blake2b/ref
-                   "${SRC}/crypto_generichash/crypto_generichash.c"
+    ${SRC}/crypto_hash
-                   "${SRC}/crypto_generichash/blake2b/generichash_blake2.c"
+    ${SRC}/crypto_hash/sha256
-                   "${SRC}/crypto_generichash/blake2b/ref/blake2b-compress-avx2.c"
+    ${SRC}/crypto_hash/sha512
-                   "${SRC}/crypto_generichash/blake2b/ref/blake2b-compress-ref.c"
+    ${SRC}/crypto_kdf/blake2b
-                   "${SRC}/crypto_generichash/blake2b/ref/blake2b-compress-sse41.c"
+    ${SRC}/crypto_kdf
-                   "${SRC}/crypto_generichash/blake2b/ref/blake2b-compress-ssse3.c"
+    ${SRC}/crypto_kx
-                   "${SRC}/crypto_generichash/blake2b/ref/blake2b-ref.c"
+    ${SRC}/crypto_onetimeauth
-                   "${SRC}/crypto_generichash/blake2b/ref/generichash_blake2b.c"
+    ${SRC}/crypto_onetimeauth/poly1305
-                   "${SRC}/crypto_hash/crypto_hash.c"
+    ${SRC}/crypto_onetimeauth/poly1305/donna
-                   "${SRC}/crypto_hash/sha256/hash_sha256.c"
+    ${SRC}/crypto_pwhash/argon2
-                   "${SRC}/crypto_hash/sha256/cp/hash_sha256_cp.c"
+    ${SRC}/crypto_pwhash
-                   "${SRC}/crypto_hash/sha512/hash_sha512.c"
+    ${SRC}/crypto_pwhash/scryptsalsa208sha256
-                   "${SRC}/crypto_hash/sha512/cp/hash_sha512_cp.c"
+    ${SRC}/crypto_pwhash/scryptsalsa208sha256/nosse
-                   "${SRC}/crypto_kdf/crypto_kdf.c"
+    ${SRC}/crypto_scalarmult
-                   "${SRC}/crypto_kdf/blake2b/kdf_blake2b.c"
+    ${SRC}/crypto_scalarmult/curve25519
-                   "${SRC}/crypto_kx/crypto_kx.c"
+    ${SRC}/crypto_scalarmult/curve25519/ref10
-                   "${SRC}/crypto_onetimeauth/crypto_onetimeauth.c"
+    ${SRC}/crypto_secretbox
-                   "${SRC}/crypto_onetimeauth/poly1305/onetimeauth_poly1305.c"
+    ${SRC}/crypto_secretbox/xsalsa20poly1305
-                   "${SRC}/crypto_onetimeauth/poly1305/donna/poly1305_donna.c"
+    ${SRC}/crypto_shorthash
-                   "${SRC}/crypto_onetimeauth/poly1305/sse2/poly1305_sse2.c"
+    ${SRC}/crypto_shorthash/siphash24
-                   "${SRC}/crypto_pwhash/crypto_pwhash.c"
+    ${SRC}/crypto_shorthash/siphash24/ref
-                   "${SRC}/crypto_pwhash/argon2/argon2-core.c"
+    ${SRC}/crypto_sign
-                   "${SRC}/crypto_pwhash/argon2/argon2-encoding.c"
+    ${SRC}/crypto_sign/ed25519
-                   "${SRC}/crypto_pwhash/argon2/argon2-fill-block-ref.c"
+    ${SRC}/crypto_sign/ed25519/ref10
-                   "${SRC}/crypto_pwhash/argon2/argon2-fill-block-ssse3.c"
+    ${SRC}/crypto_stream/chacha20
-                   "${SRC}/crypto_pwhash/argon2/argon2.c"
+    ${SRC}/crypto_stream/chacha20/ref
-                   "${SRC}/crypto_pwhash/argon2/blake2b-long.c"
+    ${SRC}/crypto_stream
-                   "${SRC}/crypto_pwhash/argon2/pwhash_argon2i.c"
+    ${SRC}/crypto_stream/salsa20
-                   "${SRC}/crypto_pwhash/scryptsalsa208sha256/crypto_scrypt-common.c"
+    ${SRC}/crypto_stream/salsa20/ref
-                   "${SRC}/crypto_pwhash/scryptsalsa208sha256/pbkdf2-sha256.c"
+    ${SRC}/crypto_stream/xsalsa20
-                   "${SRC}/crypto_pwhash/scryptsalsa208sha256/pwhash_scryptsalsa208sha256.c"
+    ${SRC}/crypto_verify/sodium
-                   "${SRC}/crypto_pwhash/scryptsalsa208sha256/scrypt_platform.c"
+    ${SRC}/randombytes
-                   "${SRC}/crypto_pwhash/scryptsalsa208sha256/nosse/pwhash_scryptsalsa208sha256_nosse.c"
+    ${SRC}/sodium
-                   "${SRC}/crypto_pwhash/scryptsalsa208sha256/sse/pwhash_scryptsalsa208sha256_sse.c"
+    )
-                   "${SRC}/crypto_scalarmult/crypto_scalarmult.c"
-                   "${SRC}/crypto_scalarmult/curve25519/scalarmult_curve25519.c"
-                   "${SRC}/crypto_scalarmult/curve25519/donna_c64/curve25519_donna_c64.c"
-                   "${SRC}/crypto_scalarmult/curve25519/ref10/x25519_ref10.c"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/consts.S"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/curve25519_sandy2x.c"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/fe51_invert.c"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/fe51_mul.S"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/fe51_nsquare.S"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/fe51_pack.S"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/fe_frombytes_sandy2x.c"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/ladder.S"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/ladder_base.S"
-                   "${SRC}/crypto_scalarmult/curve25519/sandy2x/sandy2x.S"
-                   "${SRC}/crypto_secretbox/crypto_secretbox.c"
-                   "${SRC}/crypto_secretbox/crypto_secretbox_easy.c"
-                   "${SRC}/crypto_secretbox/xchacha20poly1305/secretbox_xchacha20poly1305.c"
-                   "${SRC}/crypto_secretbox/xsalsa20poly1305/secretbox_xsalsa20poly1305.c"
-                   "${SRC}/crypto_shorthash/crypto_shorthash.c"
-                   "${SRC}/crypto_shorthash/siphash24/shorthash_siphash24.c"
-                   "${SRC}/crypto_shorthash/siphash24/shorthash_siphashx24.c"
-                   "${SRC}/crypto_shorthash/siphash24/ref/shorthash_siphash24_ref.c"
-                   "${SRC}/crypto_shorthash/siphash24/ref/shorthash_siphashx24_ref.c"
-                   "${SRC}/crypto_sign/crypto_sign.c"
-                   "${SRC}/crypto_sign/ed25519/sign_ed25519.c"
-                   "${SRC}/crypto_sign/ed25519/ref10/keypair.c"
-                   "${SRC}/crypto_sign/ed25519/ref10/obsolete.c"
-                   "${SRC}/crypto_sign/ed25519/ref10/open.c"
-                   "${SRC}/crypto_sign/ed25519/ref10/sign.c"
-                   "${SRC}/crypto_stream/crypto_stream.c"
-                   "${SRC}/crypto_stream/aes128ctr/stream_aes128ctr.c"
-                   "${SRC}/crypto_stream/aes128ctr/nacl/afternm_aes128ctr.c"
-                   "${SRC}/crypto_stream/aes128ctr/nacl/beforenm_aes128ctr.c"
-                   "${SRC}/crypto_stream/aes128ctr/nacl/consts_aes128ctr.c"
-                   "${SRC}/crypto_stream/aes128ctr/nacl/int128_aes128ctr.c"
-                   "${SRC}/crypto_stream/aes128ctr/nacl/stream_aes128ctr_nacl.c"
-                   "${SRC}/crypto_stream/aes128ctr/nacl/xor_afternm_aes128ctr.c"
-                   "${SRC}/crypto_stream/chacha20/stream_chacha20.c"
-                   "${SRC}/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-avx2.c"
-                   "${SRC}/crypto_stream/chacha20/dolbeau/chacha20_dolbeau-ssse3.c"
-                   "${SRC}/crypto_stream/chacha20/ref/chacha20_ref.c"
-                   "${SRC}/crypto_stream/salsa20/stream_salsa20.c"
-                   "${SRC}/crypto_stream/salsa20/ref/salsa20_ref.c"
-                   "${SRC}/crypto_stream/salsa20/xmm6/salsa20_xmm6-asm.S"
-                   "${SRC}/crypto_stream/salsa20/xmm6/salsa20_xmm6.c"
-                   "${SRC}/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-avx2.c"
-                   "${SRC}/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-sse2.c"
-                   "${SRC}/crypto_stream/salsa2012/stream_salsa2012.c"
-                   "${SRC}/crypto_stream/salsa2012/ref/stream_salsa2012_ref.c"
-                   "${SRC}/crypto_stream/salsa208/stream_salsa208.c"
-                   "${SRC}/crypto_stream/salsa208/ref/stream_salsa208_ref.c"
-                   "${SRC}/crypto_stream/xchacha20/stream_xchacha20.c"
-                   "${SRC}/crypto_stream/xsalsa20/stream_xsalsa20.c"
-                   "${SRC}/crypto_verify/sodium/verify.c"
-                   "${SRC}/randombytes/randombytes.c"
-                   "${SRC}/randombytes/nativeclient/randombytes_nativeclient.c"
-                   "${SRC}/randombytes/salsa20/randombytes_salsa20_random.c"
-                   "${SRC}/randombytes/sysrandom/randombytes_sysrandom.c"
-                   "${SRC}/sodium/core.c"
-                   "${SRC}/sodium/runtime.c"
-                   "${SRC}/sodium/utils.c"
-                   "${SRC}/sodium/version.c"
-                   "port/randombytes_esp8266.c")
 
 if(CONFIG_LIBSODIUM_USE_MBEDTLS_SHA)
-    list(APPEND COMPONENT_SRCS "port/crypto_hash_mbedtls/crypto_hash_sha256_mbedtls.c"
+    set(COMPONENT_SRCDIRS ${COMPONENT_SRCDIRS}
-                   "port/crypto_hash_mbedtls/crypto_hash_sha512_mbedtls.c")
+        port/crypto_hash_mbedtls
+        )
 else()
-    list(APPEND COMPONENT_SRCS "${SRC}/crypto_hash/sha256/cp/hash_sha256_cp.c"
+    set(COMPONENT_SRCDIRS ${COMPONENT_SRCDIRS}
-                   "${SRC}/crypto_hash/sha512/cp/hash_sha512_cp.c")
+        ${SRC}/crypto_hash/sha256/cp
+        ${SRC}/crypto_hash/sha512/cp
+        )
 endif()
 
 set(COMPONENT_ADD_INCLUDEDIRS ${SRC}/include port_include)
 set(COMPONENT_PRIV_INCLUDEDIRS ${SRC}/include/sodium port_include/sodium port)
 
+set(COMPONENT_REQUIRES "tcp_transport")
+
 register_component()
 
 component_compile_definitions(
@@ -139,8 +83,11 @@ component_compile_definitions(
     __STDC_CONSTANT_MACROS
     )
 
+component_compile_options(-Wno-unknown-pragmas)
+
 # patch around warnings in third-party files
 set_source_files_properties(
+    ${SRC}/crypto_pwhash/argon2/argon2-fill-block-ref.c
     ${SRC}/crypto_pwhash/argon2/pwhash_argon2i.c
     ${SRC}/crypto_pwhash/argon2/argon2-core.c
     ${SRC}/crypto_pwhash/scryptsalsa208sha256/pwhash_scryptsalsa208sha256.c
@@ -152,15 +99,10 @@ set_source_files_properties(
     PROPERTIES COMPILE_FLAGS
     -Wno-unused-variable
     )
-set_source_files_properties(
-    ${SRC}/crypto_pwhash/argon2/argon2-fill-block-ref.c
-    PROPERTIES COMPILE_FLAGS
-    -Wno-unknown-pragmas
-    )
 
+# Temporary suppress "fallthrough" warnings until they are fixed in libsodium repo
 set_source_files_properties(
     ${SRC}/crypto_shorthash/siphash24/ref/shorthash_siphashx24_ref.c
     ${SRC}/crypto_shorthash/siphash24/ref/shorthash_siphash24_ref.c
     PROPERTIES COMPILE_FLAGS
-    -Wno-implicit-fallthrough
+    -Wno-implicit-fallthrough)
-    )

components/log/CMakeLists.txt

@@ -0,0 +1,4 @@
+set(COMPONENT_SRCDIRS ".")
+set(COMPONENT_ADD_INCLUDEDIRS "include")
+set(COMPONENT_REQUIRES)
+register_component()

components/lwip/CMakeLists.txt

@@ -1,23 +1,42 @@
 set(COMPONENT_ADD_INCLUDEDIRS
-    include/lwip/apps
+    "include/lwip"
-    lwip/src/include
+    "include/lwip/apps"
-    lwip/src/include/posix
+    "lwip/src/include"
-    port/esp8266/include)
+    "lwip/src/include/posix"
+    "port/esp8266/include"
+    "port/esp8266/include/port"
+    )
 
 set(COMPONENT_SRCDIRS
-    apps/dhcpserver
+    "apps/dhcpserver"
-    apps/multi-threads
+    "apps/multi-threads"
-    lwip/src/api
+    "apps/ping"
-    lwip/src/apps/sntp
+    "lwip/src/api"
-    lwip/src/core
+    "lwip/src/apps/sntp"
-    lwip/src/core/ipv4
+    "lwip/src/core"
-    lwip/src/core/ipv6
+    "lwip/src/core/ipv4"
-    lwip/src/netif
+    "lwip/src/core/ipv6"
-    port/esp8266/freertos
+    "lwip/src/netif"
-    port/esp8266/netif)
+    "port/esp8266/freertos"
+    "port/esp8266/netif"
+    )
 
-set(COMPONENT_REQUIRES tcpip_adapter esp8266 freertos)
+if(CONFIG_LWIP_SOCKET_MULTITHREAD)
+set(COMPONENT_OBJEXCLUDE lwip/src/api/sockets.c)
+endif()
+
+if(CONFIG_USING_ESP_VFS)
+set(COMPONENT_SRCDIRS ${COMPONENT_SRCDIRS} "port")
+endif()
+
+set(COMPONENT_PRIV_REQUIRES tcpip_adapter)
 
 register_component()
 
 component_compile_options(-Wno-address)
+
+# patch around warnings in third-party files
+set_source_files_properties(lwip/src/apps/sntp/sntp.c lwip/src/core/ipv4/ip4.c
+    PROPERTIES COMPILE_FLAGS
+    -Wno-implicit-function-declaration
+    )

components/mdns/CMakeLists.txt

@@ -1,9 +1,13 @@
+if(CONFIG_ENABLE_MDNS)
 set(COMPONENT_SRCS "src/mdns.c"
                    "src/mdns_console.c"
                    "src/mdns_networking.c")
-set(COMPONENT_ADD_INCLUDEDIRS "include")
+
 set(COMPONENT_PRIV_INCLUDEDIRS "private_include")
-set(COMPONENT_REQUIRES lwip mbedtls console tcpip_adapter)
+endif()
+
+set(COMPONENT_ADD_INCLUDEDIRS "include")
+set(COMPONENT_REQUIRES "lwip" "ssl" "tcpip_adapter")
 
 register_component()
 

components/mqtt/CMakeLists.txt

@@ -1,27 +1,30 @@
 if(CONFIG_MQTT_USING_IBM)
 set(COMPONENT_ADD_INCLUDEDIRS
-    paho/MQTTClient-C/src
+    "ibm-mqtt/MQTTClient-C/src"
-    paho/MQTTClient-C/src/FreeRTOS
+    "ibm-mqtt/MQTTClient-C/src/FreeRTOS"
-    paho/MQTTPacket/src)
+    "ibm-mqtt/MQTTPacket/src")
 
 set(COMPONENT_SRCDIRS
-    paho/MQTTClient-C/src
+    "ibm-mqtt/MQTTClient-C/src"
-    paho/MQTTClient-C/src/FreeRTOS
+    "ibm-mqtt/MQTTClient-C/src/FreeRTOS"
-    paho/MQTTPacket/src)
+    "ibm-mqtt/MQTTPacket/src")
 
-set(COMPONENT_REQUIRES freertos lwip ssl)
+endif()
-elif (CONFIG_MQTT_USING_ESP)
+
-set(COMPONENT_ADD_INCLUDEDIRS esp-mqtt/include)
+if (CONFIG_MQTT_USING_ESP)
+set(COMPONENT_ADD_INCLUDEDIRS "esp-mqtt/include")
 set(COMPONENT_PRIV_INCLUDEDIRS "esp-mqtt/lib/include")
 set(COMPONENT_SRCS "esp-mqtt/mqtt_client.c"
                    "esp-mqtt/lib/mqtt_msg.c"
                    "esp-mqtt/lib/mqtt_outbox.c"
-                   "esp-mqtt/lib/platform_esp32_idf.c")
+                   "esp-mqtt/lib/platform_idf.c")
 
-set(COMPONENT_REQUIRES lwip http_parser mbedtls tcp_transport)
 endif()
 
+set(COMPONENT_REQUIRES lwip http_parser ssl tcp_transport freertos lwip ssl)
+
 register_component()
 
-target_compile_options(${COMPONENT_NAME} PUBLIC -DMQTT_TASK)
+if(CONFIG_MQTT_USING_IBM)
-
+target_compile_options(${COMPONENT_NAME} PUBLIC -DMQTT_TASK -DMQTTCLIENT_PLATFORM_HEADER=MQTTFreeRTOS.h)
+endif()

components/newlib/CMakeLists.txt

@@ -0,0 +1,28 @@
+set(COMPONENT_SRCDIRS newlib/port)
+set(COMPONENT_ADD_INCLUDEDIRS newlib/include newlib/port/include)
+
+
+if(CONFIG_NEWLIB_NANO_FORMAT)
+    set(LIBC c_nano)
+else()
+    set(LIBC c)
+endif()
+
+set(LIBM m)
+
+set(COMPONENT_REQUIRES vfs)  # for sys/ioctl.h
+
+register_component()
+
+target_compile_definitions(newlib PUBLIC
+    -D_CLOCKS_PER_SEC_=CONFIG_FREERTOS_HZ -D_POSIX_THREADS=1 -D_UNIX98_THREAD_MUTEX_ATTRIBUTES=1
+    )
+
+if(CONFIG_ENABLE_PTHREAD)
+    target_compile_definitions(newlib PUBLIC
+        -D_POSIX_THREADS=1 -D_UNIX98_THREAD_MUTEX_AT
+        )
+endif()
+
+target_link_libraries(newlib "-L ${CMAKE_CURRENT_SOURCE_DIR}/newlib/lib")
+target_link_libraries(newlib "${LIBC}" "${LIBM}")

components/nvs_flash/CMakeLists.txt

@@ -0,0 +1,12 @@
+set(COMPONENT_SRCDIRS "src")
+set(COMPONENT_ADD_INCLUDEDIRS "include")
+
+set(COMPONENT_REQUIRES "spi_flash" "util")
+
+register_component()
+
+target_link_libraries(nvs_flash stdc++)
+
+component_compile_definitions(
+    NVS_CRC_HEADER_FILE="crc.h"
+)

components/partition_table/CMakeLists.txt

@@ -0,0 +1,76 @@
+register_config_only_component()
+
+if(NOT BOOTLOADER_BUILD)
+
+set(partition_csv "${PARTITION_CSV_PATH}")
+
+if(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)
+    set(unsigned_partition_bin "partition-table-unsigned.bin")
+    set(final_partition_bin "partition-table.bin")
+    set(final_partition_target "sign_partition_table")
+else()
+    set(unsigned_partition_bin "partition-table.bin")
+    set(final_partition_bin "partition-table.bin")
+    set(final_partition_target "build_partition_table")
+endif()
+
+if(CONFIG_PARTITION_TABLE_MD5)
+    set(md5_opt --disable-md5sum)
+endif()
+
+if(CONFIG_ESPTOOLPY_FLASHSIZE)
+    set(flashsize_opt --flash-size ${CONFIG_ESPTOOLPY_FLASHSIZE})
+endif()
+
+set(PARTITION_MD5_OPT "--disable-md5sum")
+set(PARTITION_FLASHSIZE_OPT "--flash-size" "${CONFIG_ESPTOOLPY_FLASHSIZE}")
+set(PARTITION_TABLE_OFFSET "${CONFIG_PARTITION_TABLE_OFFSET}")
+set(PARTITION_TABLE_OFFSET_ARG "--offset" "${PARTITION_TABLE_OFFSET}")
+
+add_custom_command(OUTPUT "${unsigned_partition_bin}"
+    COMMAND "${PYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/gen_esp32part.py" -q
+    ${PARTITION_MD5_OPT} ${PARTITION_FLASHSIZE_OPT} ${PARTITION_TABLE_OFFSET_ARG}
+    ${partition_csv} ${unsigned_partition_bin}
+    DEPENDS ${partition_csv} "${CMAKE_CURRENT_SOURCE_DIR}/gen_esp32part.py"
+    VERBATIM)
+
+# Add signing steps
+if(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)
+    get_filename_component(secure_boot_signing_key
+        "${CONFIG_SECURE_BOOT_SIGNING_KEY}"
+        ABSOLUTE BASE_DIR "${PROJECT_PATH}")
+
+    add_custom_command(OUTPUT "${final_partition_bin}"
+        COMMAND "${PYTHON}" "${ESPSECUREPY}" sign_data --keyfile "${secure_boot_signing_key}"
+        -o "${final_partition_bin}" "${unsigned_partition_bin}"
+        DEPENDS "${unsigned_partition_bin}"
+        VERBATIM)
+
+endif()
+
+if(EXISTS ${partition_csv})
+    add_custom_target(partition_table ALL DEPENDS "${final_partition_bin}")
+else()
+    # This is a bit of a hack: If the partition input CSV is not found, create a phony partition_table target that
+    # fails the build. Have it also touch CMakeCache.txt to cause a cmake run next time
+    # (to pick up a new CSV if one exists, etc.)
+    #
+    # This is because partition CSV is required at CMake runtime (to generate metadata files with flashing data, etc) but we can't
+    # fail the build if it is not found, because the "menuconfig" target may be required to fix the problem. CMAKE_CONFIGURE_DEPENDS
+    # only works for files which exist at CMake runtime.
+    add_custom_target(partition_table ALL
+        COMMAND ${CMAKE_COMMAND} -E echo "Partition table CSV ${partition_csv} does not exist. Either change partition table in menuconfig or create this input file."
+        COMMAND ${CMAKE_COMMAND} -E touch "${CMAKE_BINARY_DIR}/CMakeCache.txt"
+        COMMAND ${CMAKE_COMMAND} -P ${IDF_PATH}/tools/cmake/scripts/fail.cmake)
+endif()
+
+add_dependencies(bootloader partition_table)
+add_dependencies(app partition_table)
+
+# Use global properties ESPTOOL_WRITE_FLASH_ARGS to pass this info to build
+# the list of esptool write arguments for flashing
+set_property(GLOBAL APPEND_STRING PROPERTY
+    ESPTOOL_WRITE_FLASH_ARGS
+    "${PARTITION_TABLE_OFFSET} ${final_partition_bin} ")
+
+endif()

components/partition_table/Makefile.projbuild

@@ -83,7 +83,7 @@ partition_table-flash: $(PARTITION_TABLE_BIN)
 partition_table-clean:
 	rm -f $(PARTITION_TABLE_BIN)
 
-global-macro: partition_table_get_info
+make_prepare: partition_table_get_info
 
 clean: partition_table-clean
 

components/partition_table/project_include.cmake

@@ -0,0 +1,65 @@
+if(NOT BOOTLOADER_BUILD)
+
+set(PARTITION_TABLE_OFFSET ${CONFIG_PARTITION_TABLE_OFFSET})
+
+# Set PARTITION_CSV_PATH to the configured partition CSV file
+# absolute path
+if(CONFIG_PARTITION_TABLE_CUSTOM)
+    # Custom filename expands any path relative to the project
+    get_filename_component(PARTITION_CSV_PATH "${CONFIG_PARTITION_TABLE_FILENAME}" ABSOLUTE BASE_DIR "${PROJECT_PATH}")
+
+    if(NOT EXISTS "${PARTITION_CSV_PATH}")
+        message(WARNING "Partition table CSV file ${PARTITION_CSV_PATH} not found. "
+            "Change custom partition CSV path in menuconfig.")
+        # Note: partition_table CMakeLists.txt contains some logic to create a dummy
+        # partition_table target in this case, see comments in that file.
+    endif()
+else()
+    # Other .csv files are always in the component directory
+    get_filename_component(PARTITION_CSV_PATH "${COMPONENT_PATH}/${CONFIG_PARTITION_TABLE_FILENAME}" ABSOLUTE)
+
+    if(NOT EXISTS "${PARTITION_CSV_PATH}")
+        message(FATAL_ERROR "Internal error, built-in ${PARTITION_CSV_PATH} not found.")
+    endif()
+endif()
+
+# need to re-run CMake if the partition CSV changes, as the offsets/sizes of partitions may change
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${PARTITION_CSV_PATH})
+
+# Parse the partition table to get variable partition offsets & sizes which must be known at CMake runtime
+function(get_partition_info variable get_part_info_args)
+    separate_arguments(get_part_info_args)
+    execute_process(COMMAND ${PYTHON}
+        ${COMPONENT_PATH}/parttool.py -q
+        --partition-table-offset ${PARTITION_TABLE_OFFSET}
+        ${get_part_info_args}
+        ${PARTITION_CSV_PATH}
+        OUTPUT_VARIABLE result
+        RESULT_VARIABLE exit_code
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if(NOT ${exit_code} EQUAL 0 AND NOT ${exit_code} EQUAL 1)
+        # can't fail here as it would prevent the user from running 'menuconfig' again
+        message(WARNING "parttool.py execution failed (${result}), problem with partition CSV file (see above)")
+    endif()
+    set(${variable} ${result} PARENT_SCOPE)
+endfunction()
+
+if(CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION)
+    get_partition_info(PHY_PARTITION_OFFSET "--type data --subtype phy --offset")
+    set(PHY_PARTITION_BIN_FILE "${CMAKE_BINARY_DIR}/esp32/phy_init_data.bin")
+endif()
+
+get_partition_info(APP_PARTITION_OFFSET "--default-boot-partition --offset")
+
+get_partition_info(APP_PARTITION_SIZE "--default-boot-partition --size")
+
+get_partition_info(OTADATA_PARTITION_OFFSET "--type data --subtype ota --offset")
+
+get_partition_info(OTADATA_PARTITION_SIZE "--type data --subtype ota --size")
+
+set(APP_OFFSET ${APP_PARTITION_OFFSET})
+set(APP_SIZE ${APP_PARTITION_SIZE})
+
+set(CFLAGS "${CFLAGS}" "-DAPP_OFFSET=${APP_OFFSET}" "-DAPP_SIZE=${APP_SIZE}")
+
+endif()

components/protobuf-c/CMakeLists.txt

@@ -1,4 +1,4 @@
-set(COMPONENT_ADD_INCLUDEDIRS protobuf-c)
+set(COMPONENT_ADD_INCLUDEDIRS "protobuf-c")
 set(COMPONENT_SRCS "protobuf-c/protobuf-c/protobuf-c.c")
 
 register_component()

components/protocomm/CMakeLists.txt

@@ -1,3 +1,4 @@
+if(CONFIG_ENABLE_UNIFIED_PROVISIONING)
 set(COMPONENT_ADD_INCLUDEDIRS include/common
                               include/security
                               include/transports)
@@ -10,7 +11,8 @@ set(COMPONENT_SRCS  "src/common/protocomm.c"
                     "proto-c/sec1.pb-c.c"
                     "proto-c/session.pb-c.c"
                     "src/transports/protocomm_httpd.c")
+endif()
 
-set(COMPONENT_PRIV_REQUIRES protobuf-c mbedtls wifi_provisioning)
+set(COMPONENT_PRIV_REQUIRES protobuf-c ssl esp_http_server http_parser)
 
 register_component()

components/pthread/CMakeLists.txt

@@ -0,0 +1,11 @@
+if(CONFIG_ENABLE_PTHREAD)
+    set(COMPONENT_SRCDIRS "src")
+    set(COMPONENT_ADD_INCLUDEDIRS "include")
+    set(COMPONENT_REQUIRES)
+endif()
+
+register_component()
+
+if(CONFIG_ENABLE_STATIC_TASK_CLEAN_UP_HOOK)
+    target_link_libraries(pthread "-Wl,--wrap=vPortCleanUpTCB")
+endif()

components/smartconfig_ack/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(COMPONENT_SRCDIRS ".")
+set(COMPONENT_ADD_INCLUDEDIRS "include")
+
+set(COMPONENT_PRIV_REQUIRES lwip tcpip_adapter)
+
+register_component()

components/spi_flash/CMakeLists.txt

@@ -0,0 +1,14 @@
+if(BOOTLOADER_BUILD)
+    # Bootloader needs SPIUnlock from this file, but doesn't
+    # need other parts of this component
+    set(COMPONENT_SRCDIRS src)
+    set(COMPONENT_PRIV_REQUIRES "bootloader_support")
+else()
+    set(COMPONENT_SRCDIRS src)
+    set(COMPONENT_PRIV_REQUIRES "esp8266" "freertos" "bootloader_support")
+endif()
+
+set(COMPONENT_ADD_INCLUDEDIRS include)
+set(COMPONENT_REQUIRES)
+
+register_component()

components/spi_ram/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(COMPONENT_SRCDIRS ".")
+set(COMPONENT_ADD_INCLUDEDIRS "include")
+
+set(COMPONENT_REQUIRES)
+
+register_component()

components/spiffs/CMakeLists.txt

@@ -10,8 +10,8 @@ set(COMPONENT_SRCS "esp_spiffs.c"
                    "spiffs/src/spiffs_nucleus.c")
 endif()
 
-set(COMPONENT_REQUIRES spi_flash)
+set(COMPONENT_REQUIRES  "freertos" "spi_flash")
-set(COMPONENT_PRIV_REQUIRES bootloader_support)
+set(COMPONENT_PRIV_REQUIRES "esp8266" "bootloader_support")
 
 register_component()
 

components/ssl/CMakeLists.txt

@@ -1,6 +1,7 @@
 
 if(CONFIG_SSL_USING_WOLFSSL)
-set(COMPONENT_ADD_INCLUDEDIRS include wolfssl/include wolfssl/wolfssl wolfssl/wolfssl/wolfssl)
+set(COMPONENT_ADD_INCLUDEDIRS wolfssl/include wolfssl/wolfssl wolfssl/wolfssl/wolfssl)
+set(COMPONENT_SRCDIRS "wolfssl/source")
 else()
 if(CONFIG_SSL_USING_MBEDTLS)
 set(COMPONENT_ADD_INCLUDEDIRS
@@ -19,21 +20,21 @@ set(COMPONENT_SRCDIRS
     mbedtls/mbedtls/library
     mbedtls/port/esp8266)
 else()
-set(COMPONENT_ADD_INCLUDEDIRS include axtls/include)
+set(COMPONENT_ADD_INCLUDEDIRS axtls/include)
 
 set(COMPONENT_SRCDIRS axtls/source/ssl axtls/source/crypto)
 endif()
 endif()
 
-set(COMPONENT_REQUIRES lwip esp8266)
+set(COMPONENT_REQUIRES "lwip" "esp8266" "util")
 
 register_component()
 
 if(CONFIG_SSL_USING_WOLFSSL)
 target_compile_options(${COMPONENT_NAME} PUBLIC -DWOLFSSL_USER_SETTINGS)
 
-target_link_libraries(ssl "-L ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/lib")
+target_link_libraries(${COMPONENT_NAME} "-L ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/lib")
-target_link_libraries(ssl wolfssl)
+target_link_libraries(${COMPONENT_NAME} wolfssl)
 else()
 if(CONFIG_SSL_USING_MBEDTLS)
 target_compile_options(${COMPONENT_NAME} PUBLIC -DMBEDTLS_CONFIG_FILE="mbedtls/esp_config.h")

components/ssl/wolfssl/source/cmake_compiling.c

@@ -0,0 +1 @@
+// Just for passing cmake project compiling.

components/tcp_transport/CMakeLists.txt

@@ -6,6 +6,6 @@ set(COMPONENT_SRCS "transport.c"
 
 set(COMPONENT_ADD_INCLUDEDIRS "include")
 
-set(COMPONENT_REQUIRES lwip esp-tls)
+set(COMPONENT_REQUIRES "lwip" "esp-tls")
 
 register_component()

components/tcpip_adapter/CMakeLists.txt

@@ -1,8 +1,8 @@
-set(COMPONENT_SRCDIRS .)
+set(COMPONENT_SRCDIRS ".")
-set(COMPONENT_ADD_INCLUDEDIRS include)
+set(COMPONENT_ADD_INCLUDEDIRS "include")
 
-set(COMPONENT_PRIV_REQUIRES lwip)
+set(COMPONENT_REQUIRES "lwip")
 
 register_component()
 
-component_compile_options(-DLWIP_OPEN_SRC)
+component_compile_options("-DLWIP_OPEN_SRC")

components/util/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(COMPONENT_SRCDIRS "src")
+set(COMPONENT_ADD_INCLUDEDIRS "include")
+
+set(COMPONENT_REQUIRES)
+
+register_component()
+
+if(NOT IS_BOOTLOADER_BUILD)
+target_compile_definitions(${COMPONENT_NAME} PRIVATE -DUSING_IBUS_FASTER_GET)
+endif()

components/vfs/CMakeLists.txt

@@ -1,3 +1,4 @@
+
 if(CONFIG_USING_ESP_VFS)
     set(COMPONENT_SRCS "vfs.c"
                        "vfs_uart.c")
@@ -7,6 +8,6 @@ endif()
 
 set(COMPONENT_ADD_INCLUDEDIRS "include")
 
-set(COMPONENT_REQUIRES)
+set(COMPONENT_REQUIRES "lwip")
 
 register_component()

components/wpa_supplicant/CMakeLists.txt

@@ -0,0 +1,14 @@
+set(COMPONENT_SRCDIRS "src/crypto" "port")
+set(COMPONENT_ADD_INCLUDEDIRS "include" "port/include")
+
+set(COMPONENT_REQUIRES "")
+set(COMPONENT_PRIV_REQUIRES "freertos" "heap" "newlib" "util")
+
+register_component()
+
+component_compile_options(-Wno-strict-aliasing)
+component_compile_definitions(
+    __ets__
+    EMBEDDED_SUPP
+    ESPRESSIF_USE
+    )

docs/conf_common.py

@@ -74,7 +74,6 @@ confgen_args = [sys.executable,
                 "../../tools/kconfig_new/confgen.py",
                 "--kconfig", "../../Kconfig",
                 "--config", temp_sdkconfig_path,
-                "--create-config-if-missing",
                 "--env", "COMPONENT_KCONFIGS={}".format(kconfigs),
                 "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(kconfig_projbuilds),
                 "--env", "IDF_PATH={}".format(idf_path),

examples/protocols/openssl_demo/CMakeLists.txt

@@ -2,7 +2,5 @@
 # in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.5)
 
-set(MAIN_SRCS main/user_main.c main/openssl_demo.c)
-
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(openssl_demo)

examples/protocols/openssl_demo/main/CMakeLists.txt

@@ -0,0 +1,3 @@
+set(COMPONENT_SRCS "openssl_demo_example_main.c")
+
+register_component()

make/component_wrapper.mk

@@ -212,7 +212,7 @@ build: $(COMPONENT_LIBRARY)
 $(COMPONENT_LIBRARY): $(COMPONENT_OBJS) $(COMPONENT_EMBED_OBJS)
 	$(summary) AR $(patsubst $(PWD)/%,%,$(CURDIR))/$@
 	rm -f $@
-	$(AR) cru $@ $^
+	$(AR) $(ARFLAGS) $@ $^
 endif
 
 # If COMPONENT_OWNCLEANTARGET is not set, define a phony clean target

make/project.mk

@@ -13,7 +13,7 @@
 .PHONY: build-components menuconfig defconfig all build clean all_binaries check-submodules size size-components size-files size-symbols list-components
 
 MAKECMDGOALS ?= all
-all: all_binaries
+all: all_binaries | check_python_dependencies
 # see below for recipe of 'all' target
 #
 # # other components will add dependencies to 'all_binaries'. The
@@ -34,6 +34,7 @@ help:
 	@echo "make size-components, size-files - Finer-grained memory footprints"
 	@echo "make size-symbols - Per symbol memory footprint. Requires COMPONENT=<component>"
 	@echo "make erase_flash - Erase entire flash contents"
+	@echo "make erase_ota - Erase ota_data partition. After that will boot first bootable partition (factory or OTAx)."
 	@echo "make monitor - Run idf_monitor tool to monitor serial output from app"
 	@echo "make simple_monitor - Monitor serial output on terminal console"
 	@echo "make list-components - List all components in the project"
@@ -42,12 +43,16 @@ help:
 	@echo "make app-flash - Flash just the app"
 	@echo "make app-clean - Clean just the app"
 	@echo "make print_flash_cmd - Print the arguments for esptool when flash"
+	@echo "make check_python_dependencies - Check that the required python packages are installed"
 	@echo ""
 	@echo "See also 'make bootloader', 'make bootloader-flash', 'make bootloader-clean', "
 	@echo "'make partition_table', etc, etc."
 
+# prepare for the global varible for compiling
+make_prepare:
+
 # Non-interactive targets. Mostly, those for which you do not need to build a binary
-NON_INTERACTIVE_TARGET += defconfig clean% %clean help list-components print_flash_cmd
+NON_INTERACTIVE_TARGET += defconfig clean% %clean help list-components print_flash_cmd check_python_dependencies
 
 # dependency checks
 ifndef MAKE_RESTARTS
@@ -131,6 +136,9 @@ ifndef COMPONENT_DIRS
 EXTRA_COMPONENT_DIRS ?=
 COMPONENT_DIRS := $(PROJECT_PATH)/components $(EXTRA_COMPONENT_DIRS) $(IDF_PATH)/components $(PROJECT_PATH)/main
 endif
+# Make sure that every directory in the list is an absolute path without trailing slash.
+# This is necessary to split COMPONENT_DIRS into SINGLE_COMPONENT_DIRS and MULTI_COMPONENT_DIRS below. 
+COMPONENT_DIRS := $(foreach cd,$(COMPONENT_DIRS),$(abspath $(cd)))
 export COMPONENT_DIRS
 
 ifdef SRCDIRS
@@ -138,41 +146,65 @@ $(warning SRCDIRS variable is deprecated. These paths can be added to EXTRA_COMP
 COMPONENT_DIRS += $(abspath $(SRCDIRS))
 endif
 
-# The project Makefile can define a list of components, but if it does not do this we just take all available components
+# List of component directories, i.e. directories which contain a component.mk file 
-# in the component dirs. A component is COMPONENT_DIRS directory, or immediate subdirectory,
+SINGLE_COMPONENT_DIRS := $(abspath $(dir $(dir $(foreach cd,$(COMPONENT_DIRS),\
+                             $(wildcard $(cd)/component.mk)))))
+
+# List of components directories, i.e. directories which may contain components 
+MULTI_COMPONENT_DIRS := $(filter-out $(SINGLE_COMPONENT_DIRS),$(COMPONENT_DIRS))
+
+# The project Makefile can define a list of components, but if it does not do this
+# we just take all available components in the component dirs.
+# A component is COMPONENT_DIRS directory, or immediate subdirectory,
 # which contains a component.mk file.
 #
 # Use the "make list-components" target to debug this step.
 ifndef COMPONENTS
 # Find all component names. The component names are the same as the
 # directories they're in, so /bla/components/mycomponent/component.mk -> mycomponent.
-COMPONENTS := $(dir $(foreach cd,$(COMPONENT_DIRS),                           \
+# We need to do this for MULTI_COMPONENT_DIRS only, since SINGLE_COMPONENT_DIRS
-					$(wildcard $(cd)/*/component.mk) $(wildcard $(cd)/component.mk) \
+# are already known to contain component.mk.
-				))
+COMPONENTS := $(dir $(foreach cd,$(MULTI_COMPONENT_DIRS),$(wildcard $(cd)/*/component.mk))) \
+              $(SINGLE_COMPONENT_DIRS)
 COMPONENTS := $(sort $(foreach comp,$(COMPONENTS),$(lastword $(subst /, ,$(comp)))))
 endif
-# After a full manifest of component names is determined, subtract the ones explicitly omitted by the project Makefile.
+# After a full manifest of component names is determined, subtract the ones explicitly
+# omitted by the project Makefile.
+EXCLUDE_COMPONENTS ?=
 ifdef EXCLUDE_COMPONENTS
-COMPONENTS := $(filter-out $(EXCLUDE_COMPONENTS), $(COMPONENTS))
+COMPONENTS := $(filter-out $(subst ",,$(EXCLUDE_COMPONENTS)), $(COMPONENTS))
+# to keep syntax highlighters happy: "))
 endif
 export COMPONENTS
 
 # Resolve all of COMPONENTS into absolute paths in COMPONENT_PATHS.
+# For each entry in COMPONENT_DIRS:
+# - either this is directory with multiple components, in which case check that
+#   a subdirectory with component name exists, and it contains a component.mk file.
+# - or, this is a directory of a single component, in which case the name of this
+#   directory has to match the component name
 #
 # If a component name exists in multiple COMPONENT_DIRS, we take the first match.
 #
 # NOTE: These paths must be generated WITHOUT a trailing / so we
 # can use $(notdir x) to get the component name.
-COMPONENT_PATHS := $(foreach comp,$(COMPONENTS),$(firstword $(foreach cd,$(COMPONENT_DIRS),$(wildcard $(dir $(cd))$(comp) $(cd)/$(comp)))))
+COMPONENT_PATHS := $(foreach comp,$(COMPONENTS),\
+                        $(firstword $(foreach cd,$(COMPONENT_DIRS),\
+                            $(if $(findstring $(cd),$(MULTI_COMPONENT_DIRS)),\
+                                 $(abspath $(dir $(wildcard $(cd)/$(comp)/component.mk))),)\
+                            $(if $(findstring $(cd),$(SINGLE_COMPONENT_DIRS)),\
+                                 $(if $(filter $(comp),$(notdir $(cd))),$(cd),),)\
+                   )))
 export COMPONENT_PATHS
 
 TEST_COMPONENTS ?=
+TEST_EXCLUDE_COMPONENTS ?=
 TESTS_ALL ?=
 
 # If TESTS_ALL set to 1, set TEST_COMPONENTS_LIST to all components.
 # Otherwise, use the list supplied in TEST_COMPONENTS.
 ifeq ($(TESTS_ALL),1)
-TEST_COMPONENTS_LIST := $(COMPONENTS)
+TEST_COMPONENTS_LIST := $(filter-out $(TEST_EXCLUDE_COMPONENTS), $(COMPONENTS))
 else
 TEST_COMPONENTS_LIST := $(TEST_COMPONENTS)
 endif
@@ -192,8 +224,6 @@ COMPONENT_LDFLAGS :=
 COMPONENT_SUBMODULES :=
 COMPONENT_LIBRARIES :=
 
-global-macro:
-
 # COMPONENT_PROJECT_VARS is the list of component_project_vars.mk generated makefiles
 # for each component.
 #
@@ -345,7 +375,9 @@ else
 CXXFLAGS += -fno-exceptions
 endif
 
-export CFLAGS CPPFLAGS CXXFLAGS
+ARFLAGS := cru
+
+export CFLAGS CPPFLAGS CXXFLAGS ARFLAGS
 
 # Set default values that were not previously defined
 CC ?= gcc
@@ -372,6 +404,14 @@ OBJCOPY := $(call dequote,$(CONFIG_TOOLPREFIX))objcopy
 SIZE := $(call dequote,$(CONFIG_TOOLPREFIX))size
 export CC CXX LD AR OBJCOPY SIZE
 
+COMPILER_VERSION_STR := $(shell $(CC) -dumpversion)
+COMPILER_VERSION_NUM := $(subst .,,$(COMPILER_VERSION_STR))
+GCC_NOT_5_2_0 := $(shell expr $(COMPILER_VERSION_STR) != "5.2.0")
+export COMPILER_VERSION_STR COMPILER_VERSION_NUM GCC_NOT_5_2_0
+
+CPPFLAGS += -DGCC_NOT_5_2_0=$(GCC_NOT_5_2_0)
+export CPPFLAGS
+
 PYTHON=$(call dequote,$(CONFIG_PYTHON))
 
 # the app is the main executable built by the project
@@ -407,7 +447,7 @@ $(APP_ELF): $(foreach libcomp,$(COMPONENT_LIBRARIES),$(BUILD_DIR_BASE)/$(libcomp
 	$(summary) LD $(patsubst $(PWD)/%,%,$@)
 	$(CC) $(LDFLAGS) -o $@ -Wl,-Map=$(APP_MAP)
 
-app: $(APP_BIN)
+app: $(APP_BIN) partition_table_get_info
 ifeq ("$(CONFIG_SECURE_BOOT_ENABLED)$(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)","y") # secure boot enabled, but remote sign app image
 	@echo "App built but not signed. Signing step via espsecure.py:"
 	@echo "espsecure.py sign_data --keyfile KEYFILE $(APP_BIN)"
@@ -418,6 +458,14 @@ else
 	@echo $(ESPTOOLPY_WRITE_FLASH) $(APP_OFFSET) $(APP_BIN)
 endif
 
+.PHONY: check_python_dependencies
+
+# Notify users when some of the required python packages are not installed
+check_python_dependencies:
+ifndef IS_BOOTLOADER_BUILD
+	$(PYTHON) $(IDF_PATH)/tools/check_python_dependencies.py
+endif
+
 all_binaries: $(APP_BIN)
 
 $(BUILD_DIR_BASE):
@@ -439,7 +487,7 @@ endef
 define GenerateComponentTargets
 .PHONY: component-$(2)-build component-$(2)-clean
 
-component-$(2)-build: check-submodules global-macro $(call prereq_if_explicit, component-$(2)-clean) | $(BUILD_DIR_BASE)/$(2)
+component-$(2)-build: check-submodules make_prepare $(call prereq_if_explicit, component-$(2)-clean) | $(BUILD_DIR_BASE)/$(2)
 	$(call ComponentMake,$(1),$(2)) build
 
 component-$(2)-clean: | $(BUILD_DIR_BASE)/$(2) $(BUILD_DIR_BASE)/$(2)/component_project_vars.mk
@@ -473,16 +521,16 @@ app-clean: $(addprefix component-,$(addsuffix -clean,$(notdir $(COMPONENT_PATHS)
 	$(summary) RM $(APP_ELF)
 	rm -f $(APP_ELF) $(APP_BIN) $(APP_MAP)
 
-size: $(APP_ELF)
+size: $(APP_ELF) | check_python_dependencies
 	$(PYTHON) $(IDF_PATH)/tools/idf_size.py $(APP_MAP)
 
-size-files: $(APP_ELF)
+size-files: $(APP_ELF) | check_python_dependencies
 	$(PYTHON) $(IDF_PATH)/tools/idf_size.py --files $(APP_MAP)
 
-size-components: $(APP_ELF)
+size-components: $(APP_ELF) | check_python_dependencies
 	$(PYTHON) $(IDF_PATH)/tools/idf_size.py --archives $(APP_MAP)
 
-size-symbols: $(APP_ELF)
+size-symbols: $(APP_ELF) | check_python_dependencies
 ifndef COMPONENT
 	$(error "ERROR: Please enter the component to look symbols for, e.g. COMPONENT=heap")
 else
@@ -513,7 +561,7 @@ check-submodules: $(IDF_PATH)/$(1)/.git
 $(IDF_PATH)/$(1)/.git:
 	@echo "WARNING: Missing submodule $(1)..."
 	[ -e ${IDF_PATH}/.git ] || ( echo "ERROR: esp-idf must be cloned from git to work."; exit 1)
-	[ -x $$(which git) ] || ( echo "ERROR: Need to run 'git submodule init $(1)' in esp-idf root directory."; exit 1)
+	[ -x "$(shell which git)" ] || ( echo "ERROR: Need to run 'git submodule init $(1)' in esp-idf root directory."; exit 1)
 	@echo "Attempting 'git submodule update --init $(1)' in esp-idf root directory..."
 	cd ${IDF_PATH} && git submodule update --init $(1)
 
@@ -535,45 +583,56 @@ list-components:
 	$(info COMPONENT_DIRS (components searched for here))
 	$(foreach cd,$(COMPONENT_DIRS),$(info $(cd)))
 	$(info $(call dequote,$(SEPARATOR)))
-	$(info COMPONENTS (list of component names))
+	$(info TEST_COMPONENTS (list of test component names))
-	$(info $(COMPONENTS))
+	$(info $(TEST_COMPONENTS_LIST))
 	$(info $(call dequote,$(SEPARATOR)))
-	$(info EXCLUDE_COMPONENTS (list of excluded names))
+	$(info TEST_EXCLUDE_COMPONENTS (list of test excluded names))
-	$(info $(if $(EXCLUDE_COMPONENTS),$(EXCLUDE_COMPONENTS),(none provided)))	
+	$(info $(if $(EXCLUDE_COMPONENTS) || $(TEST_EXCLUDE_COMPONENTS),$(EXCLUDE_COMPONENTS) $(TEST_EXCLUDE_COMPONENTS),(none provided)))
 	$(info $(call dequote,$(SEPARATOR)))
 	$(info COMPONENT_PATHS (paths to all components):)
 	$(foreach cp,$(COMPONENT_PATHS),$(info $(cp)))
 
 # print flash command, so users can dump this to config files and download somewhere without idf
-print_flash_cmd: global-macro
+print_flash_cmd: partition_table_get_info blank_ota_data
 	echo $(ESPTOOL_WRITE_FLASH_OPTIONS) $(ESPTOOL_ALL_FLASH_ARGS) | sed -e 's:'$(PWD)/build/'::g'
 
 # Check toolchain version using the output of xtensa-esp32-elf-gcc --version command.
 # The output normally looks as follows
-#     xtensa-esp32-elf-gcc (crosstool-NG crosstool-ng-1.22.0-59-ga194053) 4.8.5
+#     xtensa-esp32-elf-gcc (crosstool-NG crosstool-ng-1.22.0-80-g6c4433a) 5.2.0
-# The part in brackets is extracted into TOOLCHAIN_COMMIT_DESC variable,
+# The part in brackets is extracted into TOOLCHAIN_COMMIT_DESC variable
-# the part after the brackets is extracted into TOOLCHAIN_GCC_VER.
 ifdef CONFIG_TOOLPREFIX
 ifndef MAKE_RESTARTS
-TOOLCHAIN_COMMIT_DESC := $(shell $(CC) --version | sed -E -n 's|.*crosstool-ng-([0-9]+).([0-9]+).([0-9]+)-([0-9]+)-g([0-9a-f]{7}).*|\1.\2.\3-\4-g\5|gp')
+
-TOOLCHAIN_GCC_VER := $(shell $(CC) --version | sed -E -n 's|.*gcc.*\ \(.*\)\ (.*)|\1|gp')
+TOOLCHAIN_HEADER := $(shell $(CC) --version | head -1)
+TOOLCHAIN_PATH := $(shell which $(CC))
+TOOLCHAIN_COMMIT_DESC := $(shell $(CC) --version | sed -E -n 's|.*\(crosstool-NG (.*)\).*|\1|gp')
+TOOLCHAIN_GCC_VER := $(COMPILER_VERSION_STR)
 
 # Officially supported version(s)
-SUPPORTED_TOOLCHAIN_COMMIT_DESC ?= 1.22.0-80-g6c4433a
+include $(IDF_PATH)/tools/toolchain_versions.mk
-SUPPORTED_TOOLCHAIN_GCC_VERSIONS ?= 5.2.0
+
+ifndef IS_BOOTLOADER_BUILD
+$(info Toolchain path: $(TOOLCHAIN_PATH))
+endif
 
 ifdef TOOLCHAIN_COMMIT_DESC
-ifneq ($(TOOLCHAIN_COMMIT_DESC), $(SUPPORTED_TOOLCHAIN_COMMIT_DESC))
+ifeq (,$(findstring $(SUPPORTED_TOOLCHAIN_COMMIT_DESC),$(TOOLCHAIN_COMMIT_DESC)))
 $(info WARNING: Toolchain version is not supported: $(TOOLCHAIN_COMMIT_DESC))
 $(info Expected to see version: $(SUPPORTED_TOOLCHAIN_COMMIT_DESC))
 $(info Please check ESP-IDF setup instructions and update the toolchain, or proceed at your own risk.)
-$(info Please download and use the toolchain from the URL of README.md)
+else
+ifndef IS_BOOTLOADER_BUILD
+$(info Toolchain version: $(TOOLCHAIN_COMMIT_DESC))
+endif
 endif
 ifeq (,$(findstring $(TOOLCHAIN_GCC_VER), $(SUPPORTED_TOOLCHAIN_GCC_VERSIONS)))
 $(info WARNING: Compiler version is not supported: $(TOOLCHAIN_GCC_VER))
 $(info Expected to see version(s): $(SUPPORTED_TOOLCHAIN_GCC_VERSIONS))
 $(info Please check ESP-IDF setup instructions and update the toolchain, or proceed at your own risk.)
-$(info Please download and use the toolchain from the URL of README.md)
+else
+ifndef IS_BOOTLOADER_BUILD
+$(info Compiler version: $(TOOLCHAIN_GCC_VER))
+endif
 endif
 else
 $(info WARNING: Failed to find Xtensa toolchain, may need to alter PATH or set one in the configuration menu)

make/project_config.mk

@@ -15,11 +15,11 @@ SDKCONFIG ?= $(PROJECT_PATH)/sdkconfig
 # overrides (usually used for esp-idf examples)
 SDKCONFIG_DEFAULTS ?= $(PROJECT_PATH)/sdkconfig.defaults
 
-# Workaround to run make parallel (-j). mconf and conf cannot be made simultaneously
+# Workaround to run make parallel (-j). mconf-idf and conf-idf cannot be made simultaneously
-$(KCONFIG_TOOL_DIR)/mconf: $(KCONFIG_TOOL_DIR)/conf
+$(KCONFIG_TOOL_DIR)/mconf-idf: $(KCONFIG_TOOL_DIR)/conf-idf
 
 # reset MAKEFLAGS as the menuconfig makefile uses implicit compile rules
-$(KCONFIG_TOOL_DIR)/mconf $(KCONFIG_TOOL_DIR)/conf: $(wildcard $(KCONFIG_TOOL_DIR)/*.c)
+$(KCONFIG_TOOL_DIR)/mconf-idf $(KCONFIG_TOOL_DIR)/conf-idf: $(wildcard $(KCONFIG_TOOL_DIR)/*.c)
 	MAKEFLAGS="" CC=$(HOSTCC) LD=$(HOSTLD) \
 	$(MAKE) -C $(KCONFIG_TOOL_DIR)
 
@@ -36,13 +36,14 @@ $(SDKCONFIG): defconfig
 endif
 endif
 
-# macro for the commands to run kconfig tools conf or mconf.
+# macro for the commands to run kconfig tools conf-idf or mconf-idf.
 # $1 is the name (& args) of the conf tool to run
 define RunConf
 	mkdir -p $(BUILD_DIR_BASE)/include/config
 	cd $(BUILD_DIR_BASE); KCONFIG_AUTOHEADER=$(abspath $(BUILD_DIR_BASE)/include/sdkconfig.h) \
 	COMPONENT_KCONFIGS="$(COMPONENT_KCONFIGS)" KCONFIG_CONFIG=$(SDKCONFIG) \
 	COMPONENT_KCONFIGS_PROJBUILD="$(COMPONENT_KCONFIGS_PROJBUILD)" \
+	IDF_CMAKE=n \
 	$(KCONFIG_TOOL_DIR)/$1 $(IDF_PATH)/Kconfig
 endef
 
@@ -58,7 +59,7 @@ ifndef MAKE_RESTARTS
 # depend on any prerequisite that may cause a make restart as part of
 # the prerequisite's own recipe.
 
-menuconfig: $(KCONFIG_TOOL_DIR)/mconf
+menuconfig: $(KCONFIG_TOOL_DIR)/mconf-idf
 	$(summary) MENUCONFIG
 ifdef BATCH_BUILD
 	@echo "Can't run interactive configuration inside non-interactive build process."
@@ -67,25 +68,25 @@ ifdef BATCH_BUILD
 	@echo "See esp-idf documentation for more details."
 	@exit 1
 else
-	$(call RunConf,mconf)
+	$(call RunConf,mconf-idf)
 endif
 
 # defconfig creates a default config, based on SDKCONFIG_DEFAULTS if present
-defconfig: $(KCONFIG_TOOL_DIR)/conf
+defconfig: $(KCONFIG_TOOL_DIR)/conf-idf
 	$(summary) DEFCONFIG
 ifneq ("$(wildcard $(SDKCONFIG_DEFAULTS))","")
 	cat $(SDKCONFIG_DEFAULTS) >> $(SDKCONFIG)  # append defaults to sdkconfig, will override existing values
 endif
-	$(call RunConf,conf --olddefconfig)
+	$(call RunConf,conf-idf --olddefconfig)
 
 # if neither defconfig or menuconfig are requested, use the GENCONFIG rule to
 # ensure generated config files are up to date
-$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(KCONFIG_TOOL_DIR)/conf $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig)
+$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(KCONFIG_TOOL_DIR)/conf-idf $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig)
 	$(summary) GENCONFIG
 ifdef BATCH_BUILD  # can't prompt for new config values like on terminal
-	$(call RunConf,conf --olddefconfig)
+	$(call RunConf,conf-idf --olddefconfig)
 endif
-	$(call RunConf,conf --silentoldconfig)
+	$(call RunConf,conf-idf --silentoldconfig)
 	touch $(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h  # ensure newer than sdkconfig
 
 else  # "$(MAKE_RESTARTS)" != ""

requirements.txt

@@ -0,0 +1,10 @@
+# This is a list of python packages needed for ESP-IDF. This file is used with pip.
+# Please see the Get Started section of the ESP-IDF Programming Guide for further information.
+#
+setuptools
+# The setuptools package is required to install source distributions and on some systems is not installed by default.
+# Please keep it as the first item of this list.
+#
+pyserial>=3.0
+future>=0.15.2
+cryptography>=2.1.4

tools/cmake/components.cmake

@@ -16,13 +16,20 @@ endfunction()
 #
 function(register_component)
     get_filename_component(component_dir ${CMAKE_CURRENT_LIST_FILE} DIRECTORY)
-    get_filename_component(component ${component_dir} NAME)
+    set(component ${COMPONENT_NAME})
 
     spaces2list(COMPONENT_SRCDIRS)
     spaces2list(COMPONENT_ADD_INCLUDEDIRS)
+    spaces2list(COMPONENT_SRCEXCLUDE)
+
+    if(COMPONENT_SRCDIRS)
+        # Warn user if both COMPONENT_SRCDIRS and COMPONENT_SRCS are set
+        if(COMPONENT_SRCS)
+            message(WARNING "COMPONENT_SRCDIRS and COMPONENT_SRCS are both set, COMPONENT_SRCS will be ignored")
+        endif()
+
+        set(COMPONENT_SRCS "")
 
-    # Add to COMPONENT_SRCS by globbing in COMPONENT_SRCDIRS
-    if(NOT COMPONENT_SRCS)
         foreach(dir ${COMPONENT_SRCDIRS})
             get_filename_component(abs_dir ${dir} ABSOLUTE BASE_DIR ${component_dir})
             if(NOT IS_DIRECTORY ${abs_dir})
@@ -39,6 +46,17 @@ function(register_component)
         endforeach()
     endif()
 
+    # Remove COMPONENT_SRCEXCLUDE matches
+    foreach(exclude ${COMPONENT_SRCEXCLUDE})
+        get_filename_component(exclude "${exclude}" ABSOLUTE ${component_dir})
+        foreach(src ${COMPONENT_SRCS})
+            get_filename_component(abs_src "${src}" ABSOLUTE ${component_dir})
+            if("${exclude}" STREQUAL "${abs_src}")  # compare as canonical paths
+                list(REMOVE_ITEM COMPONENT_SRCS "${src}")
+            endif()
+        endforeach()
+    endforeach()
+
     # add as a PUBLIC library (if there are source files) or INTERFACE (if header only)
     if(COMPONENT_SRCS OR embed_binaries)
         add_library(${component} STATIC ${COMPONENT_SRCS})
@@ -84,6 +102,11 @@ function(register_component)
         endif()
         target_include_directories(${component} PRIVATE ${abs_dir})
     endforeach()
+
+    if(component IN_LIST BUILD_TEST_COMPONENTS)
+        target_link_libraries(${component} "-L${CMAKE_CURRENT_BINARY_DIR}")
+        target_link_libraries(${component} "-Wl,--whole-archive -l${component} -Wl,--no-whole-archive")
+    endif()
 endfunction()
 
 function(register_config_only_component)
@@ -135,7 +158,7 @@ function(components_finish_registration)
 
             get_target_property(a_type ${a} TYPE)
             if(${a_type} MATCHES .+_LIBRARY)
-                set(COMPONENT_LIBRARIES "${COMPONENT_LIBRARIES};${a}")
+                list(APPEND COMPONENT_LIBRARIES ${a})
             endif()
         endif()
     endforeach()

tools/cmake/convert_to_cmake.py

@@ -42,7 +42,7 @@ def get_make_variables(path, makefile="Makefile", expected_failure=False, variab
     result = {}
     BUILT_IN_VARS = set(["MAKEFILE_LIST", "SHELL", "CURDIR", "MAKEFLAGS"])
 
-    for line in output.decode().split("\n"):
+    for line in output.decode('utf-8').split("\n"):
         if line.startswith("# makefile"):  # this line appears before any variable defined in the makefile itself
             next_is_makefile = True
         elif next_is_makefile:
@@ -82,10 +82,20 @@ def get_component_variables(project_path, component_path):
             if src is not None:
                 srcs.append(src)
         make_vars["COMPONENT_SRCS"] = " ".join(srcs)
-    else:  # Use COMPONENT_SRCDIRS
+    else:
-        make_vars["COMPONENT_SRCDIRS"] = make_vars.get("COMPONENT_SRCDIRS", ".")
+        component_srcs = list()
+        for component_srcdir in make_vars.get("COMPONENT_SRCDIRS", ".").split(" "):
+            component_srcdir_path = os.path.abspath(os.path.join(component_path, component_srcdir))
+            
+            srcs = list()
+            srcs += glob.glob(os.path.join(component_srcdir_path, "*.[cS]"))
+            srcs += glob.glob(os.path.join(component_srcdir_path, "*.cpp"))
+            srcs = [('"%s"' % str(os.path.relpath(s, component_path))) for s in srcs]
 
         make_vars["COMPONENT_ADD_INCLUDEDIRS"] = make_vars.get("COMPONENT_ADD_INCLUDEDIRS", "include")
+        component_srcs += srcs
+        make_vars["COMPONENT_SRCS"] = " ".join(component_srcs)
+
 
     return make_vars
 
@@ -106,32 +116,10 @@ def convert_project(project_path):
 
     component_paths = project_vars["COMPONENT_PATHS"].split(" ")
 
-    # "main" component is made special in cmake, so extract it from the component_paths list
-    try:
-        main_component_path = [ p for p in component_paths if os.path.basename(p) == "main" ][0]
-        if debug:
-            print("Found main component %s"  % main_component_path)
-        main_vars = get_component_variables(project_path, main_component_path)
-    except IndexError:
-        print("WARNING: Project has no 'main' component, but CMake-based system requires at least one file in MAIN_SRCS...")
-        main_vars = { "COMPONENT_SRCS" : ""} # dummy for MAIN_SRCS
-
-    # Remove main component from list of components we're converting to cmake
-    component_paths = [ p for p in component_paths if os.path.basename(p) != "main" ]
-
     # Convert components as needed
     for p in component_paths:
         convert_component(project_path, p)
 
-    # Look up project variables before we start writing the file, so nothing
-    # is created if there is an error
-
-    main_srcs = main_vars["COMPONENT_SRCS"].split(" ")
-    # convert from component-relative to absolute paths
-    main_srcs = [ os.path.normpath(os.path.join(main_component_path, m)) for m in main_srcs ]
-    # convert to make relative to the project directory
-    main_srcs = [ os.path.relpath(m, project_path) for m in main_srcs ]
-
     project_name = project_vars["PROJECT_NAME"]
 
     # Generate the project CMakeLists.txt file
@@ -139,12 +127,11 @@ def convert_project(project_path):
         f.write("""
 # (Automatically converted from project Makefile by convert_to_cmake.py.)
 
-# The following four lines of boilerplate have to be in your project's CMakeLists
+# The following lines of boilerplate have to be in your project's CMakeLists
 # in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.5)
 
 """)
-        f.write("set(MAIN_SRCS %s)\n" % " ".join(main_srcs))
         f.write("""
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 """)
@@ -164,16 +151,6 @@ def convert_component(project_path, component_path):
     # Look up all the variables before we start writing the file, so it's not
     # created if there's an erro
     component_srcs = v.get("COMPONENT_SRCS", None)
-    component_srcdirs = None
-    if component_srcs is not None:
-        # see if we should be using COMPONENT_SRCS or COMPONENT_SRCDIRS, if COMPONENT_SRCS is everything in SRCDIRS
-        component_allsrcs = []
-        for d in v.get("COMPONENT_SRCDIRS", "").split(" "):
-            component_allsrcs += glob.glob(os.path.normpath(os.path.join(component_path, d, "*.[cS]")))
-            component_allsrcs += glob.glob(os.path.normpath(os.path.join(component_path, d, "*.cpp")))
-        abs_component_srcs = [os.path.normpath(os.path.join(component_path, p)) for p in component_srcs.split(" ")]
-        if set(component_allsrcs) == set(abs_component_srcs):
-            component_srcdirs = v.get("COMPONENT_SRCDIRS")
 
     component_add_includedirs = v["COMPONENT_ADD_INCLUDEDIRS"]
     cflags = v.get("CFLAGS", None)
@@ -185,10 +162,7 @@ def convert_component(project_path, component_path):
         f.write("set(COMPONENT_REQUIRES "")\n")
         f.write("set(COMPONENT_PRIV_REQUIRES "")\n\n")
 
-        if component_srcdirs is not None:
+        if component_srcs is not None:
-            f.write("set(COMPONENT_SRCDIRS %s)\n\n" % component_srcdirs)
-            f.write("register_component()\n")
-        elif component_srcs is not None:
             f.write("set(COMPONENT_SRCS %s)\n\n" % component_srcs)
             f.write("register_component()\n")
         else:

tools/cmake/crosstool_version_check.cmake

@@ -17,7 +17,6 @@ function(crosstool_version_check expected_ctng_version)
         OUTPUT_QUIET)
 
     string(REGEX MATCH "crosstool-ng-[0-9a-g\\.-]+" ctng_version "${toolchain_stderr}")
-    string(REPLACE "crosstool-ng-" "" ctng_version "${ctng_version}")
     # We use FIND to match version instead of STREQUAL because some toolchains are built
     # with longer git hash strings than others. This will match any version which starts with
     # the expected version string.
@@ -30,3 +29,21 @@ function(crosstool_version_check expected_ctng_version)
             "doesn't match supported version ${expected_ctng_version}. ${ctng_version_warning}")
     endif()
 endfunction()
+
+function(get_expected_ctng_version _toolchain_ver _gcc_ver)
+    file(STRINGS ${IDF_PATH}/tools/toolchain_versions.mk config_contents)
+    foreach(name_and_value ${config_contents})
+        # Strip spaces
+        string(REPLACE " " "" name_and_value ${name_and_value})
+        # Find variable name
+        string(REGEX MATCH "^[^=]+" name ${name_and_value})
+        # Find the value
+        string(REPLACE "${name}=" "" value ${name_and_value})
+        # Getting values
+        if("${name}" STREQUAL "SUPPORTED_TOOLCHAIN_COMMIT_DESC")
+            set("${_toolchain_ver}" "${value}" PARENT_SCOPE)
+        elseif("${name}" STREQUAL "SUPPORTED_TOOLCHAIN_GCC_VERSIONS")
+            set(${_gcc_ver} "${value}" PARENT_SCOPE)
+        endif()
+    endforeach()
+endfunction()

tools/cmake/idf_functions.cmake

@@ -15,15 +15,22 @@ macro(idf_set_global_variables)
 
     # Commmon components, required by every component in the build
     #
-    set_default(COMPONENT_REQUIRES_COMMON "cxx esp32 newlib freertos heap log soc")
+    set_default(COMPONENT_REQUIRES_COMMON "esp8266 newlib freertos heap log")
 
     # PROJECT_PATH has the path to the IDF project (top-level cmake directory)
     #
     # (cmake calls this CMAKE_SOURCE_DIR, keeping old name for compatibility.)
     set(PROJECT_PATH "${CMAKE_SOURCE_DIR}")
 
-    # Note: Unlike older build system, "main" is no longer a component. See build docs for details.
+    if(MAIN_SRCS)
-    set_default(COMPONENT_DIRS "${PROJECT_PATH}/components ${EXTRA_COMPONENT_DIRS} ${IDF_PATH}/components")
+        message(WARNING "main is now a component, use of MAIN_SRCS is deprecated")
+        set_default(COMPONENT_DIRS "${PROJECT_PATH}/components ${EXTRA_COMPONENT_DIRS} \
+                                    ${IDF_PATH}/components")
+    else()
+        set_default(COMPONENT_DIRS "${PROJECT_PATH}/components ${EXTRA_COMPONENT_DIRS} \
+                                    ${IDF_PATH}/components ${PROJECT_PATH}/main")
+    endif()
+
     spaces2list(COMPONENT_DIRS)
 
     spaces2list(COMPONENTS)
@@ -33,6 +40,13 @@ macro(idf_set_global_variables)
 
     # path to idf.py tool
     set(IDFTOOL ${PYTHON} "${IDF_PATH}/tools/idf.py")
+
+    # Temporary trick to support both gcc5 and gcc8 builds
+    if(CMAKE_C_COMPILER_VERSION VERSION_EQUAL 5.2.0)
+        set(GCC_NOT_5_2_0 0)
+    else()
+        set(GCC_NOT_5_2_0 1)
+    endif()
 endmacro()
 
 # Add all the IDF global compiler & preprocessor options
@@ -51,14 +65,16 @@ function(idf_set_global_compiler_options)
         add_compile_options(-Og)
     endif()
 
-    add_c_compile_options(-std=gnu99)
+    # Note: the visual studio generator doesn't support this syntax
+    add_compile_options("$<$<COMPILE_LANGUAGE:C>:-std=gnu99>")
 
-    add_cxx_compile_options(-std=gnu++11 -fno-rtti)
+    add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:-std=gnu++11>")
+    add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:-fno-rtti>")
 
     if(CONFIG_CXX_EXCEPTIONS)
-        add_cxx_compile_options(-fexceptions)
+        add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:-fexceptions>")
     else()
-        add_cxx_compile_options(-fno-exceptions)
+        add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions>")
     endif()
 
     # Default compiler configuration
@@ -75,10 +91,17 @@ function(idf_set_global_compiler_options)
         -Wextra
         -Wno-unused-parameter
         -Wno-sign-compare)
-    add_c_compile_options(
+    add_compile_options("$<$<COMPILE_LANGUAGE:C>:-Wno-old-style-declaration>")
-        -Wno-old-style-declaration
+
+    if(CONFIG_DISABLE_GCC8_WARNINGS)
+        add_compile_options(
+            -Wno-parentheses
+            -Wno-sizeof-pointer-memaccess
+            -Wno-clobbered
         )
 
+    endif()
+
     # Stack protection
     if(NOT BOOTLOADER_BUILD)
         if(CONFIG_STACK_CHECK_NORM)
@@ -98,8 +121,6 @@ function(idf_set_global_compiler_options)
     # go into the final binary so have no impact on size)
     add_compile_options(-ggdb)
 
-    add_compile_options("-I${CMAKE_BINARY_DIR}") # for sdkconfig.h
-
     # Enable ccache if it's on the path
     if(NOT CCACHE_DISABLE)
         find_program(CCACHE_FOUND ccache)
@@ -109,6 +130,8 @@ function(idf_set_global_compiler_options)
         endif()
     endif()
 
+    # Temporary trick to support both gcc5 and gcc8 builds
+    add_definitions(-DGCC_NOT_5_2_0=${GCC_NOT_5_2_0})
 endfunction()
 
 
@@ -121,15 +144,17 @@ function(idf_verify_environment)
 
     # Check toolchain is configured properly in cmake
     if(NOT ( ${CMAKE_SYSTEM_NAME} STREQUAL "Generic" AND ${CMAKE_C_COMPILER} MATCHES xtensa))
-        message(FATAL_ERROR "Internal error, toolchain has not been set correctly by project")
+        message(FATAL_ERROR "Internal error, toolchain has not been set correctly by project "
+            "(or an invalid CMakeCache.txt file has been generated somehow)")
     endif()
 
     #
     # Warn if the toolchain version doesn't match
     #
     # TODO: make these platform-specific for diff toolchains
-    #gcc_version_check("5.2.0")
+    get_expected_ctng_version(expected_toolchain expected_gcc)
-    #crosstool_version_check("1.22.0-80-g6c4433a")
+    gcc_version_check("${expected_gcc}")
+    crosstool_version_check("${expected_toolchain}")
 
 endfunction()
 
@@ -141,8 +166,22 @@ endfunction()
 function(idf_add_executable)
     set(exe_target ${PROJECT_NAME}.elf)
 
+    if(MAIN_SRCS)
         spaces2list(MAIN_SRCS)
-    add_executable(${exe_target} "${MAIN_SRCS}")
+        add_executable(${exe_target} ${MAIN_SRCS})
+    else()
+        # Create a dummy file to work around CMake requirement of having a source
+        # file while adding an executable
+        add_executable(${exe_target} "${CMAKE_CURRENT_BINARY_DIR}/dummy_main_src.c")
+        add_custom_command(OUTPUT dummy_main_src.c
+            COMMAND ${CMAKE_COMMAND} -E touch dummy_main_src.c
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+            VERBATIM)
+
+        add_custom_target(dummy_main_src DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/dummy_main_src.c)
+
+        add_dependencies(${exe_target} dummy_main_src)
+    endif()
 
     add_map_file(${exe_target})
 endfunction()
@@ -196,7 +235,11 @@ endfunction()
 # Running git_describe() here automatically triggers rebuilds
 # if the ESP-IDF git version changes
 function(idf_get_git_revision)
+    if(EXISTS "${IDF_PATH}/version.txt")
+        file(STRINGS "${IDF_PATH}/version.txt" IDF_VER)
+    else()
         git_describe(IDF_VER "${IDF_PATH}")
+    endif()
     add_definitions(-DIDF_VER=\"${IDF_VER}\")
     git_submodule_check("${IDF_PATH}")
     set(IDF_VER ${IDF_VER} PARENT_SCOPE)

tools/cmake/kconfig.cmake

@@ -1,27 +1,46 @@
 include(ExternalProject)
 
 macro(kconfig_set_variables)
+    set(CONFIG_DIR ${CMAKE_BINARY_DIR}/config)
     set_default(SDKCONFIG ${PROJECT_PATH}/sdkconfig)
-    set(SDKCONFIG_HEADER ${CMAKE_BINARY_DIR}/sdkconfig.h)
+    set(SDKCONFIG_HEADER ${CONFIG_DIR}/sdkconfig.h)
-    set(SDKCONFIG_CMAKE ${CMAKE_BINARY_DIR}/sdkconfig.cmake)
+    set(SDKCONFIG_CMAKE ${CONFIG_DIR}/sdkconfig.cmake)
-    set(SDKCONFIG_JSON ${CMAKE_BINARY_DIR}/sdkconfig.json)
+    set(SDKCONFIG_JSON ${CONFIG_DIR}/sdkconfig.json)
+    set(KCONFIG_JSON_MENUS ${CONFIG_DIR}/kconfig_menus.json)
 
     set(ROOT_KCONFIG ${IDF_PATH}/Kconfig)
 
     set_default(SDKCONFIG_DEFAULTS "${SDKCONFIG}.defaults")
+
+    # ensure all source files can include sdkconfig.h
+    include_directories("${CONFIG_DIR}")
 endmacro()
 
 if(CMAKE_HOST_WIN32)
-    # Prefer a prebuilt mconf on Windows
+    # Prefer a prebuilt mconf-idf on Windows
+    if(DEFINED ENV{MSYSTEM})
         find_program(WINPTY winpty)
+    else()
+        unset(WINPTY CACHE)  # in case previous CMake run was in a tty and this one is not
+    endif()
+    find_program(MCONF mconf-idf)
+
+    # Fall back to the old binary which was called 'mconf' not 'mconf-idf'
+    if(NOT MCONF)
         find_program(MCONF mconf)
+        if(MCONF)
+            message(WARNING "Falling back to mconf binary '${MCONF}' not mconf-idf. "
+                "This is probably because an old version of IDF mconf is installed and this is fine. "
+                "However if there are config problems please check the Getting Started guide for your platform.")
+        endif()
+    endif()
 
     if(NOT MCONF)
         find_program(NATIVE_GCC gcc)
         if(NOT NATIVE_GCC)
             message(FATAL_ERROR
-                "Windows requires a prebuilt ESP-IDF-specific mconf for your platform "
+                "Windows requires a prebuilt mconf-idf for your platform "
-                "on the PATH, or an MSYS2 version of gcc on the PATH to build mconf. "
+                "on the PATH, or an MSYS2 version of gcc on the PATH to build mconf-idf. "
                 "Consult the setup docs for ESP-IDF on Windows.")
         endif()
     elseif(WINPTY)
@@ -32,29 +51,38 @@ endif()
 if(NOT MCONF)
     # Use the existing Makefile to build mconf (out of tree) when needed
     #
-    set(MCONF kconfig_bin/mconf)
+    set(MCONF kconfig_bin/mconf-idf)
 
-    externalproject_add(mconf
+    externalproject_add(mconf-idf
         SOURCE_DIR ${IDF_PATH}/tools/kconfig
         CONFIGURE_COMMAND ""
         BINARY_DIR "kconfig_bin"
-        BUILD_COMMAND make -f ${IDF_PATH}/tools/kconfig/Makefile mconf
+        BUILD_COMMAND make -f ${IDF_PATH}/tools/kconfig/Makefile mconf-idf
         BUILD_BYPRODUCTS ${MCONF}
         INSTALL_COMMAND ""
         EXCLUDE_FROM_ALL 1
         )
-    set(menuconfig_depends DEPENDS mconf)
+
+    file(GLOB mconf_srcfiles ${IDF_PATH}/tools/kconfig/*.c)
+    externalproject_add_stepdependencies(mconf-idf build
+        ${mconf_srcfiles}
+        ${IDF_PATH}/tools/kconfig/Makefile
+        ${CMAKE_CURRENT_LIST_FILE})
+    unset(mconf_srcfiles)
+
+    set(menuconfig_depends DEPENDS mconf-idf)
+
 endif()
 
 # Find all Kconfig files for all components
 function(kconfig_process_config)
-    file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/include/config")
+    file(MAKE_DIRECTORY "${CONFIG_DIR}")
     set(kconfigs)
     set(kconfigs_projbuild)
 
     # Find Kconfig and Kconfig.projbuild for each component as applicable
     # if any of these change, cmake should rerun
-    foreach(dir ${BUILD_COMPONENT_PATHS} "${CMAKE_SOURCE_DIR}/main")
+    foreach(dir ${BUILD_COMPONENT_PATHS})
         file(GLOB kconfig "${dir}/Kconfig")
         if(kconfig)
             set(kconfigs "${kconfigs} ${kconfig}")
@@ -81,11 +109,11 @@ function(kconfig_process_config)
         --kconfig ${ROOT_KCONFIG}
         --config ${SDKCONFIG}
         ${defaults_arg}
-        --create-config-if-missing
         --env "COMPONENT_KCONFIGS=${kconfigs}"
-        --env "COMPONENT_KCONFIGS_PROJBUILD=${kconfigs_projbuild}")
+        --env "COMPONENT_KCONFIGS_PROJBUILD=${kconfigs_projbuild}"
+        --env "IDF_CMAKE=y")
 
-    # Generate the menuconfig target (uses C-based mconf tool, either prebuilt or via mconf target above)
+    # Generate the menuconfig target (uses C-based mconf-idf tool, either prebuilt or via mconf-idf target above)
     add_custom_target(menuconfig
         ${menuconfig_depends}
         # create any missing config file, with defaults if necessary
@@ -93,20 +121,44 @@ function(kconfig_process_config)
         COMMAND ${CMAKE_COMMAND} -E env
         "COMPONENT_KCONFIGS=${kconfigs}"
         "COMPONENT_KCONFIGS_PROJBUILD=${kconfigs_projbuild}"
+        "IDF_CMAKE=y"
         "KCONFIG_CONFIG=${SDKCONFIG}"
         ${MCONF} ${ROOT_KCONFIG}
         VERBATIM
         USES_TERMINAL)
 
+    # Custom target to run confserver.py from the build tool
+    add_custom_target(confserver
+        COMMAND ${CMAKE_COMMAND} -E env
+        "COMPONENT_KCONFIGS=${kconfigs}"
+        "COMPONENT_KCONFIGS_PROJBUILD=${kconfigs_projbuild}"
+        ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confserver.py
+        --kconfig ${IDF_PATH}/Kconfig --config ${SDKCONFIG}
+        VERBATIM
+        USES_TERMINAL)
+
     # Generate configuration output via confgen.py
     # makes sdkconfig.h and skdconfig.cmake
     #
     # This happens during the cmake run not during the build
-    execute_process(COMMAND ${confgen_basecommand}
+    if(NOT BOOTLOADER_BUILD)
+        execute_process(
+            COMMAND ${confgen_basecommand}
             --output header ${SDKCONFIG_HEADER}
             --output cmake ${SDKCONFIG_CMAKE}
             --output json ${SDKCONFIG_JSON}
+            --output json_menus ${KCONFIG_JSON_MENUS}
+            --output config ${SDKCONFIG} # only generate config at the top-level project
             RESULT_VARIABLE config_result)
+    else()
+        execute_process(
+            COMMAND ${confgen_basecommand}
+            --output header ${SDKCONFIG_HEADER}
+            --output cmake ${SDKCONFIG_CMAKE}
+            --output json ${SDKCONFIG_JSON}
+            --output json_menus ${KCONFIG_JSON_MENUS}
+            RESULT_VARIABLE config_result)
+    endif()
     if(config_result)
         message(FATAL_ERROR "Failed to run confgen.py (${confgen_basecommand}). Error ${config_result}")
     endif()

tools/cmake/project.cmake

@@ -6,11 +6,12 @@ cmake_minimum_required(VERSION 3.5)
 set(IDF_PATH "$ENV{IDF_PATH}")
 if(NOT IDF_PATH)
     # Documentation says you should set IDF_PATH in your environment, but we
-    # can infer it here if it's not set.
+    # can infer it relative to tools/cmake directory if it's not set.
-    set(IDF_PATH ${CMAKE_CURRENT_LIST_DIR})
+    get_filename_component(IDF_PATH "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE)
 endif()
 file(TO_CMAKE_PATH "${IDF_PATH}" IDF_PATH)
-set($ENV{IDF_PATH} "${IDF_PATH}")
+set(ENV{IDF_PATH} ${IDF_PATH})
+
 
 #
 # Load cmake modules
@@ -28,6 +29,15 @@ include(idf_functions)
 
 set_default(PYTHON "python")
 
+if(NOT PYTHON_DEPS_CHECKED AND NOT BOOTLOADER_BUILD)
+    message(STATUS "Checking Python dependencies...")
+    execute_process(COMMAND "${PYTHON}" "${IDF_PATH}/tools/check_python_dependencies.py"
+        RESULT_VARIABLE result)
+    if(NOT result EQUAL 0)
+        message(FATAL_ERROR "Some Python dependencies must be installed. Check above message for details.")
+    endif()
+endif()
+
 # project
 #
 # This macro wraps the cmake 'project' command to add
@@ -46,20 +56,24 @@ macro(project name)
     # Set global variables used by rest of the build
     idf_set_global_variables()
 
-    # Establish dependencies for components in the build
+    # Sort the components list, as it may be found via filesystem
-    # (this happens before we even generate config...)
+    # traversal and therefore in a non-deterministic order
-    if(COMPONENTS)
+    list(SORT COMPONENTS)
-        # Make sure if an explicit list of COMPONENTS is given, it contains the "common" component requirements
+
-        # (otherwise, if COMPONENTS is empty then all components will be included in the build.)
-        set(COMPONENTS "${COMPONENTS} ${COMPONENT_REQUIRES_COMMON}")
-    endif()
     execute_process(COMMAND "${CMAKE_COMMAND}"
         -D "COMPONENTS=${COMPONENTS}"
+        -D "COMPONENT_REQUIRES_COMMON=${COMPONENT_REQUIRES_COMMON}"
+        -D "EXCLUDE_COMPONENTS=${EXCLUDE_COMPONENTS}"
+        -D "TEST_COMPONENTS=${TEST_COMPONENTS}"
+        -D "TEST_EXCLUDE_COMPONENTS=${TEST_EXCLUDE_COMPONENTS}"
+        -D "TESTS_ALL=${TESTS_ALL}"
         -D "DEPENDENCIES_FILE=${CMAKE_BINARY_DIR}/component_depends.cmake"
         -D "COMPONENT_DIRS=${COMPONENT_DIRS}"
         -D "BOOTLOADER_BUILD=${BOOTLOADER_BUILD}"
+        -D "IDF_PATH=${IDF_PATH}"
+        -D "DEBUG=${DEBUG}"
         -P "${IDF_PATH}/tools/cmake/scripts/expand_requirements.cmake"
-        WORKING_DIRECTORY "${IDF_PATH}/tools/cmake")
+        WORKING_DIRECTORY "${PROJECT_PATH}")
     include("${CMAKE_BINARY_DIR}/component_depends.cmake")
 
     # We now have the following component-related variables:
@@ -73,6 +87,14 @@ macro(project name)
     unset(BUILD_COMPONENTS_SPACES)
     message(STATUS "Component paths: ${BUILD_COMPONENT_PATHS}")
 
+    # Print list of test components
+    if(TESTS_ALL EQUAL 1 OR TEST_COMPONENTS)
+        string(REPLACE ";" " " BUILD_TEST_COMPONENTS_SPACES "${BUILD_TEST_COMPONENTS}")
+        message(STATUS "Test component names: ${BUILD_TEST_COMPONENTS_SPACES}")
+        unset(BUILD_TEST_COMPONENTS_SPACES)
+        message(STATUS "Test component paths: ${BUILD_TEST_COMPONENT_PATHS}")
+    endif()
+
     kconfig_set_variables()
 
     kconfig_process_config()
@@ -83,7 +105,6 @@ macro(project name)
     # Now the configuration is loaded, set the toolchain appropriately
     #
     # TODO: support more toolchains than just ESP32
-    #set(CMAKE_TOOLCHAIN_FILE $ENV{IDF_PATH}/tools/cmake/toolchain-esp32.cmake)
     set(CMAKE_TOOLCHAIN_FILE $ENV{IDF_PATH}/tools/cmake/toolchain-esp8266.cmake)
 
     # Declare the actual cmake-level project
@@ -106,14 +127,24 @@ macro(project name)
 
     # Include any top-level project_include.cmake files from components
     foreach(component ${BUILD_COMPONENT_PATHS})
+        set(COMPONENT_PATH "${component}")
         include_if_exists("${component}/project_include.cmake")
+        unset(COMPONENT_PATH)
     endforeach()
 
     #
     # Add each component to the build as a library
     #
     foreach(COMPONENT_PATH ${BUILD_COMPONENT_PATHS})
+        list(FIND BUILD_TEST_COMPONENT_PATHS ${COMPONENT_PATH} idx)
+
+        if(NOT idx EQUAL -1)
+            list(GET BUILD_TEST_COMPONENTS ${idx} test_component)
+            set(COMPONENT_NAME ${test_component})
+        else()
             get_filename_component(COMPONENT_NAME ${COMPONENT_PATH} NAME)
+        endif()
+
         add_subdirectory(${COMPONENT_PATH} ${COMPONENT_NAME})
     endforeach()
     unset(COMPONENT_NAME)

tools/cmake/run_cmake_lint.sh

@@ -15,18 +15,11 @@ if [ -z "${IDF_PATH}" ]; then
     exit 3
 fi
 
-# exclusions include some third-party directories which contain upstream
+cd "$IDF_PATH"
-# CMakeLists files
+
-find ${IDF_PATH} \
+# Only list the "main" IDF repo, don't check any files in submodules (which may contain
-     -name build -prune \
+# third party CMakeLists.txt)
-     -o -name third_party -prune \
+ git ls-tree --full-tree --name-only -r HEAD | grep -v "/third_party/" | grep "^CMakeLists.txt$\|\.cmake$" \
-     \
+    | xargs cmakelint --linelength=120 --spaces=4
-     -o -name 'nghttp2' -prune \
-     -o -name 'cJSON' -prune \
-     -o -name 'Findsodium.cmake' -prune \
-     \
-     -o -name CMakeLists.txt -print0 \
-     -o -name '*.cmake' -print0 \
-    | xargs -0 cmakelint --linelength=120 --spaces=4
 
 

tools/cmake/scripts/data_file_embed_asm.cmake

@@ -67,6 +67,7 @@ endif()
 append_line(" */")
 
 append_line(".data")
+append_line(".section .rodata.embedded")
 append_identifier("${varname}")
 append_identifier("_binary_${varname}_start" "for objcopy compatibility")
 append("${data}")

tools/cmake/scripts/expand_requirements.cmake

@@ -4,6 +4,8 @@
 # Parameters:
 # - COMPONENTS = Space-separated list of initial components to include in the build.
 #   Can be empty, in which case all components are in the build.
+# - COMPONENT_REQUIRES_COMMON = Components to always include in the build, and treated as dependencies
+#   of all other components.
 # - DEPENDENCIES_FILE = Path of generated cmake file which will contain the expanded dependencies for these
 #   components.
 # - COMPONENT_DIRS = List of paths to search for all components.
@@ -13,9 +15,22 @@
 # components required for the build, and the get_component_requirements() function to return each component's
 # recursively expanded requirements.
 #
+# BUILD_COMPONENTS & BUILD_COMPONENT_PATHS will be ordered in a best-effort way so that dependencies are listed first.
+# (Note that IDF supports cyclic dependencies, and dependencies in a cycle have ordering guarantees.)
+#
+# Determinism:
+#
+# Given the the same list of names in COMPONENTS (regardless of order), and an identical value of
+# COMPONENT_REQUIRES_COMMON, and all the same COMPONENT_REQUIRES & COMPONENT_PRIV_REQUIRES values in
+# each component, then the output of BUILD_COMPONENTS should always be in the same
+# order.
+#
+# BUILD_COMPONENT_PATHS will be in the same component order as BUILD_COMPONENTS, even if the
+# actual component paths are different due to different paths.
+#
 # TODO: Error out if a component requirement is missing
 cmake_minimum_required(VERSION 3.5)
-include("utilities.cmake")
+include("${IDF_PATH}/tools/cmake/utilities.cmake")
 
 if(NOT DEPENDENCIES_FILE)
     message(FATAL_ERROR "DEPENDENCIES_FILE must be set.")
@@ -26,6 +41,8 @@ if(NOT COMPONENT_DIRS)
 endif()
 spaces2list(COMPONENT_DIRS)
 
+spaces2list(COMPONENT_REQUIRES_COMMON)
+
 function(debug message)
     if(DEBUG)
         message(STATUS "${message}")
@@ -37,8 +54,16 @@ endfunction()
 # (expand_component_requirements() includes the component CMakeLists.txt, which then sets its component variables,
 # calls this dummy macro, and immediately exits again.)
 macro(register_component)
+    if(COMPONENT STREQUAL main AND NOT COMPONENT_REQUIRES)
+        set(main_component_requires ${COMPONENTS})
+        list(REMOVE_ITEM main_component_requires "main")
+
+        set_property(GLOBAL PROPERTY "${COMPONENT}_REQUIRES" "${main_component_requires}")
+    else()
         spaces2list(COMPONENT_REQUIRES)
         set_property(GLOBAL PROPERTY "${COMPONENT}_REQUIRES" "${COMPONENT_REQUIRES}")
+    endif()
+
     spaces2list(COMPONENT_PRIV_REQUIRES)
     set_property(GLOBAL PROPERTY "${COMPONENT}_PRIV_REQUIRES" "${COMPONENT_PRIV_REQUIRES}")
 
@@ -57,14 +82,14 @@ endmacro()
 # return the path to the component in 'variable'
 #
 # Fatal error is printed if the component is not found.
-function(find_component_path find_name component_paths variable)
+function(find_component_path find_name components component_paths variable)
-    foreach(path ${component_paths})
+    list(FIND components ${find_name} idx)
-        get_filename_component(name "${path}" NAME)
+    if(NOT idx EQUAL -1)
-        if("${name}" STREQUAL "${find_name}")
+        list(GET component_paths ${idx} path)
         set("${variable}" "${path}" PARENT_SCOPE)
         return()
+    else()
     endif()
-    endforeach()
     # TODO: find a way to print the dependency chain that lead to this not-found component
     message(WARNING "Required component ${find_name} is not found in any of the provided COMPONENT_DIRS")
 endfunction()
@@ -75,10 +100,11 @@ endfunction()
 #
 # component_paths contains only unique component names. Directories
 # earlier in the component_dirs list take precedence.
-function(components_find_all component_dirs component_paths component_names)
+function(components_find_all component_dirs component_paths component_names test_component_names)
     # component_dirs entries can be files or lists of files
     set(paths "")
     set(names "")
+    set(test_names "")
 
     # start by expanding the component_dirs list with all subdirectories
     foreach(dir ${component_dirs})
@@ -91,15 +117,22 @@ function(components_find_all component_dirs component_paths component_names)
 
     # Look for a component in each component_dirs entry
     foreach(dir ${component_dirs})
+        debug("Looking for CMakeLists.txt in ${dir}")
         file(GLOB component "${dir}/CMakeLists.txt")
         if(component)
+            debug("CMakeLists.txt file ${component}")
             get_filename_component(component "${component}" DIRECTORY)
             get_filename_component(name "${component}" NAME)
             if(NOT name IN_LIST names)
-                set(names "${names};${name}")
+                list(APPEND names "${name}")
-                set(paths "${paths};${component}")
+                list(APPEND paths "${component}")
-            endif()
 
+                # Look for test component directory
+                file(GLOB test "${component}/test/CMakeLists.txt")
+                if(test)
+                    list(APPEND test_names "${name}")
+                endif()
+            endif()
         else()  # no CMakeLists.txt file
             # test for legacy component.mk and warn
             file(GLOB legacy_component "${dir}/component.mk")
@@ -109,26 +142,28 @@ function(components_find_all component_dirs component_paths component_names)
                     "Component will be skipped.")
             endif()
         endif()
-
     endforeach()
 
     set(${component_paths} ${paths} PARENT_SCOPE)
     set(${component_names} ${names} PARENT_SCOPE)
+    set(${test_component_names} ${test_names} PARENT_SCOPE)
 endfunction()
 
+
 # expand_component_requirements: Recursively expand a component's requirements,
 # setting global properties BUILD_COMPONENTS & BUILD_COMPONENT_PATHS and
 # also invoking the components to call register_component() above,
 # which will add per-component global properties with dependencies, etc.
 function(expand_component_requirements component)
-    get_property(build_components GLOBAL PROPERTY BUILD_COMPONENTS)
+    get_property(seen_components GLOBAL PROPERTY SEEN_COMPONENTS)
-    if(${component} IN_LIST build_components)
+    if(component IN_LIST seen_components)
-        return()  # already added this component
+        return()  # already added, or in process of adding, this component
     endif()
+    set_property(GLOBAL APPEND PROPERTY SEEN_COMPONENTS ${component})
 
-    find_component_path("${component}" "${ALL_COMPONENT_PATHS}" component_path)
+    find_component_path("${component}" "${ALL_COMPONENTS}" "${ALL_COMPONENT_PATHS}" COMPONENT_PATH)
-    debug("Expanding dependencies of ${component} @ ${component_path}")
+    debug("Expanding dependencies of ${component} @ ${COMPONENT_PATH}")
-    if(NOT component_path)
+    if(NOT COMPONENT_PATH)
         set_property(GLOBAL APPEND PROPERTY COMPONENTS_NOT_FOUND ${component})
         return()
     endif()
@@ -138,43 +173,124 @@ function(expand_component_requirements component)
     unset(COMPONENT_REQUIRES)
     unset(COMPONENT_PRIV_REQUIRES)
     set(COMPONENT ${component})
-    include(${component_path}/CMakeLists.txt)
+    include(${COMPONENT_PATH}/CMakeLists.txt)
-
-    set_property(GLOBAL APPEND PROPERTY BUILD_COMPONENT_PATHS ${component_path})
-    set_property(GLOBAL APPEND PROPERTY BUILD_COMPONENTS ${component})
 
     get_property(requires GLOBAL PROPERTY "${component}_REQUIRES")
     get_property(requires_priv GLOBAL PROPERTY "${component}_PRIV_REQUIRES")
-    foreach(req ${requires} ${requires_priv})
+
+    # Recurse dependencies first, so that they appear first in the list (when possible)
+    foreach(req ${COMPONENT_REQUIRES_COMMON} ${requires} ${requires_priv})
         expand_component_requirements(${req})
     endforeach()
+
+    list(FIND TEST_COMPONENTS ${component} idx)
+
+    if(NOT idx EQUAL -1)
+        list(GET TEST_COMPONENTS ${idx} test_component)
+        list(GET TEST_COMPONENT_PATHS ${idx} test_component_path)
+        set_property(GLOBAL APPEND PROPERTY BUILD_TEST_COMPONENTS ${test_component})
+        set_property(GLOBAL APPEND PROPERTY BUILD_TEST_COMPONENT_PATHS ${test_component_path})
+    endif()
+
+    # Now append this component to the full list (after its dependencies)
+    set_property(GLOBAL APPEND PROPERTY BUILD_COMPONENT_PATHS ${COMPONENT_PATH})
+    set_property(GLOBAL APPEND PROPERTY BUILD_COMPONENTS ${component})
 endfunction()
 
+# filter_components_list: Filter the components included in the build
+# as specified by the user. Or, in the case of unit testing, filter out
+# the test components to be built.
+macro(filter_components_list)
+    spaces2list(COMPONENTS)
+    spaces2list(EXCLUDE_COMPONENTS)
+    spaces2list(TEST_COMPONENTS)
+    spaces2list(TEST_EXCLUDE_COMPONENTS)
+
+    list(LENGTH ALL_COMPONENTS all_components_length)
+    math(EXPR all_components_length "${all_components_length} - 1")
+
+    foreach(component_idx RANGE 0 ${all_components_length})
+        list(GET ALL_COMPONENTS ${component_idx} component)
+        list(GET ALL_COMPONENT_PATHS ${component_idx} component_path)
+
+        if(COMPONENTS)
+            if(${component} IN_LIST COMPONENTS)
+                set(add_component 1)
+            else()
+                set(add_component 0)
+            endif()
+        else()
+            set(add_component 1)
+
+        endif()
+
+        if(NOT ${component} IN_LIST EXCLUDE_COMPONENTS AND add_component EQUAL 1)
+            list(APPEND components ${component})
+            list(APPEND component_paths ${component_path})
+
+            if(TESTS_ALL EQUAL 1 OR TEST_COMPONENTS)
+                if(NOT TESTS_ALL EQUAL 1 AND TEST_COMPONENTS)
+                    if(${component} IN_LIST TEST_COMPONENTS)
+                        set(add_test_component 1)
+                    else()
+                        set(add_test_component 0)
+                    endif()
+                else()
+                    set(add_test_component 1)
+                endif()
+
+                if(${component} IN_LIST ALL_TEST_COMPONENTS)
+                    if(NOT ${component} IN_LIST TEST_EXCLUDE_COMPONENTS AND add_test_component EQUAL 1)
+                        list(APPEND test_components ${component}_test)
+                        list(APPEND test_component_paths ${component_path}/test)
+
+                        list(APPEND components ${component}_test)
+                        list(APPEND component_paths ${component_path}/test)
+                    endif()
+                endif()
+            endif()
+        endif()
+    endforeach()
+
+    set(COMPONENTS ${components})
+
+    set(TEST_COMPONENTS ${test_components})
+    set(TEST_COMPONENT_PATHS ${test_component_paths})
+
+    list(APPEND ALL_COMPONENTS "${TEST_COMPONENTS}")
+    list(APPEND ALL_COMPONENT_PATHS "${TEST_COMPONENT_PATHS}")
+endmacro()
 
 # Main functionality goes here
-
 # Find every available component in COMPONENT_DIRS, save as ALL_COMPONENT_PATHS and ALL_COMPONENTS
-components_find_all("${COMPONENT_DIRS}" ALL_COMPONENT_PATHS ALL_COMPONENTS)
+components_find_all("${COMPONENT_DIRS}" ALL_COMPONENT_PATHS ALL_COMPONENTS ALL_TEST_COMPONENTS)
 
-if(NOT COMPONENTS)
+filter_components_list()
-    set(COMPONENTS "${ALL_COMPONENTS}")
-endif()
-spaces2list(COMPONENTS)
 
 debug("ALL_COMPONENT_PATHS ${ALL_COMPONENT_PATHS}")
 debug("ALL_COMPONENTS ${ALL_COMPONENTS}")
+debug("ALL_TEST_COMPONENTS ${ALL_TEST_COMPONENTS}")
 
+set_property(GLOBAL PROPERTY SEEN_COMPONENTS "")  # anti-infinite-recursion
 set_property(GLOBAL PROPERTY BUILD_COMPONENTS "")
 set_property(GLOBAL PROPERTY BUILD_COMPONENT_PATHS "")
+set_property(GLOBAL PROPERTY BUILD_TEST_COMPONENTS "")
+set_property(GLOBAL PROPERTY BUILD_TEST_COMPONENT_PATHS "")
 set_property(GLOBAL PROPERTY COMPONENTS_NOT_FOUND "")
 
+# Indicate that the component CMakeLists.txt is being included in the early expansion phase of the build,
+# and might not want to execute particular operations.
+set(CMAKE_BUILD_EARLY_EXPANSION 1)
 foreach(component ${COMPONENTS})
     debug("Expanding initial component ${component}")
     expand_component_requirements(${component})
 endforeach()
+unset(CMAKE_BUILD_EARLY_EXPANSION)
 
 get_property(build_components GLOBAL PROPERTY BUILD_COMPONENTS)
 get_property(build_component_paths GLOBAL PROPERTY BUILD_COMPONENT_PATHS)
+get_property(build_test_components GLOBAL PROPERTY BUILD_TEST_COMPONENTS)
+get_property(build_test_component_paths GLOBAL PROPERTY BUILD_TEST_COMPONENT_PATHS)
 get_property(not_found GLOBAL PROPERTY COMPONENTS_NOT_FOUND)
 
 debug("components in build: ${build_components}")
@@ -182,12 +298,14 @@ debug("components in build: ${build_component_paths}")
 debug("components not found: ${not_found}")
 
 function(line contents)
-    file(APPEND "${DEPENDENCIES_FILE}" "${contents}\n")
+    file(APPEND "${DEPENDENCIES_FILE}.tmp" "${contents}\n")
 endfunction()
 
-file(WRITE "${DEPENDENCIES_FILE}" "# Component requirements generated by expand_requirements.cmake\n\n")
+file(WRITE "${DEPENDENCIES_FILE}.tmp" "# Component requirements generated by expand_requirements.cmake\n\n")
 line("set(BUILD_COMPONENTS ${build_components})")
 line("set(BUILD_COMPONENT_PATHS ${build_component_paths})")
+line("set(BUILD_TEST_COMPONENTS ${build_test_components})")
+line("set(BUILD_TEST_COMPONENT_PATHS ${build_test_component_paths})")
 line("")
 
 line("# get_component_requirements: Generated function to read the dependencies of a given component.")
@@ -214,3 +332,7 @@ endforeach()
 
 line("  message(FATAL_ERROR \"Component not found: \${component}\")")
 line("endfunction()")
+
+# only replace DEPENDENCIES_FILE if it has changed (prevents ninja/make build loops.)
+execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${DEPENDENCIES_FILE}.tmp" "${DEPENDENCIES_FILE}")
+execute_process(COMMAND ${CMAKE_COMMAND} -E remove "${DEPENDENCIES_FILE}.tmp")

tools/cmake/scripts/fail.cmake

@@ -0,0 +1,4 @@
+# 'cmake -E' doesn't have a way to fail outright, so run this script
+# with 'cmake -P' to fail a build.
+message(FATAL_ERROR "Failing the build (see errors on lines above)")
+

tools/cmake/third_party/GetGitRevisionDescription.cmake

@@ -109,7 +109,9 @@ function(git_describe _var _repo_dir)
 
 	execute_process(COMMAND
 		"${GIT_EXECUTABLE}"
-		describe
+		"-C"
+		${_repo_dir}
+		describe --tag
 		${hash}
 		${ARGN}
 		WORKING_DIRECTORY

tools/cmake/toolchain-esp32.cmake

@@ -1,7 +0,0 @@
-set(CMAKE_SYSTEM_NAME Generic)
-
-set(CMAKE_C_COMPILER xtensa-esp32-elf-gcc)
-set(CMAKE_CXX_COMPILER xtensa-esp32-elf-g++)
-set(CMAKE_ASM_COMPILER xtensa-esp32-elf-gcc)
-
-set(CMAKE_EXE_LINKER_FLAGS "-nostdlib" CACHE STRING "Linker Base Flags")

tools/cmake/toolchain-esp8266.cmake

@@ -3,6 +3,5 @@ set(CMAKE_SYSTEM_NAME Generic)
 set(CMAKE_C_COMPILER xtensa-lx106-elf-gcc)
 set(CMAKE_CXX_COMPILER xtensa-lx106-elf-g++)
 set(CMAKE_ASM_COMPILER xtensa-lx106-elf-gcc)
-set(CMAKE_OBJCOPY_COMPILER xtensa-lx106-elf-objcopy)
 
 set(CMAKE_EXE_LINKER_FLAGS "-nostdlib" CACHE STRING "Linker Base Flags")

tools/cmake/utilities.cmake

@@ -9,7 +9,7 @@
 #
 function(set_default variable default_value)
     if(NOT ${variable})
-        if($ENV{${variable}})
+        if(DEFINED ENV{${variable}} AND NOT "$ENV{${variable}}" STREQUAL "")
             set(${variable} $ENV{${variable}} PARENT_SCOPE)
         else()
             set(${variable} ${default_value} PARENT_SCOPE)
@@ -31,7 +31,6 @@ function(spaces2list variable_name)
     set("${variable_name}" "${tmp}" PARENT_SCOPE)
 endfunction()
 
-
 # lines2list
 #
 # Take a variable with multiple lines of output in it, convert it
@@ -74,30 +73,6 @@ function(move_if_different source destination)
 
 endfunction()
 
-
-# add_compile_options variant for C++ code only
-#
-# This adds global options, set target properties for
-# component-specific flags
-function(add_cxx_compile_options)
-    foreach(option ${ARGV})
-        # note: the Visual Studio Generator doesn't support this...
-        add_compile_options($<$<COMPILE_LANGUAGE:CXX>:${option}>)
-    endforeach()
-endfunction()
-
-# add_compile_options variant for C code only
-#
-# This adds global options, set target properties for
-# component-specific flags
-function(add_c_compile_options)
-    foreach(option ${ARGV})
-        # note: the Visual Studio Generator doesn't support this...
-        add_compile_options($<$<COMPILE_LANGUAGE:C>:${option}>)
-    endforeach()
-endfunction()
-
-
 # target_add_binary_data adds binary data into the built target,
 # by converting it to a generated source file which is then compiled
 # to a binary object as part of the build
@@ -132,7 +107,7 @@ endmacro()
 # Append a single line to the file specified
 # The line ending is determined by the host OS
 function(file_append_line file line)
-    if(ENV{MSYSTEM} OR CMAKE_HOST_WIN32)
+    if(DEFINED ENV{MSYSTEM} OR CMAKE_HOST_WIN32)
         set(line_ending "\r\n")
     else() # unix
         set(line_ending "\n")
@@ -179,3 +154,14 @@ function(make_json_list list variable)
     string(REPLACE ";" "\", \"" result "[ \"${list}\" ]")
     set("${variable}" "${result}" PARENT_SCOPE)
 endfunction()
+
+# add_prefix
+#
+# Adds a prefix to each item in the specified list.
+#
+function(add_prefix var prefix)
+    foreach(elm ${ARGN})
+        list(APPEND newlist "${prefix}${elm}")
+    endforeach()
+    set(${var} "${newlist}" PARENT_SCOPE)
+endfunction()

tools/idf.py

@@ -21,6 +21,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+# WARNING: we don't check for Python build-time dependencies until
+# check_environment() function below. If possible, avoid importing
+# any external libraries here - put in external script, or import in
+# their specific function instead.
 import sys
 import argparse
 import os
@@ -41,7 +46,7 @@ class FatalError(RuntimeError):
 PYTHON=sys.executable
 
 # note: os.environ changes don't automatically propagate to child processes,
-# you have to pass this in explicitly
+# you have to pass env=os.environ explicitly anywhere that we create a process
 os.environ["PYTHON"]=sys.executable
 
 # Make flavors, across the various kinds of Windows environments & POSIX...
@@ -71,14 +76,13 @@ def _run_tool(tool_name, args, cwd):
         return arg
     display_args = " ".join(quote_arg(arg) for arg in args)
     print("Running %s in directory %s" % (tool_name, quote_arg(cwd)))
-    print('Executing "%s"...' % display_args)
+    print('Executing "%s"...' % str(display_args))
     try:
         # Note: we explicitly pass in os.environ here, as we may have set IDF_PATH there during startup
         subprocess.check_call(args, env=os.environ, cwd=cwd)
     except subprocess.CalledProcessError as e:
         raise FatalError("%s failed with exit code %d" % (tool_name, e.returncode))
 
-
 def check_environment():
     """
     Verify the environment contains the top-level tools we need to operate
@@ -95,8 +99,18 @@ def check_environment():
             print("WARNING: IDF_PATH environment variable is set to %s but idf.py path indicates IDF directory %s. Using the environment variable directory, but results may be unexpected..."
                   % (set_idf_path, detected_idf_path))
     else:
+        print("Setting IDF_PATH environment variable: %s" % detected_idf_path)
         os.environ["IDF_PATH"] = detected_idf_path
 
+    # check Python dependencies
+    print("Checking Python dependencies...")
+    try:
+        subprocess.check_call([ os.environ["PYTHON"],
+                                os.path.join(os.environ["IDF_PATH"], "tools", "check_python_dependencies.py")],
+                              env=os.environ)
+    except subprocess.CalledProcessError:
+        raise SystemExit(1)
+
 def executable_exists(args):
     try:
         subprocess.check_output(args)
@@ -136,18 +150,21 @@ def _ensure_build_directory(args, always_run_cmake=False):
     # Verify/create the build directory
     build_dir = args.build_dir
     if not os.path.isdir(build_dir):
-        os.mkdir(build_dir)
+        os.makedirs(build_dir)
     cache_path = os.path.join(build_dir, "CMakeCache.txt")
     if not os.path.exists(cache_path) or always_run_cmake:
         if args.generator is None:
             args.generator = detect_cmake_generator()
         try:
-            cmake_args = ["cmake", "-G", args.generator]
+            cmake_args = ["cmake", "-G", args.generator, "-DPYTHON_DEPS_CHECKED=1"]
             if not args.no_warnings:
                 cmake_args += [ "--warn-uninitialized" ]
             if args.no_ccache:
                 cmake_args += [ "-DCCACHE_DISABLE=1" ]
+            if args.define_cache_entry:
+                cmake_args +=  ["-D" + d for d in args.define_cache_entry]
             cmake_args += [ project_dir]
+
             _run_tool("cmake", cmake_args, cwd=args.build_dir)
         except:
             # don't allow partially valid CMakeCache.txt files,
@@ -204,6 +221,7 @@ def build_target(target_name, args):
     """
     _ensure_build_directory(args)
     generator_cmd = GENERATOR_CMDS[args.generator]
+
     if not args.no_ccache:
         # Setting CCACHE_BASEDIR & CCACHE_NO_HASHDIR ensures that project paths aren't stored in the ccache entries
         # (this means ccache hits can be shared between different projects. It may mean that some debug information
@@ -221,8 +239,9 @@ def build_target(target_name, args):
 
 def _get_esptool_args(args):
     esptool_path = os.path.join(os.environ["IDF_PATH"], "components/esptool_py/esptool/esptool.py")
+    if args.port is None:
+        args.port = get_default_serial_port()
     result = [ PYTHON, esptool_path ]
-    if args.port is not None:
     result += [ "-p", args.port ]
     result += [ "-b", str(args.baud) ]
     return result
@@ -241,17 +260,17 @@ def flash(action, args):
     esptool_args += [ "write_flash", "@"+flasher_args_path ]
     _run_tool("esptool.py", esptool_args, args.build_dir)
 
-
 def erase_flash(action, args):
     esptool_args = _get_esptool_args(args)
     esptool_args += [ "erase_flash" ]
     _run_tool("esptool.py", esptool_args, args.build_dir)
 
-
 def monitor(action, args):
     """
     Run idf_monitor.py to watch build output
     """
+    if args.port is None:
+        args.port = get_default_serial_port()
     desc_path = os.path.join(args.build_dir, "project_description.json")
     if not os.path.exists(desc_path):
         _ensure_build_directory(args)
@@ -267,9 +286,13 @@ def monitor(action, args):
         monitor_args += [ "-p", args.port ]
     monitor_args += [ "-b", project_desc["monitor_baud"] ]
     monitor_args += [ elf_file ]
-    if "MSYSTEM" is os.environ:
+
+    idf_py = [ PYTHON ] + get_commandline_options()  # commands to re-run idf.py
+    monitor_args += [ "-m", " ".join("'%s'" % a for a in idf_py) ]
+
+    if "MSYSTEM" in os.environ:
         monitor_args = [ "winpty" ] + monitor_args
-    _run_tool("idf_monitor", monitor_args, args.build_dir)
+    _run_tool("idf_monitor", monitor_args, args.project_dir)
 
 
 def clean(action, args):
@@ -305,43 +328,157 @@ def fullclean(action, args):
         else:
             os.remove(f)
 
+def print_closing_message(args):
+    # print a closing message of some kind
+    #
+    if "flash" in str(args.actions):
+        print("Done")
+        return
+
+    # Otherwise, if we built any binaries print a message about
+    # how to flash them
+    def print_flashing_message(title, key):
+        print("\n%s build complete. To flash, run this command:" % title)
+
+        with open(os.path.join(args.build_dir, "flasher_args.json")) as f:
+            flasher_args = json.load(f)
+
+        def flasher_path(f):
+            return os.path.relpath(os.path.join(args.build_dir, f))
+
+        if key != "project":  # flashing a single item
+            cmd = ""
+            if key == "bootloader":  # bootloader needs --flash-mode, etc to be passed in
+                cmd = " ".join(flasher_args["write_flash_args"]) + " "
+
+            cmd += flasher_args[key]["offset"] + " "
+            cmd += flasher_path(flasher_args[key]["file"])
+        else:  # flashing the whole project
+            cmd = " ".join(flasher_args["write_flash_args"]) + " "
+            flash_items = sorted(((o,f) for (o,f) in flasher_args["flash_files"].items() if len(o) > 0),
+                                 key = lambda x: int(x[0], 0))
+            for o,f in flash_items:
+                cmd += o + " " + flasher_path(f) + " "
+
+        print("%s -p %s -b %s write_flash %s" % (
+            os.path.relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]),
+            args.port or "(PORT)",
+            args.baud,
+            cmd.strip()))
+        print("or run 'idf.py -p %s %s'" % (args.port or "(PORT)", key + "-flash" if key != "project" else "flash",))
+
+    if "all" in args.actions or "build" in args.actions:
+        print_flashing_message("Project", "project")
+    else:
+        if "app" in args.actions:
+            print_flashing_message("App", "app")
+        if "partition_table" in args.actions:
+            print_flashing_message("Partition Table", "partition_table")
+        if "bootloader" in args.actions:
+            print_flashing_message("Bootloader", "bootloader")
+
 ACTIONS = {
     # action name : ( function (or alias), dependencies, order-only dependencies )
     "all" :                  ( build_target,    [], [ "reconfigure", "menuconfig", "clean", "fullclean" ] ),
     "build":                 ( "all",           [], [] ),  # build is same as 'all' target
     "clean":                 ( clean,           [], [ "fullclean" ] ),
     "fullclean":             ( fullclean,       [], [] ),
-    "reconfigure":           ( reconfigure,  [], [] ),
+    "reconfigure":           ( reconfigure,     [], [ "menuconfig" ] ),
     "menuconfig":            ( build_target,    [], [] ),
-    "size":                  ( build_target, [], [ "app" ] ),
+    "defconfig":             ( build_target,    [], [] ),
-    "size-components":       ( build_target, [], [ "app" ] ),
+    "confserver":            ( build_target,    [], [] ),
-    "size-files":            ( build_target, [], [ "app" ] ),
+    "size":                  ( build_target,    [ "app" ], [] ),
+    "size-components":       ( build_target,    [ "app" ], [] ),
+    "size-files":            ( build_target,    [ "app" ], [] ),
     "bootloader":            ( build_target,    [], [] ),
     "bootloader-clean":      ( build_target,    [], [] ),
-    "bootloader-flash":      ( flash,        [ "bootloader" ], [] ),
+    "bootloader-flash":      ( flash,           [ "bootloader" ], [ "erase_flash"] ),
     "app":                   ( build_target,    [], [ "clean", "fullclean", "reconfigure" ] ),
-    "app-flash":             ( flash,        [], [ "app" ]),
+    "app-flash":             ( flash,           [ "app" ], [ "erase_flash"]),
     "partition_table":       ( build_target,    [], [ "reconfigure" ] ),
-    "partition_table-flash": ( flash,        [ "partition_table" ], []),
+    "partition_table-flash": ( flash,           [ "partition_table" ], [ "erase_flash" ]),
-    "flash":                 ( flash,        [ "all" ], [ ] ),
+    "flash":                 ( flash,           [ "all" ], [ "erase_flash" ] ),
     "erase_flash":           ( erase_flash,     [], []),
     "monitor":               ( monitor,         [], [ "flash", "partition_table-flash", "bootloader-flash", "app-flash" ]),
 }
 
+def get_commandline_options():
+    """ Return all the command line options up to but not including the action """
+    result = []
+    for a in sys.argv:
+        if a in ACTIONS.keys():
+            break
+        else:
+            result.append(a)
+    return result
+
+def get_default_serial_port():
+    """ Return a default serial port. esptool can do this (smarter), but it can create
+    inconsistencies where esptool.py uses one port and idf_monitor uses another.
+
+    Same logic as esptool.py search order, reverse sort by name and choose the first port.
+    """
+    # Import is done here in order to move it after the check_environment() ensured that pyserial has been installed
+    import serial.tools.list_ports
+
+    ports = list(reversed(sorted(
+        p.device for p in serial.tools.list_ports.comports() )))
+    try:
+        print ("Choosing default port %s (use '-p PORT' option to set a specific serial port)" % ports[0])
+        return ports[0]
+    except IndexError:
+        raise RuntimeError("No serial ports found. Connect a device, or use '-p PORT' option to set a specific port.")
+
+# Import the actions, arguments extension file
+if os.path.exists(os.path.join(os.getcwd(), "idf_ext.py")):
+    sys.path.append(os.getcwd())
+    try:
+        from idf_ext import add_action_extensions, add_argument_extensions
+    except ImportError as e:
+        print("Error importing extension file idf_ext.py. Skipping.")
+        print("Please make sure that it contains implementations (even if they're empty implementations) of")
+        print("add_action_extensions and add_argument_extensions.")
 
 def main():
+    if sys.version_info[0] != 2 or sys.version_info[1] != 7:
+        print("Note: You are using Python %d.%d.%d. Python 3 support is new, please report any problems "
+              "you encounter. Search for 'Setting the Python Interpreter' in the ESP-IDF docs if you want to use "
+              "Python 2.7." % sys.version_info[:3])
+
+    # Add actions extensions
+    try:
+        add_action_extensions({
+                    "build_target": build_target,
+                    "reconfigure" : reconfigure,
+                    "flash" : flash,
+                    "monitor" : monitor,
+                    "clean" : clean,
+                    "fullclean" : fullclean
+                    }, ACTIONS)
+    except NameError:
+        pass
+
     parser = argparse.ArgumentParser(description='ESP-IDF build management tool')
-    parser.add_argument('-p', '--port', help="Serial port", default=None)
+    parser.add_argument('-p', '--port', help="Serial port",
-    parser.add_argument('-b', '--baud', help="Baud rate", default=460800)
+                        default=os.environ.get('ESPPORT', None))
+    parser.add_argument('-b', '--baud', help="Baud rate",
+                        default=os.environ.get('ESPBAUD', 460800))
     parser.add_argument('-C', '--project-dir', help="Project directory", default=os.getcwd())
     parser.add_argument('-B', '--build-dir', help="Build directory", default=None)
     parser.add_argument('-G', '--generator', help="Cmake generator", choices=GENERATOR_CMDS.keys())
     parser.add_argument('-n', '--no-warnings', help="Disable Cmake warnings", action="store_true")
     parser.add_argument('-v', '--verbose', help="Verbose build output", action="store_true")
+    parser.add_argument('-D', '--define-cache-entry', help="Create a cmake cache entry", nargs='+')
     parser.add_argument('--no-ccache', help="Disable ccache. Otherwise, if ccache is available on the PATH then it will be used for faster builds.", action="store_true")
     parser.add_argument('actions', help="Actions (build targets or other operations)", nargs='+',
                         choices=ACTIONS.keys())
 
+    # Add arguments extensions
+    try:
+        add_argument_extensions(parser)
+    except NameError:
+        pass
+
     args = parser.parse_args()
 
     check_environment()
@@ -374,13 +511,28 @@ def main():
 
         completed_actions.add(action)
 
-    while len(args.actions) > 0:
+    actions = list(args.actions)
-        execute_action(args.actions[0], args.actions[1:])
+    while len(actions) > 0:
-        args.actions.pop(0)
+        execute_action(actions[0], actions[1:])
+        actions.pop(0)
 
+    print_closing_message(args)
 
 if __name__ == "__main__":
     try:
+        # On MSYS2 we need to run idf.py with "winpty" in order to be able to cancel the subprocesses properly on
+        # keyboard interrupt (CTRL+C).
+        # Using an own global variable for indicating that we are running with "winpty" seems to be the most suitable
+        # option as os.environment['_'] contains "winpty" only when it is run manually from console.
+        WINPTY_VAR = 'WINPTY'
+        WINPTY_EXE = 'winpty'
+        if ('MSYSTEM' in os.environ) and (not os.environ['_'].endswith(WINPTY_EXE) and WINPTY_VAR not in os.environ):
+            os.environ[WINPTY_VAR] = '1'    # the value is of no interest to us
+            # idf.py calls itself with "winpty" and WINPTY global variable set
+            ret = subprocess.call([WINPTY_EXE, sys.executable] + sys.argv, env=os.environ)
+            if ret:
+                raise SystemExit(ret)
+        else:
             main()
     except FatalError as e:
         print(e)

tools/kconfig/.gitignore

@@ -16,7 +16,9 @@ gconf.glade.h
 # configuration programs
 #
 conf
+conf-idf
 mconf
+mconf-idf
 nconf
 qconf
 gconf
@@ -24,7 +26,9 @@ kxgettext
 
 # configuration programs, Windows
 conf.exe
+conf-idf.exe
 mconf.exe
+mconf-idf.exe
 nconf.exe
 qconf.exe
 gconf.exe

tools/kconfig/Makefile

@@ -43,7 +43,7 @@ endif
 endif # MING32
 endif # MSYSTEM
 
-default: mconf conf
+default: mconf-idf conf-idf
 
 xconfig: qconf
 	$< $(silent) $(Kconfig)
@@ -51,41 +51,41 @@ xconfig: qconf
 gconfig: gconf
 	$< $(silent) $(Kconfig)
 
-menuconfig: mconf
+menuconfig: mconf-idf
 	$< $(silent) $(Kconfig)
 
-config: conf
+config: conf-idf
 	$< $(silent) --oldaskconfig $(Kconfig)
 
 nconfig: nconf
 	$< $(silent) $(Kconfig)
 
-silentoldconfig: conf
+silentoldconfig: conf-idf
 	mkdir -p include/config include/generated
 	$< $(silent) --$@ $(Kconfig)
 
-localyesconfig localmodconfig: streamline_config.pl conf
+localyesconfig localmodconfig: streamline_config.pl conf-idf
 	mkdir -p include/config include/generated
 	perl $< --$@ . $(Kconfig) > .tmp.config
 	if [ -f .config ]; then 					\
 			cmp -s .tmp.config .config ||			\
 			(mv -f .config .config.old.1;			\
 			 mv -f .tmp.config .config;			\
-			 conf $(silent) --silentoldconfig $(Kconfig); \
+			 conf-idf $(silent) --silentoldconfig $(Kconfig); \
 			 mv -f .config.old.1 .config.old)		\
 	else								\
 			mv -f .tmp.config .config;			\
-			conf $(silent) --silentoldconfig $(Kconfig); \
+			conf-idf $(silent) --silentoldconfig $(Kconfig); \
 	fi
 	rm -f .tmp.config
 
 
-# These targets map 1:1 to the commandline options of 'conf'
+# These targets map 1:1 to the commandline options of 'conf-idf'
 simple-targets := oldconfig allnoconfig allyesconfig allmodconfig \
 	alldefconfig randconfig listnewconfig olddefconfig
 PHONY += $(simple-targets)
 
-$(simple-targets): conf
+$(simple-targets): conf-idf
 	$< $(silent) --$@ $(Kconfig)
 
 PHONY += oldnoconfig savedefconfig defconfig
@@ -95,10 +95,10 @@ PHONY += oldnoconfig savedefconfig defconfig
 # counter-intuitive name.
 oldnoconfig: olddefconfig
 
-savedefconfig: conf
+savedefconfig: conf-idf
 	$< $(silent) --$@=defconfig $(Kconfig)
 
-defconfig: conf
+defconfig: conf-idf
 ifeq ($(KBUILD_DEFCONFIG),)
 	$< $(silent) --defconfig $(Kconfig)
 else
@@ -111,12 +111,12 @@ else
 endif
 endif
 
-%_defconfig: conf
+%_defconfig: conf-idf
 	$< $(silent) --defconfig=arch/$(SRCARCH)/configs/$@ $(Kconfig)
 
 configfiles=$(wildcard $(srctree)/kernel/configs/$@ $(srctree)/arch/$(SRCARCH)/configs/$@)
 
-%.config: conf
+%.config: conf-idf
 	$(if $(call configfiles),, $(error No configuration exists for this target on this architecture))
 	$(CONFIG_SHELL) $(srctree)/scripts/kconfig/merge_config.sh -m .config $(configfiles)
 	+yes "" | $(MAKE) -f $(srctree)/Makefile oldconfig
@@ -165,7 +165,7 @@ check-lxdialog  := $(SRCDIR)/lxdialog/check-lxdialog.sh
 # Use recursively expanded variables so we do not call gcc unless
 # we really need to do so. (Do not call gcc as part of make mrproper)
 CFLAGS += $(shell $(CONFIG_SHELL) $(check-lxdialog) -ccflags) \
-                    -DLOCALE -MD
+                    -DLOCALE -MMD
 
 %.o: $(SRCDIR)/%.c
 	$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
@@ -178,10 +178,10 @@ lxdialog/%.o: $(SRCDIR)/lxdialog/%.c
 
 # ===========================================================================
 # Shared Makefile for the various kconfig executables:
-# conf:	  Used for defconfig, oldconfig and related targets
+# conf-idf:	  Used for defconfig, oldconfig and related targets
 # nconf:  Used for the nconfig target.
 #         Utilizes ncurses
-# mconf:  Used for the menuconfig target
+# mconf-idf:  Used for the menuconfig target
 #         Utilizes the lxdialog package
 # qconf:  Used for the xconfig target
 #         Based on Qt which needs to be installed to compile it
@@ -200,14 +200,15 @@ qconf-cxxobjs	:= qconf.o
 qconf-objs	:= zconf.tab.o
 gconf-objs	:= gconf.o zconf.tab.o
 
-hostprogs-y := conf nconf mconf kxgettext qconf gconf
+hostprogs-y := conf-idf nconf mconf-idf kxgettext qconf gconf
 
 all-objs := $(conf-objs) $(mconf-objs) $(lxdialog)
 all-deps := $(all-objs:.o=.d)
 
 clean-files	:= qconf.moc .tmp_qtcheck .tmp_gtkcheck
 clean-files	+= zconf.tab.c zconf.lex.c zconf.hash.c gconf.glade.h
-clean-files += $(all-objs) $(all-deps) conf mconf
+clean-files += $(all-objs) $(all-deps) conf-idf mconf-idf conf mconf
+# (note: cleans both mconf & conf (old names) and conf-idf & mconf-idf (new names))
 
 # Check that we have the required ncurses stuff installed for lxdialog (menuconfig)
 PHONY += dochecklxdialog
@@ -324,10 +325,10 @@ gconf.glade.h: gconf.glade
 	gconf.glade
 
 
-mconf: lxdialog $(mconf-objs)
+mconf-idf: lxdialog $(mconf-objs)
 	$(CC) -o $@ $(mconf-objs) $(LOADLIBES_mconf)
 
-conf: $(conf-objs)
+conf-idf: $(conf-objs)
 	$(CC) -o $@ $(conf-objs) $(LOADLIBES_conf)
 
 zconf.tab.c: zconf.lex.c

tools/kconfig/mconf.c

@@ -980,11 +980,17 @@ static int handle_exit(void)
 		}
 		/* fall through */
 	case -1:
-		if (!silent)
+		if (!silent) {
+			const char *is_cmake = getenv("IDF_CMAKE");
+			const char *build_msg;
+			if (is_cmake && is_cmake[0] == 'y')
+				build_msg = _("Ready to use CMake (or 'idf.py build') to build the project.");
+			else
+				build_msg = _("Execute 'make' to start the build or try 'make help'.");
 			printf(_("\n\n"
 					 "*** End of the configuration.\n"
-				 "*** Execute 'make' to start the build or try 'make help'."
+					 "*** %s\n\n"), build_msg);
-				 "\n\n"));
+		}
 		res = 0;
 		break;
 	default:

tools/kconfig_new/confgen.py

@@ -20,6 +20,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import print_function
 import argparse
 import sys
 import os
@@ -29,9 +30,13 @@ import json
 
 import gen_kconfig_doc
 import kconfiglib
+import pprint
 
 __version__ = "0.1"
 
+if not "IDF_CMAKE" in os.environ:
+    os.environ["IDF_CMAKE"] = ""
+
 def main():
     parser = argparse.ArgumentParser(description='confgen.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0]))
 
@@ -45,10 +50,6 @@ def main():
                         nargs='?',
                         default=None)
 
-    parser.add_argument('--create-config-if-missing',
-                        help='If set, a new config file will be saved if the old one is not found',
-                        action='store_true')
-
     parser.add_argument('--kconfig',
                         help='KConfig file with config item definitions',
                         required=True)
@@ -65,7 +66,7 @@ def main():
 
     for fmt, filename in args.output:
         if not fmt in OUTPUT_FORMATS.keys():
-            print("Format '%s' not recognised. Known formats: %s" % (fmt, OUTPUT_FORMATS))
+            print("Format '%s' not recognised. Known formats: %s" % (fmt, OUTPUT_FORMATS.keys()))
             sys.exit(1)
 
     try:
@@ -78,6 +79,8 @@ def main():
         os.environ[name] = value
 
     config = kconfiglib.Kconfig(args.kconfig)
+    config.disable_redun_warnings()
+    config.disable_override_warnings()
 
     if args.defaults is not None:
         # always load defaults first, so any items which are not defined in that config
@@ -86,15 +89,11 @@ def main():
             raise RuntimeError("Defaults file not found: %s" % args.defaults)
         config.load_config(args.defaults)
 
-    if args.config is not None:
+    # If config file previously exists, load it
-        if os.path.exists(args.config):
+    if args.config and os.path.exists(args.config):
-            config.load_config(args.config)
+        config.load_config(args.config, replace=False)
-        elif args.create_config_if_missing:
-            print("Creating config file %s..." % args.config)
-            config.write_config(args.config)
-        elif args.default is None:
-            raise RuntimeError("Config file not found: %s" % args.config)
 
+    # Output the files specified in the arguments
     for output_type, filename in args.output:
         temp_file = tempfile.mktemp(prefix="confgen_tmp")
         try:
@@ -150,9 +149,8 @@ def write_cmake(config, filename):
                     prefix, sym.name, val))
         config.walk_menu(write_node)
 
-def write_json(config, filename):
+def get_json_values(config):
     config_dict = {}
-
     def write_node(node):
         sym = node.item
         if not isinstance(sym, kconfiglib.Symbol):
@@ -168,12 +166,98 @@ def write_json(config, filename):
                 val = int(val)
             config_dict[sym.name] = val
     config.walk_menu(write_node)
+    return config_dict
+
+def write_json(config, filename):
+    config_dict = get_json_values(config)
     with open(filename, "w") as f:
         json.dump(config_dict, f, indent=4, sort_keys=True)
 
+def write_json_menus(config, filename):
+    result = []  # root level items
+    node_lookup = {}  # lookup from MenuNode to an item in result
+
+    def write_node(node):
+        try:
+            json_parent = node_lookup[node.parent]["children"]
+        except KeyError:
+            assert not node.parent in node_lookup  # if fails, we have a parent node with no "children" entity (ie a bug)
+            json_parent = result  # root level node
+
+        # node.kconfig.y means node has no dependency,
+        if node.dep is node.kconfig.y:
+            depends = None
+        else:
+            depends = kconfiglib.expr_str(node.dep)
+
+        try:
+            is_menuconfig = node.is_menuconfig
+        except AttributeError:
+            is_menuconfig = False
+
+        new_json = None
+        if node.item == kconfiglib.MENU or is_menuconfig:
+            new_json = { "type" : "menu",
+                         "title" : node.prompt[0],
+                         "depends_on": depends,
+                         "children": []
+            }
+            if is_menuconfig:
+                sym = node.item
+                new_json["name"] = sym.name
+                new_json["help"] = node.help
+                new_json["is_menuconfig"] = is_menuconfig
+                greatest_range = None
+                if len(sym.ranges) > 0:
+                    # Note: Evaluating the condition using kconfiglib's expr_value
+                    # should have one condition which is true
+                    for min_range, max_range, cond_expr in sym.ranges:
+                        if kconfiglib.expr_value(cond_expr):
+                            greatest_range = [min_range, max_range]
+                new_json["range"] = greatest_range
+
+        elif isinstance(node.item, kconfiglib.Symbol):
+            sym = node.item
+            greatest_range = None
+            if len(sym.ranges) > 0:
+                # Note: Evaluating the condition using kconfiglib's expr_value
+                # should have one condition which is true
+                for min_range, max_range, cond_expr in sym.ranges:
+                    if kconfiglib.expr_value(cond_expr):
+                        greatest_range = [int(min_range.str_value), int(max_range.str_value)]
+
+            new_json = {
+                "type" : kconfiglib.TYPE_TO_STR[sym.type],
+                "name" : sym.name,
+                "title": node.prompt[0] if node.prompt else None,
+                "depends_on" : depends,
+                "help": node.help,
+                "range" : greatest_range,
+                "children": [],
+            }
+        elif isinstance(node.item, kconfiglib.Choice):
+            choice = node.item
+            new_json = {
+                "type": "choice",
+                "title": node.prompt[0],
+                "name": choice.name,
+                "depends_on" : depends,
+                "help": node.help,
+                "children": []
+            }
+
+        if new_json:
+            json_parent.append(new_json)
+            node_lookup[node] = new_json
+
+    config.walk_menu(write_node)
+    with open(filename, "w") as f:
+        f.write(json.dumps(result, sort_keys=True, indent=4))
+
 def update_if_changed(source, destination):
     with open(source, "r") as f:
         source_contents = f.read()
+
     if os.path.exists(destination):
         with open(destination, "r") as f:
             dest_contents = f.read()
@@ -190,6 +274,7 @@ OUTPUT_FORMATS = {
     "cmake" : write_cmake,
     "docs" : gen_kconfig_doc.write_docs,
     "json" : write_json,
+    "json_menus" : write_json_menus,
     }
 
 class FatalError(RuntimeError):

tools/kconfig_new/confserver.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+#
+# Long-running server process uses stdin & stdout to communicate JSON
+# with a caller
+#
+from __future__ import print_function
+import argparse
+import json
+import kconfiglib
+import os
+import sys
+import confgen
+from confgen import FatalError, __version__
+
+def main():
+    parser = argparse.ArgumentParser(description='confserver.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0]))
+
+    parser.add_argument('--config',
+                       help='Project configuration settings',
+                       required=True)
+
+    parser.add_argument('--kconfig',
+                        help='KConfig file with config item definitions',
+                        required=True)
+
+    parser.add_argument('--env', action='append', default=[],
+                        help='Environment to set when evaluating the config file', metavar='NAME=VAL')
+
+    args = parser.parse_args()
+
+    try:
+       args.env = [ (name,value) for (name,value) in ( e.split("=",1) for e in args.env) ]
+    except ValueError:
+       print("--env arguments must each contain =. To unset an environment variable, use 'ENV='")
+       sys.exit(1)
+
+    for name, value in args.env:
+        os.environ[name] = value
+
+    print("Server running, waiting for requests on stdin...", file=sys.stderr)
+    run_server(args.kconfig, args.config)
+
+
+def run_server(kconfig, sdkconfig):
+    config = kconfiglib.Kconfig(kconfig)
+    config.load_config(sdkconfig)
+
+    config_dict = confgen.get_json_values(config)
+    ranges_dict = get_ranges(config)
+    json.dump({"version": 1, "values" : config_dict, "ranges" : ranges_dict}, sys.stdout)
+    print("\n")
+
+    while True:
+        line = sys.stdin.readline()
+        if not line:
+            break
+        req = json.loads(line)
+        before = confgen.get_json_values(config)
+        before_ranges = get_ranges(config)
+
+        if "load" in req:  # if we're loading a different sdkconfig, response should have all items in it
+            before = {}
+            before_ranges = {}
+
+            # if no new filename is supplied, use existing sdkconfig path, otherwise update the path
+            if req["load"] is None:
+                req["load"] = sdkconfig
+            else:
+                sdkconfig = req["load"]
+
+        if "save" in req:
+            if req["save"] is None:
+                req["save"] = sdkconfig
+            else:
+                sdkconfig = req["save"]
+
+        error = handle_request(config, req)
+
+        after = confgen.get_json_values(config)
+        after_ranges = get_ranges(config)
+
+        values_diff = diff(before, after)
+        ranges_diff = diff(before_ranges, after_ranges)
+        response = {"version" : 1, "values" : values_diff, "ranges" : ranges_diff}
+        if error:
+            for e in error:
+                print("Error: %s" % e, file=sys.stderr)
+            response["error"] = error
+        json.dump(response, sys.stdout)
+        print("\n")
+
+
+def handle_request(config, req):
+    if not "version" in req:
+        return [ "All requests must have a 'version'" ]
+    if int(req["version"]) != 1:
+        return [ "Only version 1 requests supported" ]
+
+    error = []
+
+    if "load" in req:
+        print("Loading config from %s..." % req["load"], file=sys.stderr)
+        try:
+            config.load_config(req["load"])
+        except Exception as e:
+            error += [ "Failed to load from %s: %s" % (req["load"], e) ]
+
+    if "set" in req:
+        handle_set(config, error, req["set"])
+
+    if "save" in req:
+        try:
+            print("Saving config to %s..." % req["save"], file=sys.stderr)
+            confgen.write_config(config, req["save"])
+        except Exception as e:
+            error += [ "Failed to save to %s: %s" % (req["save"], e) ]
+
+    return error
+
+def handle_set(config, error, to_set):
+    missing = [ k for k in to_set if not k in config.syms ]
+    if missing:
+        error.append("The following config symbol(s) were not found: %s" % (", ".join(missing)))
+    # replace name keys with the full config symbol for each key:
+    to_set = dict((config.syms[k],v) for (k,v) in to_set.items() if not k in missing)
+
+    # Work through the list of values to set, noting that
+    # some may not be immediately applicable (maybe they depend
+    # on another value which is being set). Therefore, defer
+    # knowing if any value is unsettable until then end
+
+    while len(to_set):
+        set_pass = [ (k,v) for (k,v) in to_set.items() if k.visibility ]
+        if not set_pass:
+            break  # no visible keys left
+        for (sym,val) in set_pass:
+            if sym.type in (kconfiglib.BOOL, kconfiglib.TRISTATE):
+                if val == True:
+                    sym.set_value(2)
+                elif val == False:
+                    sym.set_value(0)
+                else:
+                    error.append("Boolean symbol %s only accepts true/false values" % sym.name)
+            else:
+                sym.set_value(str(val))
+            print("Set %s" % sym.name)
+            del to_set[sym]
+
+    if len(to_set):
+        error.append("The following config symbol(s) were not visible so were not updated: %s" % (", ".join(s.name for s in to_set)))
+
+
+
+def diff(before, after):
+    """
+    Return a dictionary with the difference between 'before' and 'after' (either with the new value if changed,
+    or None as the value if a key in 'before' is missing in 'after'
+    """
+    diff = dict((k,v) for (k,v) in after.items() if before.get(k, None) != v)
+    hidden = dict((k,None) for k in before if k not in after)
+    diff.update(hidden)
+    return diff
+
+
+def get_ranges(config):
+    ranges_dict = {}
+    def handle_node(node):
+        sym = node.item
+        if not isinstance(sym, kconfiglib.Symbol):
+            return
+        active_range = sym.active_range
+        if active_range[0] is not None:
+            ranges_dict[sym.name] = active_range
+
+    config.walk_menu(handle_node)
+    return ranges_dict
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except FatalError as e:
+        print("A fatal error occurred: %s" % e, file=sys.stderr)
+        sys.exit(2)
+

tools/kconfig_new/gen_kconfig_doc.py

@@ -20,7 +20,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import print_function
 import os
+import re
 import kconfiglib
 
 # Indentation to be used in the generated file
@@ -40,6 +42,11 @@ def write_docs(config, filename):
     with open(filename, "w") as f:
         config.walk_menu(lambda node: write_menu_item(f, node))
 
+def node_is_menu(node):
+    try:
+        return node.item == kconfiglib.MENU or node.is_menuconfig
+    except AttributeError:
+        return False  # not all MenuNodes have is_menuconfig for some reason
 
 def get_breadcrumbs(node):
     # this is a bit wasteful as it recalculates each time, but still...
@@ -47,12 +54,26 @@ def get_breadcrumbs(node):
     node = node.parent
     while node.parent:
         if node.prompt:
-            result = [ node.prompt[0] ] + result
+            result = [ ":ref:`%s`" % get_link_anchor(node) ] + result
         node = node.parent
     return " > ".join(result)
 
+def get_link_anchor(node):
+    try:
+        return "CONFIG_%s" % node.item.name
+    except AttributeError:
+        assert(node_is_menu(node))  # only menus should have no item.name
+
+    # for menus, build a link anchor out of the parents
+    result = []
+    while node.parent:
+        if node.prompt:
+            result = [ re.sub(r"[^a-zA-z0-9]+", "-", node.prompt[0]) ] + result
+        node = node.parent
+    result = "-".join(result).lower()
+    return result
+
 def get_heading_level(node):
-    # bit wasteful also
     result = INITIAL_HEADING_LEVEL
     node = node.parent
     while node.parent:
@@ -71,42 +92,41 @@ def format_rest_text(text, indent):
     text += '\n'
     return text
 
-def write_menu_item(f, node):
+def node_should_write(node):
     if not node.prompt:
-        return  # Don't do anything for invisible menu items
+        return False  # Don't do anything for invisible menu items
 
     if isinstance(node.parent.item, kconfiglib.Choice):
-        return  # Skip choice nodes, they are handled as part of the parent (see below)
+        return False  # Skip choice nodes, they are handled as part of the parent (see below)
+
+    return True
+
+def write_menu_item(f, node):
+    if not node_should_write(node):
+        return
 
     try:
         name = node.item.name
     except AttributeError:
         name = None
 
-    try:
+    is_menu = node_is_menu(node)
-        is_menu = node.item == kconfiglib.MENU or node.is_menuconfig
-    except AttributeError:
-        is_menu = False  # not all MenuNodes have is_menuconfig for some reason
 
     ## Heading
     if name:
-        title = name
+        title = 'CONFIG_%s' % name
-        # add link target so we can use :ref:`CONFIG_FOO`
-        f.write('.. _CONFIG_%s:\n\n' % name)
     else:
+        # if no symbol name, use the prompt as the heading
         title = node.prompt[0]
 
-    # if no symbol name, use the prompt as the heading
+    f.write(".. _%s:\n\n" % get_link_anchor(node))
-    if True or is_menu:
     f.write('%s\n' % title)
     f.write(HEADING_SYMBOLS[get_heading_level(node)] * len(title))
     f.write('\n\n')
-    else:
-        f.write('**%s**\n\n\n' % title)
 
     if name:
         f.write('%s%s\n\n' % (INDENT, node.prompt[0]))
-        f.write('%s:emphasis:`Found in: %s`\n\n' % (INDENT, get_breadcrumbs(node)))
+        f.write('%s:emphasis:`Found in:` %s\n\n' % (INDENT, get_breadcrumbs(node)))
 
     try:
         if node.help:
@@ -131,6 +151,21 @@ def write_menu_item(f, node):
 
         f.write('\n\n')
 
+    if is_menu:
+        # enumerate links to child items
+        first = True
+        child = node.list
+        while child:
+            try:
+                if node_should_write(child):
+                    if first:
+                        f.write("Contains:\n\n")
+                        first = False
+                    f.write('- :ref:`%s`\n' % get_link_anchor(child))
+            except AttributeError:
+                pass
+            child = child.next
+        f.write('\n')
 
 if __name__ == '__main__':
     print("Run this via 'confgen.py --output doc FILENAME'")

tools/kconfig_new/kconfiglib.py

@@ -500,6 +500,8 @@ class Kconfig(object):
     __slots__ = (
         "_choices",
         "_print_undef_assign",
+        "_print_override",
+        "_print_redun_assign",
         "_print_warnings",
         "_set_re_match",
         "_unset_re_match",
@@ -575,6 +577,7 @@ class Kconfig(object):
 
         self._print_warnings = warn
         self._print_undef_assign = False
+        self._print_redun_assign = self._print_override = True
 
         self.syms = {}
         self.const_syms = {}
@@ -754,6 +757,9 @@ class Kconfig(object):
                         continue
 
                     if sym.orig_type in (BOOL, TRISTATE):
+                        if val == "":
+                            val = "n"  # C implementation allows 'blank' for 'no'
+
                         # The C implementation only checks the first character
                         # to the right of '=', for whatever reason
                         if not ((sym.orig_type == BOOL and
@@ -823,10 +829,12 @@ class Kconfig(object):
                         display_val = val
                         display_user_val = sym.user_value
 
-                    self._warn('{} set more than once. Old value: "{}", new '
+                    msg = '{} set more than once. Old value: "{}", new value: "{}".'.format(name, display_user_val, display_val)
-                               'value: "{}".'
+
-                               .format(name, display_user_val, display_val),
+                    if display_user_val == display_val:
-                               filename, linenr)
+                        self._warn_redun_assign(msg, filename, linenr)
+                    else:
+                        self._warn_override(msg, filename, linenr)
 
                 sym.set_value(val)
 
@@ -924,7 +932,7 @@ class Kconfig(object):
 
             def write_node(node):
                 item = node.item
-                if isinstance(item, Symbol):
+                if isinstance(item, Symbol) and item.env_var is None:
                     config_string = item.config_string
                     if config_string:
                         write(config_string)
@@ -1054,6 +1062,36 @@ class Kconfig(object):
         """
         self._print_undef_assign = False
 
+    def enable_redun_warnings(self):
+        """
+        Enables warnings for redundant assignments to symbols. Printed to
+        stderr. Enabled by default.
+        """
+        self._print_redun_assign = True
+
+    def disable_redun_warnings(self):
+        """
+        See enable_redun_warnings().
+        """
+        self._print_redun_assign = False
+
+    def enable_override_warnings(self):
+        """
+        Enables warnings for duplicated assignments in .config files that set
+        different values (e.g. CONFIG_FOO=m followed by CONFIG_FOO=y, where
+        the last value set is used).
+
+        These warnings are enabled by default. Disabling them might be helpful
+        in certain cases when merging configurations.
+        """
+        self._print_override = True
+
+    def disable_override_warnings(self):
+        """
+        See enable_override_warnings().
+        """
+        self._print_override = False
+
     def __repr__(self):
         """
         Returns a string with information about the Kconfig object when it is
@@ -1068,6 +1106,8 @@ class Kconfig(object):
             "warnings " + ("enabled" if self._print_warnings else "disabled"),
             "undef. symbol assignment warnings " +
                 ("enabled" if self._print_undef_assign else "disabled"),
+            "redundant symbol assignment warnings " +
+                ("enabled" if self._print_redun_assign else "disabled")
         )))
 
     #
@@ -2147,6 +2187,19 @@ class Kconfig(object):
             'attempt to assign the value "{}" to the undefined symbol {}' \
             .format(val, name), filename, linenr)
 
+    def _warn_redun_assign(self, msg, filename=None, linenr=None):
+        """
+        See the class documentation.
+        """
+        if self._print_redun_assign:
+            _stderr_msg("warning: " + msg, filename, linenr)
+
+    def _warn_override(self, msg, filename=None, linenr=None):
+        """
+        See the class documentation.
+        """
+        if self._print_override:
+            _stderr_msg("warning: " + msg, filename, linenr)
 
 class Symbol(object):
     """
@@ -2419,24 +2472,11 @@ class Symbol(object):
             base = _TYPE_TO_BASE[self.orig_type]
 
             # Check if a range is in effect
-            for low_expr, high_expr, cond in self.ranges:
+            low, high = self.active_range
-                if expr_value(cond):
-                    has_active_range = True
-
-                    # The zeros are from the C implementation running strtoll()
-                    # on empty strings
-                    low = int(low_expr.str_value, base) if \
-                      _is_base_n(low_expr.str_value, base) else 0
-                    high = int(high_expr.str_value, base) if \
-                      _is_base_n(high_expr.str_value, base) else 0
-
-                    break
-            else:
-                has_active_range = False
 
             if vis and self.user_value is not None and \
                _is_base_n(self.user_value, base) and \
-               (not has_active_range or
+               (low is None or
                 low <= int(self.user_value, base) <= high):
 
                 # If the user value is well-formed and satisfies range
@@ -2463,7 +2503,7 @@ class Symbol(object):
                     val_num = 0  # strtoll() on empty string
 
                 # This clamping procedure runs even if there's no default
-                if has_active_range:
+                if low is not None:
                     clamp = None
                     if val_num < low:
                         clamp = low
@@ -2714,6 +2754,28 @@ class Symbol(object):
             if self._is_user_assignable():
                 self._rec_invalidate()
 
+    @property
+    def active_range(self):
+        """
+        Returns a tuple of (low, high) integer values if a range
+        limit is active for this symbol, or (None, None) if no range
+        limit exists.
+        """
+        base = _TYPE_TO_BASE[self.orig_type]
+
+        for low_expr, high_expr, cond in self.ranges:
+            if expr_value(cond):
+                # The zeros are from the C implementation running strtoll()
+                # on empty strings
+                low = int(low_expr.str_value, base) if \
+                    _is_base_n(low_expr.str_value, base) else 0
+                high = int(high_expr.str_value, base) if \
+                    _is_base_n(high_expr.str_value, base) else 0
+
+                return (low, high)
+        return (None, None)
+
+
     def __repr__(self):
         """
         Returns a string with information about the symbol (including its name,

tools/kconfig_new/test/Kconfig

@@ -0,0 +1,44 @@
+menu "Test config"
+
+config TEST_BOOL
+  bool "Test boolean"
+  default n
+
+config TEST_CHILD_BOOL
+  bool "Test boolean"
+  depends on TEST_BOOL
+  default y
+
+config TEST_CHILD_STR
+  string "Test str"
+  depends on TEST_BOOL
+  default "OHAI!"
+
+choice TEST_CHOICE
+  prompt "Some choice"
+  default CHOICE_A
+
+config CHOICE_A
+  bool "A"
+
+config CHOICE_B
+  bool "B"
+
+endchoice
+
+config DEPENDS_ON_CHOICE
+   string "Depends on choice"
+   default "Depends on A" if CHOICE_A
+   default "Depends on B" if CHOICE_B
+   default "WAT"
+
+config SOME_UNRELATED_THING
+   bool "Some unrelated thing"
+
+config TEST_CONDITIONAL_RANGES
+   int "Something with a range"
+   range 0 100 if TEST_BOOL
+   range 0 10
+   default 1
+
+endmenu

tools/kconfig_new/test/sdkconfig

@@ -0,0 +1 @@
+CONFIG_SOME_UNRELATED_THING=y

tools/kconfig_new/test/test_confserver.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import os
+import sys
+import threading
+import time
+import json
+import argparse
+import shutil
+import tempfile
+
+import pexpect
+
+sys.path.append("..")
+import confserver
+
+def create_server_thread(*args):
+    t = threading.Thread()
+
+def parse_testcases():
+    with open("testcases.txt", "r") as f:
+        cases = [ l for l in f.readlines() if len(l.strip()) > 0 ]
+    # Each 3 lines in the file should be formatted as:
+    # * Description of the test change
+    # * JSON "changes" to send to the server
+    # * Result JSON to expect back from the server
+    if len(cases) % 3 != 0:
+        print("Warning: testcases.txt has wrong number of non-empty lines (%d). Should be 3 lines per test case, always." % len(cases))
+
+    for i in range(0, len(cases), 3):
+        desc = cases[i]
+        send = cases[i+1]
+        expect = cases[i+2]
+        if not desc.startswith("* "):
+            raise RuntimeError("Unexpected description at line %d: '%s'" % (i+1, desc))
+        if not send.startswith("> "):
+            raise RuntimeError("Unexpected send at line %d: '%s'" % (i+2, send))
+        if not expect.startswith("< "):
+            raise RuntimeError("Unexpected expect at line %d: '%s'" % (i+3, expect))
+        desc = desc[2:]
+        send = json.loads(send[2:])
+        expect = json.loads(expect[2:])
+        yield (desc, send, expect)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--logfile', type=argparse.FileType('w'), help='Optional session log of the interactions with confserver.py')
+    args = parser.parse_args()
+
+    try:
+        # set up temporary file to use as sdkconfig copy
+        with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_sdkconfig:
+            temp_sdkconfig_path = os.path.join(tempfile.gettempdir(), temp_sdkconfig.name)
+            with open("sdkconfig") as orig:
+                temp_sdkconfig.write(orig.read())
+
+        cmdline = "../confserver.py --kconfig Kconfig --config %s" % temp_sdkconfig_path
+        print("Running: %s" % cmdline)
+        p = pexpect.spawn(cmdline, timeout=0.5)
+        p.logfile = args.logfile
+        p.setecho(False)
+
+        def expect_json():
+            # run p.expect() to expect a json object back, and return it as parsed JSON
+            p.expect("{.+}\r\n")
+            return json.loads(p.match.group(0).strip().decode())
+
+        p.expect("Server running.+\r\n")
+        initial = expect_json()
+        print("Initial: %s" % initial)
+        cases = parse_testcases()
+
+        for (desc, send, expected) in cases:
+            print(desc)
+            req = { "version" : "1", "set" : send }
+            req = json.dumps(req)
+            print("Sending: %s" % (req))
+            p.send("%s\n" % req)
+            readback = expect_json()
+            print("Read back: %s" % (json.dumps(readback)))
+            if readback.get("version", None) != 1:
+                raise RuntimeError('Expected {"version" : 1} in response')
+            for expect_key in expected.keys():
+                read_vals = readback[expect_key]
+                exp_vals = expected[expect_key]
+                if read_vals != exp_vals:
+                    expect_diff = dict((k,v) for (k,v) in exp_vals.items() if not k in read_vals or v != read_vals[k])
+                    raise RuntimeError("Test failed! Was expecting %s: %s" % (expect_key, json.dumps(expect_diff)))
+            print("OK")
+
+        print("Testing load/save...")
+        before = os.stat(temp_sdkconfig_path).st_mtime
+        p.send("%s\n" % json.dumps({ "version" : "1", "save" : temp_sdkconfig_path }))
+        save_result = expect_json()
+        print("Save result: %s" % (json.dumps(save_result)))
+        assert len(save_result["values"]) == 0
+        assert len(save_result["ranges"]) == 0
+        after = os.stat(temp_sdkconfig_path).st_mtime
+        assert after > before
+
+        p.send("%s\n" % json.dumps({ "version" : "1", "load" : temp_sdkconfig_path }))
+        load_result = expect_json()
+        print("Load result: %s" % (json.dumps(load_result)))
+        assert len(load_result["values"]) > 0  # loading same file should return all config items
+        assert len(load_result["ranges"]) > 0
+        print("Done. All passed.")
+
+    finally:
+        try:
+            os.remove(temp_sdkconfig_path)
+        except OSError:
+            pass
+
+if __name__ == "__main__":
+    main()
+

tools/kconfig_new/test/testcases.txt

@@ -0,0 +1,31 @@
+* Set TEST_BOOL, showing child items
+> { "TEST_BOOL" : true }
+< { "values" : { "TEST_BOOL" : true, "TEST_CHILD_STR" : "OHAI!", "TEST_CHILD_BOOL" : true }, "ranges": {"TEST_CONDITIONAL_RANGES": [0, 100]} }
+
+* Set TEST_CHILD_STR
+> { "TEST_CHILD_STR" : "Other value" }
+< { "values" : { "TEST_CHILD_STR" : "Other value" } }
+
+* Clear TEST_BOOL, hiding child items
+> { "TEST_BOOL" : false }
+< { "values" : { "TEST_BOOL" : false, "TEST_CHILD_STR" : null, "TEST_CHILD_BOOL" : null },  "ranges": {"TEST_CONDITIONAL_RANGES": [0, 10]} }
+
+* Set TEST_CHILD_BOOL, invalid as parent is disabled
+> { "TEST_CHILD_BOOL" : false }
+< { "values" : { } }
+
+* Set TEST_BOOL & TEST_CHILD_STR together
+> { "TEST_BOOL" : true, "TEST_CHILD_STR" : "New value" }
+< { "values" : { "TEST_BOOL" : true, "TEST_CHILD_STR" : "New value", "TEST_CHILD_BOOL" : true } }
+
+* Set choice
+> { "CHOICE_B" : true }
+< { "values" : { "CHOICE_B" : true, "CHOICE_A" : false, "DEPENDS_ON_CHOICE" : "Depends on B" } }
+
+* Set string which depends on choice B
+> { "DEPENDS_ON_CHOICE" : "oh, really?" }
+< { "values" : { "DEPENDS_ON_CHOICE" : "oh, really?" } }
+
+* Try setting boolean values to invalid types
+> { "CHOICE_A" : 11, "TEST_BOOL" : "false" }
+< { "values" : { } }

tools/toolchain_versions.mk

@@ -1,6 +1,6 @@
-SUPPORTED_TOOLCHAIN_COMMIT_DESC = crosstool-ng-1.22.0-80-g6c4433a
+SUPPORTED_TOOLCHAIN_COMMIT_DESC = crosstool-ng-1.22.0-92-g8facf4c
 SUPPORTED_TOOLCHAIN_GCC_VERSIONS = 5.2.0
 
-CURRENT_TOOLCHAIN_COMMIT_DESC = crosstool-ng-1.22.0-80-g6c4433a
+CURRENT_TOOLCHAIN_COMMIT_DESC = crosstool-ng-1.22.0-92-g8facf4c
-CURRENT_TOOLCHAIN_COMMIT_DESC_SHORT = 1.22.0-80-g6c4433a
+CURRENT_TOOLCHAIN_COMMIT_DESC_SHORT = 1.22.0-92-g8facf4c
 CURRENT_TOOLCHAIN_GCC_VERSION = 5.2.0

tools/windows/tool_setup/idf_tool_setup.iss

@@ -41,7 +41,7 @@ Name: python32; Description: Download and Run Python 2.7.14 Installer and instal
 Name: python64; Description: Download and Run Python 2.7.14 Installer and install pyserial; GroupDescription: "Other Required Tools:"; Check: IsWin64 and not Python27Installed
 
 [Files]
-Components: toolchain; Source: "input\xtensa-esp32-elf\*"; DestDir: "{app}\toolchain\"; Flags: recursesubdirs;
+Components: toolchain; Source: "input\xtensa-esp8266-elf\*"; DestDir: "{app}\toolchain\"; Flags: recursesubdirs;
 Components: mconf; Source: "input\mconf-v4.6.0.0-idf-20180319-win32\*"; DestDir: "{app}\mconf\";
 Components: ninja; Source: "input\ninja.exe"; DestDir: "{app}";