diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..1b481064 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,82 @@ +cmake_minimum_required(VERSION 3.5) +project(esp-idf C CXX ASM) + +unset(compile_options) +unset(c_compile_options) +unset(cxx_compile_options) +unset(compile_definitions) + +# Add the following build specifications here, since these seem to be dependent +# on config values on the root Kconfig. + +if(CONFIG_COMPILER_OPTIMIZATION_LEVEL_RELEASE) + list(APPEND compile_options "-Os") +else() + list(APPEND compile_options "-Og") +endif() + +if(CONFIG_COMPILER_CXX_EXCEPTIONS) + list(APPEND cxx_compile_options "-fexceptions") +else() + list(APPEND cxx_compile_options "-fno-exceptions") +endif() + +if(CONFIG_COMPILER_DISABLE_GCC8_WARNINGS) + list(APPEND compile_options "-Wno-parentheses" + "-Wno-sizeof-pointer-memaccess" + "-Wno-clobbered") + + # doesn't use GCC_NOT_5_2_0 because idf_set_global_variables was not called before + if(GCC_NOT_5_2_0) + list(APPEND compile_options "-Wno-format-overflow" + "-Wno-stringop-truncation" + "-Wno-misleading-indentation" + "-Wno-cast-function-type" + "-Wno-implicit-fallthrough" + "-Wno-unused-const-variable" + "-Wno-switch-unreachable" + "-Wno-format-truncation" + "-Wno-memset-elt-size" + "-Wno-int-in-bool-context") + endif() +endif() + +if(CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE) + list(APPEND compile_definitions "-DNDEBUG") +endif() + +if(CONFIG_COMPILER_STACK_CHECK_MODE_NORM) + list(APPEND compile_options "-fstack-protector") +elseif(CONFIG_COMPILER_STACK_CHECK_MODE_STRONG) + list(APPEND compile_options "-fstack-protector-strong") +elseif(CONFIG_COMPILER_STACK_CHECK_MODE_ALL) + list(APPEND compile_options "-fstack-protector-all") +endif() + + +idf_build_set_property(COMPILE_OPTIONS "${compile_options}" APPEND) +idf_build_set_property(C_COMPILE_OPTIONS "${c_compile_options}" APPEND) +idf_build_set_property(CXX_COMPILE_OPTIONS "${cxx_compile_options}" APPEND) +idf_build_set_property(COMPILE_DEFINITIONS "${compile_definitions}" APPEND) + +idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + +# Add each component as a subdirectory, processing each component's CMakeLists.txt +foreach(component_target ${build_component_targets}) + __component_get_property(dir ${component_target} COMPONENT_DIR) + __component_get_property(_name ${component_target} COMPONENT_NAME) + __component_get_property(prefix ${component_target} __PREFIX) + __component_get_property(alias ${component_target} COMPONENT_ALIAS) + set(COMPONENT_NAME ${_name}) + set(COMPONENT_DIR ${dir}) + set(COMPONENT_ALIAS ${alias}) + set(COMPONENT_PATH ${dir}) # for backward compatibility only, COMPONENT_DIR is preferred + idf_build_get_property(build_prefix __PREFIX) + set(__idf_component_context 1) + if(NOT prefix STREQUAL build_prefix) + add_subdirectory(${dir} ${prefix}_${_name}) + else() + add_subdirectory(${dir} ${_name}) + endif() + set(__idf_component_context 0) +endforeach() \ No newline at end of file diff --git a/Kconfig b/Kconfig index 6199e5af..b79553ea 100644 --- a/Kconfig +++ b/Kconfig @@ -4,22 +4,23 @@ # mainmenu "Espressif IoT Development Framework Configuration" -choice TARGET_PLATFORM +choice IDF_TARGET bool "Espressif target platform choose" default IDF_TARGET_ESP8266 help Choose the specific target platform which you will use. -config IDF_TARGET_ESP32 - bool "esp32" config IDF_TARGET_ESP8266 bool "esp8266" endchoice -menu "SDK tool configuration" -config TOOLPREFIX +config IDF_TARGET + string + default "esp8266" if IDF_TARGET_ESP8266 + +menu "SDK tool configuration" +config SDK_TOOLPREFIX string - default "xtensa-esp32-elf-" if IDF_TARGET_ESP32 default "xtensa-lx106-elf-" if IDF_TARGET_ESP8266 help The prefix/path that is used to call the toolchain. The default setting assumes diff --git a/make/README b/make/README deleted file mode 100644 index ddc1b7d1..00000000 --- a/make/README +++ /dev/null @@ -1,10 +0,0 @@ -README - -The SDK uses 'make' of esp-idf, and changed things are following: - -1. remove the "toolchain" warning in file project.mk at line 543 - -Information of esp-idf is following: - -commit: 48c3ad37 - diff --git a/make/common.mk b/make/common.mk index 2eb0f0d3..bb0000a1 100644 --- a/make/common.mk +++ b/make/common.mk @@ -2,6 +2,8 @@ # and component makefiles (component_wrapper.mk) # +PYTHON=$(call dequote,$(CONFIG_SDK_PYTHON)) + # Include project config makefile, if it exists. # # (Note that we only rebuild this makefile automatically for some @@ -32,10 +34,13 @@ details := @true MAKEFLAGS += --silent endif # $(V)==1 -ifdef CONFIG_MAKE_WARN_UNDEFINED_VARIABLES +ifdef CONFIG_SDK_MAKE_WARN_UNDEFINED_VARIABLES MAKEFLAGS += --warn-undefined-variables endif +# Get version variables +include $(IDF_PATH)/make/version.mk + # General make utilities # convenience variable for printing an 80 asterisk wide separator line diff --git a/make/component_wrapper.mk b/make/component_wrapper.mk index a5450747..278452ef 100644 --- a/make/component_wrapper.mk +++ b/make/component_wrapper.mk @@ -46,6 +46,10 @@ COMPONENT_EMBED_TXTFILES ?= COMPONENT_ADD_INCLUDEDIRS = include COMPONENT_ADD_LDFLAGS = -l$(COMPONENT_NAME) +# Name of the linker fragment files this component presents to the Linker +# script generator +COMPONENT_ADD_LDFRAGMENTS ?= + # Define optional compiling macros define compile_exclude COMPONENT_OBJEXCLUDE += $(1) @@ -87,7 +91,9 @@ COMPONENT_SUBMODULES ?= COMPILING_COMPONENT_PATH := $(COMPONENT_PATH) define includeCompBuildMakefile -$(if $(V),$(info including $(1)/Makefile.componentbuild...)) +ifeq ("$(V)","1") +$$(info including $(1)/Makefile.componentbuild...) +endif COMPONENT_PATH := $(1) include $(1)/Makefile.componentbuild endef @@ -193,8 +199,8 @@ component_project_vars.mk:: @echo 'COMPONENT_LINKER_DEPS += $(call MakeVariablePath,$(call resolvepath,$(COMPONENT_ADD_LINKER_DEPS),$(COMPONENT_PATH)))' >> $@ @echo 'COMPONENT_SUBMODULES += $(call MakeVariablePath,$(abspath $(addprefix $(COMPONENT_PATH)/,$(COMPONENT_SUBMODULES))))' >> $@ @echo 'COMPONENT_LIBRARIES += $(COMPONENT_NAME)' >> $@ + @echo 'COMPONENT_LDFRAGMENTS += $(call MakeVariablePath,$(abspath $(addprefix $(COMPONENT_PATH)/,$(COMPONENT_ADD_LDFRAGMENTS))))' >> $@ @echo 'component-$(COMPONENT_NAME)-build: $(addprefix component-,$(addsuffix -build,$(COMPONENT_DEPENDS)))' >> $@ - ################################################################################ # 5) Where COMPONENT_OWNBUILDTARGET / COMPONENT_OWNCLEANTARGET # is not set by component.mk, define default build, clean, etc. targets @@ -212,7 +218,7 @@ build: $(COMPONENT_LIBRARY) $(COMPONENT_LIBRARY): $(COMPONENT_OBJS) $(COMPONENT_EMBED_OBJS) $(summary) AR $(patsubst $(PWD)/%,%,$(CURDIR))/$@ rm -f $@ - $(AR) $(ARFLAGS) $@ $^ + $(AR) $(ARFLAGS) $@ $(COMPONENT_OBJS) $(COMPONENT_EMBED_OBJS) endif # If COMPONENT_OWNCLEANTARGET is not set, define a phony clean target @@ -332,7 +338,7 @@ clean: $(summary) RM component_project_vars.mk rm -f component_project_vars.mk -component_project_vars.mk:: # no need to add variables via component.mk +component_project_vars.mk:: # no need to add variables via component.mk @echo '# COMPONENT_CONFIG_ONLY target sets no variables here' > $@ endif # COMPONENT_CONFIG_ONLY diff --git a/make/ldgen.mk b/make/ldgen.mk new file mode 100644 index 00000000..8519a013 --- /dev/null +++ b/make/ldgen.mk @@ -0,0 +1,50 @@ +# Makefile to support the linker script generation mechanism +LDGEN_FRAGMENT_FILES = $(COMPONENT_LDFRAGMENTS) +LDGEN_LIBRARIES=$(foreach libcomp,$(COMPONENT_LIBRARIES),$(BUILD_DIR_BASE)/$(libcomp)/lib$(libcomp).a) + +# Target to generate linker script generator from fragments presented by each of +# the components +ifeq ($(OS),Windows_NT) +define ldgen_process_template +$(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk + printf "$(foreach info,$(LDGEN_LIBRARIES),$(subst \,/,$(shell cygpath -w $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen_libraries + +$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries + @echo 'Generating $(notdir $(2))' + $(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \ + --input $(1) \ + --config $(SDKCONFIG) \ + --fragments $(LDGEN_FRAGMENT_FILES) \ + --libraries-file $(BUILD_DIR_BASE)/ldgen_libraries \ + --output $(2) \ + --kconfig $(IDF_PATH)/Kconfig \ + --env "COMPONENT_KCONFIGS=$(foreach k, $(COMPONENT_KCONFIGS), $(shell cygpath -w $(k)))" \ + --env "COMPONENT_KCONFIGS_PROJBUILD=$(foreach k, $(COMPONENT_KCONFIGS_PROJBUILD), $(shell cygpath -w $(k)))" \ + --env "IDF_CMAKE=n" \ + --objdump $(OBJDUMP) +endef +else # Windows_NT +define ldgen_process_template +$(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk + printf "$(foreach library,$(LDGEN_LIBRARIES),$(library)\n)" > $(BUILD_DIR_BASE)/ldgen_libraries + +$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries + @echo 'Generating $(notdir $(2))' + $(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \ + --input $(1) \ + --config $(SDKCONFIG) \ + --fragments $(LDGEN_FRAGMENT_FILES) \ + --libraries-file $(BUILD_DIR_BASE)/ldgen_libraries \ + --output $(2) \ + --kconfig $(IDF_PATH)/Kconfig \ + --env "COMPONENT_KCONFIGS=$(COMPONENT_KCONFIGS)" \ + --env "COMPONENT_KCONFIGS_PROJBUILD=$(COMPONENT_KCONFIGS_PROJBUILD)" \ + --env "IDF_CMAKE=n" \ + --objdump $(OBJDUMP) +endef +endif # Windows_NT + +define ldgen_create_commands +ldgen-clean: + rm -f $(BUILD_DIR_BASE)/ldgen_libraries +endef \ No newline at end of file diff --git a/make/project.mk b/make/project.mk index 5c5902d3..7cb31396 100644 --- a/make/project.mk +++ b/make/project.mk @@ -21,6 +21,8 @@ all: all_binaries | check_python_dependencies # target can build everything without triggering the per-component "to # flash..." output targets.) +make_prepare: + help: @echo "Welcome to Espressif IDF build system. Some useful make targets:" @echo "" @@ -34,7 +36,8 @@ help: @echo "make size-components, size-files - Finer-grained memory footprints" @echo "make size-symbols - Per symbol memory footprint. Requires COMPONENT=" @echo "make erase_flash - Erase entire flash contents" - @echo "make erase_ota - Erase ota_data partition. After that will boot first bootable partition (factory or OTAx)." + @echo "make erase_otadata - Erase ota_data partition; First bootable partition (factory or OTAx) will be used on next boot." + @echo " This assumes this project's partition table is the one flashed on the device." @echo "make monitor - Run idf_monitor tool to monitor serial output from app" @echo "make simple_monitor - Monitor serial output on terminal console" @echo "make list-components - List all components in the project" @@ -48,9 +51,6 @@ help: @echo "See also 'make bootloader', 'make bootloader-flash', 'make bootloader-clean', " @echo "'make partition_table', etc, etc." -# prepare for the global varible for compiling -make_prepare: - # Non-interactive targets. Mostly, those for which you do not need to build a binary NON_INTERACTIVE_TARGET += defconfig clean% %clean help list-components print_flash_cmd check_python_dependencies @@ -94,6 +94,15 @@ ifndef IDF_PATH $(error IDF_PATH variable is not set to a valid directory.) endif +ifdef IDF_TARGET +ifneq ($(IDF_TARGET),esp32) +$(error GNU Make based build system only supports esp32 target, but IDF_TARGET is set to $(IDF_TARGET)) +endif +else +export IDF_TARGET := esp32 +endif + + ifneq ("$(IDF_PATH)","$(SANITISED_IDF_PATH)") # implies IDF_PATH was overriden on make command line. # Due to the way make manages variables, this is hard to account for @@ -119,7 +128,7 @@ export PROJECT_PATH endif # A list of the "common" makefiles, to use as a target dependency -COMMON_MAKEFILES := $(abspath $(IDF_PATH)/make/project.mk $(IDF_PATH)/make/common.mk $(IDF_PATH)/make/component_wrapper.mk $(firstword $(MAKEFILE_LIST))) +COMMON_MAKEFILES := $(abspath $(IDF_PATH)/make/project.mk $(IDF_PATH)/make/common.mk $(IDF_PATH)/make/version.mk $(IDF_PATH)/make/component_wrapper.mk $(firstword $(MAKEFILE_LIST))) export COMMON_MAKEFILES # The directory where we put all objects/libraries/binaries. The project Makefile can @@ -212,6 +221,66 @@ endif TEST_COMPONENT_PATHS := $(foreach comp,$(TEST_COMPONENTS_LIST),$(firstword $(foreach dir,$(COMPONENT_DIRS),$(wildcard $(dir)/$(comp)/test)))) TEST_COMPONENT_NAMES := $(foreach comp,$(TEST_COMPONENT_PATHS),$(lastword $(subst /, ,$(dir $(comp))))_test) +# Set default values that were not previously defined +CC ?= gcc +LD ?= ld +AR ?= ar +OBJCOPY ?= objcopy +OBJDUMP ?= objdump +SIZE ?= size + +# Set host compiler and binutils +HOSTCC := $(CC) +HOSTLD := $(LD) +HOSTAR := $(AR) +HOSTOBJCOPY := $(OBJCOPY) +HOSTSIZE := $(SIZE) +export HOSTCC HOSTLD HOSTAR HOSTOBJCOPY SIZE + +# Set variables common to both project & component (includes config) +include $(IDF_PATH)/make/common.mk + +# Notify users when some of the required python packages are not installed +.PHONY: check_python_dependencies +check_python_dependencies: +ifndef IS_BOOTLOADER_BUILD + $(PYTHON) $(IDF_PATH)/tools/check_python_dependencies.py +endif + +# include the config generation targets (dependency: COMPONENT_PATHS) +# +# (bootloader build doesn't need this, config is exported from top-level) +ifndef IS_BOOTLOADER_BUILD +include $(IDF_PATH)/make/project_config.mk +endif + +##################################################################### +# If SDKCONFIG_MAKEFILE hasn't been generated yet (detected if no +# CONFIG_IDF_TARGET), stop the Makefile pass now to allow config to +# be created. make will build SDKCONFIG_MAKEFILE and restart, +# reevaluating everything from the top. +# +# This is important so config is present when the +# component_project_vars.mk files are generated. +# +# (After both files exist, if SDKCONFIG_MAKEFILE is updated then the +# normal dependency relationship will trigger a regeneration of +# component_project_vars.mk) +# +##################################################################### +ifndef CONFIG_IDF_TARGET +ifdef IS_BOOTLOADER_BUILD # we expect config to always have been expanded by top level project +$(error "Internal error: config has not been passed correctly to bootloader subproject") +endif +ifdef MAKE_RESTARTS +$(warning "Config was not evaluated after the first pass of 'make'") +endif +else # CONFIG_IDF_TARGET +##################################################################### +# Config is valid, can include rest of the Project Makefile +##################################################################### + + # Initialise project-wide variables which can be added to by # each component. # @@ -223,6 +292,7 @@ COMPONENT_INCLUDES := COMPONENT_LDFLAGS := COMPONENT_SUBMODULES := COMPONENT_LIBRARIES := +COMPONENT_LDFRAGMENTS := # COMPONENT_PROJECT_VARS is the list of component_project_vars.mk generated makefiles # for each component. @@ -242,9 +312,6 @@ COMPONENT_INCLUDES += $(abspath $(BUILD_DIR_BASE)/include/) export COMPONENT_INCLUDES -# Set variables common to both project & component -include $(IDF_PATH)/make/common.mk - all: ifdef CONFIG_SECURE_BOOT_ENABLED @echo "(Secure boot enabled, so bootloader not flashed automatically. See 'make bootloader' output)" @@ -262,10 +329,11 @@ endif # If we have `version.txt` then prefer that for extracting IDF version ifeq ("$(wildcard ${IDF_PATH}/version.txt)","") -IDF_VER := $(shell cd ${IDF_PATH} && git describe --always --tags --dirty) +IDF_VER_T := $(shell cd ${IDF_PATH} && git describe --always --tags --dirty) else -IDF_VER := `cat ${IDF_PATH}/version.txt` +IDF_VER_T := $(shell cat ${IDF_PATH}/version.txt) endif +IDF_VER := $(shell echo "$(IDF_VER_T)" | cut -c 1-31) # Set default LDFLAGS EXTRA_LDFLAGS ?= @@ -295,6 +363,10 @@ LDFLAGS ?= -nostdlib \ CPPFLAGS ?= EXTRA_CPPFLAGS ?= CPPFLAGS := -DESP_PLATFORM -D IDF_VER=\"$(IDF_VER)\" -MMD -MP $(CPPFLAGS) $(EXTRA_CPPFLAGS) +PROJECT_VER ?= +export IDF_VER +export PROJECT_NAME +export PROJECT_VER # Warnings-related flags relevant both for C and C++ COMMON_WARNING_FLAGS = -Wall -Werror=all \ @@ -305,12 +377,29 @@ COMMON_WARNING_FLAGS = -Wall -Werror=all \ -Wextra \ -Wno-unused-parameter -Wno-sign-compare -ifdef CONFIG_WARN_WRITE_STRINGS +ifdef CONFIG_COMPILER_DISABLE_GCC8_WARNINGS +COMMON_WARNING_FLAGS += -Wno-parentheses \ + -Wno-sizeof-pointer-memaccess \ + -Wno-clobbered \ + -Wno-format-overflow \ + -Wno-stringop-truncation \ + -Wno-misleading-indentation \ + -Wno-cast-function-type \ + -Wno-implicit-fallthrough \ + -Wno-unused-const-variable \ + -Wno-switch-unreachable \ + -Wno-format-truncation \ + -Wno-memset-elt-size \ + -Wno-int-in-bool-context +endif + +ifdef CONFIG_COMPILER_WARN_WRITE_STRINGS COMMON_WARNING_FLAGS += -Wwrite-strings -endif #CONFIG_WARN_WRITE_STRINGS +endif #CONFIG_COMPILER_WARN_WRITE_STRINGS # Flags which control code generation and dependency generation, both for C and C++ COMMON_FLAGS = \ + -Wno-frame-address \ -ffunction-sections -fdata-sections \ -fstrict-volatile-bitfields \ -mlongcalls \ @@ -318,28 +407,31 @@ COMMON_FLAGS = \ ifndef IS_BOOTLOADER_BUILD # stack protection (only one option can be selected in menuconfig) -ifdef CONFIG_STACK_CHECK_NORM +ifdef CONFIG_COMPILER_STACK_CHECK_MODE_NORM COMMON_FLAGS += -fstack-protector endif -ifdef CONFIG_STACK_CHECK_STRONG +ifdef CONFIG_COMPILER_STACK_CHECK_MODE_STRONG COMMON_FLAGS += -fstack-protector-strong endif -ifdef CONFIG_STACK_CHECK_ALL +ifdef CONFIG_COMPILER_STACK_CHECK_MODE_ALL COMMON_FLAGS += -fstack-protector-all endif endif # Optimization flags are set based on menuconfig choice -ifdef CONFIG_OPTIMIZATION_LEVEL_RELEASE +ifdef CONFIG_COMPILER_OPTIMIZATION_LEVEL_RELEASE OPTIMIZATION_FLAGS = -Os else OPTIMIZATION_FLAGS = -Og endif -ifdef CONFIG_OPTIMIZATION_ASSERTIONS_DISABLED +ifdef CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE CPPFLAGS += -DNDEBUG endif +# IDF uses some GNU extension from libc +CPPFLAGS += -D_GNU_SOURCE + # Enable generation of debugging symbols # (we generate even in Release mode, as this has no impact on final binary size.) DEBUG_FLAGS ?= -ggdb @@ -369,7 +461,7 @@ CXXFLAGS := $(strip \ $(CXXFLAGS) \ $(EXTRA_CXXFLAGS)) -ifdef CONFIG_CXX_EXCEPTIONS +ifdef CONFIG_COMPILER_CXX_EXCEPTIONS CXXFLAGS += -fexceptions else CXXFLAGS += -fno-exceptions @@ -379,30 +471,16 @@ ARFLAGS := cru export CFLAGS CPPFLAGS CXXFLAGS ARFLAGS -# Set default values that were not previously defined -CC ?= gcc -LD ?= ld -AR ?= ar -OBJCOPY ?= objcopy -SIZE ?= size - -# Set host compiler and binutils -HOSTCC := $(CC) -HOSTLD := $(LD) -HOSTAR := $(AR) -HOSTOBJCOPY := $(OBJCOPY) -HOSTSIZE := $(SIZE) -export HOSTCC HOSTLD HOSTAR HOSTOBJCOPY SIZE - # Set target compiler. Defaults to whatever the user has # configured as prefix + ye olde gcc commands -CC := $(call dequote,$(CONFIG_TOOLPREFIX))gcc -CXX := $(call dequote,$(CONFIG_TOOLPREFIX))c++ -LD := $(call dequote,$(CONFIG_TOOLPREFIX))ld -AR := $(call dequote,$(CONFIG_TOOLPREFIX))ar -OBJCOPY := $(call dequote,$(CONFIG_TOOLPREFIX))objcopy -SIZE := $(call dequote,$(CONFIG_TOOLPREFIX))size -export CC CXX LD AR OBJCOPY SIZE +CC := $(call dequote,$(CONFIG_SDK_TOOLPREFIX))gcc +CXX := $(call dequote,$(CONFIG_SDK_TOOLPREFIX))c++ +LD := $(call dequote,$(CONFIG_SDK_TOOLPREFIX))ld +AR := $(call dequote,$(CONFIG_SDK_TOOLPREFIX))ar +OBJCOPY := $(call dequote,$(CONFIG_SDK_TOOLPREFIX))objcopy +OBJDUMP := $(call dequote,$(CONFIG_SDK_TOOLPREFIX))objdump +SIZE := $(call dequote,$(CONFIG_SDK_TOOLPREFIX))size +export CC CXX LD AR OBJCOPY OBJDUMP SIZE COMPILER_VERSION_STR := $(shell $(CC) -dumpversion) COMPILER_VERSION_NUM := $(subst .,,$(COMPILER_VERSION_STR)) @@ -412,17 +490,23 @@ export COMPILER_VERSION_STR COMPILER_VERSION_NUM GCC_NOT_5_2_0 CPPFLAGS += -DGCC_NOT_5_2_0=$(GCC_NOT_5_2_0) export CPPFLAGS -PYTHON=$(call dequote,$(CONFIG_PYTHON)) # the app is the main executable built by the project APP_ELF:=$(BUILD_DIR_BASE)/$(PROJECT_NAME).elf APP_MAP:=$(APP_ELF:.elf=.map) APP_BIN:=$(APP_ELF:.elf=.bin) +# include linker script generation utils makefile +include $(IDF_PATH)/make/ldgen.mk + +$(eval $(call ldgen_create_commands)) + # Include any Makefile.projbuild file letting components add # configuration at the project level define includeProjBuildMakefile -$(if $(V),$$(info including $(1)/Makefile.projbuild...)) +ifeq ("$(V)","1") +$$(info including $(1)/Makefile.projbuild...) +endif COMPONENT_PATH := $(1) include $(1)/Makefile.projbuild endef @@ -430,13 +514,6 @@ $(foreach componentpath,$(COMPONENT_PATHS), \ $(if $(wildcard $(componentpath)/Makefile.projbuild), \ $(eval $(call includeProjBuildMakefile,$(componentpath))))) -# once we know component paths, we can include the config generation targets -# -# (bootloader build doesn't need this, config is exported from top-level) -ifndef IS_BOOTLOADER_BUILD -include $(IDF_PATH)/make/project_config.mk -endif - # ELF depends on the library archive files for COMPONENT_LIBRARIES # the rules to build these are emitted as part of GenerateComponentTarget below # @@ -458,14 +535,6 @@ else @echo $(ESPTOOLPY_WRITE_FLASH) $(APP_OFFSET) $(APP_BIN) endif -.PHONY: check_python_dependencies - -# Notify users when some of the required python packages are not installed -check_python_dependencies: -ifndef IS_BOOTLOADER_BUILD - $(PYTHON) $(IDF_PATH)/tools/check_python_dependencies.py -endif - all_binaries: $(APP_BIN) $(BUILD_DIR_BASE): @@ -541,7 +610,7 @@ endif # _config-clean), so config remains valid during all component clean # targets config-clean: app-clean bootloader-clean -clean: app-clean bootloader-clean config-clean +clean: app-clean bootloader-clean config-clean ldgen-clean # phony target to check if any git submodule listed in COMPONENT_SUBMODULES are missing # or out of date, and exit if so. Components can add paths to this variable. @@ -560,8 +629,8 @@ define GenerateSubmoduleCheckTarget check-submodules: $(IDF_PATH)/$(1)/.git $(IDF_PATH)/$(1)/.git: @echo "WARNING: Missing submodule $(1)..." - [ -e ${IDF_PATH}/.git ] || ( echo "ERROR: esp-idf must be cloned from git to work."; exit 1) - [ -x "$(shell which git)" ] || ( echo "ERROR: Need to run 'git submodule init $(1)' in esp-idf root directory."; exit 1) + [ -e ${IDF_PATH}/.git ] || { echo "ERROR: esp-idf must be cloned from git to work."; exit 1; } + [ -x "$(shell which git)" ] || { echo "ERROR: Need to run 'git submodule init $(1)' in esp-idf root directory."; exit 1; } @echo "Attempting 'git submodule update --init $(1)' in esp-idf root directory..." cd ${IDF_PATH} && git submodule update --init $(1) @@ -600,7 +669,7 @@ print_flash_cmd: partition_table_get_info blank_ota_data # The output normally looks as follows # xtensa-esp32-elf-gcc (crosstool-NG crosstool-ng-1.22.0-80-g6c4433a) 5.2.0 # The part in brackets is extracted into TOOLCHAIN_COMMIT_DESC variable -ifdef CONFIG_TOOLPREFIX +ifdef CONFIG_SDK_TOOLPREFIX ifndef MAKE_RESTARTS TOOLCHAIN_HEADER := $(shell $(CC) --version | head -1) @@ -639,4 +708,8 @@ $(info WARNING: Failed to find Xtensa toolchain, may need to alter PATH or set o endif # TOOLCHAIN_COMMIT_DESC endif #MAKE_RESTARTS -endif #CONFIG_TOOLPREFIX +endif #CONFIG_SDK_TOOLPREFIX + +##################################################################### +endif #CONFIG_IDF_TARGET + diff --git a/make/project_config.mk b/make/project_config.mk index 50cf139e..e590b401 100644 --- a/make/project_config.mk +++ b/make/project_config.mk @@ -4,6 +4,12 @@ COMPONENT_KCONFIGS := $(foreach component,$(COMPONENT_PATHS),$(wildcard $(component)/Kconfig)) COMPONENT_KCONFIGS_PROJBUILD := $(foreach component,$(COMPONENT_PATHS),$(wildcard $(component)/Kconfig.projbuild)) +ifeq ($(OS),Windows_NT) +# kconfiglib requires Windows-style paths for kconfig files +COMPONENT_KCONFIGS := $(shell cygpath -w $(COMPONENT_KCONFIGS)) +COMPONENT_KCONFIGS_PROJBUILD := $(shell cygpath -w $(COMPONENT_KCONFIGS_PROJBUILD)) +endif + #For doing make menuconfig etc KCONFIG_TOOL_DIR=$(IDF_PATH)/tools/kconfig @@ -36,8 +42,24 @@ $(SDKCONFIG): defconfig endif endif +# macro for running confgen.py +define RunConfGen + mkdir -p $(BUILD_DIR_BASE)/include/config + $(PYTHON) $(IDF_PATH)/tools/kconfig_new/confgen.py \ + --kconfig $(IDF_PATH)/Kconfig \ + --config $(SDKCONFIG) \ + --env "COMPONENT_KCONFIGS=$(strip $(COMPONENT_KCONFIGS))" \ + --env "COMPONENT_KCONFIGS_PROJBUILD=$(strip $(COMPONENT_KCONFIGS_PROJBUILD))" \ + --env "IDF_CMAKE=n" \ + --output config ${SDKCONFIG} \ + --output makefile $(SDKCONFIG_MAKEFILE) \ + --output header $(BUILD_DIR_BASE)/include/sdkconfig.h +endef + # macro for the commands to run kconfig tools conf-idf or mconf-idf. # $1 is the name (& args) of the conf tool to run +# Note: Currently only mconf-idf is used for compatibility with the CMake build system. The header file used is also +# the same. define RunConf mkdir -p $(BUILD_DIR_BASE)/include/config cd $(BUILD_DIR_BASE); KCONFIG_AUTOHEADER=$(abspath $(BUILD_DIR_BASE)/include/sdkconfig.h) \ @@ -59,7 +81,7 @@ ifndef MAKE_RESTARTS # depend on any prerequisite that may cause a make restart as part of # the prerequisite's own recipe. -menuconfig: $(KCONFIG_TOOL_DIR)/mconf-idf +menuconfig: $(KCONFIG_TOOL_DIR)/mconf-idf | check_python_dependencies $(summary) MENUCONFIG ifdef BATCH_BUILD @echo "Can't run interactive configuration inside non-interactive build process." @@ -68,25 +90,26 @@ ifdef BATCH_BUILD @echo "See esp-idf documentation for more details." @exit 1 else + $(call RunConfGen) + # RunConfGen before mconf-idf ensures that deprecated options won't be ignored (they've got renamed) $(call RunConf,mconf-idf) + # RunConfGen after mconf-idf ensures that deprecated options are appended to $(SDKCONFIG) for backward compatibility + $(call RunConfGen) endif # defconfig creates a default config, based on SDKCONFIG_DEFAULTS if present -defconfig: $(KCONFIG_TOOL_DIR)/conf-idf +defconfig: | check_python_dependencies $(summary) DEFCONFIG ifneq ("$(wildcard $(SDKCONFIG_DEFAULTS))","") cat $(SDKCONFIG_DEFAULTS) >> $(SDKCONFIG) # append defaults to sdkconfig, will override existing values endif - $(call RunConf,conf-idf --olddefconfig) + $(call RunConfGen) # if neither defconfig or menuconfig are requested, use the GENCONFIG rule to # ensure generated config files are up to date -$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(KCONFIG_TOOL_DIR)/conf-idf $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig) +$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | check_python_dependencies $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig) $(summary) GENCONFIG -ifdef BATCH_BUILD # can't prompt for new config values like on terminal - $(call RunConf,conf-idf --olddefconfig) -endif - $(call RunConf,conf-idf --silentoldconfig) + $(call RunConfGen) touch $(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h # ensure newer than sdkconfig else # "$(MAKE_RESTARTS)" != "" diff --git a/make/version.mk b/make/version.mk new file mode 100644 index 00000000..9dcda422 --- /dev/null +++ b/make/version.mk @@ -0,0 +1,3 @@ +IDF_VERSION_MAJOR := 4 +IDF_VERSION_MINOR := 0 +IDF_VERSION_PATCH := 0 diff --git a/tools/README b/tools/README deleted file mode 100644 index fffe7191..00000000 --- a/tools/README +++ /dev/null @@ -1,11 +0,0 @@ -README - -Information of 'idf_monitor.py' is following: - -URL: https://github.com/espressif/esp-idf -tag: v3.1-dev - -Information of 'cmake', 'kconfig', 'kconfig_new' and 'idf.py' is following: - -commit: 48c3ad37 - diff --git a/tools/check_kconfigs.py b/tools/check_kconfigs.py new file mode 100755 index 00000000..bdc3953e --- /dev/null +++ b/tools/check_kconfigs.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +from __future__ import unicode_literals +import os +import sys +import re +import argparse +from io import open + +# regular expression for matching Kconfig files +RE_KCONFIG = r'^Kconfig(\.projbuild)?(\.in)?$' + +# ouput file with suggestions will get this suffix +OUTPUT_SUFFIX = '.new' + +# ignored directories (makes sense only when run on IDF_PATH) +# Note: IGNORE_DIRS is a tuple in order to be able to use it directly with the startswith() built-in function which +# accepts tuples but no lists. +IGNORE_DIRS = ( + # Kconfigs from submodules need to be ignored: + os.path.join('components', 'mqtt', 'esp-mqtt'), + # Test Kconfigs are also ignored + os.path.join('tools', 'ldgen', 'test', 'data'), + os.path.join('tools', 'kconfig_new', 'test'), +) + +SPACES_PER_INDENT = 4 + +CONFIG_NAME_MAX_LENGTH = 40 + +CONFIG_NAME_MIN_PREFIX_LENGTH = 3 + +# The checker will not fail if it encounters this string (it can be used for temporarily resolve conflicts) +RE_NOERROR = re.compile(r'\s+#\s+NOERROR\s+$') + +# list or rules for lines +LINE_ERROR_RULES = [ + # (regular expression for finding, error message, correction) + (re.compile(r'\t'), 'tabulators should be replaced by spaces', r' ' * SPACES_PER_INDENT), + (re.compile(r'\s+\n'), 'trailing whitespaces should be removed', r'\n'), + (re.compile(r'.{120}'), 'line should be shorter than 120 characters', None), + # "\" is not recognized due to a bug in tools/kconfig/zconf.l. The bug was fixed but the rebuild of + # mconf-idf is not enforced and an incorrect version is supplied with all previous IDF versions. Backslashes + # cannot be enabled unless everybody updates mconf-idf. + (re.compile(r'\\\n'), 'line cannot be wrapped by backslash', None), +] + + +class InputError(RuntimeError): + """ + Represents and error on the input + """ + def __init__(self, path, line_number, error_msg, suggested_line): + super(InputError, self).__init__('{}:{}: {}'.format(path, line_number, error_msg)) + self.suggested_line = suggested_line + + +class BaseChecker(object): + """ + Base class for all checker objects + """ + def __init__(self, path_in_idf): + self.path_in_idf = path_in_idf + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + pass + + +class SourceChecker(BaseChecker): + # allow to source only files which will be also checked by the script + # Note: The rules are complex and the LineRuleChecker cannot be used + def process_line(self, line, line_number): + m = re.search(r'^\s*source(\s*)"([^"]+)"', line) + if m: + if len(m.group(1)) == 0: + raise InputError(self.path_in_idf, line_number, '"source" has to been followed by space', + line.replace('source', 'source ')) + path = m.group(2) + if path in ['$COMPONENT_KCONFIGS_PROJBUILD', '$COMPONENT_KCONFIGS']: + pass + elif not path.endswith('/Kconfig.in') and path != 'Kconfig.in': + raise InputError(self.path_in_idf, line_number, "only Kconfig.in can be sourced", + line.replace(path, os.path.join(os.path.dirname(path), 'Kconfig.in'))) + + +class LineRuleChecker(BaseChecker): + """ + checks LINE_ERROR_RULES for each line + """ + def process_line(self, line, line_number): + suppress_errors = RE_NOERROR.search(line) is not None + errors = [] + for rule in LINE_ERROR_RULES: + m = rule[0].search(line) + if m: + if suppress_errors: + # just print but no failure + e = InputError(self.path_in_idf, line_number, rule[1], line) + print(e) + else: + errors.append(rule[1]) + if rule[2]: + line = rule[0].sub(rule[2], line) + if len(errors) > 0: + raise InputError(self.path_in_idf, line_number, "; ".join(errors), line) + + +class IndentAndNameChecker(BaseChecker): + """ + checks the indentation of each line and configuration names + """ + def __init__(self, path_in_idf, debug=False): + super(IndentAndNameChecker, self).__init__(path_in_idf) + self.debug = debug + self.min_prefix_length = CONFIG_NAME_MIN_PREFIX_LENGTH + + # stack of the nested menuconfig items, e.g. ['mainmenu', 'menu', 'config'] + self.level_stack = [] + + # stack common prefixes of configs + self.prefix_stack = [] + + # if the line ends with '\' then we force the indent of the next line + self.force_next_indent = 0 + + # menu items which increase the indentation of the next line + self.re_increase_level = re.compile(r'''^\s* + ( + (menu(?!config)) + |(mainmenu) + |(choice) + |(config) + |(menuconfig) + |(help) + |(if) + |(source) + ) + ''', re.X) + + # closing menu items which decrease the indentation + self.re_decrease_level = re.compile(r'''^\s* + ( + (endmenu) + |(endchoice) + |(endif) + ) + ''', re.X) + + # matching beginning of the closing menuitems + self.pair_dic = {'endmenu': 'menu', + 'endchoice': 'choice', + 'endif': 'if', + } + + # regex for config names + self.re_name = re.compile(r'''^ + ( + (?:config) + |(?:menuconfig) + |(?:choice) + + )\s+ + (\w+) + ''', re.X) + + # regex for new prefix stack + self.re_new_stack = re.compile(r'''^ + ( + (?:menu(?!config)) + |(?:mainmenu) + |(?:choice) + + ) + ''', re.X) + + def __exit__(self, type, value, traceback): + super(IndentAndNameChecker, self).__exit__(type, value, traceback) + if len(self.prefix_stack) > 0: + self.check_common_prefix('', 'EOF') + if len(self.prefix_stack) != 0: + if self.debug: + print(self.prefix_stack) + raise RuntimeError("Prefix stack should be empty. Perhaps a menu/choice hasn't been closed") + + def del_from_level_stack(self, count): + """ delete count items from the end of the level_stack """ + if count > 0: + # del self.level_stack[-0:] would delete everything and we expect not to delete anything for count=0 + del self.level_stack[-count:] + + def update_level_for_inc_pattern(self, new_item): + if self.debug: + print('level+', new_item, ': ', self.level_stack, end=' -> ') + # "config" and "menuconfig" don't have a closing pair. So if new_item is an item which need to be indented + # outside the last "config" or "menuconfig" then we need to find to a parent where it belongs + if new_item in ['config', 'menuconfig', 'menu', 'choice', 'if', 'source']: + # item is not belonging to a previous "config" or "menuconfig" so need to indent to parent + for i, item in enumerate(reversed(self.level_stack)): + if item in ['menu', 'mainmenu', 'choice', 'if']: + # delete items ("config", "menuconfig", "help") until the appropriate parent + self.del_from_level_stack(i) + break + else: + # delete everything when configs are at top level without a parent menu, mainmenu... + self.del_from_level_stack(len(self.level_stack)) + + self.level_stack.append(new_item) + if self.debug: + print(self.level_stack) + # The new indent is for the next line. Use the old one for the current line: + return len(self.level_stack) - 1 + + def update_level_for_dec_pattern(self, new_item): + if self.debug: + print('level-', new_item, ': ', self.level_stack, end=' -> ') + target = self.pair_dic[new_item] + for i, item in enumerate(reversed(self.level_stack)): + # find the matching beginning for the closing item in reverse-order search + # Note: "menuconfig", "config" and "help" don't have closing pairs and they are also on the stack. Now they + # will be deleted together with the "menu" or "choice" we are closing. + if item == target: + i += 1 # delete also the matching beginning + if self.debug: + print('delete ', i, end=' -> ') + self.del_from_level_stack(i) + break + if self.debug: + print(self.level_stack) + return len(self.level_stack) + + def check_name_and_update_prefix(self, line, line_number): + m = self.re_name.search(line) + if m: + name = m.group(2) + name_length = len(name) + + if name_length > CONFIG_NAME_MAX_LENGTH: + raise InputError(self.path_in_idf, line_number, + '{} is {} characters long and it should be {} at most' + ''.format(name, name_length, CONFIG_NAME_MAX_LENGTH), + line + '\n') # no suggested correction for this + if len(self.prefix_stack) == 0: + self.prefix_stack.append(name) + elif self.prefix_stack[-1] is None: + self.prefix_stack[-1] = name + else: + # this has nothing common with paths but the algorithm can be used for this also + self.prefix_stack[-1] = os.path.commonprefix([self.prefix_stack[-1], name]) + if self.debug: + print('prefix+', self.prefix_stack) + m = self.re_new_stack.search(line) + if m: + self.prefix_stack.append(None) + if self.debug: + print('prefix+', self.prefix_stack) + + def check_common_prefix(self, line, line_number): + common_prefix = self.prefix_stack.pop() + if self.debug: + print('prefix-', self.prefix_stack) + if common_prefix is None: + return + common_prefix_len = len(common_prefix) + if common_prefix_len < self.min_prefix_length: + raise InputError(self.path_in_idf, line_number, + 'The common prefix for the config names of the menu ending at this line is "{}". ' + 'All config names in this menu should start with the same prefix of {} characters ' + 'or more.'.format(common_prefix, self.min_prefix_length), + line) # no suggested correction for this + if len(self.prefix_stack) > 0: + parent_prefix = self.prefix_stack[-1] + if parent_prefix is None: + # propagate to parent level where it will influence the prefix checking with the rest which might + # follow later on that level + self.prefix_stack[-1] = common_prefix + else: + if len(self.level_stack) > 0 and self.level_stack[-1] in ['mainmenu', 'menu']: + # the prefix from menu is not required to propagate to the children + return + if not common_prefix.startswith(parent_prefix): + raise InputError(self.path_in_idf, line_number, + 'Common prefix "{}" should start with {}' + ''.format(common_prefix, parent_prefix), + line) # no suggested correction for this + + def process_line(self, line, line_number): + stripped_line = line.strip() + if len(stripped_line) == 0: + self.force_next_indent = 0 + return + current_level = len(self.level_stack) + m = re.search(r'\S', line) # indent found as the first non-space character + if m: + current_indent = m.start() + else: + current_indent = 0 + + if current_level > 0 and self.level_stack[-1] == 'help': + if current_indent >= current_level * SPACES_PER_INDENT: + # this line belongs to 'help' + self.force_next_indent = 0 + return + + if self.force_next_indent > 0: + if current_indent != self.force_next_indent: + raise InputError(self.path_in_idf, line_number, + 'Indentation consists of {} spaces instead of {}'.format(current_indent, + self.force_next_indent), + (' ' * self.force_next_indent) + line.lstrip()) + else: + if not stripped_line.endswith('\\'): + self.force_next_indent = 0 + return + + elif stripped_line.endswith('\\') and stripped_line.startswith(('config', 'menuconfig', 'choice')): + raise InputError(self.path_in_idf, line_number, + 'Line-wrap with backslash is not supported here', + line) # no suggestion for this + + self.check_name_and_update_prefix(stripped_line, line_number) + + m = self.re_increase_level.search(line) + if m: + current_level = self.update_level_for_inc_pattern(m.group(1)) + else: + m = self.re_decrease_level.search(line) + if m: + new_item = m.group(1) + current_level = self.update_level_for_dec_pattern(new_item) + if new_item not in ['endif']: + # endif doesn't require to check the prefix because the items inside if/endif belong to the + # same prefix level + self.check_common_prefix(line, line_number) + + expected_indent = current_level * SPACES_PER_INDENT + + if stripped_line.endswith('\\'): + self.force_next_indent = expected_indent + SPACES_PER_INDENT + else: + self.force_next_indent = 0 + + if current_indent != expected_indent: + raise InputError(self.path_in_idf, line_number, + 'Indentation consists of {} spaces instead of {}'.format(current_indent, expected_indent), + (' ' * expected_indent) + line.lstrip()) + + +def valid_directory(path): + if not os.path.isdir(path): + raise argparse.ArgumentTypeError("{} is not a valid directory!".format(path)) + return path + + +def main(): + default_path = os.getenv('IDF_PATH', None) + + parser = argparse.ArgumentParser(description='Kconfig style checker') + parser.add_argument('--verbose', '-v', help='Print more information (useful for debugging)', + action='store_true', default=False) + parser.add_argument('--directory', '-d', help='Path to directory where Kconfigs should be recursively checked ' + '(for example $IDF_PATH)', + type=valid_directory, + required=default_path is None, + default=default_path) + args = parser.parse_args() + + success_couter = 0 + ignore_counter = 0 + failure = False + + # IGNORE_DIRS makes sense when the required directory is IDF_PATH + check_ignore_dirs = default_path is not None and os.path.abspath(args.directory) == os.path.abspath(default_path) + + for root, dirnames, filenames in os.walk(args.directory): + for filename in filenames: + full_path = os.path.join(root, filename) + path_in_idf = os.path.relpath(full_path, args.directory) + if re.search(RE_KCONFIG, filename): + if check_ignore_dirs and path_in_idf.startswith(IGNORE_DIRS): + print('{}: Ignored'.format(path_in_idf)) + ignore_counter += 1 + continue + suggestions_full_path = full_path + OUTPUT_SUFFIX + with open(full_path, 'r', encoding='utf-8') as f, \ + open(suggestions_full_path, 'w', encoding='utf-8', newline='\n') as f_o, \ + LineRuleChecker(path_in_idf) as line_checker, \ + SourceChecker(path_in_idf) as source_checker, \ + IndentAndNameChecker(path_in_idf, debug=args.verbose) as indent_and_name_checker: + try: + for line_number, line in enumerate(f, start=1): + try: + for checker in [line_checker, indent_and_name_checker, source_checker]: + checker.process_line(line, line_number) + # The line is correct therefore we echo it to the output file + f_o.write(line) + except InputError as e: + print(e) + failure = True + f_o.write(e.suggested_line) + except UnicodeDecodeError: + raise ValueError("The encoding of {} is not Unicode.".format(path_in_idf)) + + if failure: + print('{} has been saved with suggestions for resolving the issues. Please note that the ' + 'suggestions can be wrong and you might need to re-run the checker several times ' + 'for solving all issues'.format(path_in_idf + OUTPUT_SUFFIX)) + print('Please fix the errors and run {} for checking the correctness of ' + 'Kconfigs.'.format(os.path.relpath(os.path.abspath(__file__), args.directory))) + sys.exit(1) + else: + success_couter += 1 + print('{}: OK'.format(path_in_idf)) + try: + os.remove(suggestions_full_path) + except Exception: + # not a serious error is when the file cannot be deleted + print('{} cannot be deleted!'.format(suggestions_full_path)) + elif re.search(RE_KCONFIG, filename, re.IGNORECASE): + # On Windows Kconfig files are working with different cases! + raise ValueError('Incorrect filename of {}. The case should be "Kconfig"!'.format(path_in_idf)) + + if ignore_counter > 0: + print('{} files have been ignored.'.format(ignore_counter)) + + if success_couter > 0: + print('{} files have been successfully checked.'.format(success_couter)) + + +if __name__ == "__main__": + main() diff --git a/tools/check_python_dependencies.py b/tools/check_python_dependencies.py index 45baee62..e2b58b29 100755 --- a/tools/check_python_dependencies.py +++ b/tools/check_python_dependencies.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # # Copyright 2018 Espressif Systems (Shanghai) PTE LTD # @@ -14,63 +14,91 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import os import sys -import argparse + try: import pkg_resources -except: +except Exception: print('pkg_resources cannot be imported probably because the pip package is not installed and/or using a ' 'legacy Python interpreter. Please refer to the Get Started section of the ESP-IDF Programming Guide for ' - 'setting up the required packages.') + 'setting up the required packages.') sys.exit(1) + +def escape_backslash(path): + if sys.platform == "win32": + # escaped backslashes are necessary in order to be able to copy-paste the printed path + return path.replace("\\", "\\\\") + else: + return path + + +def is_virtualenv(): + """Detects if current python is inside virtualenv, pyvenv (python 3.4-3.5) or venv""" + + return (hasattr(sys, 'real_prefix') or + (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix)) + + if __name__ == "__main__": idf_path = os.getenv("IDF_PATH") parser = argparse.ArgumentParser(description='ESP32 Python package dependency checker') parser.add_argument('--requirements', '-r', - help='Path to the requrements file', - default=idf_path + '/requirements.txt') + help='Path to the requrements file', + default=os.path.join(idf_path, 'requirements.txt')) args = parser.parse_args() - # Special case for MINGW32 Python, needs some packages - # via MSYS2 not via pip or system breaks... - if sys.platform == "win32" and \ - os.environ.get("MSYSTEM", None) == "MINGW32" and \ - "/mingw32/bin/python" in sys.executable: - failed = False - try: - import cryptography - except ImportError: - print("Please run the following command to install MSYS2's MINGW Python cryptography package:") - print("pacman -S mingw-w64-i686-python%d-cryptography" % (sys.version_info[0],)) - failed = True - try: - import setuptools - except ImportError: - print("Please run the following command to install MSYS2's MINGW Python setuptools package:") - print("pacman -S mingw-w64-i686-python%d-setuptools" % (sys.version_info[0],)) - failed = True - if failed: - sys.exit(1) - not_satisfied = [] with open(args.requirements) as f: for line in f: line = line.strip() try: pkg_resources.require(line) - except: + except Exception: not_satisfied.append(line) if len(not_satisfied) > 0: print('The following Python requirements are not satisfied:') for requirement in not_satisfied: print(requirement) - print('Please refer to the Get Started section of the ESP-IDF Programming Guide for setting up the required ' - 'packages. Alternatively, you can run "{} -m pip install --user -r {}" for resolving the issue.' - ''.format(sys.executable, args.requirements)) + if os.environ.get('IDF_PYTHON_ENV_PATH'): + # We are running inside a private virtual environment under IDF_TOOLS_PATH, + # ask the user to run install.bat again. + if sys.platform == "win32" and not os.environ.get("MSYSTEM"): + install_script = 'install.bat' + else: + install_script = 'install.sh' + print('To install the missing packages, please run "%s"' % os.path.join(idf_path, install_script)) + elif sys.platform == "win32" and os.environ.get("MSYSTEM", None) == "MINGW32" and "/mingw32/bin/python" in sys.executable: + print("The recommended way to install a packages is via \"pacman\". Please run \"pacman -Ss \" for" + " searching the package database and if found then " + "\"pacman -S mingw-w64-i686-python{}-\" for installing it.".format(sys.version_info[0],)) + print("NOTE: You may need to run \"pacman -Syu\" if your package database is older and run twice if the " + "previous run updated \"pacman\" itself.") + print("Please read https://github.com/msys2/msys2/wiki/Using-packages for further information about using " + "\"pacman\"") + # Special case for MINGW32 Python, needs some packages + # via MSYS2 not via pip or system breaks... + for requirement in not_satisfied: + if requirement.startswith('cryptography'): + print("WARNING: The cryptography package have dependencies on system packages so please make sure " + "you run \"pacman -Syu\" followed by \"pacman -S mingw-w64-i686-python{}-cryptography\"." + "".format(sys.version_info[0],)) + continue + elif requirement.startswith('setuptools'): + print("Please run the following command to install MSYS2's MINGW Python setuptools package:") + print("pacman -S mingw-w64-i686-python{}-setuptools".format(sys.version_info[0],)) + continue + else: + print('Please refer to the Get Started section of the ESP-IDF Programming Guide for setting up the required' + ' packages.') + print('Alternatively, you can run "{} -m pip install {}-r {}" for resolving the issue.' + ''.format(escape_backslash(sys.executable), + '' if is_virtualenv() else '--user ', + escape_backslash(args.requirements))) sys.exit(1) print('Python requirements from {} are satisfied.'.format(args.requirements)) diff --git a/tools/cmake/build.cmake b/tools/cmake/build.cmake new file mode 100644 index 00000000..5383581f --- /dev/null +++ b/tools/cmake/build.cmake @@ -0,0 +1,479 @@ + +# idf_build_get_property +# +# @brief Retrieve the value of the specified property related to ESP-IDF build. +# +# @param[out] var the variable to store the value in +# @param[in] property the property to get the value of +# +# @param[in, optional] GENERATOR_EXPRESSION (option) retrieve the generator expression for the property +# instead of actual value +function(idf_build_get_property var property) + cmake_parse_arguments(_ "GENERATOR_EXPRESSION" "" "" ${ARGN}) + if(__GENERATOR_EXPRESSION) + set(val "$") + else() + get_property(val TARGET __idf_build_target PROPERTY ${property}) + endif() + set(${var} ${val} PARENT_SCOPE) +endfunction() + +# idf_build_set_property +# +# @brief Set the value of the specified property related to ESP-IDF build. The property is +# also added to the internal list of build properties if it isn't there already. +# +# @param[in] property the property to set the value of +# @param[out] value value of the property +# +# @param[in, optional] APPEND (option) append the value to the current value of the +# property instead of replacing it +function(idf_build_set_property property value) + cmake_parse_arguments(_ "APPEND" "" "" ${ARGN}) + + if(__APPEND) + set_property(TARGET __idf_build_target APPEND PROPERTY ${property} ${value}) + else() + set_property(TARGET __idf_build_target PROPERTY ${property} ${value}) + endif() + + # Keep track of set build properties so that they can be exported to a file that + # will be included in early expansion script. + idf_build_get_property(build_properties __BUILD_PROPERTIES) + if(NOT property IN_LIST build_properties) + idf_build_set_property(__BUILD_PROPERTIES "${property}" APPEND) + endif() +endfunction() + +# idf_build_unset_property +# +# @brief Unset the value of the specified property related to ESP-IDF build. Equivalent +# to setting the property to an empty string; though it also removes the property +# from the internal list of build properties. +# +# @param[in] property the property to unset the value of +function(idf_build_unset_property property) + idf_build_set_property(${property} "") # set to an empty value + idf_build_get_property(build_properties __BUILD_PROPERTIES) # remove from tracked properties + list(REMOVE_ITEM build_properties ${property}) + idf_build_set_property(__BUILD_PROPERTIES "${build_properties}") +endfunction() + +# +# Retrieve the IDF_PATH repository's version, either using a version +# file or Git revision. Sets the IDF_VER build property. +# +function(__build_get_idf_git_revision) + idf_build_get_property(idf_path IDF_PATH) + git_describe(idf_ver_git "${idf_path}") + if(EXISTS "${idf_path}/version.txt") + file(STRINGS "${idf_path}/version.txt" idf_ver_t) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${idf_path}/version.txt") + else() + set(idf_ver_t ${idf_ver_git}) + endif() + # cut IDF_VER to required 32 characters. + string(SUBSTRING "${idf_ver_t}" 0 31 idf_ver) + idf_build_set_property(COMPILE_DEFINITIONS "-DIDF_VER=\"${idf_ver}\"" APPEND) + git_submodule_check("${idf_path}") + idf_build_set_property(IDF_VER ${idf_ver}) +endfunction() + +# +# Sets initial list of build specifications (compile options, definitions, etc.) common across +# all library targets built under the ESP-IDF build system. These build specifications are added +# privately using the directory-level CMake commands (add_compile_options, include_directories, etc.) +# during component registration. +# +function(__build_set_default_build_specifications) + unset(compile_definitions) + unset(compile_options) + unset(c_compile_options) + unset(cxx_compile_options) + + list(APPEND compile_definitions "-D_GNU_SOURCE") + + list(APPEND compile_options "-ffunction-sections" + "-fdata-sections" + "-fstrict-volatile-bitfields" + "-nostdlib" + # warning-related flags + "-Wall" + "-Werror=all" + "-Wno-error=unused-function" + "-Wno-error=unused-but-set-variable" + "-Wno-error=unused-variable" + "-Wno-error=deprecated-declarations" + "-Wextra" + "-Wno-unused-parameter" + "-Wno-sign-compare" + # always generate debug symbols (even in release mode, these don't + # go into the final binary so have no impact on size + "-ggdb") + + list(APPEND c_compile_options "-std=gnu99" + "-Wno-old-style-declaration") + + list(APPEND cxx_compile_options "-std=gnu++11" + "-fno-rtti") + + idf_build_set_property(COMPILE_DEFINITIONS "${compile_definitions}" APPEND) + idf_build_set_property(COMPILE_OPTIONS "${compile_options}" APPEND) + idf_build_set_property(C_COMPILE_OPTIONS "${c_compile_options}" APPEND) + idf_build_set_property(CXX_COMPILE_OPTIONS "${cxx_compile_options}" APPEND) +endfunction() + +# +# Initialize the build. This gets called upon inclusion of idf.cmake to set internal +# properties used for the processing phase of the build. +# +function(__build_init idf_path) + # Create the build target, to which the ESP-IDF build properties, dependencies are attached to + add_custom_target(__idf_build_target) + + set_default(python "python") + + idf_build_set_property(PYTHON ${python}) + idf_build_set_property(IDF_PATH ${idf_path}) + + idf_build_set_property(__PREFIX idf) + idf_build_set_property(__CHECK_PYTHON 1) + + __build_set_default_build_specifications() + + # Add internal components to the build + idf_build_get_property(idf_path IDF_PATH) + idf_build_get_property(prefix __PREFIX) + file(GLOB component_dirs ${idf_path}/components/*) + foreach(component_dir ${component_dirs}) + get_filename_component(component_dir ${component_dir} ABSOLUTE) + __component_dir_quick_check(is_component ${component_dir}) + if(is_component) + __component_add(${component_dir} ${prefix}) + endif() + endforeach() + + # Set components required by all other components in the build + set(requires_common cxx newlib freertos heap log soc esp_rom esp_common xtensa) + idf_build_set_property(__COMPONENT_REQUIRES_COMMON "${requires_common}") + + __build_get_idf_git_revision() + __kconfig_init() +endfunction() + +# idf_build_component +# +# @brief Present a directory that contains a component to the build system. +# Relative paths are converted to absolute paths with respect to current directory. +# All calls to this command must be performed before idf_build_process. +# +# @note This command does not guarantee that the component will be processed +# during build (see the COMPONENTS argument description for command idf_build_process) +# +# @param[in] component_dir directory of the component +function(idf_build_component component_dir) + idf_build_get_property(prefix __PREFIX) + __component_add(${component_dir} ${prefix} 0) +endfunction() + +# +# Resolve the requirement component to the component target created for that component. +# +function(__build_resolve_and_add_req var component_target req type) + __component_get_target(_component_target ${req}) + if(NOT _component_target) + message(FATAL_ERROR "Failed to resolve component '${req}'.") + endif() + __component_set_property(${component_target} ${type} ${_component_target} APPEND) + set(${var} ${_component_target} PARENT_SCOPE) +endfunction() + +# +# Build a list of components (in the form of component targets) to be added to the build +# based on public and private requirements. This list is saved in an internal property, +# __BUILD_COMPONENT_TARGETS. +# +function(__build_expand_requirements component_target) + # Since there are circular dependencies, make sure that we do not infinitely + # expand requirements for each component. + idf_build_get_property(component_targets_seen __COMPONENT_TARGETS_SEEN) + __component_get_property(component_registered ${component_target} __COMPONENT_REGISTERED) + if(component_target IN_LIST component_targets_seen OR NOT component_registered) + return() + endif() + + idf_build_set_property(__COMPONENT_TARGETS_SEEN ${component_target} APPEND) + + get_property(reqs TARGET ${component_target} PROPERTY REQUIRES) + get_property(priv_reqs TARGET ${component_target} PROPERTY PRIV_REQUIRES) + + foreach(req ${reqs}) + __build_resolve_and_add_req(_component_target ${component_target} ${req} __REQUIRES) + __build_expand_requirements(${_component_target}) + endforeach() + + foreach(req ${priv_reqs}) + __build_resolve_and_add_req(_component_target ${component_target} ${req} __PRIV_REQUIRES) + __build_expand_requirements(${_component_target}) + endforeach() + + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + if(NOT component_target IN_LIST build_component_targets) + idf_build_set_property(__BUILD_COMPONENT_TARGETS ${component_target} APPEND) + endif() +endfunction() + +# +# Write a CMake file containing set build properties, owing to the fact that an internal +# list of properties is maintained in idf_build_set_property() call. This is used to convert +# those set properties to variables in the scope the output file is included in. +# +function(__build_write_properties output_file) + idf_build_get_property(build_properties __BUILD_PROPERTIES) + foreach(property ${build_properties}) + idf_build_get_property(val ${property}) + set(build_properties_text "${build_properties_text}\nset(${property} ${val})") + endforeach() + file(WRITE ${output_file} "${build_properties_text}") +endfunction() + +# +# Check if the Python interpreter used for the build has all the required modules. +# +function(__build_check_python) + idf_build_get_property(check __CHECK_PYTHON) + if(check) + idf_build_get_property(python PYTHON) + idf_build_get_property(idf_path IDF_PATH) + message(STATUS "Checking Python dependencies...") + execute_process(COMMAND "${python}" "${idf_path}/tools/check_python_dependencies.py" + RESULT_VARIABLE result) + if(NOT result EQUAL 0) + message(FATAL_ERROR "Some Python dependencies must be installed. Check above message for details.") + endif() + endif() +endfunction() + +# +# Prepare for component processing expanding each component's project include +# +macro(__build_process_project_includes) + # Include the sdkconfig cmake file, since the following operations require + # knowledge of config values. + idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE) + include(${sdkconfig_cmake}) + + # Make each build property available as a read-only variable + idf_build_get_property(build_properties __BUILD_PROPERTIES) + foreach(build_property ${build_properties}) + idf_build_get_property(val ${build_property}) + set(${build_property} "${val}") + endforeach() + + # Check that the CMake target value matches the Kconfig target value. + __target_check() + + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + + # Include each component's project_include.cmake + foreach(component_target ${build_component_targets}) + __component_get_property(dir ${component_target} COMPONENT_DIR) + __component_get_property(_name ${component_target} COMPONENT_NAME) + set(COMPONENT_NAME ${_name}) + set(COMPONENT_DIR ${dir}) + set(COMPONENT_PATH ${dir}) # this is deprecated, users are encouraged to use COMPONENT_DIR; + # retained for compatibility + if(EXISTS ${COMPONENT_DIR}/project_include.cmake) + include(${COMPONENT_DIR}/project_include.cmake) + endif() + endforeach() +endmacro() + +# +# Utility macro for setting default property value if argument is not specified +# for idf_build_process(). +# +macro(__build_set_default var default) + set(_var __${var}) + if(${_var}) + idf_build_set_property(${var} "${${_var}}") + else() + idf_build_set_property(${var} "${default}") + endif() + unset(_var) +endmacro() + +# +# Import configs as build instance properties so that they are accessible +# using idf_build_get_config(). Config has to have been generated before calling +# this command. +# +function(__build_import_configs) + # Include the sdkconfig cmake file, since the following operations require + # knowledge of config values. + idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE) + include(${sdkconfig_cmake}) + + idf_build_set_property(__CONFIG_VARIABLES "${CONFIGS_LIST}") + foreach(config ${CONFIGS_LIST}) + set_property(TARGET __idf_build_target PROPERTY ${config} "${${config}}") + endforeach() +endfunction() + +# idf_build_process +# +# @brief Main processing step for ESP-IDF build: config generation, adding components to the build, +# dependency resolution, etc. +# +# @param[in] target ESP-IDF target +# +# @param[in, optional] PROJECT_DIR (single value) directory of the main project the buildsystem +# is processed for; defaults to CMAKE_SOURCE_DIR +# @param[in, optional] PROJECT_VER (single value) version string of the main project; defaults +# to 0.0.0 +# @param[in, optional] PROJECT_NAME (single value) main project name, defaults to CMAKE_PROJECT_NAME +# @param[in, optional] SDKCONFIG (single value) sdkconfig output path, defaults to PROJECT_DIR/sdkconfig +# if PROJECT_DIR is set and CMAKE_SOURCE_DIR/sdkconfig if not +# @param[in, optional] SDKCONFIG_DEFAULTS (single value) config defaults file to use for the build; defaults +# to none (Kconfig defaults or previously generated config are used) +# @param[in, optional] BUILD_DIR (single value) directory for build artifacts; defautls to CMAKE_BINARY_DIR +# @param[in, optional] COMPONENTS (multivalue) select components to process among the components +# known by the build system +# (added via `idf_build_component`). This argument is used to trim the build. +# Other components are automatically added if they are required +# in the dependency chain, i.e. +# the public and private requirements of the components in this list +# are automatically added, and in +# turn the public and private requirements of those requirements, +# so on and so forth. If not specified, all components known to the build system +# are processed. +macro(idf_build_process target) + set(options) + set(single_value PROJECT_DIR PROJECT_VER PROJECT_NAME BUILD_DIR SDKCONFIG SDKCONFIG_DEFAULTS) + set(multi_value COMPONENTS) + cmake_parse_arguments(_ "${options}" "${single_value}" "${multi_value}" ${ARGN}) + + idf_build_set_property(BOOTLOADER_BUILD "${BOOTLOADER_BUILD}") + + # Check build target is specified. Since this target corresponds to a component + # name, the target component is automatically added to the list of common component + # requirements. + if(target STREQUAL "") + message(FATAL_ERROR "Build target not specified.") + endif() + + idf_build_set_property(IDF_TARGET ${target}) + + __build_set_default(PROJECT_DIR ${CMAKE_SOURCE_DIR}) + __build_set_default(PROJECT_NAME ${CMAKE_PROJECT_NAME}) + __build_set_default(PROJECT_VER "0.0.0") + __build_set_default(BUILD_DIR ${CMAKE_BINARY_DIR}) + + idf_build_get_property(project_dir PROJECT_DIR) + __build_set_default(SDKCONFIG "${project_dir}/sdkconfig") + + __build_set_default(SDKCONFIG_DEFAULTS "") + + # Check for required Python modules + __build_check_python() + + idf_build_set_property(__COMPONENT_REQUIRES_COMMON ${target} APPEND) + + # Perform early expansion of component CMakeLists.txt in CMake scripting mode. + # It is here we retrieve the public and private requirements of each component. + # It is also here we add the common component requirements to each component's + # own requirements. + __component_get_requirements() + + idf_build_get_property(component_targets __COMPONENT_TARGETS) + + # Finally, do component expansion. In this case it simply means getting a final list + # of build component targets given the requirements set by each component. + + # Check if we need to trim the components first, and build initial components list + # from that. + if(__COMPONENTS) + unset(component_targets) + foreach(component ${__COMPONENTS}) + __component_get_target(component_target ${component}) + if(NOT component_target) + message(FATAL_ERROR "Failed to resolve component '${component}'.") + endif() + list(APPEND component_targets ${component_target}) + endforeach() + endif() + + foreach(component_target ${component_targets}) + __build_expand_requirements(${component_target}) + endforeach() + idf_build_unset_property(__COMPONENT_TARGETS_SEEN) + + # Get a list of common component requirements in component targets form (previously + # we just have a list of component names) + idf_build_get_property(common_reqs __COMPONENT_REQUIRES_COMMON) + foreach(common_req ${common_reqs}) + __component_get_target(component_target ${common_req}) + __component_get_property(lib ${component_target} COMPONENT_LIB) + idf_build_set_property(___COMPONENT_REQUIRES_COMMON ${lib} APPEND) + endforeach() + + # Generate config values in different formats + idf_build_get_property(sdkconfig SDKCONFIG) + idf_build_get_property(sdkconfig_defaults SDKCONFIG_DEFAULTS) + __kconfig_generate_config("${sdkconfig}" "${sdkconfig_defaults}") + __build_import_configs() + + # Temporary trick to support both gcc5 and gcc8 builds + if(CMAKE_C_COMPILER_VERSION VERSION_EQUAL 5.2.0) + set(GCC_NOT_5_2_0 0 CACHE STRING "GCC is 5.2.0 version") + else() + set(GCC_NOT_5_2_0 1 CACHE STRING "GCC is not 5.2.0 version") + endif() + idf_build_set_property(COMPILE_DEFINITIONS "-DGCC_NOT_5_2_0" APPEND) + + # All targets built under this scope is with the ESP-IDF build system + set(ESP_PLATFORM 1) + idf_build_set_property(COMPILE_DEFINITIONS "-DESP_PLATFORM" APPEND) + + # Perform component processing (inclusion of project_include.cmake, adding component + # subdirectories, creating library targets, linking libraries, etc.) + __build_process_project_includes() + + idf_build_get_property(idf_path IDF_PATH) + add_subdirectory(${idf_path} ${build_dir}/esp-idf) + + unset(ESP_PLATFORM) +endmacro() + +# idf_build_executable +# +# @brief Specify the executable the build system can attach dependencies to (for generating +# files used for linking, targets which should execute before creating the specified executable, +# generating additional binary files, generating files related to flashing, etc.) +function(idf_build_executable elf) + # Propagate link dependencies from component library targets to the executable + idf_build_get_property(link_depends __LINK_DEPENDS) + set_property(TARGET ${elf} APPEND PROPERTY LINK_DEPENDS "${link_depends}") + + # Set the EXECUTABLE_NAME and EXECUTABLE properties since there are generator expression + # from components that depend on it + get_filename_component(elf_name ${elf} NAME_WE) + idf_build_set_property(EXECUTABLE_NAME ${elf_name}) + idf_build_set_property(EXECUTABLE ${elf}) + + # Add dependency of the build target to the executable + add_dependencies(${elf} __idf_build_target) +endfunction() + +# idf_build_get_config +# +# @brief Get value of specified config variable +function(idf_build_get_config var config) + cmake_parse_arguments(_ "GENERATOR_EXPRESSION" "" "" ${ARGN}) + if(__GENERATOR_EXPRESSION) + set(val "$") + else() + get_property(val TARGET __idf_build_target PROPERTY ${config}) + endif() + set(${var} ${val} PARENT_SCOPE) +endfunction() \ No newline at end of file diff --git a/tools/cmake/component.cmake b/tools/cmake/component.cmake new file mode 100644 index 00000000..6959af26 --- /dev/null +++ b/tools/cmake/component.cmake @@ -0,0 +1,541 @@ +# +# Internal function for retrieving component properties from a component target. +# +function(__component_get_property var component_target property) + get_property(val TARGET ${component_target} PROPERTY ${property}) + set(${var} "${val}" PARENT_SCOPE) +endfunction() + +# +# Internal function for setting component properties on a component target. As with build properties, +# set properties are also keeped track of. +# +function(__component_set_property component_target property val) + cmake_parse_arguments(_ "APPEND" "" "" ${ARGN}) + + if(__APPEND) + set_property(TARGET ${component_target} APPEND PROPERTY ${property} "${val}") + else() + set_property(TARGET ${component_target} PROPERTY ${property} "${val}") + endif() + + # Keep track of set component properties + __component_get_property(properties ${component_target} __COMPONENT_PROPERTIES) + if(NOT property IN_LIST properties) + __component_set_property(${component_target} __COMPONENT_PROPERTIES ${property} APPEND) + endif() +endfunction() + +# +# Given a component name or alias, get the corresponding component target. +# +function(__component_get_target var name_or_alias) + # Look at previously resolved names or aliases + idf_build_get_property(component_names_resolved __COMPONENT_NAMES_RESOLVED) + list(FIND component_names_resolved ${name_or_alias} result) + if(NOT result EQUAL -1) + # If it has been resolved before, return that value. The index is the same + # as in __COMPONENT_NAMES_RESOLVED as these are parallel lists. + idf_build_get_property(component_targets_resolved __COMPONENT_TARGETS_RESOLVED) + list(GET component_targets_resolved ${result} target) + set(${var} ${target} PARENT_SCOPE) + return() + endif() + + idf_build_get_property(component_targets __COMPONENT_TARGETS) + + # Assume first that the paramters is an alias. + string(REPLACE "::" "_" name_or_alias "${name_or_alias}") + set(component_target ___${name_or_alias}) + + if(component_target IN_LIST component_targets) + set(${var} ${component_target} PARENT_SCOPE) + set(target ${component_target}) + else() # assumption is wrong, try to look for it manually + unset(target) + foreach(component_target ${component_targets}) + __component_get_property(_component_name ${component_target} COMPONENT_NAME) + if(name_or_alias STREQUAL _component_name) + # There should only be one component of the same name + if(NOT target) + set(target ${component_target}) + else() + message(FATAL_ERROR "Multiple components with name '${name_or_alias}' found.") + return() + endif() + endif() + endforeach() + set(${var} ${target} PARENT_SCOPE) + endif() + + # Save the resolved name or alias + if(target) + idf_build_set_property(__COMPONENT_NAMES_RESOLVED ${name_or_alias} APPEND) + idf_build_set_property(__COMPONENT_TARGETS_RESOLVED ${target} APPEND) + endif() +endfunction() + +# +# Called during component registration, sets basic properties of the current component. +# +macro(__component_set_properties) + __component_get_property(type ${component_target} COMPONENT_TYPE) + + # Fill in the rest of component property + __component_set_property(${component_target} SRCS "${sources}") + __component_set_property(${component_target} INCLUDE_DIRS "${__INCLUDE_DIRS}") + + if(type STREQUAL LIBRARY) + __component_set_property(${component_target} PRIV_INCLUDE_DIRS "${__PRIV_INCLUDE_DIRS}") + endif() + + __component_set_property(${component_target} LDFRAGMENTS "${__LDFRAGMENTS}") + __component_set_property(${component_target} EMBED_FILES "${__EMBED_FILES}") + __component_set_property(${component_target} EMBED_TXTFILES "${__EMBED_TXTFILES}") + __component_set_property(${component_target} REQUIRED_IDF_TARGETS "${__REQUIRED_IDF_TARGETS}") +endmacro() + +# +# Perform a quick check if given component dir satisfies basic requirements. +# +function(__component_dir_quick_check var component_dir) + set(res 1) + get_filename_component(abs_dir ${component_dir} ABSOLUTE) + + # Check this is really a directory and that a CMakeLists.txt file for this component exists + # - warn and skip anything which isn't valid looking (probably cruft) + if(NOT IS_DIRECTORY "${abs_dir}") + message(STATUS "Unexpected file in components directory: ${abs_dir}") + set(res 0) + endif() + + get_filename_component(base_dir ${abs_dir} NAME) + string(SUBSTRING "${base_dir}" 0 1 first_char) + + if(NOT first_char STREQUAL ".") + if(NOT EXISTS "${abs_dir}/CMakeLists.txt") + message(STATUS "Component directory ${abs_dir} does not contain a CMakeLists.txt file. " + "No component will be added") + set(res 0) + endif() + else() + set(res 0) # quietly ignore dot-folders + endif() + + set(${var} ${res} PARENT_SCOPE) +endfunction() + +# +# Write a CMake file containing all component and their properties. This is possible because each component +# keeps a list of all its properties. +# +function(__component_write_properties output_file) + idf_build_get_property(component_targets __COMPONENT_TARGETS) + foreach(component_target ${component_targets}) + __component_get_property(component_properties ${component_target} __COMPONENT_PROPERTIES) + foreach(property ${component_properties}) + __component_get_property(val ${component_target} ${property}) + set(component_properties_text + "${component_properties_text}\nset(__component_${component_target}_${property} ${val})") + endforeach() + file(WRITE ${output_file} "${component_properties_text}") + endforeach() +endfunction() + +# +# Add a component to process in the build. The components are keeped tracked of in property +# __COMPONENT_TARGETS in component target form. +# +function(__component_add component_dir prefix) + # For each component, two entities are created: a component target and a component library. The + # component library is created during component registration (the actual static/interface library). + # On the other hand, component targets are created early in the build + # (during adding component as this function suggests). + # This is so that we still have a target to attach properties to up until the component registration. + # Plus, interface libraries have limitations on the types of properties that can be set on them, + # so later in the build, these component targets actually contain the properties meant for the + # corresponding component library. + idf_build_get_property(component_targets __COMPONENT_TARGETS) + get_filename_component(abs_dir ${component_dir} ABSOLUTE) + get_filename_component(base_dir ${abs_dir} NAME) + + if(NOT EXISTS "${abs_dir}/CMakeLists.txt") + message(FATAL_ERROR "Directory '${component_dir}' does not contain a component.") + endif() + + set(component_name ${base_dir}) + # The component target has three underscores as a prefix. The corresponding component library + # only has two. + set(component_target ___${prefix}_${component_name}) + + # If a component of the same name has not been added before If it has been added + # before just override the properties. As a side effect, components added later + # 'override' components added earlier. + if(NOT component_target IN_LIST component_targets) + if(NOT TARGET ${component_target}) + add_custom_target(${component_target} EXCLUDE_FROM_ALL) + endif() + idf_build_set_property(__COMPONENT_TARGETS ${component_target} APPEND) + endif() + + set(component_lib __${prefix}_${component_name}) + set(component_dir ${abs_dir}) + set(component_alias ${prefix}::${component_name}) # The 'alias' of the component library, + # used to refer to the component outside + # the build system. Users can use this name + # to resolve ambiguity with component names + # and to link IDF components to external targets. + + # Set the basic properties of the component + __component_set_property(${component_target} COMPONENT_LIB ${component_lib}) + __component_set_property(${component_target} COMPONENT_NAME ${component_name}) + __component_set_property(${component_target} COMPONENT_DIR ${component_dir}) + __component_set_property(${component_target} COMPONENT_ALIAS ${component_alias}) + __component_set_property(${component_target} __PREFIX ${prefix}) + + # Set Kconfig related properties on the component + __kconfig_component_init(${component_target}) +endfunction() + +# +# Given a component directory, get the requirements by expanding it early. The expansion is performed +# using a separate CMake script (the expansion is performed in a separate instance of CMake in scripting mode). +# +function(__component_get_requirements) + idf_build_get_property(idf_path IDF_PATH) + + idf_build_get_property(build_dir BUILD_DIR) + set(build_properties_file ${build_dir}/build_properties.temp.cmake) + set(component_properties_file ${build_dir}/component_properties.temp.cmake) + set(component_requires_file ${build_dir}/component_requires.temp.cmake) + + __build_write_properties(${build_properties_file}) + __component_write_properties(${component_properties_file}) + + execute_process(COMMAND "${CMAKE_COMMAND}" + -D "BUILD_PROPERTIES_FILE=${build_properties_file}" + -D "COMPONENT_PROPERTIES_FILE=${component_properties_file}" + -D "COMPONENT_REQUIRES_FILE=${component_requires_file}" + -P "${idf_path}/tools/cmake/scripts/component_get_requirements.cmake" + RESULT_VARIABLE result + ERROR_VARIABLE error + ) + + if(NOT result EQUAL 0) + message(FATAL_ERROR "${error}") + endif() + + include(${component_requires_file}) + + file(REMOVE ${build_properties_file}) + file(REMOVE ${component_properties_file}) + file(REMOVE ${component_requires_file}) +endfunction() + +# __component_add_sources, __component_check_target +# +# Utility macros for component registration. Adds source files and checks target requirements +# respectively. +macro(__component_add_sources sources) + set(sources "") + if(__SRCS) + if(__SRC_DIRS) + message(WARNING "SRCS and SRC_DIRS are both specified; ignoring SRC_DIRS.") + endif() + foreach(src ${__SRCS}) + get_filename_component(src "${src}" ABSOLUTE BASE_DIR ${COMPONENT_DIR}) + list(APPEND sources ${src}) + endforeach() + else() + if(__SRC_DIRS) + foreach(dir ${__SRC_DIRS}) + get_filename_component(abs_dir ${dir} ABSOLUTE BASE_DIR ${COMPONENT_DIR}) + + if(NOT IS_DIRECTORY ${abs_dir}) + message(FATAL_ERROR "SRC_DIRS entry '${dir}' does not exist.") + endif() + + file(GLOB dir_sources "${abs_dir}/*.c" "${abs_dir}/*.cpp" "${abs_dir}/*.S") + + if(dir_sources) + foreach(src ${dir_sources}) + get_filename_component(src "${src}" ABSOLUTE BASE_DIR ${COMPONENT_DIR}) + list(APPEND sources "${src}") + endforeach() + else() + message(WARNING "No source files found for SRC_DIRS entry '${dir}'.") + endif() + endforeach() + endif() + + if(__EXCLUDE_SRCS) + foreach(src ${__EXCLUDE_SRCS}) + get_filename_component(src "${src}" ABSOLUTE) + list(REMOVE_ITEM source "${src}") + endforeach() + endif() + endif() + + list(REMOVE_DUPLICATES sources) +endmacro() + +macro(__component_check_target) + if(__REQUIRED_IDF_TARGETS) + idf_build_get_property(idf_target IDF_TARGET) + if(NOT idf_target IN_LIST __REQUIRED_IDF_TARGETS) + message(FATAL_ERROR "Component ${COMPONENT_NAME} only supports targets: ${__REQUIRED_IDF_TARGETS}") + endif() + endif() +endmacro() + +# __component_set_dependencies, __component_set_all_dependencies +# +# Links public and private requirements for the currently processed component +macro(__component_set_dependencies reqs type) + foreach(req ${reqs}) + if(req IN_LIST build_component_targets) + __component_get_property(req_lib ${req} COMPONENT_LIB) + if("${type}" STREQUAL "PRIVATE") + set_property(TARGET ${component_lib} APPEND PROPERTY LINK_LIBRARIES ${req_lib}) + set_property(TARGET ${component_lib} APPEND PROPERTY INTERFACE_LINK_LIBRARIES $) + elseif("${type}" STREQUAL "PUBLIC") + set_property(TARGET ${component_lib} APPEND PROPERTY LINK_LIBRARIES ${req_lib}) + set_property(TARGET ${component_lib} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${req_lib}) + else() # INTERFACE + set_property(TARGET ${component_lib} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${req_lib}) + endif() + endif() + endforeach() +endmacro() + +macro(__component_set_all_dependencies) + __component_get_property(type ${component_target} COMPONENT_TYPE) + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + + if(NOT type STREQUAL CONFIG_ONLY) + __component_get_property(reqs ${component_target} __REQUIRES) + __component_set_dependencies("${reqs}" PUBLIC) + + __component_get_property(priv_reqs ${component_target} __PRIV_REQUIRES) + __component_set_dependencies("${priv_reqs}" PRIVATE) + else() + __component_get_property(reqs ${component_target} __REQUIRES) + __component_set_dependencies("${reqs}" INTERFACE) + endif() +endmacro() + +# idf_component_get_property +# +# @brief Retrieve the value of the specified component property +# +# @param[out] var the variable to store the value of the property in +# @param[in] component the component name or alias to get the value of the property of +# @param[in] property the property to get the value of +# +# @param[in, optional] GENERATOR_EXPRESSION (option) retrieve the generator expression for the property +# instead of actual value +function(idf_component_get_property var component property) + cmake_parse_arguments(_ "GENERATOR_EXPRESSION" "" "" ${ARGN}) + __component_get_target(component_target ${component}) + if(__GENERATOR_EXPRESSION) + set(val "$") + else() + __component_get_property(val ${component_target} ${property}) + endif() + set(${var} "${val}" PARENT_SCOPE) +endfunction() + +# idf_component_set_property +# +# @brief Set the value of the specified component property related. The property is +# also added to the internal list of component properties if it isn't there already. +# +# @param[in] component component name or alias of the component to set the property of +# @param[in] property the property to set the value of +# @param[out] value value of the property to set to +# +# @param[in, optional] APPEND (option) append the value to the current value of the +# property instead of replacing it +function(idf_component_set_property component property val) + cmake_parse_arguments(_ "APPEND" "" "" ${ARGN}) + __component_get_target(component_target ${component}) + + if(__APPEND) + __component_set_property(${component_target} ${property} "${val}" APPEND) + else() + __component_set_property(${component_target} ${property} "${val}") + endif() +endfunction() + + +# idf_component_register +# +# @brief Register a component to the build, creating component library targets etc. +# +# @param[in, optional] SRCS (multivalue) list of source files for the component +# @param[in, optional] SRC_DIRS (multivalue) list of source directories to look for source files +# in (.c, .cpp. .S); ignored when SRCS is specified. +# @param[in, optional] EXCLUDE_SRCS (multivalue) used to exclude source files for the specified +# SRC_DIRS +# @param[in, optional] INCLUDE_DIRS (multivalue) public include directories for the created component library +# @param[in, optional] PRIV_INCLUDE_DIRS (multivalue) private include directories for the created component library +# @param[in, optional] LDFRAGMENTS (multivalue) linker script fragments for the component +# @param[in, optional] REQUIRES (multivalue) publicly required components in terms of usage requirements +# @param[in, optional] PRIV_REQUIRES (multivalue) privately required components in terms of usage requirements +# or components only needed for functions/values defined in its project_include.cmake +# @param[in, optional] REQUIRED_IDF_TARGETS (multivalue) the list of IDF build targets that the component only supports +# @param[in, optional] EMBED_FILES (multivalue) list of binary files to embed with the component +# @param[in, optional] EMBED_TXTFILES (multivalue) list of text files to embed with the component +function(idf_component_register) + set(options) + set(single_value) + set(multi_value SRCS SRC_DIRS EXCLUDE_SRCS + INCLUDE_DIRS PRIV_INCLUDE_DIRS LDFRAGMENTS REQUIRES + PRIV_REQUIRES REQUIRED_IDF_TARGETS EMBED_FILES EMBED_TXTFILES) + cmake_parse_arguments(_ "${options}" "${single_value}" "${multi_value}" ${ARGN}) + + if(NOT __idf_component_context) + message(FATAL_ERROR "Called idf_component_register from a non-component directory.") + endif() + + __component_check_target() + __component_add_sources(sources) + + # Create the final target for the component. This target is the target that is + # visible outside the build system. + __component_get_target(component_target ${COMPONENT_ALIAS}) + __component_get_property(component_lib ${component_target} COMPONENT_LIB) + + # Use generator expression so that users can append/override flags even after call to + # idf_build_process + idf_build_get_property(include_directories INCLUDE_DIRECTORIES GENERATOR_EXPRESSION) + idf_build_get_property(compile_options COMPILE_OPTIONS GENERATOR_EXPRESSION) + idf_build_get_property(c_compile_options C_COMPILE_OPTIONS GENERATOR_EXPRESSION) + idf_build_get_property(cxx_compile_options CXX_COMPILE_OPTIONS GENERATOR_EXPRESSION) + idf_build_get_property(common_reqs ___COMPONENT_REQUIRES_COMMON) + + include_directories("${include_directories}") + add_compile_options("${compile_options}") + add_c_compile_options("${c_compile_options}") + add_cxx_compile_options("${cxx_compile_options}") + + # Unfortunately add_definitions() does not support generator expressions. A new command + # add_compile_definition() does but is only available on CMake 3.12 or newer. This uses + # add_compile_options(), which can add any option as the workaround. + # + # TODO: Use add_compile_definitions() once minimum supported version is 3.12 or newer. + idf_build_get_property(compile_definitions COMPILE_DEFINITIONS GENERATOR_EXPRESSION) + add_compile_options("${compile_definitions}") + + list(REMOVE_ITEM common_reqs ${component_lib}) + link_libraries(${common_reqs}) + + idf_build_get_property(config_dir CONFIG_DIR) + + # The contents of 'sources' is from the __component_add_sources call + if(sources OR __EMBED_FILES OR __EMBED_TXTFILES) + add_library(${component_lib} STATIC ${sources}) + __component_set_property(${component_target} COMPONENT_TYPE LIBRARY) + target_include_directories(${component_lib} PUBLIC ${__INCLUDE_DIRS}) + target_include_directories(${component_lib} PRIVATE ${__PRIV_INCLUDE_DIRS}) + target_include_directories(${component_lib} PUBLIC ${config_dir}) + set_target_properties(${component_lib} PROPERTIES OUTPUT_NAME ${COMPONENT_NAME}) + __ldgen_add_component(${component_lib}) + else() + add_library(${component_lib} INTERFACE) + __component_set_property(${component_target} COMPONENT_TYPE CONFIG_ONLY) + target_include_directories(${component_lib} INTERFACE ${__INCLUDE_DIRS}) + target_include_directories(${component_lib} INTERFACE ${config_dir}) + endif() + + # Alias the static/interface library created for linking to external targets. + # The alias is the :: name. + __component_get_property(component_alias ${component_target} COMPONENT_ALIAS) + add_library(${component_alias} ALIAS ${component_lib}) + + # Perform other component processing, such as embedding binaries and processing linker + # script fragments + foreach(file ${__EMBED_FILES}) + target_add_binary_data(${component_lib} "${file}" "BINARY") + endforeach() + + foreach(file ${__EMBED_TXTFILES}) + target_add_binary_data(${component_lib} "${file}" "TEXT") + endforeach() + + if(__LDFRAGMENTS) + __ldgen_add_fragment_files("${__LDFRAGMENTS}") + endif() + + # Set dependencies + __component_set_all_dependencies() + + # Add the component to built components + idf_build_set_property(__BUILD_COMPONENTS ${component_lib} APPEND) + idf_build_set_property(BUILD_COMPONENTS ${component_alias} APPEND) + + # Make the COMPONENT_LIB variable available in the component CMakeLists.txt + set(COMPONENT_LIB ${component_lib} PARENT_SCOPE) + # COMPONENT_TARGET is deprecated but is made available with same function + # as COMPONENT_LIB for compatibility. + set(COMPONENT_TARGET ${component_lib} PARENT_SCOPE) +endfunction() + +# +# Deprecated functions +# + +# register_component +# +# Compatibility function for registering 3.xx style components. +macro(register_component) + spaces2list(COMPONENT_SRCS) + spaces2list(COMPONENT_SRCDIRS) + spaces2list(COMPONENT_ADD_INCLUDEDIRS) + spaces2list(COMPONENT_PRIV_INCLUDEDIRS) + spaces2list(COMPONENT_REQUIRES) + spaces2list(COMPONENT_PRIV_REQUIRES) + spaces2list(COMPONENT_ADD_LDFRAGMENTS) + spaces2list(COMPONENT_EMBED_FILES) + spaces2list(COMPONENT_EMBED_TXTFILES) + spaces2list(COMPONENT_SRCEXCLUDE) + idf_component_register(SRCS "${COMPONENT_SRCS}" + SRC_DIRS "${COMPONENT_SRCDIRS}" + INCLUDE_DIRS "${COMPONENT_ADD_INCLUDEDIRS}" + PRIV_INCLUDE_DIRS "${COMPONENT_PRIV_INCLUDEDIRS}" + REQUIRES "${COMPONENT_REQUIRES}" + PRIV_REQUIRES "${COMPONENT_PRIV_REQUIRES}" + LDFRAGMENTS "${COMPONENT_ADD_LDFRAGMENTS}" + EMBED_FILES "${COMPONENT_EMBED_FILES}" + EMBED_TXTFILES "${COMPONENT_EMBED_TXTFILES}" + EXCLUDE_SRCS "${COMPONENT_SRCEXCLUDE}") +endmacro() + +# require_idf_targets +# +# Compatibility function for requiring IDF build targets for 3.xx style components. +function(require_idf_targets) + set(__REQUIRED_IDF_TARGETS "${ARGN}") + __component_check_target() +endfunction() + +# register_config_only_component +# +# Compatibility function for registering 3.xx style config components. +macro(register_config_only_component) + register_component() +endmacro() + +# component_compile_options +# +# Wrapper around target_compile_options that passes the component name +function(component_compile_options) + target_compile_options(${COMPONENT_LIB} PRIVATE ${ARGV}) +endfunction() + +# component_compile_definitions +# +# Wrapper around target_compile_definitions that passes the component name +function(component_compile_definitions) + target_compile_definitions(${COMPONENT_LIB} PRIVATE ${ARGV}) +endfunction() \ No newline at end of file diff --git a/tools/cmake/components.cmake b/tools/cmake/components.cmake deleted file mode 100644 index bafad5c4..00000000 --- a/tools/cmake/components.cmake +++ /dev/null @@ -1,188 +0,0 @@ -# Given a list of components in 'component_paths', filter only paths to the components -# mentioned in 'components' and return as a list in 'result_paths' -function(components_get_paths component_paths components result_paths) - set(result "") - foreach(path ${component_paths}) - get_filename_component(name "${path}" NAME) - if("${name}" IN_LIST components) - list(APPEND result "${name}") - endif() - endforeach() - set("${result_path}" "${result}" PARENT_SCOPE) -endfunction() - -# Add a component to the build, using the COMPONENT variables defined -# in the parent -# -function(register_component) - get_filename_component(component_dir ${CMAKE_CURRENT_LIST_FILE} DIRECTORY) - set(component ${COMPONENT_NAME}) - - spaces2list(COMPONENT_SRCDIRS) - spaces2list(COMPONENT_ADD_INCLUDEDIRS) - spaces2list(COMPONENT_SRCEXCLUDE) - - if(COMPONENT_SRCDIRS) - # Warn user if both COMPONENT_SRCDIRS and COMPONENT_SRCS are set - if(COMPONENT_SRCS) - message(WARNING "COMPONENT_SRCDIRS and COMPONENT_SRCS are both set, COMPONENT_SRCS will be ignored") - endif() - - set(COMPONENT_SRCS "") - - foreach(dir ${COMPONENT_SRCDIRS}) - get_filename_component(abs_dir ${dir} ABSOLUTE BASE_DIR ${component_dir}) - if(NOT IS_DIRECTORY ${abs_dir}) - message(FATAL_ERROR "${CMAKE_CURRENT_LIST_FILE}: COMPONENT_SRCDIRS entry '${dir}' does not exist") - endif() - - file(GLOB matches "${abs_dir}/*.c" "${abs_dir}/*.cpp" "${abs_dir}/*.S") - if(matches) - list(SORT matches) - set(COMPONENT_SRCS "${COMPONENT_SRCS};${matches}") - else() - message(FATAL_ERROR "${CMAKE_CURRENT_LIST_FILE}: COMPONENT_SRCDIRS entry '${dir}' has no source files") - endif() - endforeach() - endif() - - # Remove COMPONENT_SRCEXCLUDE matches - foreach(exclude ${COMPONENT_SRCEXCLUDE}) - get_filename_component(exclude "${exclude}" ABSOLUTE ${component_dir}) - foreach(src ${COMPONENT_SRCS}) - get_filename_component(abs_src "${src}" ABSOLUTE ${component_dir}) - if("${exclude}" STREQUAL "${abs_src}") # compare as canonical paths - list(REMOVE_ITEM COMPONENT_SRCS "${src}") - endif() - endforeach() - endforeach() - - # add as a PUBLIC library (if there are source files) or INTERFACE (if header only) - if(COMPONENT_SRCS OR embed_binaries) - add_library(${component} STATIC ${COMPONENT_SRCS}) - set(include_type PUBLIC) - else() - add_library(${component} INTERFACE) # header-only component - set(include_type INTERFACE) - endif() - - # binaries to embed directly in library - spaces2list(COMPONENT_EMBED_FILES) - spaces2list(COMPONENT_EMBED_TXTFILES) - foreach(embed_data ${COMPONENT_EMBED_FILES} ${COMPONENT_EMBED_TXTFILES}) - if(embed_data IN_LIST COMPONENT_EMBED_TXTFILES) - set(embed_type "TEXT") - else() - set(embed_type "BINARY") - endif() - target_add_binary_data("${component}" "${embed_data}" "${embed_type}") - endforeach() - - # add component public includes - foreach(include_dir ${COMPONENT_ADD_INCLUDEDIRS}) - get_filename_component(abs_dir ${include_dir} ABSOLUTE BASE_DIR ${component_dir}) - if(NOT IS_DIRECTORY ${abs_dir}) - message(FATAL_ERROR "${CMAKE_CURRENT_LIST_FILE}: " - "COMPONENT_ADD_INCLUDEDIRS entry '${include_dir}' not found") - endif() - target_include_directories(${component} ${include_type} ${abs_dir}) - endforeach() - - # add component private includes - foreach(include_dir ${COMPONENT_PRIV_INCLUDEDIRS}) - if(${include_type} STREQUAL INTERFACE) - message(FATAL_ERROR "${CMAKE_CURRENT_LIST_FILE} " - "sets no component source files but sets COMPONENT_PRIV_INCLUDEDIRS") - endif() - - get_filename_component(abs_dir ${include_dir} ABSOLUTE BASE_DIR ${component_dir}) - if(NOT IS_DIRECTORY ${abs_dir}) - message(FATAL_ERROR "${CMAKE_CURRENT_LIST_FILE}: " - "COMPONENT_PRIV_INCLUDEDIRS entry '${include_dir}' does not exist") - endif() - target_include_directories(${component} PRIVATE ${abs_dir}) - endforeach() - - if(component IN_LIST BUILD_TEST_COMPONENTS) - target_link_libraries(${component} "-L${CMAKE_CURRENT_BINARY_DIR}") - target_link_libraries(${component} "-Wl,--whole-archive -l${component} -Wl,--no-whole-archive") - endif() -endfunction() - -function(register_config_only_component) - get_filename_component(component_dir ${CMAKE_CURRENT_LIST_FILE} DIRECTORY) - get_filename_component(component ${component_dir} NAME) - - # No-op for now... -endfunction() - -function(add_component_dependencies target dep dep_type) - get_target_property(target_type ${target} TYPE) - get_target_property(target_imported ${target} IMPORTED) - - if(${target_type} STREQUAL STATIC_LIBRARY OR ${target_type} STREQUAL EXECUTABLE) - if(TARGET ${dep}) - # Add all compile options exported by dep into target - target_include_directories(${target} ${dep_type} - $) - target_compile_definitions(${target} ${dep_type} - $) - target_compile_options(${target} ${dep_type} - $) - endif() - endif() -endfunction() - -function(components_finish_registration) - - # have the executable target depend on all components in the build - set_target_properties(${CMAKE_PROJECT_NAME}.elf PROPERTIES INTERFACE_COMPONENT_REQUIRES "${BUILD_COMPONENTS}") - - spaces2list(COMPONENT_REQUIRES_COMMON) - - # each component should see the include directories of its requirements - # - # (we can't do this until all components are registered and targets exist in cmake, as we have - # a circular requirements graph...) - foreach(a ${BUILD_COMPONENTS}) - if(TARGET ${a}) - get_component_requirements("${a}" a_deps a_priv_deps) - list(APPEND a_priv_deps ${COMPONENT_REQUIRES_COMMON}) - foreach(b ${a_deps}) - add_component_dependencies(${a} ${b} PUBLIC) - endforeach() - - foreach(b ${a_priv_deps}) - add_component_dependencies(${a} ${b} PRIVATE) - endforeach() - - get_target_property(a_type ${a} TYPE) - if(${a_type} MATCHES .+_LIBRARY) - list(APPEND COMPONENT_LIBRARIES ${a}) - endif() - endif() - endforeach() - - # Add each component library's link-time dependencies (which are otherwise ignored) to the executable - # LINK_DEPENDS in order to trigger a re-link when needed (on Ninja/Makefile generators at least). - # (maybe this should probably be something CMake does, but it doesn't do it...) - foreach(component ${BUILD_COMPONENTS}) - if(TARGET ${component}) - get_target_property(imported ${component} IMPORTED) - get_target_property(type ${component} TYPE) - if(NOT imported) - if(${type} STREQUAL STATIC_LIBRARY OR ${type} STREQUAL EXECUTABLE) - get_target_property(link_depends "${component}" LINK_DEPENDS) - if(link_depends) - set_property(TARGET ${CMAKE_PROJECT_NAME}.elf APPEND PROPERTY LINK_DEPENDS "${link_depends}") - endif() - endif() - endif() - endif() - endforeach() - - target_link_libraries(${CMAKE_PROJECT_NAME}.elf ${COMPONENT_LIBRARIES}) - - message(STATUS "Component libraries: ${COMPONENT_LIBRARIES}") - -endfunction() diff --git a/tools/cmake/convert_to_cmake.py b/tools/cmake/convert_to_cmake.py index d792dd11..0ba7fa62 100755 --- a/tools/cmake/convert_to_cmake.py +++ b/tools/cmake/convert_to_cmake.py @@ -8,10 +8,10 @@ import subprocess import re import os.path import glob -import sys debug = False + def get_make_variables(path, makefile="Makefile", expected_failure=False, variables={}): """ Given the path to a Makefile of some kind, return a dictionary of all variables defined in this Makefile @@ -20,9 +20,9 @@ def get_make_variables(path, makefile="Makefile", expected_failure=False, variab Overrides IDF_PATH= to avoid recursively evaluating the entire project Makefile structure. """ - variable_setters = [ ("%s=%s" % (k,v)) for (k,v) in variables.items() ] + variable_setters = [("%s=%s" % (k,v)) for (k,v) in variables.items()] - cmdline = ["make", "-rpn", "-C", path, "-f", makefile ] + variable_setters + cmdline = ["make", "-rpn", "-C", path, "-f", makefile] + variable_setters if debug: print("Running %s..." % (" ".join(cmdline))) @@ -54,15 +54,16 @@ def get_make_variables(path, makefile="Makefile", expected_failure=False, variab return result + def get_component_variables(project_path, component_path): make_vars = get_make_variables(component_path, os.path.join(os.environ["IDF_PATH"], "make", "component_wrapper.mk"), expected_failure=True, - variables = { - "COMPONENT_MAKEFILE" : os.path.join(component_path, "component.mk"), - "COMPONENT_NAME" : os.path.basename(component_path), + variables={ + "COMPONENT_MAKEFILE": os.path.join(component_path, "component.mk"), + "COMPONENT_NAME": os.path.basename(component_path), "PROJECT_PATH": project_path, }) @@ -70,7 +71,7 @@ def get_component_variables(project_path, component_path): # Convert to sources def find_src(obj): obj = os.path.splitext(obj)[0] - for ext in [ "c", "cpp", "S" ]: + for ext in ["c", "cpp", "S"]: if os.path.exists(os.path.join(component_path, obj) + "." + ext): return obj + "." + ext print("WARNING: Can't find source file for component %s COMPONENT_OBJS %s" % (component_path, obj)) @@ -86,7 +87,7 @@ def get_component_variables(project_path, component_path): component_srcs = list() for component_srcdir in make_vars.get("COMPONENT_SRCDIRS", ".").split(" "): component_srcdir_path = os.path.abspath(os.path.join(component_path, component_srcdir)) - + srcs = list() srcs += glob.glob(os.path.join(component_srcdir_path, "*.[cS]")) srcs += glob.glob(os.path.join(component_srcdir_path, "*.cpp")) @@ -96,7 +97,6 @@ def get_component_variables(project_path, component_path): component_srcs += srcs make_vars["COMPONENT_SRCS"] = " ".join(component_srcs) - return make_vars @@ -111,13 +111,17 @@ def convert_project(project_path): raise RuntimeError("This project already has a CMakeLists.txt file") project_vars = get_make_variables(project_path, expected_failure=True) - if not "PROJECT_NAME" in project_vars: + if "PROJECT_NAME" not in project_vars: raise RuntimeError("PROJECT_NAME does not appear to be defined in IDF project Makefile at %s" % project_path) component_paths = project_vars["COMPONENT_PATHS"].split(" ") # Convert components as needed for p in component_paths: + if "MSYSTEM" in os.environ: + cmd = ["cygpath", "-w", p] + p = subprocess.check_output(cmd).strip() + convert_component(project_path, p) project_name = project_vars["PROJECT_NAME"] @@ -139,6 +143,7 @@ include($ENV{IDF_PATH}/tools/cmake/project.cmake) print("Converted project %s" % project_cmakelists) + def convert_component(project_path, component_path): if debug: print("Converting %s..." % (component_path)) @@ -156,19 +161,16 @@ def convert_component(project_path, component_path): cflags = v.get("CFLAGS", None) with open(cmakelists_path, "w") as f: - f.write("set(COMPONENT_ADD_INCLUDEDIRS %s)\n\n" % component_add_includedirs) - - f.write("# Edit following two lines to set component requirements (see docs)\n") - f.write("set(COMPONENT_REQUIRES "")\n") - f.write("set(COMPONENT_PRIV_REQUIRES "")\n\n") - if component_srcs is not None: - f.write("set(COMPONENT_SRCS %s)\n\n" % component_srcs) - f.write("register_component()\n") + f.write("idf_component_register(SRCS %s)\n" % component_srcs) + f.write(" INCLUDE_DIRS %s" % component_add_includedirs) + f.write(" # Edit following two lines to set component requirements (see docs)\n") + f.write(" REQUIRES "")\n") + f.write(" PRIV_REQUIRES "")\n\n") else: - f.write("register_config_only_component()\n") + f.write("idf_component_register()\n") if cflags is not None: - f.write("component_compile_options(%s)\n" % cflags) + f.write("target_compile_options(${COMPONENT_LIB} PRIVATE %s)\n" % cflags) print("Converted %s" % cmakelists_path) diff --git a/tools/cmake/crosstool_version_check.cmake b/tools/cmake/crosstool_version_check.cmake index 1180fc02..93b77d55 100644 --- a/tools/cmake/crosstool_version_check.cmake +++ b/tools/cmake/crosstool_version_check.cmake @@ -12,11 +12,11 @@ endfunction() function(crosstool_version_check expected_ctng_version) execute_process( - COMMAND ${CMAKE_C_COMPILER} -v - ERROR_VARIABLE toolchain_stderr - OUTPUT_QUIET) + COMMAND ${CMAKE_C_COMPILER} --version + OUTPUT_VARIABLE toolchain_version + ERROR_QUIET) - string(REGEX MATCH "crosstool-ng-[0-9a-g\\.-]+" ctng_version "${toolchain_stderr}") + string(REGEX REPLACE ".*(crosstool-NG ([^\)]+)).*\n" "\\2" ctng_version "${toolchain_version}") # We use FIND to match version instead of STREQUAL because some toolchains are built # with longer git hash strings than others. This will match any version which starts with # the expected version string. @@ -31,7 +31,8 @@ function(crosstool_version_check expected_ctng_version) endfunction() function(get_expected_ctng_version _toolchain_ver _gcc_ver) - file(STRINGS ${IDF_PATH}/tools/toolchain_versions.mk config_contents) + idf_build_get_property(idf_path IDF_PATH) + file(STRINGS ${idf_path}/tools/toolchain_versions.mk config_contents) foreach(name_and_value ${config_contents}) # Strip spaces string(REPLACE " " "" name_and_value ${name_and_value}) diff --git a/tools/cmake/idf.cmake b/tools/cmake/idf.cmake new file mode 100644 index 00000000..e97d83fc --- /dev/null +++ b/tools/cmake/idf.cmake @@ -0,0 +1,46 @@ +get_property(__idf_env_set GLOBAL PROPERTY __IDF_ENV_SET) +if(NOT __idf_env_set) + # Infer an IDF_PATH relative to the tools/cmake directory + get_filename_component(_idf_path "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) + file(TO_CMAKE_PATH "${_idf_path}" _idf_path) + + # Get the path set in environment + set(idf_path $ENV{IDF_PATH}) + file(TO_CMAKE_PATH "${idf_path}" idf_path) + + # Environment IDF_PATH should match the inferred IDF_PATH. If not, warn the user. + if(idf_path) + if(NOT idf_path STREQUAL _idf_path) + message(WARNING "IDF_PATH environment variable is different from inferred IDF_PATH. + Check if your project's top-level CMakeLists.txt includes the right + CMake files. Environment IDF_PATH will be used for the build.") + endif() + else() + message(WARNING "IDF_PATH environment variable not found. Setting IDF_PATH to '${_idf_path}'.") + set(idf_path ${_idf_path}) + set(ENV{IDF_PATH} ${_idf_path}) + endif() + + # Include other CMake modules required + set(CMAKE_MODULE_PATH + "${idf_path}/tools/cmake" + "${idf_path}/tools/cmake/third_party" + ${CMAKE_MODULE_PATH}) + include(build) + + set(IDF_PATH ${idf_path}) + + include(GetGitRevisionDescription) + include(git_submodules) + include(crosstool_version_check) + include(kconfig) + include(component) + include(utilities) + include(targets) + include(ldgen) + include(version) + + __build_init("${idf_path}") + + set_property(GLOBAL PROPERTY __IDF_ENV_SET 1) +endif() \ No newline at end of file diff --git a/tools/cmake/idf_functions.cmake b/tools/cmake/idf_functions.cmake deleted file mode 100644 index a034b809..00000000 --- a/tools/cmake/idf_functions.cmake +++ /dev/null @@ -1,246 +0,0 @@ -# Some IDF-specific functions and functions - -include(crosstool_version_check) - -# -# Set some variables used by rest of the build -# -# Note at the time this macro is expanded, the config is not yet -# loaded and the toolchain and project are not yet set -# -macro(idf_set_global_variables) - # Note that CONFIG_xxx is not available when this function is called - - set_default(EXTRA_COMPONENT_DIRS "") - - # Commmon components, required by every component in the build - # - set_default(COMPONENT_REQUIRES_COMMON "esp8266 newlib freertos heap log") - - # PROJECT_PATH has the path to the IDF project (top-level cmake directory) - # - # (cmake calls this CMAKE_SOURCE_DIR, keeping old name for compatibility.) - set(PROJECT_PATH "${CMAKE_SOURCE_DIR}") - - if(MAIN_SRCS) - message(WARNING "main is now a component, use of MAIN_SRCS is deprecated") - set_default(COMPONENT_DIRS "${PROJECT_PATH}/components ${EXTRA_COMPONENT_DIRS} \ - ${IDF_PATH}/components") - else() - set_default(COMPONENT_DIRS "${PROJECT_PATH}/components ${EXTRA_COMPONENT_DIRS} \ - ${IDF_PATH}/components ${PROJECT_PATH}/main") - endif() - - spaces2list(COMPONENT_DIRS) - - spaces2list(COMPONENTS) - - # Tell cmake to drop executables in the top-level build dir - set(EXECUTABLE_OUTPUT_PATH "${CMAKE_BINARY_DIR}") - - # path to idf.py tool - set(IDFTOOL ${PYTHON} "${IDF_PATH}/tools/idf.py") - - # Temporary trick to support both gcc5 and gcc8 builds - if(CMAKE_C_COMPILER_VERSION VERSION_EQUAL 5.2.0) - set(GCC_NOT_5_2_0 0) - else() - set(GCC_NOT_5_2_0 1) - endif() -endmacro() - -# Add all the IDF global compiler & preprocessor options -# (applied to all components). Some are config-dependent -# -# If you only want to set options for a particular component, -# don't call or edit this function. TODO DESCRIBE WHAT TO DO INSTEAD -# -function(idf_set_global_compiler_options) - add_definitions(-DESP_PLATFORM) - add_definitions(-DHAVE_CONFIG_H) - - if(CONFIG_OPTIMIZATION_LEVEL_RELEASE) - add_compile_options(-Os) - else() - add_compile_options(-Og) - endif() - - # Note: the visual studio generator doesn't support this syntax - add_compile_options("$<$:-std=gnu99>") - - add_compile_options("$<$:-std=gnu++11>") - add_compile_options("$<$:-fno-rtti>") - - if(CONFIG_CXX_EXCEPTIONS) - add_compile_options("$<$:-fexceptions>") - else() - add_compile_options("$<$:-fno-exceptions>") - endif() - - # Default compiler configuration - add_compile_options(-ffunction-sections -fdata-sections -fstrict-volatile-bitfields -mlongcalls -nostdlib) - - # Default warnings configuration - add_compile_options( - -Wall - -Werror=all - -Wno-error=unused-function - -Wno-error=unused-but-set-variable - -Wno-error=unused-variable - -Wno-error=deprecated-declarations - -Wextra - -Wno-unused-parameter - -Wno-sign-compare) - add_compile_options("$<$:-Wno-old-style-declaration>") - - if(CONFIG_DISABLE_GCC8_WARNINGS) - add_compile_options( - -Wno-parentheses - -Wno-sizeof-pointer-memaccess - -Wno-clobbered - ) - - endif() - - # Stack protection - if(NOT BOOTLOADER_BUILD) - if(CONFIG_STACK_CHECK_NORM) - add_compile_options(-fstack-protector) - elseif(CONFIG_STACK_CHECK_STRONG) - add_compile_options(-fstack-protector-strong) - elseif(CONFIG_STACK_CHECK_ALL) - add_compile_options(-fstack-protector-all) - endif() - endif() - - if(CONFIG_OPTIMIZATION_ASSERTIONS_DISABLED) - add_definitions(-DNDEBUG) - endif() - - # Always generate debug symbols (even in Release mode, these don't - # go into the final binary so have no impact on size) - add_compile_options(-ggdb) - - # Enable ccache if it's on the path - if(NOT CCACHE_DISABLE) - find_program(CCACHE_FOUND ccache) - if(CCACHE_FOUND) - message(STATUS "ccache will be used for faster builds") - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) - endif() - endif() - - # Temporary trick to support both gcc5 and gcc8 builds - add_definitions(-DGCC_NOT_5_2_0=${GCC_NOT_5_2_0}) -endfunction() - - -# Verify the IDF environment is configured correctly (environment, toolchain, etc) -function(idf_verify_environment) - - if(NOT CMAKE_PROJECT_NAME) - message(FATAL_ERROR "Internal error, IDF project.cmake should have set this variable already") - endif() - - # Check toolchain is configured properly in cmake - if(NOT ( ${CMAKE_SYSTEM_NAME} STREQUAL "Generic" AND ${CMAKE_C_COMPILER} MATCHES xtensa)) - message(FATAL_ERROR "Internal error, toolchain has not been set correctly by project " - "(or an invalid CMakeCache.txt file has been generated somehow)") - endif() - - # - # Warn if the toolchain version doesn't match - # - # TODO: make these platform-specific for diff toolchains - get_expected_ctng_version(expected_toolchain expected_gcc) - gcc_version_check("${expected_gcc}") - crosstool_version_check("${expected_toolchain}") - -endfunction() - -# idf_add_executable -# -# Calls add_executable to add the final project executable -# Adds .map & .bin file targets -# Sets up flash-related targets -function(idf_add_executable) - set(exe_target ${PROJECT_NAME}.elf) - - if(MAIN_SRCS) - spaces2list(MAIN_SRCS) - add_executable(${exe_target} ${MAIN_SRCS}) - else() - # Create a dummy file to work around CMake requirement of having a source - # file while adding an executable - add_executable(${exe_target} "${CMAKE_CURRENT_BINARY_DIR}/dummy_main_src.c") - add_custom_command(OUTPUT dummy_main_src.c - COMMAND ${CMAKE_COMMAND} -E touch dummy_main_src.c - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - VERBATIM) - - add_custom_target(dummy_main_src DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/dummy_main_src.c) - - add_dependencies(${exe_target} dummy_main_src) - endif() - - add_map_file(${exe_target}) -endfunction() - - -# add_map_file -# -# Set linker args for 'exe_target' to generate a linker Map file -function(add_map_file exe_target) - get_filename_component(basename ${exe_target} NAME_WE) - set(mapfile "${basename}.map") - target_link_libraries(${exe_target} "-Wl,--gc-sections -Wl,--cref -Wl,--Map=${mapfile} -Wl,--start-group") - set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY - ADDITIONAL_MAKE_CLEAN_FILES - "${CMAKE_CURRENT_BINARY_DIR}/${mapfile}") - - # add size targets, depend on map file, run idf_size.py - add_custom_target(size - DEPENDS ${exe_target} - COMMAND ${PYTHON} ${IDF_PATH}/tools/idf_size.py ${mapfile} - ) - add_custom_target(size-files - DEPENDS ${exe_target} - COMMAND ${PYTHON} ${IDF_PATH}/tools/idf_size.py --files ${mapfile} - ) - add_custom_target(size-components - DEPENDS ${exe_target} - COMMAND ${PYTHON} ${IDF_PATH}/tools/idf_size.py --archives ${mapfile} - ) - -endfunction() - -# component_compile_options -# -# Wrapper around target_compile_options that passes the component name -function(component_compile_options) - target_compile_options(${COMPONENT_NAME} PRIVATE ${ARGV}) -endfunction() - -# component_compile_definitions -# -# Wrapper around target_compile_definitions that passes the component name -function(component_compile_definitions) - target_compile_definitions(${COMPONENT_NAME} PRIVATE ${ARGV}) -endfunction() - -# idf_get_git_revision -# -# Set global IDF_VER to the git revision of ESP-IDF. -# -# Running git_describe() here automatically triggers rebuilds -# if the ESP-IDF git version changes -function(idf_get_git_revision) - if(EXISTS "${IDF_PATH}/version.txt") - file(STRINGS "${IDF_PATH}/version.txt" IDF_VER) - else() - git_describe(IDF_VER "${IDF_PATH}") - endif() - add_definitions(-DIDF_VER=\"${IDF_VER}\") - git_submodule_check("${IDF_PATH}") - set(IDF_VER ${IDF_VER} PARENT_SCOPE) -endfunction() diff --git a/tools/cmake/kconfig.cmake b/tools/cmake/kconfig.cmake index 5a87b8c0..561c95cc 100644 --- a/tools/cmake/kconfig.cmake +++ b/tools/cmake/kconfig.cmake @@ -1,182 +1,237 @@ include(ExternalProject) -macro(kconfig_set_variables) - set(CONFIG_DIR ${CMAKE_BINARY_DIR}/config) - set_default(SDKCONFIG ${PROJECT_PATH}/sdkconfig) - set(SDKCONFIG_HEADER ${CONFIG_DIR}/sdkconfig.h) - set(SDKCONFIG_CMAKE ${CONFIG_DIR}/sdkconfig.cmake) - set(SDKCONFIG_JSON ${CONFIG_DIR}/sdkconfig.json) - set(KCONFIG_JSON_MENUS ${CONFIG_DIR}/kconfig_menus.json) +function(__kconfig_init) + idf_build_get_property(idf_path IDF_PATH) + if(CMAKE_HOST_WIN32) + # Prefer a prebuilt mconf-idf on Windows + if(DEFINED ENV{MSYSTEM}) + find_program(WINPTY winpty) + else() + unset(WINPTY CACHE) # in case previous CMake run was in a tty and this one is not + endif() + unset(MCONF CACHE) # needed when MSYS and CMD is intermixed (cache could contain an incompatible path) + find_program(MCONF mconf-idf) - set(ROOT_KCONFIG ${IDF_PATH}/Kconfig) + # Fall back to the old binary which was called 'mconf' not 'mconf-idf' + if(NOT MCONF) + find_program(MCONF mconf) + if(MCONF) + message(WARNING "Falling back to mconf binary '${MCONF}' not mconf-idf. " + "This is probably because an old version of IDF mconf is installed and this is fine. " + "However if there are config problems please check the Getting Started guide for your platform.") + endif() + endif() - set_default(SDKCONFIG_DEFAULTS "${SDKCONFIG}.defaults") - - # ensure all source files can include sdkconfig.h - include_directories("${CONFIG_DIR}") -endmacro() - -if(CMAKE_HOST_WIN32) - # Prefer a prebuilt mconf-idf on Windows - if(DEFINED ENV{MSYSTEM}) - find_program(WINPTY winpty) - else() - unset(WINPTY CACHE) # in case previous CMake run was in a tty and this one is not - endif() - find_program(MCONF mconf-idf) - - # Fall back to the old binary which was called 'mconf' not 'mconf-idf' - if(NOT MCONF) - find_program(MCONF mconf) - if(MCONF) - message(WARNING "Falling back to mconf binary '${MCONF}' not mconf-idf. " - "This is probably because an old version of IDF mconf is installed and this is fine. " - "However if there are config problems please check the Getting Started guide for your platform.") + if(NOT MCONF) + find_program(NATIVE_GCC gcc) + if(NOT NATIVE_GCC) + message(FATAL_ERROR + "Windows requires a prebuilt mconf-idf for your platform " + "on the PATH, or an MSYS2 version of gcc on the PATH to build mconf-idf. " + "Consult the setup docs for ESP-IDF on Windows.") + endif() + elseif(WINPTY) + set(MCONF "\"${WINPTY}\" \"${MCONF}\"") endif() endif() if(NOT MCONF) - find_program(NATIVE_GCC gcc) - if(NOT NATIVE_GCC) - message(FATAL_ERROR - "Windows requires a prebuilt mconf-idf for your platform " - "on the PATH, or an MSYS2 version of gcc on the PATH to build mconf-idf. " - "Consult the setup docs for ESP-IDF on Windows.") - endif() - elseif(WINPTY) - set(MCONF "${WINPTY}" "${MCONF}") + # Use the existing Makefile to build mconf (out of tree) when needed + # + set(MCONF ${CMAKE_BINARY_DIR}/kconfig_bin/mconf-idf) + set(src_path ${idf_path}/tools/kconfig) + + # note: we preemptively remove any build files from the src dir + # as we're building out of tree, but don't want build system to + # #include any from there that were previously build with/for make + externalproject_add(mconf-idf + SOURCE_DIR ${src_path} + CONFIGURE_COMMAND "" + BINARY_DIR "kconfig_bin" + BUILD_COMMAND rm -f ${src_path}/zconf.lex.c ${src_path}/zconf.hash.c + COMMAND make -f ${src_path}/Makefile mconf-idf + BUILD_BYPRODUCTS ${MCONF} + INSTALL_COMMAND "" + EXCLUDE_FROM_ALL 1 + ) + + file(GLOB mconf_srcfiles ${src_path}/*.c) + list(REMOVE_ITEM mconf_srcfiles "${src_path}/zconf.lex.c" "${src_path}/zconf.hash.c") + externalproject_add_stepdependencies(mconf-idf build + ${mconf_srcfiles} + ${src_path}/Makefile + ${CMAKE_CURRENT_LIST_FILE}) + unset(mconf_srcfiles) + unset(src_path) + + set(menuconfig_depends DEPENDS mconf-idf) endif() -endif() -if(NOT MCONF) - # Use the existing Makefile to build mconf (out of tree) when needed - # - set(MCONF kconfig_bin/mconf-idf) + idf_build_set_property(__MCONF ${MCONF}) + idf_build_set_property(__MENUCONFIG_DEPENDS "${menuconfig_depends}") - externalproject_add(mconf-idf - SOURCE_DIR ${IDF_PATH}/tools/kconfig - CONFIGURE_COMMAND "" - BINARY_DIR "kconfig_bin" - BUILD_COMMAND make -f ${IDF_PATH}/tools/kconfig/Makefile mconf-idf - BUILD_BYPRODUCTS ${MCONF} - INSTALL_COMMAND "" - EXCLUDE_FROM_ALL 1 - ) + idf_build_get_property(idf_path IDF_PATH) + idf_build_set_property(__ROOT_KCONFIG ${idf_path}/Kconfig) + idf_build_set_property(__OUTPUT_SDKCONFIG 1) +endfunction() - file(GLOB mconf_srcfiles ${IDF_PATH}/tools/kconfig/*.c) - externalproject_add_stepdependencies(mconf-idf build - ${mconf_srcfiles} - ${IDF_PATH}/tools/kconfig/Makefile - ${CMAKE_CURRENT_LIST_FILE}) - unset(mconf_srcfiles) +# +# Initialize Kconfig-related properties for components. +# This function assumes that all basic properties of the components have been +# set prior to calling it. +# +function(__kconfig_component_init component_target) + __component_get_property(component_dir ${component_target} COMPONENT_DIR) + file(GLOB kconfig "${component_dir}/Kconfig") + __component_set_property(${component_target} KCONFIG "${kconfig}") + file(GLOB kconfig "${component_dir}/Kconfig.projbuild") + __component_set_property(${component_target} KCONFIG_PROJBUILD "${kconfig}") +endfunction() - set(menuconfig_depends DEPENDS mconf-idf) - -endif() - -# Find all Kconfig files for all components -function(kconfig_process_config) - file(MAKE_DIRECTORY "${CONFIG_DIR}") - set(kconfigs) - set(kconfigs_projbuild) - - # Find Kconfig and Kconfig.projbuild for each component as applicable - # if any of these change, cmake should rerun - foreach(dir ${BUILD_COMPONENT_PATHS}) - file(GLOB kconfig "${dir}/Kconfig") - if(kconfig) - set(kconfigs "${kconfigs} ${kconfig}") - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${kconfig}) - endif() - file(GLOB kconfig ${dir}/Kconfig.projbuild) - if(kconfig) - set(kconfigs_projbuild "${kconfigs_projbuild} ${kconfig}") - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${kconfig}) +# +# Generate the config files and create config related targets and configure +# dependencies. +# +function(__kconfig_generate_config sdkconfig sdkconfig_defaults) + # List all Kconfig and Kconfig.projbuild in known components + idf_build_get_property(component_targets __COMPONENT_TARGETS) + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + foreach(component_target ${component_targets}) + if(component_target IN_LIST build_component_targets) + __component_get_property(kconfig ${component_target} KCONFIG) + __component_get_property(kconfig_projbuild ${component_target} KCONFIG_PROJBUILD) + if(kconfig) + list(APPEND kconfigs ${kconfig}) + endif() + if(kconfig_projbuild) + list(APPEND kconfig_projbuilds ${kconfig_projbuild}) + endif() endif() endforeach() - if(EXISTS ${SDKCONFIG_DEFAULTS}) - set(defaults_arg --defaults "${SDKCONFIG_DEFAULTS}") + # Store the list version of kconfigs and kconfig_projbuilds + idf_build_set_property(KCONFIGS "${kconfigs}") + idf_build_set_property(KCONFIG_PROJBUILDS "${kconfig_projbuilds}") + + idf_build_get_property(idf_target IDF_TARGET) + idf_build_get_property(idf_path IDF_PATH) + + string(REPLACE ";" " " kconfigs "${kconfigs}") + string(REPLACE ";" " " kconfig_projbuilds "${kconfig_projbuilds}") + + # Place config-related environment arguments into config.env file + # to work around command line length limits for execute_process + # on Windows & CMake < 3.11 + set(config_env_path "${CMAKE_CURRENT_BINARY_DIR}/config.env") + configure_file("${idf_path}/tools/kconfig_new/config.env.in" ${config_env_path}) + idf_build_set_property(CONFIG_ENV_PATH ${config_env_path}) + + if(sdkconfig_defaults) + set(defaults_arg --defaults "${sdkconfig_defaults}") endif() - # Set these in the parent scope, so that they can be written to project_description.json - set(kconfigs "${kconfigs}") - set(COMPONENT_KCONFIGS "${kconfigs}" PARENT_SCOPE) - set(COMPONENT_KCONFIGS_PROJBUILD "${kconfigs_projbuild}" PARENT_SCOPE) + if(EXISTS "${sdkconfig_defaults}.${idf_target}") + list(APPEND defaults_arg --defaults "${sdkconfig_defaults}.${idf_target}") + endif() + + idf_build_get_property(root_kconfig __ROOT_KCONFIG) + idf_build_get_property(python PYTHON) set(confgen_basecommand - ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confgen.py - --kconfig ${ROOT_KCONFIG} - --config ${SDKCONFIG} + ${python} ${idf_path}/tools/kconfig_new/confgen.py + --kconfig ${root_kconfig} + --config ${sdkconfig} ${defaults_arg} - --env "COMPONENT_KCONFIGS=${kconfigs}" - --env "COMPONENT_KCONFIGS_PROJBUILD=${kconfigs_projbuild}" - --env "IDF_CMAKE=y") + --env-file ${config_env_path}) + + idf_build_get_property(build_dir BUILD_DIR) + set(config_dir ${build_dir}/config) + file(MAKE_DIRECTORY "${config_dir}") + + # Generate the config outputs + set(sdkconfig_cmake ${config_dir}/sdkconfig.cmake) + set(sdkconfig_header ${config_dir}/sdkconfig.h) + set(sdkconfig_json ${config_dir}/sdkconfig.json) + set(sdkconfig_json_menus ${config_dir}/kconfig_menus.json) + + idf_build_get_property(output_sdkconfig __OUTPUT_SDKCONFIG) + if(output_sdkconfig) + execute_process( + COMMAND ${confgen_basecommand} + --output header ${sdkconfig_header} + --output cmake ${sdkconfig_cmake} + --output json ${sdkconfig_json} + --output json_menus ${sdkconfig_json_menus} + --output config ${sdkconfig} + RESULT_VARIABLE config_result) + else() + execute_process( + COMMAND ${confgen_basecommand} + --output header ${sdkconfig_header} + --output cmake ${sdkconfig_cmake} + --output json ${sdkconfig_json} + --output json_menus ${sdkconfig_json_menus} + RESULT_VARIABLE config_result) + endif() + + if(config_result) + message(FATAL_ERROR "Failed to run confgen.py (${confgen_basecommand}). Error ${config_result}") + endif() + + # Add the generated config header to build specifications. + idf_build_set_property(INCLUDE_DIRECTORIES ${config_dir} APPEND) + + # When sdkconfig file changes in the future, trigger a cmake run + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${sdkconfig}") + + # Ditto if either of the generated files are missing/modified (this is a bit irritating as it means + # you can't edit these manually without them being regenerated, but I don't know of a better way...) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${sdkconfig_header}") + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${sdkconfig_cmake}") + + # Or if the config generation tool changes + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${idf_path}/tools/kconfig_new/confgen.py") + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${idf_path}/tools/kconfig_new/kconfiglib.py") + + set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY + ADDITIONAL_MAKE_CLEAN_FILES "${sdkconfig_header}" "${sdkconfig_cmake}") + + idf_build_set_property(SDKCONFIG_HEADER ${sdkconfig_header}) + idf_build_set_property(SDKCONFIG_JSON ${sdkconfig_json}) + idf_build_set_property(SDKCONFIG_CMAKE ${sdkconfig_cmake}) + idf_build_set_property(SDKCONFIG_JSON_MENUS ${sdkconfig_json_menus}) + idf_build_set_property(CONFIG_DIR ${config_dir}) + + idf_build_get_property(menuconfig_depends __MENUCONFIG_DEPENDS) + + idf_build_get_property(mconf __MCONF) # Generate the menuconfig target (uses C-based mconf-idf tool, either prebuilt or via mconf-idf target above) add_custom_target(menuconfig ${menuconfig_depends} # create any missing config file, with defaults if necessary - COMMAND ${confgen_basecommand} --output config ${SDKCONFIG} + COMMAND ${confgen_basecommand} --env "IDF_TARGET=${idf_target}" --output config ${sdkconfig} COMMAND ${CMAKE_COMMAND} -E env "COMPONENT_KCONFIGS=${kconfigs}" - "COMPONENT_KCONFIGS_PROJBUILD=${kconfigs_projbuild}" + "COMPONENT_KCONFIGS_PROJBUILD=${kconfig_projbuilds}" "IDF_CMAKE=y" - "KCONFIG_CONFIG=${SDKCONFIG}" - ${MCONF} ${ROOT_KCONFIG} - VERBATIM - USES_TERMINAL) + "KCONFIG_CONFIG=${sdkconfig}" + "IDF_TARGET=${idf_target}" + ${mconf} ${root_kconfig} + # VERBATIM cannot be used here because it cannot handle ${mconf}="winpty mconf-idf" and the escaping must be + # done manually + USES_TERMINAL + # additional run of confgen esures that the deprecated options will be inserted into sdkconfig (for backward + # compatibility) + COMMAND ${confgen_basecommand} --env "IDF_TARGET=${idf_target}" --output config ${sdkconfig} + ) # Custom target to run confserver.py from the build tool add_custom_target(confserver - COMMAND ${CMAKE_COMMAND} -E env - "COMPONENT_KCONFIGS=${kconfigs}" - "COMPONENT_KCONFIGS_PROJBUILD=${kconfigs_projbuild}" - ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confserver.py - --kconfig ${IDF_PATH}/Kconfig --config ${SDKCONFIG} + COMMAND ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confserver.py + --env-file ${config_env_path} + --kconfig ${IDF_PATH}/Kconfig + --config ${sdkconfig} VERBATIM USES_TERMINAL) - - # Generate configuration output via confgen.py - # makes sdkconfig.h and skdconfig.cmake - # - # This happens during the cmake run not during the build - if(NOT BOOTLOADER_BUILD) - execute_process( - COMMAND ${confgen_basecommand} - --output header ${SDKCONFIG_HEADER} - --output cmake ${SDKCONFIG_CMAKE} - --output json ${SDKCONFIG_JSON} - --output json_menus ${KCONFIG_JSON_MENUS} - --output config ${SDKCONFIG} # only generate config at the top-level project - RESULT_VARIABLE config_result) - else() - execute_process( - COMMAND ${confgen_basecommand} - --output header ${SDKCONFIG_HEADER} - --output cmake ${SDKCONFIG_CMAKE} - --output json ${SDKCONFIG_JSON} - --output json_menus ${KCONFIG_JSON_MENUS} - RESULT_VARIABLE config_result) - endif() - if(config_result) - message(FATAL_ERROR "Failed to run confgen.py (${confgen_basecommand}). Error ${config_result}") - endif() - - # When sdkconfig file changes in the future, trigger a cmake run - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${SDKCONFIG}") - - # Ditto if either of the generated files are missing/modified (this is a bit irritating as it means - # you can't edit these manually without them being regenerated, but I don't know of a better way...) - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${SDKCONFIG_HEADER}") - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${SDKCONFIG_CMAKE}") - - # Or if the config generation tool changes - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${IDF_PATH}/tools/kconfig_new/confgen.py") - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${IDF_PATH}/tools/kconfig_new/kconfiglib.py") - - set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY - ADDITIONAL_MAKE_CLEAN_FILES - "${SDKCONFIG_HEADER}" "${SDKCONFIG_CMAKE}") - endfunction() diff --git a/tools/cmake/ldgen.cmake b/tools/cmake/ldgen.cmake new file mode 100644 index 00000000..22d7fe67 --- /dev/null +++ b/tools/cmake/ldgen.cmake @@ -0,0 +1,77 @@ +# Utilities for supporting linker script generation in the build system + +# __ldgen_add_fragment_files +# +# Add one or more linker fragment files, and append it to the list of fragment +# files found so far. +function(__ldgen_add_fragment_files fragment_files) + spaces2list(fragment_files) + + foreach(fragment_file ${fragment_files}) + get_filename_component(abs_path ${fragment_file} ABSOLUTE) + list(APPEND _fragment_files ${abs_path}) + endforeach() + + idf_build_set_property(__LDGEN_FRAGMENT_FILES "${_fragment_files}" APPEND) +endfunction() + +# __ldgen_add_component +# +# Generate sections info for specified target to be used in linker script generation +function(__ldgen_add_component component_lib) + idf_build_set_property(__LDGEN_LIBRARIES "$" APPEND) + idf_build_set_property(__LDGEN_DEPENDS ${component_lib} APPEND) +endfunction() + +# __ldgen_process_template +# +# Passes a linker script template to the linker script generation tool for +# processing +function(__ldgen_process_template template output) + idf_build_get_property(idf_target IDF_TARGET) + idf_build_get_property(idf_path IDF_PATH) + + idf_build_get_property(build_dir BUILD_DIR) + idf_build_get_property(ldgen_libraries __LDGEN_LIBRARIES GENERATOR_EXPRESSION) + file(GENERATE OUTPUT ${build_dir}/ldgen_libraries.in CONTENT $) + file(GENERATE OUTPUT ${build_dir}/ldgen_libraries INPUT ${build_dir}/ldgen_libraries.in) + + set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES + "${build_dir}/ldgen_libraries.in" + "${build_dir}/ldgen_libraries") + + idf_build_get_property(ldgen_fragment_files __LDGEN_FRAGMENT_FILES GENERATOR_EXPRESSION) + idf_build_get_property(ldgen_depends __LDGEN_DEPENDS GENERATOR_EXPRESSION) + # Create command to invoke the linker script generator tool. + idf_build_get_property(sdkconfig SDKCONFIG) + idf_build_get_property(root_kconfig __ROOT_KCONFIG) + idf_build_get_property(kconfigs KCONFIGS) + idf_build_get_property(kconfig_projbuilds KCONFIG_PROJBUILDS) + + idf_build_get_property(python PYTHON) + + string(REPLACE ";" " " kconfigs "${kconfigs}") + string(REPLACE ";" " " kconfig_projbuilds "${kconfig_projbuilds}") + + idf_build_get_property(config_env_path CONFIG_ENV_PATH) + + add_custom_command( + OUTPUT ${output} + COMMAND ${python} ${idf_path}/tools/ldgen/ldgen.py + --config ${sdkconfig} + --fragments "$" + --input ${template} + --output ${output} + --kconfig ${root_kconfig} + --env-file "${config_env_path}" + --libraries-file ${build_dir}/ldgen_libraries + --objdump ${CMAKE_OBJDUMP} + DEPENDS ${template} ${ldgen_fragment_files} ${ldgen_depends} ${SDKCONFIG} + ) + + get_filename_component(_name ${output} NAME) + add_custom_target(__ldgen_output_${_name} DEPENDS ${output}) + add_dependencies(__idf_build_target __ldgen_output_${_name}) + idf_build_set_property(__LINK_DEPENDS ${output} APPEND) +endfunction() diff --git a/tools/cmake/project.cmake b/tools/cmake/project.cmake index dc0b21b2..ff9b75ec 100644 --- a/tools/cmake/project.cmake +++ b/tools/cmake/project.cmake @@ -1,172 +1,435 @@ # Designed to be included from an IDF app's CMakeLists.txt file -# cmake_minimum_required(VERSION 3.5) -# Set IDF_PATH, as nothing else will work without this -set(IDF_PATH "$ENV{IDF_PATH}") -if(NOT IDF_PATH) - # Documentation says you should set IDF_PATH in your environment, but we - # can infer it relative to tools/cmake directory if it's not set. - get_filename_component(IDF_PATH "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) -endif() -file(TO_CMAKE_PATH "${IDF_PATH}" IDF_PATH) -set(ENV{IDF_PATH} ${IDF_PATH}) - - -# -# Load cmake modules -# -set(CMAKE_MODULE_PATH - "${IDF_PATH}/tools/cmake" - "${IDF_PATH}/tools/cmake/third_party" - ${CMAKE_MODULE_PATH}) -include(GetGitRevisionDescription) -include(utilities) -include(components) -include(kconfig) -include(git_submodules) -include(idf_functions) +# The mere inclusion of this CMake file sets up some interal build properties. +# These properties can be modified in between this inclusion the the idf_build_process +# call. +include(${CMAKE_CURRENT_LIST_DIR}/idf.cmake) +set(IDFTOOL ${PYTHON} "${IDF_PATH}/tools/idf.py") +# Internally, the Python interpreter is already set to 'python'. Re-set here +# to be absolutely sure. set_default(PYTHON "python") +idf_build_set_property(PYTHON ${PYTHON}) -if(NOT PYTHON_DEPS_CHECKED AND NOT BOOTLOADER_BUILD) - message(STATUS "Checking Python dependencies...") - execute_process(COMMAND "${PYTHON}" "${IDF_PATH}/tools/check_python_dependencies.py" - RESULT_VARIABLE result) - if(NOT result EQUAL 0) - message(FATAL_ERROR "Some Python dependencies must be installed. Check above message for details.") - endif() +# On processing, checking Python required modules can be turned off if it was +# already checked externally. +if(PYTHON_DEPS_CHECKED) + idf_build_set_property(__CHECK_PYTHON 0) endif() -# project +# Initialize build target for this build using the environment variable or +# value passed externally. +__target_init() + # -# This macro wraps the cmake 'project' command to add -# all of the IDF-specific functionality required +# Get the project version from either a version file or the Git revision. This is passed +# to the idf_build_process call. Dependencies are also set here for when the version file +# changes (if it is used). # -# Implementation Note: This macro wraps 'project' on purpose, because cmake has -# some backwards-compatible magic where if you don't call "project" in the -# top-level CMakeLists file, it will call it implicitly. However, the implicit -# project will not have CMAKE_TOOLCHAIN_FILE set and therefore tries to -# create a native build project. -# -# Therefore, to keep all the IDF "build magic", the cleanest way is to keep the -# top-level "project" call but customize it to do what we want in the IDF build. -# -macro(project name) - # Set global variables used by rest of the build - idf_set_global_variables() - - # Sort the components list, as it may be found via filesystem - # traversal and therefore in a non-deterministic order - list(SORT COMPONENTS) - - execute_process(COMMAND "${CMAKE_COMMAND}" - -D "COMPONENTS=${COMPONENTS}" - -D "COMPONENT_REQUIRES_COMMON=${COMPONENT_REQUIRES_COMMON}" - -D "EXCLUDE_COMPONENTS=${EXCLUDE_COMPONENTS}" - -D "TEST_COMPONENTS=${TEST_COMPONENTS}" - -D "TEST_EXCLUDE_COMPONENTS=${TEST_EXCLUDE_COMPONENTS}" - -D "TESTS_ALL=${TESTS_ALL}" - -D "DEPENDENCIES_FILE=${CMAKE_BINARY_DIR}/component_depends.cmake" - -D "COMPONENT_DIRS=${COMPONENT_DIRS}" - -D "BOOTLOADER_BUILD=${BOOTLOADER_BUILD}" - -D "IDF_PATH=${IDF_PATH}" - -D "DEBUG=${DEBUG}" - -P "${IDF_PATH}/tools/cmake/scripts/expand_requirements.cmake" - WORKING_DIRECTORY "${PROJECT_PATH}") - include("${CMAKE_BINARY_DIR}/component_depends.cmake") - - # We now have the following component-related variables: - # COMPONENTS is the list of initial components set by the user (or empty to include all components in the build). - # BUILD_COMPONENTS is the list of components to include in the build. - # BUILD_COMPONENT_PATHS is the paths to all of these components. - - # Print list of components - string(REPLACE ";" " " BUILD_COMPONENTS_SPACES "${BUILD_COMPONENTS}") - message(STATUS "Component names: ${BUILD_COMPONENTS_SPACES}") - unset(BUILD_COMPONENTS_SPACES) - message(STATUS "Component paths: ${BUILD_COMPONENT_PATHS}") - - # Print list of test components - if(TESTS_ALL EQUAL 1 OR TEST_COMPONENTS) - string(REPLACE ";" " " BUILD_TEST_COMPONENTS_SPACES "${BUILD_TEST_COMPONENTS}") - message(STATUS "Test component names: ${BUILD_TEST_COMPONENTS_SPACES}") - unset(BUILD_TEST_COMPONENTS_SPACES) - message(STATUS "Test component paths: ${BUILD_TEST_COMPONENT_PATHS}") - endif() - - kconfig_set_variables() - - kconfig_process_config() - - # Include sdkconfig.cmake so rest of the build knows the configuration - include(${SDKCONFIG_CMAKE}) - - # Now the configuration is loaded, set the toolchain appropriately - # - # TODO: support more toolchains than just ESP32 - set(CMAKE_TOOLCHAIN_FILE $ENV{IDF_PATH}/tools/cmake/toolchain-esp8266.cmake) - - # Declare the actual cmake-level project - _project(${name} ASM C CXX) - - # generate compile_commands.json (needs to come after project) - set(CMAKE_EXPORT_COMPILE_COMMANDS 1) - - # Verify the environment is configured correctly - idf_verify_environment() - - # Add some idf-wide definitions - idf_set_global_compiler_options() - - # Check git revision (may trigger reruns of cmake) - ## sets IDF_VER to IDF git revision - idf_get_git_revision() - ## if project uses git, retrieve revision - git_describe(PROJECT_VER "${CMAKE_CURRENT_SOURCE_DIR}") - - # Include any top-level project_include.cmake files from components - foreach(component ${BUILD_COMPONENT_PATHS}) - set(COMPONENT_PATH "${component}") - include_if_exists("${component}/project_include.cmake") - unset(COMPONENT_PATH) - endforeach() - - # - # Add each component to the build as a library - # - foreach(COMPONENT_PATH ${BUILD_COMPONENT_PATHS}) - list(FIND BUILD_TEST_COMPONENT_PATHS ${COMPONENT_PATH} idx) - - if(NOT idx EQUAL -1) - list(GET BUILD_TEST_COMPONENTS ${idx} test_component) - set(COMPONENT_NAME ${test_component}) +function(__project_get_revision var) + set(_project_path "${CMAKE_CURRENT_LIST_DIR}") + if(NOT DEFINED PROJECT_VER) + if(EXISTS "${_project_path}/version.txt") + file(STRINGS "${_project_path}/version.txt" PROJECT_VER) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_project_path}/version.txt") else() - get_filename_component(COMPONENT_NAME ${COMPONENT_PATH} NAME) + git_describe(PROJECT_VER_GIT "${_project_path}") + if(PROJECT_VER_GIT) + set(PROJECT_VER ${PROJECT_VER_GIT}) + else() + message(STATUS "Project is not inside a git repository, \ + will not use 'git describe' to determine PROJECT_VER.") + set(PROJECT_VER "1") + endif() endif() + endif() + message(STATUS "Project version: ${PROJECT_VER}") + set(${var} "${PROJECT_VER}" PARENT_SCOPE) +endfunction() - add_subdirectory(${COMPONENT_PATH} ${COMPONENT_NAME}) +# +# Output the built components to the user. Generates files for invoking idf_monitor.py +# that doubles as an overview of some of the more important build properties. +# +function(__project_info test_components) + idf_build_get_property(prefix __PREFIX) + idf_build_get_property(_build_components BUILD_COMPONENTS) + idf_build_get_property(build_dir BUILD_DIR) + idf_build_get_property(idf_path IDF_PATH) + + list(SORT _build_components) + + unset(build_components) + unset(build_component_paths) + + foreach(build_component ${_build_components}) + __component_get_target(component_target "${build_component}") + __component_get_property(_name ${component_target} COMPONENT_NAME) + __component_get_property(_prefix ${component_target} __PREFIX) + __component_get_property(_alias ${component_target} COMPONENT_ALIAS) + __component_get_property(_dir ${component_target} COMPONENT_DIR) + + if(_alias IN_LIST test_components) + list(APPEND test_component_paths ${_dir}) + else() + if(_prefix STREQUAL prefix) + set(component ${_name}) + else() + set(component ${_alias}) + endif() + list(APPEND build_components ${component}) + list(APPEND build_component_paths ${_dir}) + endif() endforeach() - unset(COMPONENT_NAME) - unset(COMPONENT_PATH) - # - # Add the app executable to the build (has name of PROJECT.elf) - # - idf_add_executable() + set(PROJECT_NAME ${CMAKE_PROJECT_NAME}) + idf_build_get_property(PROJECT_PATH PROJECT_DIR) + idf_build_get_property(BUILD_DIR BUILD_DIR) + idf_build_get_property(SDKCONFIG SDKCONFIG) + idf_build_get_property(SDKCONFIG_DEFAULTS SDKCONFIG_DEFAULTS) + idf_build_get_property(PROJECT_EXECUTABLE EXECUTABLE) + set(PROJECT_BIN ${CMAKE_PROJECT_NAME}.bin) + idf_build_get_property(IDF_VER IDF_VER) + + idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE) + include(${sdkconfig_cmake}) + idf_build_get_property(COMPONENT_KCONFIGS KCONFIGS) + idf_build_get_property(COMPONENT_KCONFIGS_PROJBUILD KCONFIG_PROJBUILDS) # Write project description JSON file - make_json_list("${BUILD_COMPONENTS}" build_components_json) - make_json_list("${BUILD_COMPONENT_PATHS}" build_component_paths_json) - configure_file("${IDF_PATH}/tools/cmake/project_description.json.in" - "${CMAKE_BINARY_DIR}/project_description.json") - unset(build_components_json) - unset(build_component_paths_json) + idf_build_get_property(build_dir BUILD_DIR) + make_json_list("${build_components};${test_components}" build_components_json) + make_json_list("${build_component_paths};${test_component_paths}" build_component_paths_json) + configure_file("${idf_path}/tools/cmake/project_description.json.in" + "${build_dir}/project_description.json") + # We now have the following component-related variables: # - # Finish component registration (add cross-dependencies, make - # executable dependent on all components) + # build_components is the list of components to include in the build. + # build_component_paths is the paths to all of these components, obtained from the component dependencies file. # - components_finish_registration() + # Print the list of found components and test components + string(REPLACE ";" " " build_components "${build_components}") + string(REPLACE ";" " " build_component_paths "${build_component_paths}") + message(STATUS "Components: ${build_components}") + message(STATUS "Component paths: ${build_component_paths}") + if(test_components) + string(REPLACE ";" " " test_components "${test_components}") + string(REPLACE ";" " " test_component_paths "${test_component_paths}") + message(STATUS "Test components: ${test_components}") + message(STATUS "Test component paths: ${test_component_paths}") + endif() +endfunction() + +function(__project_init components_var test_components_var) + # Use EXTRA_CFLAGS, EXTRA_CXXFLAGS and EXTRA_CPPFLAGS to add more priority options to the compiler + # EXTRA_CPPFLAGS is used for both C and C++ + # Unlike environments' CFLAGS/CXXFLAGS/CPPFLAGS which work for both host and target build, + # these works only for target build + set(extra_cflags "$ENV{EXTRA_CFLAGS}") + set(extra_cxxflags "$ENV{EXTRA_CXXFLAGS}") + set(extra_cppflags "$ENV{EXTRA_CPPFLAGS}") + + spaces2list(extra_cflags) + spaces2list(extra_cxxflags) + spaces2list(extra_cppflags) + + idf_build_set_property(C_COMPILE_OPTIONS "${extra_cflags}" APPEND) + idf_build_set_property(CXX_COMPILE_OPTIONS "${extra_cxxflags}" APPEND) + idf_build_set_property(COMPILE_OPTIONS "${extra_cppflags}" APPEND) + + function(__project_component_dir component_dir) + get_filename_component(component_dir "${component_dir}" ABSOLUTE) + if(EXISTS ${component_dir}/CMakeLists.txt) + idf_build_component(${component_dir}) + else() + file(GLOB component_dirs ${component_dir}/*) + foreach(component_dir ${component_dirs}) + if(EXISTS ${component_dir}/CMakeLists.txt) + get_filename_component(base_dir ${component_dir} NAME) + __component_dir_quick_check(is_component ${component_dir}) + if(is_component) + idf_build_component(${component_dir}) + endif() + endif() + endforeach() + endif() + endfunction() + + # Add component directories to the build, given the component filters, exclusions + # extra directories, etc. passed from the root CMakeLists.txt. + if(COMPONENT_DIRS) + # User wants to fully override where components are pulled from. + spaces2list(COMPONENT_DIRS) + idf_build_set_property(__COMPONENT_TARGETS "") + foreach(component_dir ${COMPONENT_DIRS}) + __project_component_dir(${component_dir}) + endforeach() + else() + # Look for components in the usual places: CMAKE_CURRENT_LIST_DIR/main, + # CMAKE_CURRENT_LIST_DIR/components, and the extra component dirs + if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/main") + __project_component_dir("${CMAKE_CURRENT_LIST_DIR}/main") + endif() + + __project_component_dir("${CMAKE_CURRENT_LIST_DIR}/components") + + spaces2list(EXTRA_COMPONENT_DIRS) + foreach(component_dir ${EXTRA_COMPONENT_DIRS}) + __project_component_dir("${component_dir}") + endforeach() + endif() + + spaces2list(COMPONENTS) + spaces2list(EXCLUDE_COMPONENTS) + idf_build_get_property(component_targets __COMPONENT_TARGETS) + foreach(component_target ${component_targets}) + __component_get_property(component_name ${component_target} COMPONENT_NAME) + set(include 1) + if(COMPONENTS AND NOT component_name IN_LIST COMPONENTS) + set(include 0) + endif() + if(EXCLUDE_COMPONENTS AND component_name IN_LIST EXCLUDE_COMPONENTS) + set(include 0) + endif() + if(include) + list(APPEND components ${component_name}) + endif() + endforeach() + + if(TESTS_ALL OR BUILD_TESTS OR TEST_COMPONENTS OR TEST_EXCLUDE_COMPONENTS) + spaces2list(TEST_COMPONENTS) + spaces2list(TEST_EXCLUDE_COMPONENTS) + idf_build_get_property(component_targets __COMPONENT_TARGETS) + foreach(component_target ${component_targets}) + __component_get_property(component_dir ${component_target} COMPONENT_DIR) + __component_get_property(component_name ${component_target} COMPONENT_NAME) + if(component_name IN_LIST components) + set(include 1) + if(TEST_COMPONENTS AND NOT component_name IN_LIST TEST_COMPONENTS) + set(include 0) + endif() + if(TEST_EXCLUDE_COMPONENTS AND component_name IN_LIST TEST_EXCLUDE_COMPONENTS) + set(include 0) + endif() + if(include AND EXISTS ${component_dir}/test) + __component_add(${component_dir}/test ${component_name}) + list(APPEND test_components ${component_name}::test) + endif() + endif() + endforeach() + endif() + + set(${components_var} "${components}" PARENT_SCOPE) + set(${test_components_var} "${test_components}" PARENT_SCOPE) +endfunction() + +# Trick to temporarily redefine project(). When functions are overriden in CMake, the originals can still be accessed +# using an underscore prefixed function of the same name. The following lines make sure that __project calls +# the original project(). See https://cmake.org/pipermail/cmake/2015-October/061751.html. +function(project) +endfunction() + +function(_project) +endfunction() + +macro(project project_name) + # Initialize project, preparing COMPONENTS argument for idf_build_process() + # call later using external COMPONENT_DIRS, COMPONENTS_DIRS, EXTRA_COMPONENTS_DIR, + # EXTRA_COMPONENTS_DIRS, COMPONENTS, EXLUDE_COMPONENTS, TEST_COMPONENTS, + # TEST_EXLUDE_COMPONENTS, TESTS_ALL, BUILD_TESTS + __project_init(components test_components) + + __target_set_toolchain() + + if(CCACHE_ENABLE) + find_program(CCACHE_FOUND ccache) + if(CCACHE_FOUND) + message(STATUS "ccache will be used for faster builds") + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) + else() + message(WARNING "enabled ccache in build but ccache program not found") + endif() + endif() + + # The actual call to project() + __project(${project_name} C CXX ASM) + + # Generate compile_commands.json (needs to come after project call). + set(CMAKE_EXPORT_COMPILE_COMMANDS 1) + + # Since components can import third-party libraries, the original definition of project() should be restored + # before the call to add components to the build. + function(project) + set(project_ARGV ARGV) + __project(${${project_ARGV}}) + + # Set the variables that project() normally sets, documented in the + # command's docs. + # + # https://cmake.org/cmake/help/v3.5/command/project.html + # + # There is some nuance when it comes to setting version variables in terms of whether + # CMP0048 is set to OLD or NEW. However, the proper behavior should have bee already handled by the original + # project call, and we're just echoing the values those variables were set to. + set(PROJECT_NAME "${PROJECT_NAME}" PARENT_SCOPE) + set(PROJECT_BINARY_DIR "${PROJECT_BINARY_DIR}" PARENT_SCOPE) + set(PROJECT_SOURCE_DIR "${PROJECT_SOURCE_DIR}" PARENT_SCOPE) + set(PROJECT_VERSION "${PROJECT_VERSION}" PARENT_SCOPE) + set(PROJECT_VERSION_MAJOR "${PROJECT_VERSION_MAJOR}" PARENT_SCOPE) + set(PROJECT_VERSION_MINOR "${PROJECT_VERSION_MINOR}" PARENT_SCOPE) + set(PROJECT_VERSION_PATCH "${PROJECT_VERSION_PATCH}" PARENT_SCOPE) + set(PROJECT_VERSION_TWEAK "${PROJECT_VERSION_TWEAK}" PARENT_SCOPE) + + set(${PROJECT_NAME}_BINARY_DIR "${${PROJECT_NAME}_BINARY_DIR}" PARENT_SCOPE) + set(${PROJECT_NAME}_SOURCE_DIR "${${PROJECT_NAME}_SOURCE_DIR}" PARENT_SCOPE) + set(${PROJECT_NAME}_VERSION "${${PROJECT_NAME}_VERSION}" PARENT_SCOPE) + set(${PROJECT_NAME}_VERSION_MAJOR "${${PROJECT_NAME}_VERSION_MAJOR}" PARENT_SCOPE) + set(${PROJECT_NAME}_VERSION_MINOR "${${PROJECT_NAME}_VERSION_MINOR}" PARENT_SCOPE) + set(${PROJECT_NAME}_VERSION_PATCH "${${PROJECT_NAME}_VERSION_PATCH}" PARENT_SCOPE) + set(${PROJECT_NAME}_VERSION_TWEAK "${${PROJECT_NAME}_VERSION_TWEAK}" PARENT_SCOPE) + endfunction() + + # Prepare the following arguments for the idf_build_process() call using external + # user values: + # + # SDKCONFIG_DEFAULTS is from external SDKCONFIG_DEFAULTS + # SDKCONFIG is from external SDKCONFIG + # BUILD_DIR is set to project binary dir + # + # PROJECT_NAME is taken from the passed name from project() call + # PROJECT_DIR is set to the current directory + # PROJECT_VER is from the version text or git revision of the current repo + if(SDKCONFIG_DEFAULTS) + get_filename_component(sdkconfig_defaults "${SDKCONFIG_DEFAULTS}" ABSOLUTE) + if(NOT EXISTS "${sdkconfig_defaults}") + message(FATAL_ERROR "SDKCONFIG_DEFAULTS '${sdkconfig_defaults}' does not exist.") + endif() + else() + if(EXISTS "${CMAKE_SOURCE_DIR}/sdkconfig.defaults") + set(sdkconfig_defaults "${CMAKE_SOURCE_DIR}/sdkconfig.defaults") + else() + set(sdkconfig_defaults "") + endif() + endif() + + if(SDKCONFIG) + get_filename_component(sdkconfig "${SDKCONFIG}" ABSOLUTE) + if(NOT EXISTS "${sdkconfig}") + message(FATAL_ERROR "SDKCONFIG '${sdkconfig}' does not exist.") + endif() + set(sdkconfig ${SDKCONFIG}) + else() + set(sdkconfig "${CMAKE_CURRENT_LIST_DIR}/sdkconfig") + endif() + + if(BUILD_DIR) + get_filename_component(build_dir "${BUILD_DIR}" ABSOLUTE) + if(NOT EXISTS "${build_dir}") + message(FATAL_ERROR "BUILD_DIR '${build_dir}' does not exist.") + endif() + else() + set(build_dir ${CMAKE_BINARY_DIR}) + endif() + + __project_get_revision(project_ver) + + message(STATUS "Building ESP-IDF components for target ${IDF_TARGET}") + + idf_build_process(${IDF_TARGET} + SDKCONFIG_DEFAULTS "${sdkconfig_defaults}" + SDKCONFIG ${sdkconfig} + BUILD_DIR ${build_dir} + PROJECT_NAME ${CMAKE_PROJECT_NAME} + PROJECT_DIR ${CMAKE_CURRENT_LIST_DIR} + PROJECT_VER "${project_ver}" + COMPONENTS "${components};${test_components}") + + # Special treatment for 'main' component for standard projects (not part of core build system). + # Have it depend on every other component in the build. This is + # a convenience behavior for the standard project; thus is done outside of the core build system + # so that it treats components equally. + # + # This behavior should only be when user did not set REQUIRES/PRIV_REQUIRES manually. + idf_build_get_property(build_components BUILD_COMPONENTS) + if(idf::main IN_LIST build_components) + __component_get_target(main_target idf::main) + __component_get_property(reqs ${main_target} REQUIRES) + __component_get_property(priv_reqs ${main_target} PRIV_REQUIRES) + idf_build_get_property(common_reqs __COMPONENT_REQUIRES_COMMON) + if(reqs STREQUAL common_reqs AND NOT priv_reqs) #if user has not set any requirements + list(REMOVE_ITEM build_components idf::main) + __component_get_property(lib ${main_target} COMPONENT_LIB) + set_property(TARGET ${lib} APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${build_components}") + get_property(type TARGET ${lib} PROPERTY TYPE) + if(type STREQUAL STATIC_LIBRARY) + set_property(TARGET ${lib} APPEND PROPERTY LINK_LIBRARIES "${build_components}") + endif() + endif() + endif() + + set(project_elf ${CMAKE_PROJECT_NAME}.elf) + + # Create a dummy file to work around CMake requirement of having a source + # file while adding an executable + set(project_elf_src ${CMAKE_BINARY_DIR}/project_elf_src.c) + add_custom_command(OUTPUT ${project_elf_src} + COMMAND ${CMAKE_COMMAND} -E touch ${project_elf_src} + VERBATIM) + add_custom_target(_project_elf_src DEPENDS "${project_elf_src}") + add_executable(${project_elf} "${project_elf_src}") + add_dependencies(${project_elf} _project_elf_src) + + if(test_components) + target_link_libraries(${project_elf} "-Wl,--whole-archive") + foreach(test_component ${test_components}) + if(TARGET ${test_component}) + target_link_libraries(${project_elf} ${test_component}) + endif() + endforeach() + target_link_libraries(${project_elf} "-Wl,--no-whole-archive") + endif() + + idf_build_get_property(build_components BUILD_COMPONENTS) + if(test_components) + list(REMOVE_ITEM build_components ${test_components}) + endif() + target_link_libraries(${project_elf} ${build_components}) + + set(mapfile "${CMAKE_BINARY_DIR}/${CMAKE_PROJECT_NAME}.map") + target_link_libraries(${project_elf} "-Wl,--cref -Wl,--Map=${mapfile}") + + set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY + ADDITIONAL_MAKE_CLEAN_FILES + "${mapfile}" "${project_elf_src}") + + idf_build_get_property(idf_path IDF_PATH) + idf_build_get_property(python PYTHON) + + set(idf_size ${python} ${idf_path}/tools/idf_size.py) + if(DEFINED OUTPUT_JSON AND OUTPUT_JSON) + list(APPEND idf_size "--json") + endif() + + # Add size targets, depend on map file, run idf_size.py + add_custom_target(size + DEPENDS ${project_elf} + COMMAND ${idf_size} ${mapfile} + ) + add_custom_target(size-files + DEPENDS ${project_elf} + COMMAND ${idf_size} --files ${mapfile} + ) + add_custom_target(size-components + DEPENDS ${project_elf} + COMMAND ${idf_size} --archives ${mapfile} + ) + + unset(idf_size) + + idf_build_executable(${project_elf}) + + __project_info("${test_components}") endmacro() diff --git a/tools/cmake/project_description.json.in b/tools/cmake/project_description.json.in index 878dce3b..e0a62ee5 100644 --- a/tools/cmake/project_description.json.in +++ b/tools/cmake/project_description.json.in @@ -1,14 +1,14 @@ { "project_name": "${PROJECT_NAME}", "project_path": "${PROJECT_PATH}", - "build_dir": "${CMAKE_BINARY_DIR}", + "build_dir": "${BUILD_DIR}", "config_file": "${SDKCONFIG}", "config_defaults": "${SDKCONFIG_DEFAULTS}", - "app_elf": "${PROJECT_NAME}.elf", - "app_bin": "${PROJECT_NAME}.bin", + "app_elf": "${PROJECT_EXECUTABLE}", + "app_bin": "${PROJECT_BIN}", "git_revision": "${IDF_VER}", "phy_data_partition": "${CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION}", - "monitor_baud" : "${CONFIG_MONITOR_BAUD}", + "monitor_baud" : "${CONFIG_ESPTOOLPY_MONITOR_BAUD}", "config_environment" : { "COMPONENT_KCONFIGS" : "${COMPONENT_KCONFIGS}", "COMPONENT_KCONFIGS_PROJBUILD" : "${COMPONENT_KCONFIGS_PROJBUILD}" diff --git a/tools/cmake/scripts/component_get_requirements.cmake b/tools/cmake/scripts/component_get_requirements.cmake new file mode 100644 index 00000000..24637008 --- /dev/null +++ b/tools/cmake/scripts/component_get_requirements.cmake @@ -0,0 +1,103 @@ +include("${BUILD_PROPERTIES_FILE}") +include("${COMPONENT_PROPERTIES_FILE}") + +function(idf_build_get_property var property) + cmake_parse_arguments(_ "GENERATOR_EXPRESSION" "" "" ${ARGN}) + if(__GENERATOR_EXPRESSION) + message(FATAL_ERROR "Getting build property generator expression not + supported before idf_component_register().") + endif() + set(${var} ${${property}} PARENT_SCOPE) +endfunction() + +idf_build_get_property(idf_path IDF_PATH) +include(${idf_path}/tools/cmake/utilities.cmake) + +function(__component_get_property var component_target property) + set(_property __component_${component_target}_${property}) + set(${var} ${${_property}} PARENT_SCOPE) +endfunction() + +macro(require_idf_targets) +endmacro() + +macro(idf_component_register) + set(options) + set(single_value) + set(multi_value SRCS SRC_DIRS EXCLUDE_SRCS + INCLUDE_DIRS PRIV_INCLUDE_DIRS LDFRAGMENTS REQUIRES + PRIV_REQUIRES REQUIRED_IDF_TARGETS EMBED_FILES EMBED_TXTFILES) + cmake_parse_arguments(_ "${options}" "${single_value}" "${multi_value}" "${ARGN}") + set(__component_requires "${__REQUIRES}") + set(__component_priv_requires "${__PRIV_REQUIRES}") + set(__component_registered 1) + return() +endmacro() + +macro(register_component) + set(__component_requires "${COMPONENT_REQUIRES}") + set(__component_priv_requires "${COMPONENT_PRIV_REQUIRES}") + set(__component_registered 1) + return() +endmacro() + +macro(register_config_only_component) + register_component() +endmacro() + +idf_build_get_property(__common_reqs __COMPONENT_REQUIRES_COMMON) +idf_build_get_property(__component_targets __COMPONENT_TARGETS) + +function(__component_get_requirements) + # This is in a function (separate variable context) so that variables declared + # and set by the included CMakeLists.txt does not bleed into the next inclusion. + # We are only interested in the public and private requirements of components + __component_get_property(__component_dir ${__component_target} COMPONENT_DIR) + __component_get_property(__component_name ${__component_target} COMPONENT_NAME) + set(COMPONENT_NAME ${__component_name}) + set(COMPONENT_DIR ${__component_dir}) + set(COMPONENT_PATH ${__component_dir}) # for backward compatibility only, COMPONENT_DIR is preferred + include(${__component_dir}/CMakeLists.txt OPTIONAL) + + spaces2list(__component_requires) + spaces2list(__component_priv_requires) + + set(__component_requires "${__component_requires}" PARENT_SCOPE) + set(__component_priv_requires "${__component_priv_requires}" PARENT_SCOPE) + set(__component_registered ${__component_registered} PARENT_SCOPE) +endfunction() + +set(CMAKE_BUILD_EARLY_EXPANSION 1) +foreach(__component_target ${__component_targets}) + set(__component_requires "") + set(__component_priv_requires "") + set(__component_registered 0) + + __component_get_requirements() + + list(APPEND __component_requires "${__common_reqs}") + + # Remove duplicates and the component itself from its requirements + __component_get_property(__component_alias ${__component_target} COMPONENT_ALIAS) + __component_get_property(__component_name ${__component_target} COMPONENT_NAME) + + # Prevent component from linking to itself. + if(__component_requires) + list(REMOVE_DUPLICATES __component_requires) + list(REMOVE_ITEM __component_requires ${__component_alias} ${__component_name}) + endif() + + if(__component_requires) + list(REMOVE_DUPLICATES __component_priv_requires) + list(REMOVE_ITEM __component_priv_requires ${__component_alias} ${__component_name}) + endif() + + set(__contents +"__component_set_property(${__component_target} REQUIRES \"${__component_requires}\") +__component_set_property(${__component_target} PRIV_REQUIRES \"${__component_priv_requires}\") +__component_set_property(${__component_target} __COMPONENT_REGISTERED ${__component_registered})" + ) + set(__component_requires_contents "${__component_requires_contents}\n${__contents}") +endforeach() + +file(WRITE ${COMPONENT_REQUIRES_FILE} "${__component_requires_contents}") \ No newline at end of file diff --git a/tools/cmake/scripts/expand_requirements.cmake b/tools/cmake/scripts/expand_requirements.cmake deleted file mode 100644 index 8197016f..00000000 --- a/tools/cmake/scripts/expand_requirements.cmake +++ /dev/null @@ -1,338 +0,0 @@ -# expand_requires.cmake is a utility cmake script to expand component requirements early in the build, -# before the components are ready to be included. -# -# Parameters: -# - COMPONENTS = Space-separated list of initial components to include in the build. -# Can be empty, in which case all components are in the build. -# - COMPONENT_REQUIRES_COMMON = Components to always include in the build, and treated as dependencies -# of all other components. -# - DEPENDENCIES_FILE = Path of generated cmake file which will contain the expanded dependencies for these -# components. -# - COMPONENT_DIRS = List of paths to search for all components. -# - DEBUG = Set -DDEBUG=1 to debug component lists in the build. -# -# If successful, DEPENDENCIES_FILE can be expanded to set BUILD_COMPONENTS & BUILD_COMPONENT_PATHS with all -# components required for the build, and the get_component_requirements() function to return each component's -# recursively expanded requirements. -# -# BUILD_COMPONENTS & BUILD_COMPONENT_PATHS will be ordered in a best-effort way so that dependencies are listed first. -# (Note that IDF supports cyclic dependencies, and dependencies in a cycle have ordering guarantees.) -# -# Determinism: -# -# Given the the same list of names in COMPONENTS (regardless of order), and an identical value of -# COMPONENT_REQUIRES_COMMON, and all the same COMPONENT_REQUIRES & COMPONENT_PRIV_REQUIRES values in -# each component, then the output of BUILD_COMPONENTS should always be in the same -# order. -# -# BUILD_COMPONENT_PATHS will be in the same component order as BUILD_COMPONENTS, even if the -# actual component paths are different due to different paths. -# -# TODO: Error out if a component requirement is missing -cmake_minimum_required(VERSION 3.5) -include("${IDF_PATH}/tools/cmake/utilities.cmake") - -if(NOT DEPENDENCIES_FILE) - message(FATAL_ERROR "DEPENDENCIES_FILE must be set.") -endif() - -if(NOT COMPONENT_DIRS) - message(FATAL_ERROR "COMPONENT_DIRS variable must be set") -endif() -spaces2list(COMPONENT_DIRS) - -spaces2list(COMPONENT_REQUIRES_COMMON) - -function(debug message) - if(DEBUG) - message(STATUS "${message}") - endif() -endfunction() - -# Dummy register_component used to save requirements variables as global properties, for later expansion -# -# (expand_component_requirements() includes the component CMakeLists.txt, which then sets its component variables, -# calls this dummy macro, and immediately exits again.) -macro(register_component) - if(COMPONENT STREQUAL main AND NOT COMPONENT_REQUIRES) - set(main_component_requires ${COMPONENTS}) - list(REMOVE_ITEM main_component_requires "main") - - set_property(GLOBAL PROPERTY "${COMPONENT}_REQUIRES" "${main_component_requires}") - else() - spaces2list(COMPONENT_REQUIRES) - set_property(GLOBAL PROPERTY "${COMPONENT}_REQUIRES" "${COMPONENT_REQUIRES}") - endif() - - spaces2list(COMPONENT_PRIV_REQUIRES) - set_property(GLOBAL PROPERTY "${COMPONENT}_PRIV_REQUIRES" "${COMPONENT_PRIV_REQUIRES}") - - # This is tricky: we override register_component() so it returns out of the component CMakeLists.txt - # (as we're declaring it as a macro not a function, so it doesn't have its own scope.) - # - # This means no targets are defined, and the component expansion ends early. - return() -endmacro() - -macro(register_config_only_component) - register_component() -endmacro() - -# Given a component name (find_name) and a list of component paths (component_paths), -# return the path to the component in 'variable' -# -# Fatal error is printed if the component is not found. -function(find_component_path find_name components component_paths variable) - list(FIND components ${find_name} idx) - if(NOT idx EQUAL -1) - list(GET component_paths ${idx} path) - set("${variable}" "${path}" PARENT_SCOPE) - return() - else() - endif() - # TODO: find a way to print the dependency chain that lead to this not-found component - message(WARNING "Required component ${find_name} is not found in any of the provided COMPONENT_DIRS") -endfunction() - -# components_find_all: Search 'component_dirs' for components and return them -# as a list of names in 'component_names' and a list of full paths in -# 'component_paths' -# -# component_paths contains only unique component names. Directories -# earlier in the component_dirs list take precedence. -function(components_find_all component_dirs component_paths component_names test_component_names) - # component_dirs entries can be files or lists of files - set(paths "") - set(names "") - set(test_names "") - - # start by expanding the component_dirs list with all subdirectories - foreach(dir ${component_dirs}) - # Iterate any subdirectories for values - file(GLOB subdirs LIST_DIRECTORIES true "${dir}/*") - foreach(subdir ${subdirs}) - set(component_dirs "${component_dirs};${subdir}") - endforeach() - endforeach() - - # Look for a component in each component_dirs entry - foreach(dir ${component_dirs}) - debug("Looking for CMakeLists.txt in ${dir}") - file(GLOB component "${dir}/CMakeLists.txt") - if(component) - debug("CMakeLists.txt file ${component}") - get_filename_component(component "${component}" DIRECTORY) - get_filename_component(name "${component}" NAME) - if(NOT name IN_LIST names) - list(APPEND names "${name}") - list(APPEND paths "${component}") - - # Look for test component directory - file(GLOB test "${component}/test/CMakeLists.txt") - if(test) - list(APPEND test_names "${name}") - endif() - endif() - else() # no CMakeLists.txt file - # test for legacy component.mk and warn - file(GLOB legacy_component "${dir}/component.mk") - if(legacy_component) - get_filename_component(legacy_component "${legacy_component}" DIRECTORY) - message(WARNING "Component ${legacy_component} contains old-style component.mk but no CMakeLists.txt. " - "Component will be skipped.") - endif() - endif() - endforeach() - - set(${component_paths} ${paths} PARENT_SCOPE) - set(${component_names} ${names} PARENT_SCOPE) - set(${test_component_names} ${test_names} PARENT_SCOPE) -endfunction() - - -# expand_component_requirements: Recursively expand a component's requirements, -# setting global properties BUILD_COMPONENTS & BUILD_COMPONENT_PATHS and -# also invoking the components to call register_component() above, -# which will add per-component global properties with dependencies, etc. -function(expand_component_requirements component) - get_property(seen_components GLOBAL PROPERTY SEEN_COMPONENTS) - if(component IN_LIST seen_components) - return() # already added, or in process of adding, this component - endif() - set_property(GLOBAL APPEND PROPERTY SEEN_COMPONENTS ${component}) - - find_component_path("${component}" "${ALL_COMPONENTS}" "${ALL_COMPONENT_PATHS}" COMPONENT_PATH) - debug("Expanding dependencies of ${component} @ ${COMPONENT_PATH}") - if(NOT COMPONENT_PATH) - set_property(GLOBAL APPEND PROPERTY COMPONENTS_NOT_FOUND ${component}) - return() - endif() - - # include the component CMakeLists.txt to expand its properties - # into the global cache (via register_component(), above) - unset(COMPONENT_REQUIRES) - unset(COMPONENT_PRIV_REQUIRES) - set(COMPONENT ${component}) - include(${COMPONENT_PATH}/CMakeLists.txt) - - get_property(requires GLOBAL PROPERTY "${component}_REQUIRES") - get_property(requires_priv GLOBAL PROPERTY "${component}_PRIV_REQUIRES") - - # Recurse dependencies first, so that they appear first in the list (when possible) - foreach(req ${COMPONENT_REQUIRES_COMMON} ${requires} ${requires_priv}) - expand_component_requirements(${req}) - endforeach() - - list(FIND TEST_COMPONENTS ${component} idx) - - if(NOT idx EQUAL -1) - list(GET TEST_COMPONENTS ${idx} test_component) - list(GET TEST_COMPONENT_PATHS ${idx} test_component_path) - set_property(GLOBAL APPEND PROPERTY BUILD_TEST_COMPONENTS ${test_component}) - set_property(GLOBAL APPEND PROPERTY BUILD_TEST_COMPONENT_PATHS ${test_component_path}) - endif() - - # Now append this component to the full list (after its dependencies) - set_property(GLOBAL APPEND PROPERTY BUILD_COMPONENT_PATHS ${COMPONENT_PATH}) - set_property(GLOBAL APPEND PROPERTY BUILD_COMPONENTS ${component}) -endfunction() - -# filter_components_list: Filter the components included in the build -# as specified by the user. Or, in the case of unit testing, filter out -# the test components to be built. -macro(filter_components_list) - spaces2list(COMPONENTS) - spaces2list(EXCLUDE_COMPONENTS) - spaces2list(TEST_COMPONENTS) - spaces2list(TEST_EXCLUDE_COMPONENTS) - - list(LENGTH ALL_COMPONENTS all_components_length) - math(EXPR all_components_length "${all_components_length} - 1") - - foreach(component_idx RANGE 0 ${all_components_length}) - list(GET ALL_COMPONENTS ${component_idx} component) - list(GET ALL_COMPONENT_PATHS ${component_idx} component_path) - - if(COMPONENTS) - if(${component} IN_LIST COMPONENTS) - set(add_component 1) - else() - set(add_component 0) - endif() - else() - set(add_component 1) - - endif() - - if(NOT ${component} IN_LIST EXCLUDE_COMPONENTS AND add_component EQUAL 1) - list(APPEND components ${component}) - list(APPEND component_paths ${component_path}) - - if(TESTS_ALL EQUAL 1 OR TEST_COMPONENTS) - if(NOT TESTS_ALL EQUAL 1 AND TEST_COMPONENTS) - if(${component} IN_LIST TEST_COMPONENTS) - set(add_test_component 1) - else() - set(add_test_component 0) - endif() - else() - set(add_test_component 1) - endif() - - if(${component} IN_LIST ALL_TEST_COMPONENTS) - if(NOT ${component} IN_LIST TEST_EXCLUDE_COMPONENTS AND add_test_component EQUAL 1) - list(APPEND test_components ${component}_test) - list(APPEND test_component_paths ${component_path}/test) - - list(APPEND components ${component}_test) - list(APPEND component_paths ${component_path}/test) - endif() - endif() - endif() - endif() - endforeach() - - set(COMPONENTS ${components}) - - set(TEST_COMPONENTS ${test_components}) - set(TEST_COMPONENT_PATHS ${test_component_paths}) - - list(APPEND ALL_COMPONENTS "${TEST_COMPONENTS}") - list(APPEND ALL_COMPONENT_PATHS "${TEST_COMPONENT_PATHS}") -endmacro() - -# Main functionality goes here -# Find every available component in COMPONENT_DIRS, save as ALL_COMPONENT_PATHS and ALL_COMPONENTS -components_find_all("${COMPONENT_DIRS}" ALL_COMPONENT_PATHS ALL_COMPONENTS ALL_TEST_COMPONENTS) - -filter_components_list() - -debug("ALL_COMPONENT_PATHS ${ALL_COMPONENT_PATHS}") -debug("ALL_COMPONENTS ${ALL_COMPONENTS}") -debug("ALL_TEST_COMPONENTS ${ALL_TEST_COMPONENTS}") - -set_property(GLOBAL PROPERTY SEEN_COMPONENTS "") # anti-infinite-recursion -set_property(GLOBAL PROPERTY BUILD_COMPONENTS "") -set_property(GLOBAL PROPERTY BUILD_COMPONENT_PATHS "") -set_property(GLOBAL PROPERTY BUILD_TEST_COMPONENTS "") -set_property(GLOBAL PROPERTY BUILD_TEST_COMPONENT_PATHS "") -set_property(GLOBAL PROPERTY COMPONENTS_NOT_FOUND "") - -# Indicate that the component CMakeLists.txt is being included in the early expansion phase of the build, -# and might not want to execute particular operations. -set(CMAKE_BUILD_EARLY_EXPANSION 1) -foreach(component ${COMPONENTS}) - debug("Expanding initial component ${component}") - expand_component_requirements(${component}) -endforeach() -unset(CMAKE_BUILD_EARLY_EXPANSION) - -get_property(build_components GLOBAL PROPERTY BUILD_COMPONENTS) -get_property(build_component_paths GLOBAL PROPERTY BUILD_COMPONENT_PATHS) -get_property(build_test_components GLOBAL PROPERTY BUILD_TEST_COMPONENTS) -get_property(build_test_component_paths GLOBAL PROPERTY BUILD_TEST_COMPONENT_PATHS) -get_property(not_found GLOBAL PROPERTY COMPONENTS_NOT_FOUND) - -debug("components in build: ${build_components}") -debug("components in build: ${build_component_paths}") -debug("components not found: ${not_found}") - -function(line contents) - file(APPEND "${DEPENDENCIES_FILE}.tmp" "${contents}\n") -endfunction() - -file(WRITE "${DEPENDENCIES_FILE}.tmp" "# Component requirements generated by expand_requirements.cmake\n\n") -line("set(BUILD_COMPONENTS ${build_components})") -line("set(BUILD_COMPONENT_PATHS ${build_component_paths})") -line("set(BUILD_TEST_COMPONENTS ${build_test_components})") -line("set(BUILD_TEST_COMPONENT_PATHS ${build_test_component_paths})") -line("") - -line("# get_component_requirements: Generated function to read the dependencies of a given component.") -line("#") -line("# Parameters:") -line("# - component: Name of component") -line("# - var_requires: output variable name. Set to recursively expanded COMPONENT_REQUIRES ") -line("# for this component.") -line("# - var_private_requires: output variable name. Set to recursively expanded COMPONENT_PRIV_REQUIRES ") -line("# for this component.") -line("#") -line("# Throws a fatal error if 'componeont' is not found (indicates a build system problem).") -line("#") -line("function(get_component_requirements component var_requires var_private_requires)") -foreach(build_component ${build_components}) - get_property(reqs GLOBAL PROPERTY "${build_component}_REQUIRES") - get_property(private_reqs GLOBAL PROPERTY "${build_component}_PRIV_REQUIRES") - line(" if(\"\$\{component}\" STREQUAL \"${build_component}\")") - line(" set(\${var_requires} \"${reqs}\" PARENT_SCOPE)") - line(" set(\${var_private_requires} \"${private_reqs}\" PARENT_SCOPE)") - line(" return()") - line(" endif()") -endforeach() - -line(" message(FATAL_ERROR \"Component not found: \${component}\")") -line("endfunction()") - -# only replace DEPENDENCIES_FILE if it has changed (prevents ninja/make build loops.) -execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${DEPENDENCIES_FILE}.tmp" "${DEPENDENCIES_FILE}") -execute_process(COMMAND ${CMAKE_COMMAND} -E remove "${DEPENDENCIES_FILE}.tmp") diff --git a/tools/cmake/targets.cmake b/tools/cmake/targets.cmake new file mode 100644 index 00000000..57d3b764 --- /dev/null +++ b/tools/cmake/targets.cmake @@ -0,0 +1,69 @@ +# +# Set the target used for the standard project build. +# +macro(__target_init) + # Input is IDF_TARGET environement variable + set(env_idf_target $ENV{IDF_TARGET}) + + if(NOT env_idf_target) + # IDF_TARGET not set in environment, see if it is set in cache + if(IDF_TARGET) + set(env_idf_target ${IDF_TARGET}) + else() + set(env_idf_target esp32) + message(STATUS "IDF_TARGET not set, using default target: ${env_idf_target}") + endif() + else() + # IDF_TARGET set both in environment and in cache, must be the same + if(NOT ${IDF_TARGET} STREQUAL ${env_idf_target}) + message(FATAL_ERROR "IDF_TARGET in CMake cache does not match " + "IDF_TARGET environment variable. To change the target, clear " + "the build directory and sdkconfig file, and build the project again") + endif() + endif() + + # IDF_TARGET will be used by Kconfig, make sure it is set + set(ENV{IDF_TARGET} ${env_idf_target}) + + # Finally, set IDF_TARGET in cache + set(IDF_TARGET ${env_idf_target} CACHE STRING "IDF Build Target") +endmacro() + +# +# Check that the set build target and the config target matches. +# +function(__target_check) + # Should be called after sdkconfig CMake file has been included. + idf_build_get_property(idf_target IDF_TARGET) + if(NOT ${idf_target} STREQUAL ${CONFIG_IDF_TARGET}) + message(FATAL_ERROR "CONFIG_IDF_TARGET in sdkconfig does not match " + "IDF_TARGET environement variable. To change the target, delete " + "sdkconfig file and build the project again.") + endif() +endfunction() + +# +# Used by the project CMake file to set the toolchain before project() call. +# +macro(__target_set_toolchain) + idf_build_get_property(idf_path IDF_PATH) + # First try to load the toolchain file from the tools/cmake/directory of IDF + set(toolchain_file_global ${idf_path}/tools/cmake/toolchain-${IDF_TARGET}.cmake) + if(EXISTS ${toolchain_file_global}) + set(CMAKE_TOOLCHAIN_FILE ${toolchain_file_global}) + else() + __component_get_target(component_target ${IDF_TARGET}) + if(NOT component_target) + message(FATAL_ERROR "Unable to resolve '${IDF_TARGET}' for setting toolchain file.") + endif() + get_property(component_dir TARGET ${component_target} PROPERTY COMPONENT_DIR) + # Try to load the toolchain file from the directory of IDF_TARGET component + set(toolchain_file_component ${component_dir}/toolchain-${IDF_TARGET}.cmake) + if(EXISTS ${toolchain_file_component}) + set(CMAKE_TOOLCHAIN_FILE ${toolchain_file_component}) + else() + message(FATAL_ERROR "Toolchain file toolchain-${IDF_TARGET}.cmake not found," + "checked ${toolchain_file_global} and ${toolchain_file_component}") + endif() + endif() +endmacro() \ No newline at end of file diff --git a/tools/cmake/third_party/GetGitRevisionDescription.cmake b/tools/cmake/third_party/GetGitRevisionDescription.cmake index 8bac5008..1de63b36 100644 --- a/tools/cmake/third_party/GetGitRevisionDescription.cmake +++ b/tools/cmake/third_party/GetGitRevisionDescription.cmake @@ -111,8 +111,10 @@ function(git_describe _var _repo_dir) "${GIT_EXECUTABLE}" "-C" ${_repo_dir} - describe --tag - ${hash} + describe + "--always" + "--tags" + "--dirty" ${ARGN} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" diff --git a/tools/cmake/toolchain-esp32.cmake b/tools/cmake/toolchain-esp32.cmake new file mode 100644 index 00000000..b63ae5a8 --- /dev/null +++ b/tools/cmake/toolchain-esp32.cmake @@ -0,0 +1,11 @@ +set(CMAKE_SYSTEM_NAME Generic) + +set(CMAKE_C_COMPILER xtensa-esp32-elf-gcc) +set(CMAKE_CXX_COMPILER xtensa-esp32-elf-g++) +set(CMAKE_ASM_COMPILER xtensa-esp32-elf-gcc) + +set(CMAKE_C_FLAGS "-mlongcalls -Wno-frame-address" CACHE STRING "C Compiler Base Flags") +set(CMAKE_CXX_FLAGS "-mlongcalls -Wno-frame-address" CACHE STRING "C++ Compiler Base Flags") + +# Can be removed after gcc 5.2.0 support is removed (ref GCC_NOT_5_2_0) +set(CMAKE_EXE_LINKER_FLAGS "-nostdlib" CACHE STRING "Linker Base Flags") diff --git a/tools/cmake/toolchain-esp8266.cmake b/tools/cmake/toolchain-esp8266.cmake deleted file mode 100644 index 4f11253c..00000000 --- a/tools/cmake/toolchain-esp8266.cmake +++ /dev/null @@ -1,7 +0,0 @@ -set(CMAKE_SYSTEM_NAME Generic) - -set(CMAKE_C_COMPILER xtensa-lx106-elf-gcc) -set(CMAKE_CXX_COMPILER xtensa-lx106-elf-g++) -set(CMAKE_ASM_COMPILER xtensa-lx106-elf-gcc) - -set(CMAKE_EXE_LINKER_FLAGS "-nostdlib" CACHE STRING "Linker Base Flags") diff --git a/tools/cmake/utilities.cmake b/tools/cmake/utilities.cmake index fcbd6f85..75806130 100644 --- a/tools/cmake/utilities.cmake +++ b/tools/cmake/utilities.cmake @@ -77,21 +77,24 @@ endfunction() # by converting it to a generated source file which is then compiled # to a binary object as part of the build function(target_add_binary_data target embed_file embed_type) + idf_build_get_property(build_dir BUILD_DIR) + idf_build_get_property(idf_path IDF_PATH) get_filename_component(embed_file "${embed_file}" ABSOLUTE) get_filename_component(name "${embed_file}" NAME) - set(embed_srcfile "${CMAKE_BINARY_DIR}/${name}.S") + set(embed_srcfile "${build_dir}/${name}.S") add_custom_command(OUTPUT "${embed_srcfile}" COMMAND "${CMAKE_COMMAND}" -D "DATA_FILE=${embed_file}" -D "SOURCE_FILE=${embed_srcfile}" -D "FILE_TYPE=${embed_type}" - -P "${IDF_PATH}/tools/cmake/scripts/data_file_embed_asm.cmake" + -P "${idf_path}/tools/cmake/scripts/data_file_embed_asm.cmake" MAIN_DEPENDENCY "${embed_file}" - DEPENDENCIES "${IDF_PATH}/tools/cmake/scripts/data_file_embed_asm.cmake" - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}") + DEPENDS "${idf_path}/tools/cmake/scripts/data_file_embed_asm.cmake" + WORKING_DIRECTORY "${build_dir}" + VERBATIM) set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${embed_srcfile}") @@ -127,25 +130,42 @@ endfunction() # Automatically adds a -L search path for the containing directory (if found), # and then adds -T with the filename only. This allows INCLUDE directives to be # used to include other linker scripts in the same directory. -function(target_linker_script target) - foreach(scriptfile ${ARGN}) +function(target_linker_script target deptype scriptfiles) + cmake_parse_arguments(_ "" "PROCESS" "" ${ARGN}) + foreach(scriptfile ${scriptfiles}) get_filename_component(abs_script "${scriptfile}" ABSOLUTE) message(STATUS "Adding linker script ${abs_script}") + if(__PROCESS) + get_filename_component(output "${__PROCESS}" ABSOLUTE) + __ldgen_process_template(${abs_script} ${output}) + set(abs_script ${output}) + endif() + get_filename_component(search_dir "${abs_script}" DIRECTORY) get_filename_component(scriptname "${abs_script}" NAME) - get_target_property(link_libraries "${target}" LINK_LIBRARIES) - list(FIND "${link_libraries}" "-L ${search_dir}" found_search_dir) - if(found_search_dir EQUAL "-1") # not already added as a search path - target_link_libraries("${target}" "-L ${search_dir}") + if(deptype STREQUAL INTERFACE OR deptype STREQUAL PUBLIC) + get_target_property(link_libraries "${target}" INTERFACE_LINK_LIBRARIES) + else() + get_target_property(link_libraries "${target}" LINK_LIBRARIES) endif() - target_link_libraries("${target}" "-T ${scriptname}") + list(FIND "${link_libraries}" "-L ${search_dir}" found_search_dir) + if(found_search_dir EQUAL "-1") # not already added as a search path + target_link_libraries("${target}" "${deptype}" "-L ${search_dir}") + endif() + + target_link_libraries("${target}" "${deptype}" "-T ${scriptname}") # Note: In ESP-IDF, most targets are libraries and libary LINK_DEPENDS don't propagate to - # executable(s) the library is linked to. This is done manually in components.cmake. - set_property(TARGET "${target}" APPEND PROPERTY LINK_DEPENDS "${abs_script}") + # executable(s) the library is linked to. Attach manually to executable once it is known. + # + # Property INTERFACE_LINK_DEPENDS is available in CMake 3.13 which should propagate link + # dependencies. + if(NOT __PROCESS) + idf_build_set_property(__LINK_DEPENDS ${abs_script} APPEND) + endif() endforeach() endfunction() @@ -164,4 +184,70 @@ function(add_prefix var prefix) list(APPEND newlist "${prefix}${elm}") endforeach() set(${var} "${newlist}" PARENT_SCOPE) -endfunction() \ No newline at end of file +endfunction() + +# fail_at_build_time +# +# Creates a phony target which fails the build and touches CMakeCache.txt to cause a cmake run next time. +# +# This is used when a missing file is required at CMake runtime, but we can't fail the build if it is not found, +# because the "menuconfig" target may be required to fix the problem. +# +# We cannot use CMAKE_CONFIGURE_DEPENDS instead because it only works for files which exist at CMake runtime. +# +function(fail_at_build_time target_name message_line0) + idf_build_get_property(idf_path IDF_PATH) + set(message_lines COMMAND ${CMAKE_COMMAND} -E echo "${message_line0}") + foreach(message_line ${ARGN}) + set(message_lines ${message_lines} COMMAND ${CMAKE_COMMAND} -E echo "${message_line}") + endforeach() + # Generate a timestamp file that gets included. When deleted on build, this forces CMake + # to rerun. + string(RANDOM filename) + set(filename "${CMAKE_CURRENT_BINARY_DIR}/${filename}.cmake") + file(WRITE "${filename}" "") + include("${filename}") + add_custom_target(${target_name} ALL + ${message_lines} + COMMAND ${CMAKE_COMMAND} -E remove "${filename}" + COMMAND ${CMAKE_COMMAND} -P ${idf_path}/tools/cmake/scripts/fail.cmake + VERBATIM) +endfunction() + +function(check_exclusive_args args prefix) + set(_args ${args}) + spaces2list(_args) + set(only_arg 0) + foreach(arg ${_args}) + if(${prefix}_${arg} AND only_arg) + message(FATAL_ERROR "${args} are exclusive arguments") + endif() + + if(${prefix}_${arg}) + set(only_arg 1) + endif() + endforeach() +endfunction() + + +# add_compile_options variant for C++ code only +# +# This adds global options, set target properties for +# component-specific flags +function(add_cxx_compile_options) + foreach(option ${ARGV}) + # note: the Visual Studio Generator doesn't support this... + add_compile_options($<$:${option}>) + endforeach() +endfunction() + +# add_compile_options variant for C code only +# +# This adds global options, set target properties for +# component-specific flags +function(add_c_compile_options) + foreach(option ${ARGV}) + # note: the Visual Studio Generator doesn't support this... + add_compile_options($<$:${option}>) + endforeach() +endfunction() diff --git a/tools/cmake/version.cmake b/tools/cmake/version.cmake new file mode 100644 index 00000000..265ab33b --- /dev/null +++ b/tools/cmake/version.cmake @@ -0,0 +1,3 @@ +set(IDF_VERSION_MAJOR 4) +set(IDF_VERSION_MINOR 0) +set(IDF_VERSION_PATCH 0) diff --git a/tools/elf_to_ld.sh b/tools/elf_to_ld.sh new file mode 100755 index 00000000..138f24d6 --- /dev/null +++ b/tools/elf_to_ld.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo '/*' +echo 'ESP32 ROM address table' +echo 'Generated for ROM with MD5sum:' +md5sum $1 +echo '*/' +xtensa-esp108-elf-nm $1 | grep '[0-9a-f] [TBRD]' | while read adr ttp nm; do + if ! echo "$nm" | grep -q -e '^_bss' -e '_heap'; then + echo "PROVIDE ( $nm = 0x$adr );"; + fi +done \ No newline at end of file diff --git a/tools/esp_prov/README.md b/tools/esp_prov/README.md index 98f54f77..7f8a443b 100644 --- a/tools/esp_prov/README.md +++ b/tools/esp_prov/README.md @@ -1,18 +1,16 @@ # ESP Provisioning Tool # NAME -`esp_prov` - A python based utility for testing the provisioning examples over a host using Httpd on a soft AP interface. +`esp_prov` - A python based utility for testing the provisioning examples over a host # SYNOPSIS ``` -python esp_prov.py --ssid < AP SSID > --passphrase < AP Password > --sec_ver < Security version 0 / 1 > [ Optional parameters... ] +python esp_prov.py --transport < mode of provisioning : softap \ ble \ console > [ Optional parameters... ] ``` # DESCRIPTION -For SoftAP + HTTPD based provisioning. This assumes that the device is running in Wi-Fi SoftAP mode and hosts an HTTP server supporting specific endpoint URIs. Also client needs to connect to the device softAP network before running `esp_prov`. - Usage of `esp-prov` assumes that the provisioning app has specific protocomm endpoints active. These endpoints are active in the provisioning examples and accept specific protobuf data structures: | Endpoint Name | URI (HTTP server on ip:port) | Description | @@ -20,8 +18,10 @@ Usage of `esp-prov` assumes that the provisioning app has specific protocomm end | prov-session | http://ip:port/prov-session | Security endpoint used for session establishment | | prov-config | http://ip:port/prov-config | Endpoint used for configuring Wi-Fi credentials on device | | proto-ver | http://ip:port/proto-ver | Version endpoint for checking protocol compatibility | +| prov-scan | http://ip:port/prov-scan | Endpoint used for scanning Wi-Fi APs | | custom-config | http://ip:port/custom-config | Optional endpoint for configuring custom credentials | + # PARAMETERS * `--help` @@ -30,11 +30,20 @@ Usage of `esp-prov` assumes that the provisioning app has specific protocomm end * `--verbose`, `-v` Sets the verbosity level of output log -* `--ssid ` - For specifying the SSID of the Wi-Fi AP to which the device is to connect after provisioning +* `--transport ` + Three options are available: + * `softap` + For SoftAP + HTTPD based provisioning. This assumes that the device is running in Wi-Fi SoftAP mode and hosts an HTTP server supporting specific endpoint URIs. Also client needs to connect to the device softAP network before running `esp_prov` + * `ble` + For BLE based provisioning (Linux support only. In Windows/macOS it redirects to console). This assumes that the provisioning endpoints are active on the device with specific BLE service UUIDs + * `console` + For debugging via console based provisioning. The client->device commands are printed to STDOUT and device->client messages are accepted via STDIN. This is to be used when device is accepting provisioning commands on UART console. -* `--passphrase ` - For specifying the password of the Wi-Fi AP to which the device is to connect after provisioning +* `--ssid ` (Optional) + For specifying the SSID of the Wi-Fi AP to which the device is to connect after provisioning. If not provided, scanning is initiated and scan results, as seen by the device, are displayed, of which an SSID can be picked and the corresponding password specified. + +* `--passphrase ` (Optional) + For specifying the password of the Wi-Fi AP to which the device is to connect after provisioning. Only used when corresponding SSID is provided using `--ssid` * `--sec_ver ` For specifying version of protocomm endpoint security to use. For now two versions are supported: @@ -44,11 +53,9 @@ Usage of `esp-prov` assumes that the provisioning app has specific protocomm end * `--pop ` (Optional) For specifying optional Proof of Possession string to use for protocomm endpoint security version 1. This option is ignored when security version 0 is in use -* `--proto_ver ` (Optional) (Default `V0.1`) - For specifying version string for checking compatibility with provisioning app prior to starting provisioning process - -* `--softap_endpoint ` (Optional) (Default `192.168.4.1:80`) - For specifying the IP and port of the HTTP server on which provisioning app is running. The client must connect to the device SoftAP prior to running `esp_prov` +* `--service_name (Optional) + When transport mode is ble, this specifies the BLE device name to which connection is to be established for provisioned. + When transport mode is softap, this specifies the HTTP server hostname / IP which is running the provisioning service, on the SoftAP network of the device which is to be provisioned. This defaults to `192.168.4.1:80` if not specified * `--custom_config` (Optional) This flag assumes the provisioning app has an endpoint called `custom-config`. Use `--custom_info` and `--custom_ver` options to specify the fields accepted by this endpoint @@ -61,10 +68,12 @@ Usage of `esp-prov` assumes that the provisioning app has specific protocomm end # AVAILABILITY -`esp_prov` is intended as a cross-platform tool. +`esp_prov` is intended as a cross-platform tool, but currently BLE communication functionality is only available on Linux (via BlueZ and DBus) For android, a provisioning tool along with source code is available [here](https://github.com/espressif/esp-idf-provisioning-android) +On macOS and Windows, running with `--transport ble` option falls back to console mode, ie. write data and target UUID are printed to STDOUT and read data is input through STDIN. Users are free to use their app of choice to connect to the BLE device, send the write data to the target characteristic and read from it. + ## Dependencies This requires the following python libraries to run (included in requirements.txt): @@ -75,13 +84,23 @@ This requires the following python libraries to run (included in requirements.tx Run `pip install -r $IDF_PATH/tools/esp_prov/requirements.txt` Note : -* The packages listed in requirements.txt are limited only to the ones needed AFTER fully satisfying the requirements of ESP8266_RTOS_SDK +* The packages listed in requirements.txt are limited only to the ones needed AFTER fully satisfying the requirements of ESP-IDF +* BLE communication is only supported on Linux (via Bluez and DBus), therefore, the dependencies for this have been made optional + +## Optional Dependencies (Linux Only) + +These dependencies are for enabling communication with BLE devices using Bluez and DBus on Linux: +* `dbus-python` + +Run `pip install -r $IDF_PATH/tools/esp_prov/requirements_linux_extra.txt` # EXAMPLE USAGE Please refer to the README.md files with the examples present under `$IDF_PATH/examples/provisioning/`, namely: +* `ble_prov` * `softap_prov` * `custom_config` +* `console_prov` Each of these examples use specific options of the `esp_prov` tool and give an overview to simple as well as advanced usage scenarios. diff --git a/tools/esp_prov/esp_prov.py b/tools/esp_prov/esp_prov.py index 44306153..52fb394f 100644 --- a/tools/esp_prov/esp_prov.py +++ b/tools/esp_prov/esp_prov.py @@ -16,18 +16,39 @@ # from __future__ import print_function +from builtins import input import argparse +import textwrap import time import os import sys +import json +from getpass import getpass -idf_path = os.environ['IDF_PATH'] -sys.path.insert(0, idf_path + "/components/protocomm/python") -sys.path.insert(1, idf_path + "/tools/esp_prov") +try: + import security + import transport + import prov + +except ImportError: + idf_path = os.environ['IDF_PATH'] + sys.path.insert(0, idf_path + "/components/protocomm/python") + sys.path.insert(1, idf_path + "/tools/esp_prov") + + import security + import transport + import prov + +# Set this to true to allow exceptions to be thrown +config_throw_except = False + + +def on_except(err): + if config_throw_except: + raise RuntimeError(err) + else: + print(err) -import security -import transport -import prov def get_security(secver, pop=None, verbose=False): if secver == 1: @@ -36,146 +57,394 @@ def get_security(secver, pop=None, verbose=False): return security.Security0(verbose) return None -def get_transport(sel_transport, softap_endpoint=None): + +def get_transport(sel_transport, service_name): try: tp = None if (sel_transport == 'softap'): - tp = transport.Transport_Softap(softap_endpoint) + if service_name is None: + service_name = '192.168.4.1:80' + tp = transport.Transport_HTTP(service_name) + elif (sel_transport == 'ble'): + if service_name is None: + raise RuntimeError('"--service_name" must be specified for ble transport') + # BLE client is now capable of automatically figuring out + # the primary service from the advertisement data and the + # characteristics corresponding to each endpoint. + # Below, the service_uuid field and 16bit UUIDs in the nu_lookup + # table are provided only to support devices running older firmware, + # in which case, the automated discovery will fail and the client + # will fallback to using the provided UUIDs instead + nu_lookup = {'prov-session': 'ff51', 'prov-config': 'ff52', 'proto-ver': 'ff53'} + tp = transport.Transport_BLE(devname=service_name, + service_uuid='0000ffff-0000-1000-8000-00805f9b34fb', + nu_lookup=nu_lookup) + elif (sel_transport == 'console'): + tp = transport.Transport_Console() return tp except RuntimeError as e: - print(e) + on_except(e) return None -def version_match(tp, protover): + +def version_match(tp, protover, verbose=False): try: response = tp.send_data('proto-ver', protover) - if response != protover: + + if verbose: + print("proto-ver response : ", response) + + # First assume this to be a simple version string + if response.lower() == protover.lower(): + return True + + try: + # Else interpret this as JSON structure containing + # information with versions and capabilities of both + # provisioning service and application + info = json.loads(response) + if info['prov']['ver'].lower() == protover.lower(): + return True + + except ValueError: + # If decoding as JSON fails, it means that capabilities + # are not supported return False - return True - except RuntimeError as e: - print(e) + + except Exception as e: + on_except(e) return None + +def has_capability(tp, capability='none', verbose=False): + # Note : default value of `capability` argument cannot be empty string + # because protocomm_httpd expects non zero content lengths + try: + response = tp.send_data('proto-ver', capability) + + if verbose: + print("proto-ver response : ", response) + + try: + # Interpret this as JSON structure containing + # information with versions and capabilities of both + # provisioning service and application + info = json.loads(response) + supported_capabilities = info['prov']['cap'] + if capability.lower() == 'none': + # No specific capability to check, but capabilities + # feature is present so return True + return True + elif capability in supported_capabilities: + return True + return False + + except ValueError: + # If decoding as JSON fails, it means that capabilities + # are not supported + return False + + except RuntimeError as e: + on_except(e) + + return False + + +def get_version(tp): + response = None + try: + response = tp.send_data('proto-ver', '---') + except RuntimeError as e: + on_except(e) + response = '' + return response + + def establish_session(tp, sec): try: response = None while True: request = sec.security_session(response) - if request == None: + if request is None: break response = tp.send_data('prov-session', request) - if (response == None): + if (response is None): return False return True except RuntimeError as e: - print(e) + on_except(e) return None + def custom_config(tp, sec, custom_info, custom_ver): try: message = prov.custom_config_request(sec, custom_info, custom_ver) response = tp.send_data('custom-config', message) return (prov.custom_config_response(sec, response) == 0) except RuntimeError as e: - print(e) + on_except(e) return None + +def scan_wifi_APs(sel_transport, tp, sec): + APs = [] + group_channels = 0 + readlen = 100 + if sel_transport == 'softap': + # In case of softAP we must perform the scan on individual channels, one by one, + # so that the Wi-Fi controller gets ample time to send out beacons (necessary to + # maintain connectivity with authenticated stations. As scanning one channel at a + # time will be slow, we can group more than one channels to be scanned in quick + # succession, hence speeding up the scan process. Though if too many channels are + # present in a group, the controller may again miss out on sending beacons. Hence, + # the application must should use an optimum value. The following value usually + # works out in most cases + group_channels = 5 + elif sel_transport == 'ble': + # Read at most 4 entries at a time. This is because if we are using BLE transport + # then the response packet size should not exceed the present limit of 256 bytes of + # characteristic value imposed by protocomm_ble. This limit may be removed in the + # future + readlen = 4 + try: + message = prov.scan_start_request(sec, blocking=True, group_channels=group_channels) + start_time = time.time() + response = tp.send_data('prov-scan', message) + stop_time = time.time() + print("++++ Scan process executed in " + str(stop_time - start_time) + " sec") + prov.scan_start_response(sec, response) + + message = prov.scan_status_request(sec) + response = tp.send_data('prov-scan', message) + result = prov.scan_status_response(sec, response) + print("++++ Scan results : " + str(result["count"])) + if result["count"] != 0: + index = 0 + remaining = result["count"] + while remaining: + count = [remaining, readlen][remaining > readlen] + message = prov.scan_result_request(sec, index, count) + response = tp.send_data('prov-scan', message) + APs += prov.scan_result_response(sec, response) + remaining -= count + index += count + + except RuntimeError as e: + on_except(e) + return None + + return APs + + def send_wifi_config(tp, sec, ssid, passphrase): try: message = prov.config_set_config_request(sec, ssid, passphrase) response = tp.send_data('prov-config', message) return (prov.config_set_config_response(sec, response) == 0) except RuntimeError as e: - print(e) + on_except(e) return None + def apply_wifi_config(tp, sec): try: message = prov.config_apply_config_request(sec) response = tp.send_data('prov-config', message) - return (prov.config_set_config_response(sec, response) == 0) + return (prov.config_apply_config_response(sec, response) == 0) except RuntimeError as e: - print(e) + on_except(e) return None + def get_wifi_config(tp, sec): try: message = prov.config_get_status_request(sec) response = tp.send_data('prov-config', message) return prov.config_get_status_response(sec, response) except RuntimeError as e: - print(e) + on_except(e) return None + +def desc_format(*args): + desc = '' + for arg in args: + desc += textwrap.fill(replace_whitespace=False, text=arg) + "\n" + return desc + + if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Generate ESP prov payload") + parser = argparse.ArgumentParser(description=desc_format( + 'ESP Provisioning tool for configuring devices ' + 'running protocomm based provisioning service.', + 'See esp-idf/examples/provisioning for sample applications'), + formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("--ssid", dest = 'ssid', type = str, - help = "SSID of Wi-Fi Network", required = True) - parser.add_argument("--passphrase", dest = 'passphrase', type = str, - help = "Passphrase of Wi-Fi network", default = '') + parser.add_argument("--transport", required=True, dest='mode', type=str, + help=desc_format( + 'Mode of transport over which provisioning is to be performed.', + 'This should be one of "softap", "ble" or "console"')) - parser.add_argument("--sec_ver", dest = 'secver', type = int, - help = "Security scheme version", default = 1) - parser.add_argument("--proto_ver", dest = 'protover', type = str, - help = "Protocol version", default = 'V0.1') - parser.add_argument("--pop", dest = 'pop', type = str, - help = "Proof of possession", default = '') + parser.add_argument("--service_name", dest='name', type=str, + help=desc_format( + 'This specifies the name of the provisioning service to connect to, ' + 'depending upon the mode of transport :', + '\t- transport "ble" : The BLE Device Name', + '\t- transport "softap" : HTTP Server hostname or IP', + '\t (default "192.168.4.1:80")')) - parser.add_argument("--softap_endpoint", dest = 'softap_endpoint', type = str, - help = ", http(s):// shouldn't be included", default = '192.168.4.1:80') - + parser.add_argument("--proto_ver", dest='version', type=str, default='', + help=desc_format( + 'This checks the protocol version of the provisioning service running ' + 'on the device before initiating Wi-Fi configuration')) - parser.add_argument("--custom_config", help="Provision Custom Configuration", - action = "store_true") - parser.add_argument("--custom_info", dest = 'custom_info', type = str, - help = "Custom Config Info String", default = '') - parser.add_argument("--custom_ver", dest = 'custom_ver', type = int, - help = "Custom Config Version Number", default = 2) + parser.add_argument("--sec_ver", dest='secver', type=int, default=None, + help=desc_format( + 'Protocomm security scheme used by the provisioning service for secure ' + 'session establishment. Accepted values are :', + '\t- 0 : No security', + '\t- 1 : X25519 key exchange + AES-CTR encryption', + '\t + Authentication using Proof of Possession (PoP)', + 'In case device side application uses IDF\'s provisioning manager, ' + 'the compatible security version is automatically determined from ' + 'capabilities retrieved via the version endpoint')) + + parser.add_argument("--pop", dest='pop', type=str, default='', + help=desc_format( + 'This specifies the Proof of possession (PoP) when security scheme 1 ' + 'is used')) + + parser.add_argument("--ssid", dest='ssid', type=str, default='', + help=desc_format( + 'This configures the device to use SSID of the Wi-Fi network to which ' + 'we would like it to connect to permanently, once provisioning is complete. ' + 'If Wi-Fi scanning is supported by the provisioning service, this need not ' + 'be specified')) + + parser.add_argument("--passphrase", dest='passphrase', type=str, default='', + help=desc_format( + 'This configures the device to use Passphrase for the Wi-Fi network to which ' + 'we would like it to connect to permanently, once provisioning is complete. ' + 'If Wi-Fi scanning is supported by the provisioning service, this need not ' + 'be specified')) + + parser.add_argument("--custom_config", action="store_true", + help=desc_format( + 'This is an optional parameter, only intended for use with ' + '"examples/provisioning/custom_config"')) + parser.add_argument("--custom_info", dest='custom_info', type=str, default='', + help=desc_format( + 'Custom Config Info String. "--custom_config" must be specified for using this')) + parser.add_argument("--custom_ver", dest='custom_ver', type=int, default=2, + help=desc_format( + 'Custom Config Version Number. "--custom_config" must be specified for using this')) + + parser.add_argument("-v","--verbose", help="Increase output verbosity", action="store_true") - parser.add_argument("-v","--verbose", help = "increase output verbosity", - action = "store_true") args = parser.parse_args() - print("==== Esp_Prov Version: " + args.protover + " ====") - - security = get_security(args.secver, args.pop, args.verbose) - if security == None: - print("---- Invalid Security Version ----") + obj_transport = get_transport(args.mode.lower(), args.name) + if obj_transport is None: + print("---- Failed to establish connection ----") exit(1) - transport = get_transport('softap', args.softap_endpoint) - if transport == None: - print("---- Invalid provisioning mode ----") + # If security version not specified check in capabilities + if args.secver is None: + # First check if capabilities are supported or not + if not has_capability(obj_transport): + print('Security capabilities could not be determined. Please specify "--sec_ver" explicitly') + print("---- Invalid Security Version ----") + exit(2) + + # When no_sec is present, use security 0, else security 1 + args.secver = int(not has_capability(obj_transport, 'no_sec')) + print("Security scheme determined to be :", args.secver) + + if (args.secver != 0) and not has_capability(obj_transport, 'no_pop'): + if len(args.pop) == 0: + print("---- Proof of Possession argument not provided ----") + exit(2) + elif len(args.pop) != 0: + print("---- Proof of Possession will be ignored ----") + args.pop = '' + + obj_security = get_security(args.secver, args.pop, args.verbose) + if obj_security is None: + print("---- Invalid Security Version ----") exit(2) - print("\n==== Verifying protocol version ====") - if not version_match(transport, args.protover): - print("---- Error in protocol version matching ----") - exit(3) - print("==== Verified protocol version successfully ====") + if args.version != '': + print("\n==== Verifying protocol version ====") + if not version_match(obj_transport, args.version, args.verbose): + print("---- Error in protocol version matching ----") + exit(3) + print("==== Verified protocol version successfully ====") print("\n==== Starting Session ====") - if not establish_session(transport, security): + if not establish_session(obj_transport, obj_security): + print("Failed to establish session. Ensure that security scheme and proof of possession are correct") print("---- Error in establishing session ----") exit(4) print("==== Session Established ====") if args.custom_config: print("\n==== Sending Custom config to esp32 ====") - if not custom_config(transport, security, args.custom_info, args.custom_ver): + if not custom_config(obj_transport, obj_security, args.custom_info, args.custom_ver): print("---- Error in custom config ----") exit(5) print("==== Custom config sent successfully ====") + if args.ssid == '': + if not has_capability(obj_transport, 'wifi_scan'): + print("---- Wi-Fi Scan List is not supported by provisioning service ----") + print("---- Rerun esp_prov with SSID and Passphrase as argument ----") + exit(3) + + while True: + print("\n==== Scanning Wi-Fi APs ====") + start_time = time.time() + APs = scan_wifi_APs(args.mode.lower(), obj_transport, obj_security) + end_time = time.time() + print("\n++++ Scan finished in " + str(end_time - start_time) + " sec") + if APs is None: + print("---- Error in scanning Wi-Fi APs ----") + exit(8) + + if len(APs) == 0: + print("No APs found!") + exit(9) + + print("==== Wi-Fi Scan results ====") + print("{0: >4} {1: <33} {2: <12} {3: >4} {4: <4} {5: <16}".format( + "S.N.", "SSID", "BSSID", "CHN", "RSSI", "AUTH")) + for i in range(len(APs)): + print("[{0: >2}] {1: <33} {2: <12} {3: >4} {4: <4} {5: <16}".format( + i + 1, APs[i]["ssid"], APs[i]["bssid"], APs[i]["channel"], APs[i]["rssi"], APs[i]["auth"])) + + while True: + try: + select = int(input("Select AP by number (0 to rescan) : ")) + if select < 0 or select > len(APs): + raise ValueError + break + except ValueError: + print("Invalid input! Retry") + + if select != 0: + break + + args.ssid = APs[select - 1]["ssid"] + prompt_str = "Enter passphrase for {0} : ".format(args.ssid) + args.passphrase = getpass(prompt_str) + print("\n==== Sending Wi-Fi credential to esp32 ====") - if not send_wifi_config(transport, security, args.ssid, args.passphrase): + if not send_wifi_config(obj_transport, obj_security, args.ssid, args.passphrase): print("---- Error in send Wi-Fi config ----") exit(6) print("==== Wi-Fi Credentials sent successfully ====") print("\n==== Applying config to esp32 ====") - if not apply_wifi_config(transport, security): + if not apply_wifi_config(obj_transport, obj_security): print("---- Error in apply Wi-Fi config ----") exit(7) print("==== Apply config sent successfully ====") @@ -183,7 +452,7 @@ if __name__ == '__main__': while True: time.sleep(5) print("\n==== Wi-Fi connection state ====") - ret = get_wifi_config(transport, security) + ret = get_wifi_config(obj_transport, obj_security) if (ret == 1): continue elif (ret == 0): diff --git a/tools/esp_prov/proto/__init__.py b/tools/esp_prov/proto/__init__.py index 52141d0d..82726cd9 100644 --- a/tools/esp_prov/proto/__init__.py +++ b/tools/esp_prov/proto/__init__.py @@ -13,20 +13,32 @@ # limitations under the License. # -import imp import os + +def _load_source(name, path): + try: + from importlib.machinery import SourceFileLoader + return SourceFileLoader(name, path).load_module() + except ImportError: + # importlib.machinery doesn't exists in Python 2 so we will use imp (deprecated in Python 3) + import imp + return imp.load_source(name, path) + + idf_path = os.environ['IDF_PATH'] # protocomm component related python files generated from .proto files -constants_pb2 = imp.load_source("constants_pb2", idf_path + "/components/protocomm/python/constants_pb2.py") -sec0_pb2 = imp.load_source("sec0_pb2", idf_path + "/components/protocomm/python/sec0_pb2.py") -sec1_pb2 = imp.load_source("sec1_pb2", idf_path + "/components/protocomm/python/sec1_pb2.py") -session_pb2 = imp.load_source("session_pb2", idf_path + "/components/protocomm/python/session_pb2.py") +constants_pb2 = _load_source("constants_pb2", idf_path + "/components/protocomm/python/constants_pb2.py") +sec0_pb2 = _load_source("sec0_pb2", idf_path + "/components/protocomm/python/sec0_pb2.py") +sec1_pb2 = _load_source("sec1_pb2", idf_path + "/components/protocomm/python/sec1_pb2.py") +session_pb2 = _load_source("session_pb2", idf_path + "/components/protocomm/python/session_pb2.py") # wifi_provisioning component related python files generated from .proto files -wifi_constants_pb2 = imp.load_source("wifi_constants_pb2", idf_path + "/components/wifi_provisioning/python/wifi_constants_pb2.py") -wifi_config_pb2 = imp.load_source("wifi_config_pb2", idf_path + "/components/wifi_provisioning/python/wifi_config_pb2.py") +wifi_constants_pb2 = _load_source("wifi_constants_pb2", idf_path + "/components/wifi_provisioning/python/wifi_constants_pb2.py") +wifi_config_pb2 = _load_source("wifi_config_pb2", idf_path + "/components/wifi_provisioning/python/wifi_config_pb2.py") +wifi_scan_pb2 = _load_source("wifi_scan_pb2", idf_path + "/components/wifi_provisioning/python/wifi_scan_pb2.py") # custom_provisioning component related python files generated from .proto files -custom_config_pb2 = imp.load_source("custom_config_pb2", idf_path + "/examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py") +custom_config_pb2 = _load_source("custom_config_pb2", idf_path + + "/examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py") diff --git a/tools/esp_prov/prov/__init__.py b/tools/esp_prov/prov/__init__.py index 79708453..9c55a1fe 100644 --- a/tools/esp_prov/prov/__init__.py +++ b/tools/esp_prov/prov/__init__.py @@ -13,5 +13,6 @@ # limitations under the License. # -from .wifi_prov import * -from .custom_prov import * +from .wifi_prov import * # noqa F403 +from .custom_prov import * # noqa F403 +from .wifi_scan import * # noqa F403 diff --git a/tools/esp_prov/prov/custom_prov.py b/tools/esp_prov/prov/custom_prov.py index d5d29132..fe611dbe 100644 --- a/tools/esp_prov/prov/custom_prov.py +++ b/tools/esp_prov/prov/custom_prov.py @@ -21,9 +21,11 @@ from future.utils import tobytes import utils import proto + def print_verbose(security_ctx, data): if (security_ctx.verbose): - print("++++ " + data + " ++++") + print("++++ " + data + " ++++") + def custom_config_request(security_ctx, info, version): # Form protobuf request packet from custom-config data @@ -34,6 +36,7 @@ def custom_config_request(security_ctx, info, version): print_verbose(security_ctx, "Client -> Device (CustomConfig cmd) " + utils.str_to_hexstr(enc_cmd)) return enc_cmd + def custom_config_response(security_ctx, response_data): # Interpret protobuf response packet decrypt = security_ctx.decrypt_data(tobytes(response_data)) diff --git a/tools/esp_prov/prov/wifi_prov.py b/tools/esp_prov/prov/wifi_prov.py index ff6319b7..9140291f 100644 --- a/tools/esp_prov/prov/wifi_prov.py +++ b/tools/esp_prov/prov/wifi_prov.py @@ -21,9 +21,11 @@ from future.utils import tobytes import utils import proto + def print_verbose(security_ctx, data): if (security_ctx.verbose): - print("++++ " + data + " ++++") + print("++++ " + data + " ++++") + def config_get_status_request(security_ctx): # Form protobuf request packet for GetStatus command @@ -35,6 +37,7 @@ def config_get_status_request(security_ctx): print_verbose(security_ctx, "Client -> Device (Encrypted CmdGetStatus) " + utils.str_to_hexstr(encrypted_cfg)) return encrypted_cfg + def config_get_status_response(security_ctx, response_data): # Interpret protobuf response packet from GetStatus command decrypted_message = security_ctx.decrypt_data(tobytes(response_data)) @@ -56,6 +59,7 @@ def config_get_status_response(security_ctx, response_data): print("++++ Failure reason: " + "Incorrect SSID ++++") return cmd_resp1.resp_get_status.sta_state + def config_set_config_request(security_ctx, ssid, passphrase): # Form protobuf request packet for SetConfig command cmd = proto.wifi_config_pb2.WiFiConfigPayload() @@ -66,6 +70,7 @@ def config_set_config_request(security_ctx, ssid, passphrase): print_verbose(security_ctx, "Client -> Device (SetConfig cmd) " + utils.str_to_hexstr(enc_cmd)) return enc_cmd + def config_set_config_response(security_ctx, response_data): # Interpret protobuf response packet from SetConfig command decrypt = security_ctx.decrypt_data(tobytes(response_data)) @@ -74,6 +79,7 @@ def config_set_config_response(security_ctx, response_data): print_verbose(security_ctx, "SetConfig status " + str(cmd_resp4.resp_set_config.status)) return cmd_resp4.resp_set_config.status + def config_apply_config_request(security_ctx): # Form protobuf request packet for ApplyConfig command cmd = proto.wifi_config_pb2.WiFiConfigPayload() @@ -82,6 +88,7 @@ def config_apply_config_request(security_ctx): print_verbose(security_ctx, "Client -> Device (ApplyConfig cmd) " + utils.str_to_hexstr(enc_cmd)) return enc_cmd + def config_apply_config_response(security_ctx, response_data): # Interpret protobuf response packet from ApplyConfig command decrypt = security_ctx.decrypt_data(tobytes(response_data)) diff --git a/tools/esp_prov/prov/wifi_scan.py b/tools/esp_prov/prov/wifi_scan.py new file mode 100644 index 00000000..1b3f9135 --- /dev/null +++ b/tools/esp_prov/prov/wifi_scan.py @@ -0,0 +1,105 @@ +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# APIs for interpreting and creating protobuf packets for Wi-Fi Scanning + +from __future__ import print_function +from future.utils import tobytes + +import utils +import proto + + +def print_verbose(security_ctx, data): + if (security_ctx.verbose): + print("++++ " + data + " ++++") + + +def scan_start_request(security_ctx, blocking=True, passive=False, group_channels=5, period_ms=120): + # Form protobuf request packet for ScanStart command + cmd = proto.wifi_scan_pb2.WiFiScanPayload() + cmd.msg = proto.wifi_scan_pb2.TypeCmdScanStart + cmd.cmd_scan_start.blocking = blocking + cmd.cmd_scan_start.passive = passive + cmd.cmd_scan_start.group_channels = group_channels + cmd.cmd_scan_start.period_ms = period_ms + enc_cmd = security_ctx.encrypt_data(cmd.SerializeToString()).decode('latin-1') + print_verbose(security_ctx, "Client -> Device (Encrypted CmdScanStart) " + utils.str_to_hexstr(enc_cmd)) + return enc_cmd + + +def scan_start_response(security_ctx, response_data): + # Interpret protobuf response packet from ScanStart command + dec_resp = security_ctx.decrypt_data(tobytes(response_data)) + resp = proto.wifi_scan_pb2.WiFiScanPayload() + resp.ParseFromString(dec_resp) + print_verbose(security_ctx, "ScanStart status " + str(resp.status)) + if resp.status != 0: + raise RuntimeError + + +def scan_status_request(security_ctx): + # Form protobuf request packet for ScanStatus command + cmd = proto.wifi_scan_pb2.WiFiScanPayload() + cmd.msg = proto.wifi_scan_pb2.TypeCmdScanStatus + enc_cmd = security_ctx.encrypt_data(cmd.SerializeToString()).decode('latin-1') + print_verbose(security_ctx, "Client -> Device (Encrypted CmdScanStatus) " + utils.str_to_hexstr(enc_cmd)) + return enc_cmd + + +def scan_status_response(security_ctx, response_data): + # Interpret protobuf response packet from ScanStatus command + dec_resp = security_ctx.decrypt_data(tobytes(response_data)) + resp = proto.wifi_scan_pb2.WiFiScanPayload() + resp.ParseFromString(dec_resp) + print_verbose(security_ctx, "ScanStatus status " + str(resp.status)) + if resp.status != 0: + raise RuntimeError + return {"finished": resp.resp_scan_status.scan_finished, "count": resp.resp_scan_status.result_count} + + +def scan_result_request(security_ctx, index, count): + # Form protobuf request packet for ScanResult command + cmd = proto.wifi_scan_pb2.WiFiScanPayload() + cmd.msg = proto.wifi_scan_pb2.TypeCmdScanResult + cmd.cmd_scan_result.start_index = index + cmd.cmd_scan_result.count = count + enc_cmd = security_ctx.encrypt_data(cmd.SerializeToString()).decode('latin-1') + print_verbose(security_ctx, "Client -> Device (Encrypted CmdScanResult) " + utils.str_to_hexstr(enc_cmd)) + return enc_cmd + + +def scan_result_response(security_ctx, response_data): + # Interpret protobuf response packet from ScanResult command + dec_resp = security_ctx.decrypt_data(tobytes(response_data)) + resp = proto.wifi_scan_pb2.WiFiScanPayload() + resp.ParseFromString(dec_resp) + print_verbose(security_ctx, "ScanResult status " + str(resp.status)) + if resp.status != 0: + raise RuntimeError + authmode_str = ["Open", "WEP", "WPA_PSK", "WPA2_PSK", "WPA_WPA2_PSK", "WPA2_ENTERPRISE"] + results = [] + for entry in resp.resp_scan_result.entries: + results += [{"ssid": entry.ssid.decode('latin-1').rstrip('\x00'), + "bssid": utils.str_to_hexstr(entry.bssid.decode('latin-1')), + "channel": entry.channel, + "rssi": entry.rssi, + "auth": authmode_str[entry.auth]}] + print_verbose(security_ctx, "ScanResult SSID : " + str(results[-1]["ssid"])) + print_verbose(security_ctx, "ScanResult BSSID : " + str(results[-1]["bssid"])) + print_verbose(security_ctx, "ScanResult Channel : " + str(results[-1]["channel"])) + print_verbose(security_ctx, "ScanResult RSSI : " + str(results[-1]["rssi"])) + print_verbose(security_ctx, "ScanResult AUTH : " + str(results[-1]["auth"])) + return results diff --git a/tools/esp_prov/requirements_linux_extra.txt b/tools/esp_prov/requirements_linux_extra.txt new file mode 100644 index 00000000..555438cc --- /dev/null +++ b/tools/esp_prov/requirements_linux_extra.txt @@ -0,0 +1 @@ +dbus-python diff --git a/tools/esp_prov/security/__init__.py b/tools/esp_prov/security/__init__.py index c0bb9ffd..8a4a4fdb 100644 --- a/tools/esp_prov/security/__init__.py +++ b/tools/esp_prov/security/__init__.py @@ -13,5 +13,5 @@ # limitations under the License. # -from .security0 import * -from .security1 import * +from .security0 import * # noqa: F403, F401 +from .security1 import * # noqa: F403, F401 diff --git a/tools/esp_prov/security/security.py b/tools/esp_prov/security/security.py index 89f011ad..563a0889 100644 --- a/tools/esp_prov/security/security.py +++ b/tools/esp_prov/security/security.py @@ -15,7 +15,7 @@ # Base class for protocomm security + class Security: def __init__(self, security_session): self.security_session = security_session - diff --git a/tools/esp_prov/security/security0.py b/tools/esp_prov/security/security0.py index 8e105f6a..3e8d3536 100644 --- a/tools/esp_prov/security/security0.py +++ b/tools/esp_prov/security/security0.py @@ -19,9 +19,9 @@ from __future__ import print_function from future.utils import tobytes -import utils import proto -from .security import * +from .security import Security + class Security0(Security): def __init__(self, verbose): diff --git a/tools/esp_prov/security/security1.py b/tools/esp_prov/security/security1.py index 7e7b9601..650abd79 100644 --- a/tools/esp_prov/security/security1.py +++ b/tools/esp_prov/security/security1.py @@ -21,7 +21,7 @@ from future.utils import tobytes import utils import proto -from .security import * +from .security import Security from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes @@ -30,6 +30,7 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes import session_pb2 + # Enum for state of protocomm_security1 FSM class security_state: REQUEST1 = 0 @@ -37,6 +38,7 @@ class security_state: RESPONSE2 = 2 FINISHED = 3 + def xor(a, b): # XOR two inputs of type `bytes` ret = bytearray() @@ -50,6 +52,7 @@ def xor(a, b): # Convert bytearray to bytes return bytes(ret) + class Security1(Security): def __init__(self, pop, verbose): # Initialize state of the security1 FSM diff --git a/tools/esp_prov/transport/__init__.py b/tools/esp_prov/transport/__init__.py index 445a2ad3..907df1f3 100644 --- a/tools/esp_prov/transport/__init__.py +++ b/tools/esp_prov/transport/__init__.py @@ -13,4 +13,6 @@ # limitations under the License. # -from .transport_softap import * +from .transport_console import * # noqa: F403, F401 +from .transport_http import * # noqa: F403, F401 +from .transport_ble import * # noqa: F403, F401 diff --git a/tools/esp_prov/transport/ble_cli.py b/tools/esp_prov/transport/ble_cli.py new file mode 100644 index 00000000..ad80bf2d --- /dev/null +++ b/tools/esp_prov/transport/ble_cli.py @@ -0,0 +1,293 @@ +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function +from builtins import input +from future.utils import iteritems + +import platform + +import utils + +fallback = True + + +# Check if platform is Linux and required packages are installed +# else fallback to console mode +if platform.system() == 'Linux': + try: + import dbus + import dbus.mainloop.glib + import time + fallback = False + except ImportError: + pass + + +# -------------------------------------------------------------------- + + +# BLE client (Linux Only) using Bluez and DBus +class BLE_Bluez_Client: + def connect(self, devname, iface, chrc_names, fallback_srv_uuid): + self.devname = devname + self.srv_uuid_fallback = fallback_srv_uuid + self.chrc_names = [name.lower() for name in chrc_names] + self.device = None + self.adapter = None + self.adapter_props = None + self.services = None + self.nu_lookup = None + self.characteristics = dict() + self.srv_uuid_adv = None + + dbus.mainloop.glib.DBusGMainLoop(set_as_default=True) + bus = dbus.SystemBus() + manager = dbus.Interface(bus.get_object("org.bluez", "/"), "org.freedesktop.DBus.ObjectManager") + objects = manager.GetManagedObjects() + + for path, interfaces in iteritems(objects): + adapter = interfaces.get("org.bluez.Adapter1") + if adapter is not None: + if path.endswith(iface): + self.adapter = dbus.Interface(bus.get_object("org.bluez", path), "org.bluez.Adapter1") + self.adapter_props = dbus.Interface(bus.get_object("org.bluez", path), "org.freedesktop.DBus.Properties") + break + + if self.adapter is None: + raise RuntimeError("Bluetooth adapter not found") + + self.adapter_props.Set("org.bluez.Adapter1", "Powered", dbus.Boolean(1)) + self.adapter.StartDiscovery() + + retry = 10 + while (retry > 0): + try: + if self.device is None: + print("Connecting...") + # Wait for device to be discovered + time.sleep(5) + self._connect_() + print("Connected") + print("Getting Services...") + # Wait for services to be discovered + time.sleep(5) + self._get_services_() + return True + except Exception as e: + print(e) + retry -= 1 + print("Retries left", retry) + continue + self.adapter.StopDiscovery() + return False + + def _connect_(self): + bus = dbus.SystemBus() + manager = dbus.Interface(bus.get_object("org.bluez", "/"), "org.freedesktop.DBus.ObjectManager") + objects = manager.GetManagedObjects() + dev_path = None + for path, interfaces in iteritems(objects): + if "org.bluez.Device1" not in interfaces: + continue + if interfaces["org.bluez.Device1"].get("Name") == self.devname: + dev_path = path + break + + if dev_path is None: + raise RuntimeError("BLE device not found") + + try: + self.device = bus.get_object("org.bluez", dev_path) + try: + uuids = self.device.Get('org.bluez.Device1', 'UUIDs', + dbus_interface='org.freedesktop.DBus.Properties') + # There should be 1 service UUID in advertising data + # If bluez had cached an old version of the advertisement data + # the list of uuids may be incorrect, in which case connection + # or service discovery may fail the first time. If that happens + # the cache will be refreshed before next retry + if len(uuids) == 1: + self.srv_uuid_adv = uuids[0] + except dbus.exceptions.DBusException as e: + print(e) + + self.device.Connect(dbus_interface='org.bluez.Device1') + except Exception as e: + print(e) + self.device = None + raise RuntimeError("BLE device could not connect") + + def _get_services_(self): + bus = dbus.SystemBus() + manager = dbus.Interface(bus.get_object("org.bluez", "/"), "org.freedesktop.DBus.ObjectManager") + objects = manager.GetManagedObjects() + service_found = False + for srv_path, srv_interfaces in iteritems(objects): + if "org.bluez.GattService1" not in srv_interfaces: + continue + if not srv_path.startswith(self.device.object_path): + continue + service = bus.get_object("org.bluez", srv_path) + srv_uuid = service.Get('org.bluez.GattService1', 'UUID', + dbus_interface='org.freedesktop.DBus.Properties') + + # If service UUID doesn't match the one found in advertisement data + # then also check if it matches the fallback UUID + if srv_uuid not in [self.srv_uuid_adv, self.srv_uuid_fallback]: + continue + + nu_lookup = dict() + characteristics = dict() + for chrc_path, chrc_interfaces in iteritems(objects): + if "org.bluez.GattCharacteristic1" not in chrc_interfaces: + continue + if not chrc_path.startswith(service.object_path): + continue + chrc = bus.get_object("org.bluez", chrc_path) + uuid = chrc.Get('org.bluez.GattCharacteristic1', 'UUID', + dbus_interface='org.freedesktop.DBus.Properties') + characteristics[uuid] = chrc + for desc_path, desc_interfaces in iteritems(objects): + if "org.bluez.GattDescriptor1" not in desc_interfaces: + continue + if not desc_path.startswith(chrc.object_path): + continue + desc = bus.get_object("org.bluez", desc_path) + desc_uuid = desc.Get('org.bluez.GattDescriptor1', 'UUID', + dbus_interface='org.freedesktop.DBus.Properties') + if desc_uuid[4:8] != '2901': + continue + try: + readval = desc.ReadValue({}, dbus_interface='org.bluez.GattDescriptor1') + except dbus.exceptions.DBusException: + break + found_name = ''.join(chr(b) for b in readval).lower() + nu_lookup[found_name] = uuid + break + + match_found = True + for name in self.chrc_names: + if name not in nu_lookup: + # Endpoint name not present + match_found = False + break + + # Create lookup table only if all endpoint names found + self.nu_lookup = [None, nu_lookup][match_found] + self.characteristics = characteristics + service_found = True + + # If the service UUID matches that in the advertisement + # we can stop the search now. If it doesn't match, we + # have found the service corresponding to the fallback + # UUID, in which case don't break and keep searching + # for the advertised service + if srv_uuid == self.srv_uuid_adv: + break + + if not service_found: + self.device.Disconnect(dbus_interface='org.bluez.Device1') + if self.adapter: + self.adapter.RemoveDevice(self.device) + self.device = None + self.nu_lookup = None + self.characteristics = dict() + raise RuntimeError("Provisioning service not found") + + def get_nu_lookup(self): + return self.nu_lookup + + def has_characteristic(self, uuid): + if uuid in self.characteristics: + return True + return False + + def disconnect(self): + if self.device: + self.device.Disconnect(dbus_interface='org.bluez.Device1') + if self.adapter: + self.adapter.RemoveDevice(self.device) + self.device = None + self.nu_lookup = None + self.characteristics = dict() + if self.adapter_props: + self.adapter_props.Set("org.bluez.Adapter1", "Powered", dbus.Boolean(0)) + + def send_data(self, characteristic_uuid, data): + try: + path = self.characteristics[characteristic_uuid] + except KeyError: + raise RuntimeError("Invalid characteristic : " + characteristic_uuid) + + try: + path.WriteValue([ord(c) for c in data], {}, dbus_interface='org.bluez.GattCharacteristic1') + except dbus.exceptions.DBusException as e: + raise RuntimeError("Failed to write value to characteristic " + characteristic_uuid + ": " + str(e)) + + try: + readval = path.ReadValue({}, dbus_interface='org.bluez.GattCharacteristic1') + except dbus.exceptions.DBusException as e: + raise RuntimeError("Failed to read value from characteristic " + characteristic_uuid + ": " + str(e)) + return ''.join(chr(b) for b in readval) + + +# -------------------------------------------------------------------- + + +# Console based BLE client for Cross Platform support +class BLE_Console_Client: + def connect(self, devname, iface, chrc_names, fallback_srv_uuid): + print("BLE client is running in console mode") + print("\tThis could be due to your platform not being supported or dependencies not being met") + print("\tPlease ensure all pre-requisites are met to run the full fledged client") + print("BLECLI >> Please connect to BLE device `" + devname + "` manually using your tool of choice") + resp = input("BLECLI >> Was the device connected successfully? [y/n] ") + if resp != 'Y' and resp != 'y': + return False + print("BLECLI >> List available attributes of the connected device") + resp = input("BLECLI >> Is the service UUID '" + fallback_srv_uuid + "' listed among available attributes? [y/n] ") + if resp != 'Y' and resp != 'y': + return False + return True + + def get_nu_lookup(self): + return None + + def has_characteristic(self, uuid): + resp = input("BLECLI >> Is the characteristic UUID '" + uuid + "' listed among available attributes? [y/n] ") + if resp != 'Y' and resp != 'y': + return False + return True + + def disconnect(self): + pass + + def send_data(self, characteristic_uuid, data): + print("BLECLI >> Write following data to characteristic with UUID '" + characteristic_uuid + "' :") + print("\t>> " + utils.str_to_hexstr(data)) + print("BLECLI >> Enter data read from characteristic (in hex) :") + resp = input("\t<< ") + return utils.hexstr_to_str(resp) + + +# -------------------------------------------------------------------- + + +# Function to get client instance depending upon platform +def get_client(): + if fallback: + return BLE_Console_Client() + return BLE_Bluez_Client() diff --git a/tools/esp_prov/transport/transport.py b/tools/esp_prov/transport/transport.py index d6851375..48947178 100644 --- a/tools/esp_prov/transport/transport.py +++ b/tools/esp_prov/transport/transport.py @@ -17,6 +17,7 @@ import abc + class Transport(): @abc.abstractmethod diff --git a/tools/esp_prov/transport/transport_ble.py b/tools/esp_prov/transport/transport_ble.py new file mode 100644 index 00000000..333d9510 --- /dev/null +++ b/tools/esp_prov/transport/transport_ble.py @@ -0,0 +1,67 @@ +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function + +from .transport import Transport + +from . import ble_cli + + +class Transport_BLE(Transport): + def __init__(self, devname, service_uuid, nu_lookup): + # Expect service UUID like '0000ffff-0000-1000-8000-00805f9b34fb' + for name in nu_lookup.keys(): + # Calculate characteristic UUID for each endpoint + nu_lookup[name] = service_uuid[:4] + '{:02x}'.format( + int(nu_lookup[name], 16) & int(service_uuid[4:8], 16)) + service_uuid[8:] + + # Get BLE client module + self.cli = ble_cli.get_client() + + # Use client to connect to BLE device and bind to service + if not self.cli.connect(devname=devname, iface='hci0', + chrc_names=nu_lookup.keys(), + fallback_srv_uuid=service_uuid): + raise RuntimeError("Failed to initialize transport") + + # Irrespective of provided parameters, let the client + # generate a lookup table by reading advertisement data + # and characteristic user descriptors + self.name_uuid_lookup = self.cli.get_nu_lookup() + + # If that doesn't work, use the lookup table provided as parameter + if self.name_uuid_lookup is None: + self.name_uuid_lookup = nu_lookup + # Check if expected characteristics are provided by the service + for name in self.name_uuid_lookup.keys(): + if not self.cli.has_characteristic(self.name_uuid_lookup[name]): + raise RuntimeError("'" + name + "' endpoint not found") + + def __del__(self): + # Make sure device is disconnected before application gets closed + try: + self.disconnect() + except Exception: + pass + + def disconnect(self): + self.cli.disconnect() + + def send_data(self, ep_name, data): + # Write (and read) data to characteristic corresponding to the endpoint + if ep_name not in self.name_uuid_lookup.keys(): + raise RuntimeError("Invalid endpoint : " + ep_name) + return self.cli.send_data(self.name_uuid_lookup[ep_name], data) diff --git a/tools/esp_prov/transport/transport_console.py b/tools/esp_prov/transport/transport_console.py new file mode 100644 index 00000000..8e95141c --- /dev/null +++ b/tools/esp_prov/transport/transport_console.py @@ -0,0 +1,33 @@ +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function +from builtins import input + +import utils + +from .transport import Transport + + +class Transport_Console(Transport): + + def send_data(self, path, data, session_id=0): + print("Client->Device msg :", path, session_id, utils.str_to_hexstr(data)) + try: + resp = input("Enter device->client msg : ") + except Exception as err: + print("error:", err) + return None + return utils.hexstr_to_str(resp) diff --git a/tools/esp_prov/transport/transport_softap.py b/tools/esp_prov/transport/transport_http.py similarity index 60% rename from tools/esp_prov/transport/transport_softap.py rename to tools/esp_prov/transport/transport_http.py index 83f1818c..3c7aed01 100644 --- a/tools/esp_prov/transport/transport_softap.py +++ b/tools/esp_prov/transport/transport_http.py @@ -16,13 +16,30 @@ from __future__ import print_function from future.utils import tobytes +import socket import http.client +import ssl -from .transport import * +from .transport import Transport -class Transport_Softap(Transport): - def __init__(self, url): - self.conn = http.client.HTTPConnection(url, timeout=30) + +class Transport_HTTP(Transport): + def __init__(self, hostname, certfile=None): + try: + socket.gethostbyname(hostname.split(':')[0]) + except socket.gaierror: + raise RuntimeError("Unable to resolve hostname :" + hostname) + + if certfile is None: + self.conn = http.client.HTTPConnection(hostname, timeout=30) + else: + ssl_ctx = ssl.create_default_context(cafile=certfile) + self.conn = http.client.HTTPSConnection(hostname, context=ssl_ctx, timeout=30) + try: + print("Connecting to " + hostname) + self.conn.connect() + except Exception as err: + raise RuntimeError("Connection Failure : " + str(err)) self.headers = {"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"} def _send_post_request(self, path, data): @@ -36,4 +53,4 @@ class Transport_Softap(Transport): raise RuntimeError("Server responded with error code " + str(response.status)) def send_data(self, ep_name, data): - return self._send_post_request('/'+ ep_name, data) + return self._send_post_request('/' + ep_name, data) diff --git a/tools/esp_prov/utils/__init__.py b/tools/esp_prov/utils/__init__.py index e46d42d0..4fff04c9 100644 --- a/tools/esp_prov/utils/__init__.py +++ b/tools/esp_prov/utils/__init__.py @@ -13,4 +13,4 @@ # limitations under the License. # -from .convenience import * +from .convenience import * # noqa: F403, F401 diff --git a/tools/esp_prov/utils/convenience.py b/tools/esp_prov/utils/convenience.py index b894ad14..9fec1dea 100644 --- a/tools/esp_prov/utils/convenience.py +++ b/tools/esp_prov/utils/convenience.py @@ -15,15 +15,17 @@ # Convenience functions for commonly used data type conversions + def str_to_hexstr(string): # Form hexstr by appending ASCII codes (in hex) corresponding to # each character in the input string return ''.join('{:02x}'.format(ord(c)) for c in string) + def hexstr_to_str(hexstr): # Prepend 0 (if needed) to make the hexstr length an even number - if len(hexstr)%2 == 1: + if len(hexstr) % 2 == 1: hexstr = '0' + hexstr # Interpret consecutive pairs of hex characters as 8 bit ASCII codes # and append characters corresponding to each code to form the string - return ''.join(chr(int(hexstr[2*i:2*i+2], 16)) for i in range(len(hexstr)//2)) + return ''.join(chr(int(hexstr[2 * i: 2 * i + 2], 16)) for i in range(len(hexstr) // 2)) diff --git a/tools/format-minimal.sh b/tools/format-minimal.sh new file mode 100644 index 00000000..ff829247 --- /dev/null +++ b/tools/format-minimal.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Runs astyle with parameters which should be checked in a pre-commit hook +astyle \ + --style=otbs \ + --indent=spaces=4 \ + --convert-tabs \ + --keep-one-line-statements \ + --pad-header \ + "$@" diff --git a/tools/format.sh b/tools/format.sh new file mode 100755 index 00000000..aadb337a --- /dev/null +++ b/tools/format.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Runs astyle with the full set of formatting options +astyle \ + --style=otbs \ + --indent=spaces=4 \ + --convert-tabs \ + --align-pointer=name \ + --align-reference=name \ + --keep-one-line-statements \ + --pad-header \ + --pad-oper \ + "$@" diff --git a/tools/gen_appbin.py b/tools/gen_appbin.py deleted file mode 100644 index 26a510cd..00000000 --- a/tools/gen_appbin.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/python -# -# File : gen_appbin.py -# This file is part of Espressif's generate bin script. -# Copyright (C) 2013 - 2016, Espressif Systems -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of version 3 of the GNU General Public License as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - -"""This file is part of Espressif's generate bin script. - argv[1] is elf file name - argv[2] is version num""" - -import string -import sys -import os -import re -import binascii -import struct -import zlib - - -TEXT_ADDRESS = 0x40100000 -# app_entry = 0 -# data_address = 0x3ffb0000 -# data_end = 0x40000000 -# text_end = 0x40120000 - -CHECKSUM_INIT = 0xEF - -chk_sum = CHECKSUM_INIT -blocks = 0 - -def write_file(file_name,data): - if file_name is None: - print 'file_name cannot be none\n' - sys.exit(0) - - fp = open(file_name,'ab') - - if fp: - fp.seek(0,os.SEEK_END) - fp.write(data) - fp.close() - else: - print '%s write fail\n'%(file_name) - -def combine_bin(file_name,dest_file_name,start_offset_addr,need_chk): - global chk_sum - global blocks - if dest_file_name is None: - print 'dest_file_name cannot be none\n' - sys.exit(0) - - if file_name: - fp = open(file_name,'rb') - if fp: - ########## write text ########## - fp.seek(0,os.SEEK_END) - data_len = fp.tell() - if data_len: - if need_chk: - tmp_len = (data_len + 3) & (~3) - else: - tmp_len = (data_len + 15) & (~15) - data_bin = struct.pack(' eagle.app.sym' - else : - cmd = 'xtensa-lx106-elf-nm -g ' + elf_file + ' > eagle.app.sym' - - os.system(cmd) - - fp = file('./eagle.app.sym') - if fp is None: - print "open sym file error\n" - sys.exit(0) - - lines = fp.readlines() - fp.close() - - entry_addr = None - p = re.compile('(\w*)(\sT\s)(call_user_start)$') - for line in lines: - m = p.search(line) - if m != None: - entry_addr = m.group(1) - # print entry_addr - - if entry_addr is None: - print 'no entry point!!' - sys.exit(0) - - data_start_addr = '0' - p = re.compile('(\w*)(\sA\s)(_data_start)$') - for line in lines: - m = p.search(line) - if m != None: - data_start_addr = m.group(1) - # print data_start_addr - - rodata_start_addr = '0' - p = re.compile('(\w*)(\sA\s)(_rodata_start)$') - for line in lines: - m = p.search(line) - if m != None: - rodata_start_addr = m.group(1) - # print rodata_start_addr - - # write flash bin header - #============================ - # SPI FLASH PARAMS - #------------------- - #flash_mode= - # 0: QIO - # 1: QOUT - # 2: DIO - # 3: DOUT - #------------------- - #flash_clk_div= - # 0 : 80m / 2 - # 1 : 80m / 3 - # 2 : 80m / 4 - # 0xf: 80m / 1 - #------------------- - #flash_size_map= - # 0 : 512 KB (256 KB + 256 KB) - # 1 : 256 KB - # 2 : 1024 KB (512 KB + 512 KB) - # 3 : 2048 KB (512 KB + 512 KB) - # 4 : 4096 KB (512 KB + 512 KB) - # 5 : 2048 KB (1024 KB + 1024 KB) - # 6 : 4096 KB (1024 KB + 1024 KB) - #------------------- - # END OF SPI FLASH PARAMS - #============================ - byte2=int(flash_mode)&0xff - byte3=(((int(flash_size_map)<<4)| int(flash_clk_div))&0xff) - - if boot_mode == '2': - # write irom bin head - data_bin = struct.pack('> 8)+chr((all_bin_crc & 0x00FF0000) >> 16)+chr((all_bin_crc & 0xFF000000) >> 24)) - cmd = 'rm eagle.app.sym' - os.system(cmd) - -if __name__=='__main__': - gen_appbin() diff --git a/tools/gen_esp_err_to_name.py b/tools/gen_esp_err_to_name.py index ba77ecee..68a7d211 100755 --- a/tools/gen_esp_err_to_name.py +++ b/tools/gen_esp_err_to_name.py @@ -46,7 +46,11 @@ ignore_files = ['components/mdns/test_afl_fuzz_host/esp32_compat.h'] ignore_dirs = ('examples') # macros from here have higher priorities in case of collisions -priority_headers = ['components/esp32/include/esp_err.h'] +priority_headers = ['components/esp_common/include/esp_err.h'] + +# The following headers won't be included. This is useful if they are permanently included from esp_err_to_name.c.in. +dont_include = ['soc/soc.h', + 'esp_err.h'] err_dict = collections.defaultdict(list) # identified errors are stored here; mapped by the error code rev_err_dict = dict() # map of error string to error code @@ -265,7 +269,8 @@ def generate_c_output(fin, fout): elif re.match(r'@HEADERS@', line): for i in include_list: - fout.write("#if __has_include(\"" + i + "\")\n#include \"" + i + "\"\n#endif\n") + if i not in dont_include: + fout.write("#if __has_include(\"" + i + "\")\n#include \"" + i + "\"\n#endif\n") elif re.match(r'@ERROR_ITEMS@', line): last_file = "" for k in sorted(err_dict.keys()): @@ -319,8 +324,9 @@ def main(): idf_path = os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) parser = argparse.ArgumentParser(description='ESP32 esp_err_to_name lookup generator for esp_err_t') - parser.add_argument('--c_input', help='Path to the esp_err_to_name.c.in template input.', default=idf_path + '/components/esp8266/source/esp_err_to_name.c.in') - parser.add_argument('--c_output', help='Path to the esp_err_to_name.c output.', default=idf_path + '/components/esp8266/source/esp_err_to_name.c') + parser.add_argument('--c_input', help='Path to the esp_err_to_name.c.in template input.', + default=idf_path + '/components/esp_common/src/esp_err_to_name.c.in') + parser.add_argument('--c_output', help='Path to the esp_err_to_name.c output.', default=idf_path + '/components/esp_common/src/esp_err_to_name.c') parser.add_argument('--rst_output', help='Generate .rst output and save it into this file') args = parser.parse_args() diff --git a/tools/idf.py b/tools/idf.py index 801a4d0b..669a1889 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -7,7 +7,7 @@ # # # -# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# Copyright 2019 Espressif Systems (Shanghai) PTE LTD # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,34 +26,42 @@ # check_environment() function below. If possible, avoid importing # any external libraries here - put in external script, or import in # their specific function instead. -import sys -import argparse +import codecs +import json +import locale +import multiprocessing import os import os.path -import subprocess -import multiprocessing import re import shutil -import json +import subprocess +import sys + class FatalError(RuntimeError): """ Wrapper class for runtime errors that aren't caused by bugs in idf.py or the build proces.s """ + pass + # Use this Python interpreter for any subprocesses we launch -PYTHON=sys.executable +PYTHON = sys.executable # note: os.environ changes don't automatically propagate to child processes, # you have to pass env=os.environ explicitly anywhere that we create a process -os.environ["PYTHON"]=sys.executable +os.environ["PYTHON"] = sys.executable + +# Name of the program, normally 'idf.py'. +# Can be overridden from idf.bat using IDF_PY_PROGRAM_NAME +PROG = os.getenv("IDF_PY_PROGRAM_NAME", sys.argv[0]) # Make flavors, across the various kinds of Windows environments & POSIX... if "MSYSTEM" in os.environ: # MSYS MAKE_CMD = "make" MAKE_GENERATOR = "MSYS Makefiles" -elif os.name == 'nt': # other Windows +elif os.name == "nt": # other Windows MAKE_CMD = "mingw32-make" MAKE_GENERATOR = "MinGW Makefiles" else: @@ -62,11 +70,17 @@ else: GENERATORS = [ # ('generator name', 'build command line', 'version command line', 'verbose flag') - ("Ninja", [ "ninja" ], [ "ninja", "--version" ], "-v"), - (MAKE_GENERATOR, [ MAKE_CMD, "-j", str(multiprocessing.cpu_count()+2) ], [ "make", "--version" ], "VERBOSE=1"), - ] -GENERATOR_CMDS = dict( (a[0], a[1]) for a in GENERATORS ) -GENERATOR_VERBOSE = dict( (a[0], a[3]) for a in GENERATORS ) + ("Ninja", ["ninja"], ["ninja", "--version"], "-v"), + ( + MAKE_GENERATOR, + [MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)], + [MAKE_CMD, "--version"], + "VERBOSE=1", + ), +] +GENERATOR_CMDS = dict((a[0], a[1]) for a in GENERATORS) +GENERATOR_VERBOSE = dict((a[0], a[3]) for a in GENERATORS) + def _run_tool(tool_name, args, cwd): def quote_arg(arg): @@ -74,6 +88,7 @@ def _run_tool(tool_name, args, cwd): if " " in arg and not (arg.startswith('"') or arg.startswith("'")): return "'" + arg + "'" return arg + display_args = " ".join(quote_arg(arg) for arg in args) print("Running %s in directory %s" % (tool_name, quote_arg(cwd))) print('Executing "%s"...' % str(display_args)) @@ -83,6 +98,17 @@ def _run_tool(tool_name, args, cwd): except subprocess.CalledProcessError as e: raise FatalError("%s failed with exit code %d" % (tool_name, e.returncode)) + +def _realpath(path): + """ + Return the cannonical path with normalized case. + + It is useful on Windows to comparision paths in case-insensitive manner. + On Unix and Mac OS X it works as `os.path.realpath()` only. + """ + return os.path.normcase(os.path.realpath(path)) + + def check_environment(): """ Verify the environment contains the top-level tools we need to operate @@ -90,14 +116,17 @@ def check_environment(): (cmake will check a lot of other things) """ if not executable_exists(["cmake", "--version"]): - raise FatalError("'cmake' must be available on the PATH to use idf.py") + raise FatalError("'cmake' must be available on the PATH to use %s" % PROG) # find the directory idf.py is in, then the parent directory of this, and assume this is IDF_PATH - detected_idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "..")) + detected_idf_path = _realpath(os.path.join(os.path.dirname(__file__), "..")) if "IDF_PATH" in os.environ: - set_idf_path = os.path.realpath(os.environ["IDF_PATH"]) + set_idf_path = _realpath(os.environ["IDF_PATH"]) if set_idf_path != detected_idf_path: - print("WARNING: IDF_PATH environment variable is set to %s but idf.py path indicates IDF directory %s. Using the environment variable directory, but results may be unexpected..." - % (set_idf_path, detected_idf_path)) + print( + "WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. " + "Using the environment variable directory, but results may be unexpected..." + % (set_idf_path, PROG, detected_idf_path) + ) else: print("Setting IDF_PATH environment variable: %s" % detected_idf_path) os.environ["IDF_PATH"] = detected_idf_path @@ -105,19 +134,27 @@ def check_environment(): # check Python dependencies print("Checking Python dependencies...") try: - subprocess.check_call([ os.environ["PYTHON"], - os.path.join(os.environ["IDF_PATH"], "tools", "check_python_dependencies.py")], - env=os.environ) + subprocess.check_call( + [ + os.environ["PYTHON"], + os.path.join( + os.environ["IDF_PATH"], "tools", "check_python_dependencies.py" + ), + ], + env=os.environ, + ) except subprocess.CalledProcessError: raise SystemExit(1) + def executable_exists(args): try: subprocess.check_output(args) return True - except: + except Exception: return False + def detect_cmake_generator(): """ Find the default cmake generator, if none was specified. Raises an exception if no valid generator is found. @@ -125,7 +162,37 @@ def detect_cmake_generator(): for (generator, _, version_check, _) in GENERATORS: if executable_exists(version_check): return generator - raise FatalError("To use idf.py, either the 'ninja' or 'GNU make' build tool must be available in the PATH") + raise FatalError( + "To use %s, either the 'ninja' or 'GNU make' build tool must be available in the PATH" + % PROG + ) + + +def _strip_quotes(value, regexp=re.compile(r"^\"(.*)\"$|^'(.*)'$|^(.*)$")): + """ + Strip quotes like CMake does during parsing cache entries + """ + + return [x for x in regexp.match(value).groups() if x is not None][0].rstrip() + + +def _new_cmakecache_entries(cache_path, new_cache_entries): + if not os.path.exists(cache_path): + return True + + current_cache = parse_cmakecache(cache_path) + + if new_cache_entries: + current_cache = parse_cmakecache(cache_path) + + for entry in new_cache_entries: + key, value = entry.split("=", 1) + current_value = current_cache.get(key, None) + if current_value is None or _strip_quotes(value) != current_value: + return True + + return False + def _ensure_build_directory(args, always_run_cmake=False): """Check the build directory exists and that cmake has been run there. @@ -141,32 +208,43 @@ def _ensure_build_directory(args, always_run_cmake=False): # Verify the project directory if not os.path.isdir(project_dir): if not os.path.exists(project_dir): - raise FatalError("Project directory %s does not exist") + raise FatalError("Project directory %s does not exist" % project_dir) else: - raise FatalError("%s must be a project directory") + raise FatalError("%s must be a project directory" % project_dir) if not os.path.exists(os.path.join(project_dir, "CMakeLists.txt")): - raise FatalError("CMakeLists.txt not found in project directory %s" % project_dir) + raise FatalError( + "CMakeLists.txt not found in project directory %s" % project_dir + ) # Verify/create the build directory build_dir = args.build_dir if not os.path.isdir(build_dir): os.makedirs(build_dir) cache_path = os.path.join(build_dir, "CMakeCache.txt") - if not os.path.exists(cache_path) or always_run_cmake: + + args.define_cache_entry = list(args.define_cache_entry) + args.define_cache_entry.append("CCACHE_ENABLE=%d" % args.ccache) + + if always_run_cmake or _new_cmakecache_entries(cache_path, args.define_cache_entry): if args.generator is None: args.generator = detect_cmake_generator() try: - cmake_args = ["cmake", "-G", args.generator, "-DPYTHON_DEPS_CHECKED=1"] + cmake_args = [ + "cmake", + "-G", + args.generator, + "-DPYTHON_DEPS_CHECKED=1", + "-DESP_PLATFORM=1", + ] if not args.no_warnings: - cmake_args += [ "--warn-uninitialized" ] - if args.no_ccache: - cmake_args += [ "-DCCACHE_DISABLE=1" ] + cmake_args += ["--warn-uninitialized"] + if args.define_cache_entry: - cmake_args += ["-D" + d for d in args.define_cache_entry] - cmake_args += [ project_dir] + cmake_args += ["-D" + d for d in args.define_cache_entry] + cmake_args += [project_dir] _run_tool("cmake", cmake_args, cwd=args.build_dir) - except: + except Exception: # don't allow partially valid CMakeCache.txt files, # to keep the "should I run cmake?" logic simple if os.path.exists(cache_path): @@ -180,16 +258,22 @@ def _ensure_build_directory(args, always_run_cmake=False): except KeyError: generator = detect_cmake_generator() if args.generator is None: - args.generator = generator # reuse the previously configured generator, if none was given + args.generator = ( + generator + ) # reuse the previously configured generator, if none was given if generator != args.generator: - raise FatalError("Build is configured for generator '%s' not '%s'. Run 'idf.py fullclean' to start again." - % (generator, args.generator)) + raise FatalError( + "Build is configured for generator '%s' not '%s'. Run '%s fullclean' to start again." + % (generator, args.generator, PROG) + ) try: home_dir = cache["CMAKE_HOME_DIRECTORY"] - if os.path.normcase(os.path.realpath(home_dir)) != os.path.normcase(os.path.realpath(project_dir)): - raise FatalError("Build directory '%s' configured for project '%s' not '%s'. Run 'idf.py fullclean' to start again." - % (build_dir, os.path.realpath(home_dir), os.path.realpath(project_dir))) + if _realpath(home_dir) != _realpath(project_dir): + raise FatalError( + "Build directory '%s' configured for project '%s' not '%s'. Run '%s fullclean' to start again." + % (build_dir, _realpath(home_dir), _realpath(project_dir), PROG) + ) except KeyError: pass # if cmake failed part way, CMAKE_HOME_DIRECTORY may not be set yet @@ -207,12 +291,13 @@ def parse_cmakecache(path): for line in f: # cmake cache lines look like: CMAKE_CXX_FLAGS_DEBUG:STRING=-g # groups are name, type, value - m = re.match(r"^([^#/:=]+):([^:=]+)=(.+)\n$", line) + m = re.match(r"^([^#/:=]+):([^:=]+)=(.*)\n$", line) if m: - result[m.group(1)] = m.group(3) + result[m.group(1)] = m.group(3) return result -def build_target(target_name, args): + +def build_target(target_name, ctx, args): """ Execute the target build system to build target 'target_name' @@ -222,50 +307,65 @@ def build_target(target_name, args): _ensure_build_directory(args) generator_cmd = GENERATOR_CMDS[args.generator] - if not args.no_ccache: + if args.ccache: # Setting CCACHE_BASEDIR & CCACHE_NO_HASHDIR ensures that project paths aren't stored in the ccache entries # (this means ccache hits can be shared between different projects. It may mean that some debug information # will point to files in another project, if these files are perfect duplicates of each other.) # # It would be nicer to set these from cmake, but there's no cross-platform way to set build-time environment - #os.environ["CCACHE_BASEDIR"] = args.build_dir - #os.environ["CCACHE_NO_HASHDIR"] = "1" + # os.environ["CCACHE_BASEDIR"] = args.build_dir + # os.environ["CCACHE_NO_HASHDIR"] = "1" pass if args.verbose: - generator_cmd += [ GENERATOR_VERBOSE[args.generator] ] + generator_cmd += [GENERATOR_VERBOSE[args.generator]] _run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir) def _get_esptool_args(args): - esptool_path = os.path.join(os.environ["IDF_PATH"], "components/esptool_py/esptool/esptool.py") + esptool_path = os.path.join( + os.environ["IDF_PATH"], "components/esptool_py/esptool/esptool.py" + ) if args.port is None: args.port = get_default_serial_port() - result = [ PYTHON, esptool_path ] - result += [ "-p", args.port ] - result += [ "-b", str(args.baud) ] + result = [PYTHON, esptool_path] + result += ["-p", args.port] + result += ["-b", str(args.baud)] + + with open(os.path.join(args.build_dir, "flasher_args.json")) as f: + flasher_args = json.load(f) + + extra_esptool_args = flasher_args["extra_esptool_args"] + result += ["--after", extra_esptool_args["after"]] return result -def flash(action, args): + +def flash(action, ctx, args): """ Run esptool to flash the entire project, from an argfile generated by the build system """ flasher_args_path = { # action -> name of flasher args file generated by build system - "bootloader-flash": "flash_bootloader_args", + "bootloader-flash": "flash_bootloader_args", "partition_table-flash": "flash_partition_table_args", - "app-flash": "flash_app_args", - "flash": "flash_project_args", - }[action] + "app-flash": "flash_app_args", + "flash": "flash_project_args", + "encrypted-app-flash": "flash_encrypted_app_args", + "encrypted-flash": "flash_encrypted_project_args", + }[ + action + ] esptool_args = _get_esptool_args(args) - esptool_args += [ "write_flash", "@"+flasher_args_path ] + esptool_args += ["write_flash", "@" + flasher_args_path] _run_tool("esptool.py", esptool_args, args.build_dir) -def erase_flash(action, args): + +def erase_flash(action, ctx, args): esptool_args = _get_esptool_args(args) - esptool_args += [ "erase_flash" ] + esptool_args += ["erase_flash"] _run_tool("esptool.py", esptool_args, args.build_dir) -def monitor(action, args): + +def monitor(action, ctx, args, print_filter): """ Run idf_monitor.py to watch build output """ @@ -279,32 +379,61 @@ def monitor(action, args): elf_file = os.path.join(args.build_dir, project_desc["app_elf"]) if not os.path.exists(elf_file): - raise FatalError("ELF file '%s' not found. You need to build & flash the project before running 'monitor', and the binary on the device must match the one in the build directory exactly. Try 'idf.py flash monitor'." % elf_file) + raise FatalError( + "ELF file '%s' not found. You need to build & flash the project before running 'monitor', " + "and the binary on the device must match the one in the build directory exactly. " + "Try '%s flash monitor'." % (elf_file, PROG) + ) idf_monitor = os.path.join(os.environ["IDF_PATH"], "tools/idf_monitor.py") - monitor_args = [PYTHON, idf_monitor ] + monitor_args = [PYTHON, idf_monitor] if args.port is not None: - monitor_args += [ "-p", args.port ] - monitor_args += [ "-b", project_desc["monitor_baud"] ] - monitor_args += [ elf_file ] + monitor_args += ["-p", args.port] + monitor_args += ["-b", project_desc["monitor_baud"]] + if print_filter is not None: + monitor_args += ["--print_filter", print_filter] + monitor_args += [elf_file] - idf_py = [ PYTHON ] + get_commandline_options() # commands to re-run idf.py - monitor_args += [ "-m", " ".join("'%s'" % a for a in idf_py) ] + idf_py = [PYTHON] + get_commandline_options(ctx) # commands to re-run idf.py + monitor_args += ["-m", " ".join("'%s'" % a for a in idf_py)] if "MSYSTEM" in os.environ: - monitor_args = [ "winpty" ] + monitor_args + monitor_args = ["winpty"] + monitor_args _run_tool("idf_monitor", monitor_args, args.project_dir) -def clean(action, args): +def clean(action, ctx, args): if not os.path.isdir(args.build_dir): print("Build directory '%s' not found. Nothing to clean." % args.build_dir) return - build_target("clean", args) + build_target("clean", ctx, args) -def reconfigure(action, args): + +def reconfigure(action, ctx, args): _ensure_build_directory(args, True) -def fullclean(action, args): + +def _delete_windows_symlinks(directory): + """ + It deletes symlinks recursively on Windows. It is useful for Python 2 which doesn't detect symlinks on Windows. + """ + deleted_paths = [] + if os.name == "nt": + import ctypes + + for root, dirnames, _filenames in os.walk(directory): + for d in dirnames: + full_path = os.path.join(root, d) + try: + full_path = full_path.decode("utf-8") + except Exception: + pass + if ctypes.windll.kernel32.GetFileAttributesW(full_path) & 0x0400: + os.rmdir(full_path) + deleted_paths.append(full_path) + return deleted_paths + + +def fullclean(action, ctx, args): build_dir = args.build_dir if not os.path.isdir(build_dir): print("Build directory '%s' not found. Nothing to clean." % build_dir) @@ -314,104 +443,66 @@ def fullclean(action, args): return if not os.path.exists(os.path.join(build_dir, "CMakeCache.txt")): - raise FatalError("Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically delete files in this directory. Delete the directory manually to 'clean' it." % build_dir) - red_flags = [ "CMakeLists.txt", ".git", ".svn" ] + raise FatalError( + "Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically " + "delete files in this directory. Delete the directory manually to 'clean' it." + % build_dir + ) + red_flags = ["CMakeLists.txt", ".git", ".svn"] for red in red_flags: red = os.path.join(build_dir, red) if os.path.exists(red): - raise FatalError("Refusing to automatically delete files in directory containing '%s'. Delete files manually if you're sure." % red) + raise FatalError( + "Refusing to automatically delete files in directory containing '%s'. Delete files manually if you're sure." + % red + ) # OK, delete everything in the build directory... - for f in os.listdir(build_dir): # TODO: once we are Python 3 only, this can be os.scandir() + # Note: Python 2.7 doesn't detect symlinks on Windows (it is supported form 3.2). Tools promising to not + # follow symlinks will actually follow them. Deleting the build directory with symlinks deletes also items + # outside of this directory. + deleted_symlinks = _delete_windows_symlinks(build_dir) + if args.verbose and len(deleted_symlinks) > 1: + print( + "The following symlinks were identified and removed:\n%s" + % "\n".join(deleted_symlinks) + ) + for f in os.listdir( + build_dir + ): # TODO: once we are Python 3 only, this can be os.scandir() f = os.path.join(build_dir, f) + if args.verbose: + print("Removing: %s" % f) if os.path.isdir(f): shutil.rmtree(f) else: os.remove(f) -def print_closing_message(args): - # print a closing message of some kind - # - if "flash" in str(args.actions): - print("Done") - return - # Otherwise, if we built any binaries print a message about - # how to flash them - def print_flashing_message(title, key): - print("\n%s build complete. To flash, run this command:" % title) +def _safe_relpath(path, start=None): + """ Return a relative path, same as os.path.relpath, but only if this is possible. - with open(os.path.join(args.build_dir, "flasher_args.json")) as f: - flasher_args = json.load(f) + It is not possible on Windows, if the start directory and the path are on different drives. + """ + try: + return os.path.relpath(path, os.curdir if start is None else start) + except ValueError: + return os.path.abspath(path) - def flasher_path(f): - return os.path.relpath(os.path.join(args.build_dir, f)) - if key != "project": # flashing a single item - cmd = "" - if key == "bootloader": # bootloader needs --flash-mode, etc to be passed in - cmd = " ".join(flasher_args["write_flash_args"]) + " " - - cmd += flasher_args[key]["offset"] + " " - cmd += flasher_path(flasher_args[key]["file"]) - else: # flashing the whole project - cmd = " ".join(flasher_args["write_flash_args"]) + " " - flash_items = sorted(((o,f) for (o,f) in flasher_args["flash_files"].items() if len(o) > 0), - key = lambda x: int(x[0], 0)) - for o,f in flash_items: - cmd += o + " " + flasher_path(f) + " " - - print("%s -p %s -b %s write_flash %s" % ( - os.path.relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]), - args.port or "(PORT)", - args.baud, - cmd.strip())) - print("or run 'idf.py -p %s %s'" % (args.port or "(PORT)", key + "-flash" if key != "project" else "flash",)) - - if "all" in args.actions or "build" in args.actions: - print_flashing_message("Project", "project") - else: - if "app" in args.actions: - print_flashing_message("App", "app") - if "partition_table" in args.actions: - print_flashing_message("Partition Table", "partition_table") - if "bootloader" in args.actions: - print_flashing_message("Bootloader", "bootloader") - -ACTIONS = { - # action name : ( function (or alias), dependencies, order-only dependencies ) - "all" : ( build_target, [], [ "reconfigure", "menuconfig", "clean", "fullclean" ] ), - "build": ( "all", [], [] ), # build is same as 'all' target - "clean": ( clean, [], [ "fullclean" ] ), - "fullclean": ( fullclean, [], [] ), - "reconfigure": ( reconfigure, [], [ "menuconfig" ] ), - "menuconfig": ( build_target, [], [] ), - "defconfig": ( build_target, [], [] ), - "confserver": ( build_target, [], [] ), - "size": ( build_target, [ "app" ], [] ), - "size-components": ( build_target, [ "app" ], [] ), - "size-files": ( build_target, [ "app" ], [] ), - "bootloader": ( build_target, [], [] ), - "bootloader-clean": ( build_target, [], [] ), - "bootloader-flash": ( flash, [ "bootloader" ], [ "erase_flash"] ), - "app": ( build_target, [], [ "clean", "fullclean", "reconfigure" ] ), - "app-flash": ( flash, [ "app" ], [ "erase_flash"]), - "partition_table": ( build_target, [], [ "reconfigure" ] ), - "partition_table-flash": ( flash, [ "partition_table" ], [ "erase_flash" ]), - "flash": ( flash, [ "all" ], [ "erase_flash" ] ), - "erase_flash": ( erase_flash, [], []), - "monitor": ( monitor, [], [ "flash", "partition_table-flash", "bootloader-flash", "app-flash" ]), -} - -def get_commandline_options(): - """ Return all the command line options up to but not including the action """ +def get_commandline_options(ctx): + """ Return all the command line options up to first action """ + # This approach ignores argument parsing done Click result = [] - for a in sys.argv: - if a in ACTIONS.keys(): + + for arg in sys.argv: + if arg in ctx.command.commands_with_aliases: break - else: - result.append(a) + + result.append(arg) + return result + def get_default_serial_port(): """ Return a default serial port. esptool can do this (smarter), but it can create inconsistencies where esptool.py uses one port and idf_monitor uses another. @@ -421,102 +512,766 @@ def get_default_serial_port(): # Import is done here in order to move it after the check_environment() ensured that pyserial has been installed import serial.tools.list_ports - ports = list(reversed(sorted( - p.device for p in serial.tools.list_ports.comports() ))) + ports = list(reversed(sorted(p.device for p in serial.tools.list_ports.comports()))) try: - print ("Choosing default port %s (use '-p PORT' option to set a specific serial port)" % ports[0]) + print( + "Choosing default port %s (use '-p PORT' option to set a specific serial port)" + % ports[0].encode("ascii", "ignore") + ) return ports[0] except IndexError: - raise RuntimeError("No serial ports found. Connect a device, or use '-p PORT' option to set a specific port.") + raise RuntimeError( + "No serial ports found. Connect a device, or use '-p PORT' option to set a specific port." + ) -# Import the actions, arguments extension file -if os.path.exists(os.path.join(os.getcwd(), "idf_ext.py")): - sys.path.append(os.getcwd()) - try: - from idf_ext import add_action_extensions, add_argument_extensions - except ImportError as e: - print("Error importing extension file idf_ext.py. Skipping.") - print("Please make sure that it contains implementations (even if they're empty implementations) of") - print("add_action_extensions and add_argument_extensions.") -def main(): - if sys.version_info[0] != 2 or sys.version_info[1] != 7: - print("Note: You are using Python %d.%d.%d. Python 3 support is new, please report any problems " - "you encounter. Search for 'Setting the Python Interpreter' in the ESP-IDF docs if you want to use " - "Python 2.7." % sys.version_info[:3]) +class PropertyDict(dict): + def __init__(self, *args, **kwargs): + super(PropertyDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + +def init_cli(): + # Click is imported here to run it after check_environment() + import click + + class Task(object): + def __init__( + self, callback, name, aliases, dependencies, order_dependencies, action_args + ): + self.callback = callback + self.name = name + self.dependencies = dependencies + self.order_dependencies = order_dependencies + self.action_args = action_args + self.aliases = aliases + + def run(self, context, global_args, action_args=None): + if action_args is None: + action_args = self.action_args + + self.callback(self.name, context, global_args, **action_args) + + class Action(click.Command): + def __init__( + self, + name=None, + aliases=None, + dependencies=None, + order_dependencies=None, + **kwargs + ): + super(Action, self).__init__(name, **kwargs) + + self.name = self.name or self.callback.__name__ + + if aliases is None: + aliases = [] + self.aliases = aliases + + self.help = self.help or self.callback.__doc__ + if self.help is None: + self.help = "" + + if dependencies is None: + dependencies = [] + + if order_dependencies is None: + order_dependencies = [] + + # Show first line of help if short help is missing + self.short_help = self.short_help or self.help.split("\n")[0] + + # Add aliases to help string + if aliases: + aliases_help = "Aliases: %s." % ", ".join(aliases) + + self.help = "\n".join([self.help, aliases_help]) + self.short_help = " ".join([aliases_help, self.short_help]) + + if self.callback is not None: + callback = self.callback + + def wrapped_callback(**action_args): + return Task( + callback=callback, + name=self.name, + dependencies=dependencies, + order_dependencies=order_dependencies, + action_args=action_args, + aliases=self.aliases, + ) + + self.callback = wrapped_callback + + class Argument(click.Argument): + """Positional argument""" + + def __init__(self, **kwargs): + names = kwargs.pop("names") + super(Argument, self).__init__(names, **kwargs) + + class Scope(object): + """ + Scope for sub-command option. + possible values: + - default - only available on defined level (global/action) + - global - When defined for action, also available as global + - shared - Opposite to 'global': when defined in global scope, also available for all actions + """ + + SCOPES = ("default", "global", "shared") + + def __init__(self, scope=None): + if scope is None: + self._scope = "default" + elif isinstance(scope, str) and scope in self.SCOPES: + self._scope = scope + elif isinstance(scope, Scope): + self._scope = str(scope) + else: + raise FatalError("Unknown scope for option: %s" % scope) + + @property + def is_global(self): + return self._scope == "global" + + @property + def is_shared(self): + return self._scope == "shared" + + def __str__(self): + return self._scope + + class Option(click.Option): + """Option that knows whether it should be global""" + + def __init__(self, scope=None, **kwargs): + kwargs["param_decls"] = kwargs.pop("names") + super(Option, self).__init__(**kwargs) + + self.scope = Scope(scope) + + if self.scope.is_global: + self.help += " This option can be used at most once either globally, or for one subcommand." + + class CLI(click.MultiCommand): + """Action list contains all actions with options available for CLI""" + + def __init__(self, action_lists=None, help=None): + super(CLI, self).__init__( + chain=True, + invoke_without_command=True, + result_callback=self.execute_tasks, + context_settings={"max_content_width": 140}, + help=help, + ) + self._actions = {} + self.global_action_callbacks = [] + self.commands_with_aliases = {} + + if action_lists is None: + action_lists = [] + + shared_options = [] + + for action_list in action_lists: + # Global options + for option_args in action_list.get("global_options", []): + option = Option(**option_args) + self.params.append(option) + + if option.scope.is_shared: + shared_options.append(option) + + for action_list in action_lists: + # Global options validators + self.global_action_callbacks.extend( + action_list.get("global_action_callbacks", []) + ) + + for action_list in action_lists: + # Actions + for name, action in action_list.get("actions", {}).items(): + arguments = action.pop("arguments", []) + options = action.pop("options", []) + + if arguments is None: + arguments = [] + + if options is None: + options = [] + + self._actions[name] = Action(name=name, **action) + for alias in [name] + action.get("aliases", []): + self.commands_with_aliases[alias] = name + + for argument_args in arguments: + self._actions[name].params.append(Argument(**argument_args)) + + # Add all shared options + for option in shared_options: + self._actions[name].params.append(option) + + for option_args in options: + option = Option(**option_args) + + if option.scope.is_shared: + raise FatalError( + '"%s" is defined for action "%s". ' + ' "shared" options can be declared only on global level' % (option.name, name) + ) + + # Promote options to global if see for the first time + if option.scope.is_global and option.name not in [o.name for o in self.params]: + self.params.append(option) + + self._actions[name].params.append(option) + + def list_commands(self, ctx): + return sorted(self._actions) + + def get_command(self, ctx, name): + return self._actions.get(self.commands_with_aliases.get(name)) + + def _print_closing_message(self, args, actions): + # print a closing message of some kind + # + if "flash" in str(actions): + print("Done") + return + + # Otherwise, if we built any binaries print a message about + # how to flash them + def print_flashing_message(title, key): + print("\n%s build complete. To flash, run this command:" % title) + + with open(os.path.join(args.build_dir, "flasher_args.json")) as f: + flasher_args = json.load(f) + + def flasher_path(f): + return _safe_relpath(os.path.join(args.build_dir, f)) + + if key != "project": # flashing a single item + cmd = "" + if ( + key == "bootloader" + ): # bootloader needs --flash-mode, etc to be passed in + cmd = " ".join(flasher_args["write_flash_args"]) + " " + + cmd += flasher_args[key]["offset"] + " " + cmd += flasher_path(flasher_args[key]["file"]) + else: # flashing the whole project + cmd = " ".join(flasher_args["write_flash_args"]) + " " + flash_items = sorted( + ( + (o, f) + for (o, f) in flasher_args["flash_files"].items() + if len(o) > 0 + ), + key=lambda x: int(x[0], 0), + ) + for o, f in flash_items: + cmd += o + " " + flasher_path(f) + " " + + print( + "%s -p %s -b %s --after %s write_flash %s" + % ( + _safe_relpath( + "%s/components/esptool_py/esptool/esptool.py" + % os.environ["IDF_PATH"] + ), + args.port or "(PORT)", + args.baud, + flasher_args["extra_esptool_args"]["after"], + cmd.strip(), + ) + ) + print( + "or run 'idf.py -p %s %s'" + % ( + args.port or "(PORT)", + key + "-flash" if key != "project" else "flash", + ) + ) + + if "all" in actions or "build" in actions: + print_flashing_message("Project", "project") + else: + if "app" in actions: + print_flashing_message("App", "app") + if "partition_table" in actions: + print_flashing_message("Partition Table", "partition_table") + if "bootloader" in actions: + print_flashing_message("Bootloader", "bootloader") + + def execute_tasks(self, tasks, **kwargs): + ctx = click.get_current_context() + global_args = PropertyDict(ctx.params) + + # Set propagated global options + for task in tasks: + for key in list(task.action_args): + option = next((o for o in ctx.command.params if o.name == key), None) + if option and (option.scope.is_global or option.scope.is_shared): + local_value = task.action_args.pop(key) + global_value = global_args[key] + default = () if option.multiple else option.default + + if global_value != default and local_value != default and global_value != local_value: + raise FatalError( + 'Option "%s" provided for "%s" is already defined to a different value. ' + "This option can appear at most once in the command line." % (key, task.name) + ) + if local_value != default: + global_args[key] = local_value + + # Validate global arguments + for action_callback in ctx.command.global_action_callbacks: + action_callback(ctx, global_args, tasks) + + # very simple dependency management + completed_tasks = set() + + if not tasks: + print(ctx.get_help()) + ctx.exit() + + while tasks: + task = tasks[0] + tasks_dict = dict([(t.name, t) for t in tasks]) + + name_with_aliases = task.name + if task.aliases: + name_with_aliases += " (aliases: %s)" % ", ".join(task.aliases) + + ready_to_run = True + for dep in task.dependencies: + if dep not in completed_tasks: + print( + 'Adding %s\'s dependency "%s" to list of actions' + % (task.name, dep) + ) + dep_task = ctx.invoke(ctx.command.get_command(ctx, dep)) + + # Remove global options from dependent tasks + for key in list(dep_task.action_args): + option = next((o for o in ctx.command.params if o.name == key), None) + if option and (option.scope.is_global or option.scope.is_shared): + dep_task.action_args.pop(key) + + tasks.insert(0, dep_task) + ready_to_run = False + + for dep in task.order_dependencies: + if dep in tasks_dict.keys() and dep not in completed_tasks: + tasks.insert(0, tasks.pop(tasks.index(tasks_dict[dep]))) + ready_to_run = False + + if ready_to_run: + tasks.pop(0) + + if task.name in completed_tasks: + print( + "Skipping action that is already done: %s" + % name_with_aliases + ) + else: + print("Executing action: %s" % name_with_aliases) + task.run(ctx, global_args, task.action_args) + + completed_tasks.add(task.name) + + self._print_closing_message(global_args, completed_tasks) + + @staticmethod + def merge_action_lists(*action_lists): + merged_actions = { + "global_options": [], + "actions": {}, + "global_action_callbacks": [], + } + for action_list in action_lists: + merged_actions["global_options"].extend( + action_list.get("global_options", []) + ) + merged_actions["actions"].update(action_list.get("actions", {})) + merged_actions["global_action_callbacks"].extend( + action_list.get("global_action_callbacks", []) + ) + return merged_actions + + # That's a tiny parser that parse project-dir even before constructing + # fully featured click parser to be sure that extensions are loaded from the right place + @click.command( + add_help_option=False, + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, + ) + @click.option("-C", "--project-dir", default=os.getcwd()) + def parse_project_dir(project_dir): + return _realpath(project_dir) + + project_dir = parse_project_dir(standalone_mode=False) + + # Load base idf commands + def validate_root_options(ctx, args, tasks): + args.project_dir = _realpath(args.project_dir) + if args.build_dir is not None and args.project_dir == _realpath(args.build_dir): + raise FatalError( + "Setting the build directory to the project directory is not supported. Suggest dropping " + "--build-dir option, the default is a 'build' subdirectory inside the project directory." + ) + if args.build_dir is None: + args.build_dir = os.path.join(args.project_dir, "build") + args.build_dir = _realpath(args.build_dir) + + # Possible keys for action dict are: global_options, actions and global_action_callbacks + global_options = [ + { + "names": ["-D", "--define-cache-entry"], + "help": "Create a cmake cache entry.", + "scope": "global", + "multiple": True, + } + ] + + root_options = { + "global_options": [ + { + "names": ["-C", "--project-dir"], + "help": "Project directory.", + "type": click.Path(), + "default": os.getcwd(), + }, + { + "names": ["-B", "--build-dir"], + "help": "Build directory.", + "type": click.Path(), + "default": None, + }, + { + "names": ["-n", "--no-warnings"], + "help": "Disable Cmake warnings.", + "is_flag": True, + "default": False, + }, + { + "names": ["-v", "--verbose"], + "help": "Verbose build output.", + "is_flag": True, + "default": False, + }, + { + "names": ["--ccache/--no-ccache"], + "help": "Use ccache in build. Disabled by default.", + "is_flag": True, + "default": False, + }, + { + "names": ["-G", "--generator"], + "help": "CMake generator.", + "type": click.Choice(GENERATOR_CMDS.keys()), + }, + ], + "global_action_callbacks": [validate_root_options], + } + + build_actions = { + "actions": { + "all": { + "aliases": ["build"], + "callback": build_target, + "short_help": "Build the project.", + "help": "Build the project. This can involve multiple steps:\n\n" + + "1. Create the build directory if needed. The sub-directory 'build' is used to hold build output, " + + "although this can be changed with the -B option.\n\n" + + "2. Run CMake as necessary to configure the project and generate build files for the main build tool.\n\n" + + "3. Run the main build tool (Ninja or GNU Make). By default, the build tool is automatically detected " + + "but it can be explicitly set by passing the -G option to idf.py.\n\n", + "options": global_options, + "order_dependencies": [ + "reconfigure", + "menuconfig", + "clean", + "fullclean", + ], + }, + "menuconfig": { + "callback": build_target, + "help": 'Run "menuconfig" project configuration tool.', + "options": global_options, + }, + "confserver": { + "callback": build_target, + "help": "Run JSON configuration server.", + "options": global_options, + }, + "size": { + "callback": build_target, + "help": "Print basic size information about the app.", + "options": global_options, + "dependencies": ["app"], + }, + "size-components": { + "callback": build_target, + "help": "Print per-component size information.", + "options": global_options, + "dependencies": ["app"], + }, + "size-files": { + "callback": build_target, + "help": "Print per-source-file size information.", + "options": global_options, + "dependencies": ["app"], + }, + "bootloader": { + "callback": build_target, + "help": "Build only bootloader.", + "options": global_options, + }, + "app": { + "callback": build_target, + "help": "Build only the app.", + "order_dependencies": ["clean", "fullclean", "reconfigure"], + "options": global_options, + }, + "efuse_common_table": { + "callback": build_target, + "help": "Genereate C-source for IDF's eFuse fields.", + "order_dependencies": ["reconfigure"], + "options": global_options, + }, + "efuse_custom_table": { + "callback": build_target, + "help": "Genereate C-source for user's eFuse fields.", + "order_dependencies": ["reconfigure"], + "options": global_options, + }, + "show_efuse_table": { + "callback": build_target, + "help": "Print eFuse table.", + "order_dependencies": ["reconfigure"], + "options": global_options, + }, + "partition_table": { + "callback": build_target, + "help": "Build only partition table.", + "order_dependencies": ["reconfigure"], + "options": global_options, + }, + "erase_otadata": { + "callback": build_target, + "help": "Erase otadata partition.", + "options": global_options, + }, + "read_otadata": { + "callback": build_target, + "help": "Read otadata partition.", + "options": global_options, + }, + } + } + + clean_actions = { + "actions": { + "reconfigure": { + "callback": reconfigure, + "short_help": "Re-run CMake.", + "help": "Re-run CMake even if it doesn't seem to need re-running. This isn't necessary during normal usage, " + + "but can be useful after adding/removing files from the source tree, or when modifying CMake cache variables. " + + "For example, \"idf.py -DNAME='VALUE' reconfigure\" " + + 'can be used to set variable "NAME" in CMake cache to value "VALUE".', + "options": global_options, + "order_dependencies": ["menuconfig"], + }, + "clean": { + "callback": clean, + "short_help": "Delete build output files from the build directory.", + "help": "Delete build output files from the build directory , forcing a 'full rebuild' the next time " + + "the project is built. Cleaning doesn't delete CMake configuration output and some other files", + "order_dependencies": ["fullclean"], + }, + "fullclean": { + "callback": fullclean, + "short_help": "Delete the entire build directory contents.", + "help": "Delete the entire build directory contents. This includes all CMake configuration output." + + "The next time the project is built, CMake will configure it from scratch. " + + "Note that this option recursively deletes all files in the build directory, so use with care." + + "Project configuration is not deleted.", + }, + } + } + + baud_rate = { + "names": ["-b", "--baud"], + "help": "Baud rate.", + "scope": "global", + "envvar": "ESPBAUD", + "default": 460800, + } + + port = { + "names": ["-p", "--port"], + "help": "Serial port.", + "scope": "global", + "envvar": "ESPPORT", + "default": None, + } + + serial_actions = { + "actions": { + "flash": { + "callback": flash, + "help": "Flash the project.", + "options": global_options + [baud_rate, port], + "dependencies": ["all"], + "order_dependencies": ["erase_flash"], + }, + "erase_flash": { + "callback": erase_flash, + "help": "Erase entire flash chip.", + "options": [baud_rate, port], + }, + "monitor": { + "callback": monitor, + "help": "Display serial output.", + "options": [ + port, + { + "names": ["--print-filter", "--print_filter"], + "help": ( + "Filter monitor output.\n" + "Restrictions on what to print can be specified as a series of : items " + "where is the tag string and is a character from the set " + "{N, E, W, I, D, V, *} referring to a level. " + 'For example, "tag1:W" matches and prints only the outputs written with ' + 'ESP_LOGW("tag1", ...) or at lower verbosity level, i.e. ESP_LOGE("tag1", ...). ' + 'Not specifying a or using "*" defaults to Verbose level.\n' + 'Please see the IDF Monitor section of the ESP-IDF documentation ' + 'for a more detailed description and further examples.'), + "default": None, + }, + ], + "order_dependencies": [ + "flash", + "partition_table-flash", + "bootloader-flash", + "app-flash", + ], + }, + "partition_table-flash": { + "callback": flash, + "help": "Flash partition table only.", + "options": [baud_rate, port], + "dependencies": ["partition_table"], + "order_dependencies": ["erase_flash"], + }, + "bootloader-flash": { + "callback": flash, + "help": "Flash bootloader only.", + "options": [baud_rate, port], + "dependencies": ["bootloader"], + "order_dependencies": ["erase_flash"], + }, + "app-flash": { + "callback": flash, + "help": "Flash the app only.", + "options": [baud_rate, port], + "dependencies": ["app"], + "order_dependencies": ["erase_flash"], + }, + "encrypted-app-flash": { + "callback": flash, + "help": "Flash the encrypted app only.", + "dependencies": ["app"], + "order_dependencies": ["erase_flash"], + }, + "encrypted-flash": { + "callback": flash, + "help": "Flash the encrypted project.", + "dependencies": ["all"], + "order_dependencies": ["erase_flash"], + }, + }, + } + + base_actions = CLI.merge_action_lists( + root_options, build_actions, clean_actions, serial_actions + ) + all_actions = [base_actions] + + # Load extensions + if os.path.exists(os.path.join(project_dir, "idf_ext.py")): + sys.path.append(project_dir) + try: + from idf_ext import action_extensions + except ImportError: + print("Error importing extension file idf_ext.py. Skipping.") + print( + "Please make sure that it contains implementation (even if it's empty) of add_action_extensions" + ) # Add actions extensions try: - add_action_extensions({ - "build_target": build_target, - "reconfigure" : reconfigure, - "flash" : flash, - "monitor" : monitor, - "clean" : clean, - "fullclean" : fullclean - }, ACTIONS) + all_actions.append(action_extensions(base_actions, project_dir)) except NameError: pass - parser = argparse.ArgumentParser(description='ESP-IDF build management tool') - parser.add_argument('-p', '--port', help="Serial port", - default=os.environ.get('ESPPORT', None)) - parser.add_argument('-b', '--baud', help="Baud rate", - default=os.environ.get('ESPBAUD', 460800)) - parser.add_argument('-C', '--project-dir', help="Project directory", default=os.getcwd()) - parser.add_argument('-B', '--build-dir', help="Build directory", default=None) - parser.add_argument('-G', '--generator', help="Cmake generator", choices=GENERATOR_CMDS.keys()) - parser.add_argument('-n', '--no-warnings', help="Disable Cmake warnings", action="store_true") - parser.add_argument('-v', '--verbose', help="Verbose build output", action="store_true") - parser.add_argument('-D', '--define-cache-entry', help="Create a cmake cache entry", nargs='+') - parser.add_argument('--no-ccache', help="Disable ccache. Otherwise, if ccache is available on the PATH then it will be used for faster builds.", action="store_true") - parser.add_argument('actions', help="Actions (build targets or other operations)", nargs='+', - choices=ACTIONS.keys()) + return CLI(help="ESP-IDF build management", action_lists=all_actions) - # Add arguments extensions - try: - add_argument_extensions(parser) - except NameError: - pass - - args = parser.parse_args() +def main(): check_environment() + cli = init_cli() + cli(prog_name=PROG) - # Advanced parameter checks - if args.build_dir is not None and os.path.realpath(args.project_dir) == os.path.realpath(args.build_dir): - raise FatalError("Setting the build directory to the project directory is not supported. Suggest dropping --build-dir option, the default is a 'build' subdirectory inside the project directory.") - if args.build_dir is None: - args.build_dir = os.path.join(args.project_dir, "build") - args.build_dir = os.path.realpath(args.build_dir) - completed_actions = set() - def execute_action(action, remaining_actions): - ( function, dependencies, order_dependencies ) = ACTIONS[action] - # very simple dependency management, build a set of completed actions and make sure - # all dependencies are in it - for dep in dependencies: - if not dep in completed_actions: - execute_action(dep, remaining_actions) - for dep in order_dependencies: - if dep in remaining_actions and not dep in completed_actions: - execute_action(dep, remaining_actions) +def _valid_unicode_config(): + # Python 2 is always good + if sys.version_info[0] == 2: + return True - if action in completed_actions: - pass # we've already done this, don't do it twice... - elif function in ACTIONS: # alias of another action - execute_action(function, remaining_actions) - else: - function(action, args) + # With python 3 unicode environment is required + try: + return codecs.lookup(locale.getpreferredencoding()).name != "ascii" + except Exception: + return False - completed_actions.add(action) - actions = list(args.actions) - while len(actions) > 0: - execute_action(actions[0], actions[1:]) - actions.pop(0) +def _find_usable_locale(): + try: + locales = subprocess.Popen( + ["locale", "-a"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ).communicate()[0] + except OSError: + locales = "" + if isinstance(locales, bytes): + locales = locales.decode("ascii", "replace") + + usable_locales = [] + for line in locales.splitlines(): + locale = line.strip() + locale_name = locale.lower().replace("-", "") + + # C.UTF-8 is the best option, if supported + if locale_name == "c.utf8": + return locale + + if locale_name.endswith(".utf8"): + # Make a preference of english locales + if locale.startswith("en_"): + usable_locales.insert(0, locale) + else: + usable_locales.append(locale) + + if not usable_locales: + raise FatalError( + "Support for Unicode filenames is required, but no suitable UTF-8 locale was found on your system." + " Please refer to the manual for your operating system for details on locale reconfiguration." + ) + + return usable_locales[0] - print_closing_message(args) if __name__ == "__main__": try: @@ -524,18 +1279,37 @@ if __name__ == "__main__": # keyboard interrupt (CTRL+C). # Using an own global variable for indicating that we are running with "winpty" seems to be the most suitable # option as os.environment['_'] contains "winpty" only when it is run manually from console. - WINPTY_VAR = 'WINPTY' - WINPTY_EXE = 'winpty' - if ('MSYSTEM' in os.environ) and (not os.environ['_'].endswith(WINPTY_EXE) and WINPTY_VAR not in os.environ): - os.environ[WINPTY_VAR] = '1' # the value is of no interest to us + WINPTY_VAR = "WINPTY" + WINPTY_EXE = "winpty" + if ("MSYSTEM" in os.environ) and ( + not os.environ.get("_", "").endswith(WINPTY_EXE) and WINPTY_VAR not in os.environ + ): + os.environ[WINPTY_VAR] = "1" # the value is of no interest to us # idf.py calls itself with "winpty" and WINPTY global variable set - ret = subprocess.call([WINPTY_EXE, sys.executable] + sys.argv, env=os.environ) + ret = subprocess.call( + [WINPTY_EXE, sys.executable] + sys.argv, env=os.environ + ) if ret: raise SystemExit(ret) + + elif os.name == "posix" and not _valid_unicode_config(): + # Trying to find best utf-8 locale available on the system and restart python with it + best_locale = _find_usable_locale() + + print( + "Your environment is not configured to handle unicode filenames outside of ASCII range." + " Environment variable LC_ALL is temporary set to %s for unicode support." + % best_locale + ) + + os.environ["LC_ALL"] = best_locale + ret = subprocess.call([sys.executable] + sys.argv, env=os.environ) + if ret: + raise SystemExit(ret) + else: main() + except FatalError as e: print(e) sys.exit(2) - - diff --git a/tools/idf_monitor.py b/tools/idf_monitor.py index 86b53538..f6802061 100755 --- a/tools/idf_monitor.py +++ b/tools/idf_monitor.py @@ -3,8 +3,8 @@ # esp-idf serial output monitor tool. Does some helpful things: # - Looks up hex addresses in ELF file with addr2line # - Reset ESP32 via serial RTS line (Ctrl-T Ctrl-R) -# - Run "make flash" (Ctrl-T Ctrl-F) -# - Run "make app-flash" (Ctrl-T Ctrl-A) +# - Run flash build target to rebuild and flash entire project (Ctrl-T Ctrl-F) +# - Run app-flash build target to rebuild and flash app only (Ctrl-T Ctrl-A) # - If gdbstub output is detected, gdb is automatically loaded # # Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD @@ -28,17 +28,22 @@ # Originally released under BSD-3-Clause license. # from __future__ import print_function, division +from __future__ import unicode_literals +from builtins import chr +from builtins import object +from builtins import bytes import subprocess import argparse import codecs +import datetime import re import os try: import queue except ImportError: import Queue as queue +import shlex import time -import datetime import sys import serial import serial.tools.miniterm as miniterm @@ -46,6 +51,7 @@ import threading import ctypes import types from distutils.version import StrictVersion +from io import open key_description = miniterm.key_description @@ -56,60 +62,44 @@ CTRL_F = '\x06' CTRL_H = '\x08' CTRL_R = '\x12' CTRL_T = '\x14' +CTRL_Y = '\x19' +CTRL_P = '\x10' +CTRL_L = '\x0c' CTRL_RBRACKET = '\x1d' # Ctrl+] -# ANSI terminal codes +# ANSI terminal codes (if changed, regular expressions in LineMatcher need to be udpated) ANSI_RED = '\033[1;31m' ANSI_YELLOW = '\033[0;33m' ANSI_NORMAL = '\033[0m' + def color_print(message, color): """ Print a message to stderr with colored highlighting """ sys.stderr.write("%s%s%s\n" % (color, message, ANSI_NORMAL)) + def yellow_print(message): color_print(message, ANSI_YELLOW) + def red_print(message): color_print(message, ANSI_RED) -__version__ = "1.0" -def mkdir(path): - path=path.strip() - path=path.rstrip("\\") - isExists=os.path.exists(path) - - if not isExists: - os.makedirs(path) - -def esp_openlog(project_name): - logdir = os.getcwd() + '/esplog' - mkdir(logdir) - filename = project_name + datetime.datetime.now().strftime('%Y%m%d%H%M%S.log') - return open(logdir + '/' + filename, 'w+') +__version__ = "1.1" # Tags for tuples in queues TAG_KEY = 0 TAG_SERIAL = 1 +TAG_SERIAL_FLUSH = 2 # regex matches an potential PC value (0x4xxxxxxx) MATCH_PCADDR = re.compile(r'0x4[0-9a-f]{7}', re.IGNORECASE) DEFAULT_TOOLCHAIN_PREFIX = "xtensa-esp32-elf-" -def is_ascii(b): - if b > '\x7f': - return False - return True +DEFAULT_PRINT_FILTER = "" -def get_time_stamp(): - ct = time.time() - local_time = time.localtime(ct) - data_head = time.strftime("%Y-%m-%d %H:%M:%S", local_time) - data_secs = (ct - long(ct)) * 1000 - time_stamp = "%s.%03d" % (data_head, data_secs) - return time_stamp class StoppableThread(object): """ @@ -136,10 +126,10 @@ class StoppableThread(object): self._thread.start() def _cancel(self): - pass # override to provide cancellation functionality + pass # override to provide cancellation functionality def run(self): - pass # override for the main thread behaviour + pass # override for the main thread behaviour def _run_outer(self): try: @@ -154,14 +144,16 @@ class StoppableThread(object): self._cancel() old_thread.join() + class ConsoleReader(StoppableThread): """ Read input keys from the console and push them to the queue, until stopped. """ - def __init__(self, console, event_queue): + def __init__(self, console, event_queue, test_mode): super(ConsoleReader, self).__init__() self.console = console self.event_queue = event_queue + self.test_mode = test_mode def run(self): self.console.setup() @@ -178,6 +170,13 @@ class ConsoleReader(StoppableThread): time.sleep(0.1) if not self.alive: break + elif self.test_mode: + # In testing mode the stdin is connected to PTY but is not used for input anything. For PTY + # the canceling by fcntl.ioctl isn't working and would hang in self.console.getkey(). + # Therefore, we avoid calling it. + while self.alive: + time.sleep(0.1) + break c = self.console.getkey() except KeyboardInterrupt: c = '\x03' @@ -187,20 +186,24 @@ class ConsoleReader(StoppableThread): self.console.cleanup() def _cancel(self): - if os.name == 'posix': + if os.name == 'posix' and not self.test_mode: # this is the way cancel() is implemented in pyserial 3.3 or newer, # older pyserial (3.1+) has cancellation implemented via 'select', # which does not work when console sends an escape sequence response - # + # # even older pyserial (<3.1) does not have this method # # on Windows there is a different (also hacky) fix, applied above. # # note that TIOCSTI is not implemented in WSL / bash-on-Windows. # TODO: introduce some workaround to make it work there. - import fcntl, termios + # + # Note: This would throw exception in testing mode when the stdin is connected to PTY. + import fcntl + import termios fcntl.ioctl(self.console.fd, termios.TIOCSTI, b'\0') + class SerialReader(StoppableThread): """ Read serial data from the serial port and push to the event queue, until stopped. @@ -233,10 +236,71 @@ class SerialReader(StoppableThread): if hasattr(self.serial, 'cancel_read'): try: self.serial.cancel_read() - except: + except Exception: pass +class LineMatcher(object): + """ + Assembles a dictionary of filtering rules based on the --print_filter + argument of idf_monitor. Then later it is used to match lines and + determine whether they should be shown on screen or not. + """ + LEVEL_N = 0 + LEVEL_E = 1 + LEVEL_W = 2 + LEVEL_I = 3 + LEVEL_D = 4 + LEVEL_V = 5 + + level = {'N': LEVEL_N, 'E': LEVEL_E, 'W': LEVEL_W, 'I': LEVEL_I, 'D': LEVEL_D, + 'V': LEVEL_V, '*': LEVEL_V, '': LEVEL_V} + + def __init__(self, print_filter): + self._dict = dict() + self._re = re.compile(r'^(?:\033\[[01];?[0-9]+m?)?([EWIDV]) \([0-9]+\) ([^:]+): ') + items = print_filter.split() + if len(items) == 0: + self._dict["*"] = self.LEVEL_V # default is to print everything + for f in items: + s = f.split(r':') + if len(s) == 1: + # specifying no warning level defaults to verbose level + lev = self.LEVEL_V + elif len(s) == 2: + if len(s[0]) == 0: + raise ValueError('No tag specified in filter ' + f) + try: + lev = self.level[s[1].upper()] + except KeyError: + raise ValueError('Unknown warning level in filter ' + f) + else: + raise ValueError('Missing ":" in filter ' + f) + self._dict[s[0]] = lev + + def match(self, line): + try: + m = self._re.search(line) + if m: + lev = self.level[m.group(1)] + if m.group(2) in self._dict: + return self._dict[m.group(2)] >= lev + return self._dict.get("*", self.LEVEL_N) >= lev + except (KeyError, IndexError): + # Regular line written with something else than ESP_LOG* + # or an empty line. + pass + # We need something more than "*.N" for printing. + return self._dict.get("*", self.LEVEL_N) > self.LEVEL_N + + +class SerialStopException(Exception): + """ + This exception is used for stopping the IDF monitor in testing mode. + """ + pass + + class Monitor(object): """ Monitor application main class. @@ -246,12 +310,12 @@ class Monitor(object): Main difference is that all event processing happens in the main thread, not the worker threads. """ - def __init__(self, serial_instance, elf_file, make="make", toolchain_prefix=DEFAULT_TOOLCHAIN_PREFIX, eol="CRLF", enable_time='n', enable_savelog='n'): + def __init__(self, serial_instance, elf_file, print_filter, make="make", toolchain_prefix=DEFAULT_TOOLCHAIN_PREFIX, eol="CRLF"): super(Monitor, self).__init__() self.event_queue = queue.Queue() self.console = miniterm.Console() if os.name == 'nt': - sys.stderr = ANSIColorConverter(sys.stderr) + sys.stderr = ANSIColorConverter(sys.stderr, decode_output=True) self.console.output = ANSIColorConverter(self.console.output) self.console.byte_output = ANSIColorConverter(self.console.byte_output) @@ -259,36 +323,45 @@ class Monitor(object): # Use Console.getkey implementation from 3.3.0 (to be in sync with the ConsoleReader._cancel patch above) def getkey_patched(self): c = self.enc_stdin.read(1) - if c == unichr(0x7f): - c = unichr(8) # map the BS key (which yields DEL) to backspace + if c == chr(0x7f): + c = chr(8) # map the BS key (which yields DEL) to backspace return c - - self.console.getkey = types.MethodType(getkey_patched, self.console) - + + self.console.getkey = types.MethodType(getkey_patched, self.console) + + socket_mode = serial_instance.port.startswith("socket://") # testing hook - data from serial can make exit the monitor self.serial = serial_instance - self.console_reader = ConsoleReader(self.console, self.event_queue) + self.console_reader = ConsoleReader(self.console, self.event_queue, socket_mode) self.serial_reader = SerialReader(self.serial, self.event_queue) self.elf_file = elf_file - self.make = make + if not os.path.exists(make): + self.make = shlex.split(make) # allow for possibility the "make" arg is a list of arguments (for idf.py) + else: + self.make = make self.toolchain_prefix = toolchain_prefix self.menu_key = CTRL_T self.exit_key = CTRL_RBRACKET - self.enable_time = enable_time - self.enable_savelog = enable_savelog - self.next_line = True - if self.enable_savelog == 'y': - self.log_file = esp_openlog(os.path.splitext(os.path.basename(self.elf_file))[0]) self.translate_eol = { - "CRLF": lambda c: c.replace(b"\n", b"\r\n"), - "CR": lambda c: c.replace(b"\n", b"\r"), - "LF": lambda c: c.replace(b"\r", b"\n"), + "CRLF": lambda c: c.replace("\n", "\r\n"), + "CR": lambda c: c.replace("\n", "\r"), + "LF": lambda c: c.replace("\r", "\n"), }[eol] # internal state self._pressed_menu_key = False - self._read_line = b"" + self._last_line_part = b"" self._gdb_buffer = b"" + self._pc_address_buffer = b"" + self._line_matcher = LineMatcher(print_filter) + self._invoke_processing_last_line_timer = None + self._force_line_print = False + self._output_enabled = True + self._serial_check_exit = socket_mode + self._log_file = None + + def invoke_processing_last_line(self): + self.event_queue.put((TAG_SERIAL_FLUSH, b''), False) def main_loop(self): self.console_reader.start() @@ -300,15 +373,30 @@ class Monitor(object): self.handle_key(data) elif event_tag == TAG_SERIAL: self.handle_serial_input(data) + if self._invoke_processing_last_line_timer is not None: + self._invoke_processing_last_line_timer.cancel() + self._invoke_processing_last_line_timer = threading.Timer(0.1, self.invoke_processing_last_line) + self._invoke_processing_last_line_timer.start() + # If no futher data is received in the next short period + # of time then the _invoke_processing_last_line_timer + # generates an event which will result in the finishing of + # the last line. This is fix for handling lines sent + # without EOL. + elif event_tag == TAG_SERIAL_FLUSH: + self.handle_serial_input(data, finalize_line=True) else: raise RuntimeError("Bad event data %r" % ((event_tag,data),)) + except SerialStopException: + sys.stderr.write(ANSI_NORMAL + "Stopping condition has been received\n") finally: try: self.console_reader.stop() self.serial_reader.stop() - if self.enable_savelog == 'y': - self.log_file.close() - except: + self.stop_logging() + # Cancelling _invoke_processing_last_line_timer is not + # important here because receiving empty data doesn't matter. + self._invoke_processing_last_line_timer = None + except Exception: pass sys.stderr.write(ANSI_NORMAL + "\n") @@ -326,52 +414,87 @@ class Monitor(object): key = self.translate_eol(key) self.serial.write(codecs.encode(key)) except serial.SerialException: - pass # this shouldn't happen, but sometimes port has closed in serial thread + pass # this shouldn't happen, but sometimes port has closed in serial thread except UnicodeEncodeError: - pass # this can happen if a non-ascii character was passed, ignoring + pass # this can happen if a non-ascii character was passed, ignoring - def handle_serial_input(self, data): - # this may need to be made more efficient, as it pushes out a byte - # at a time to the console - for b in data: - if is_ascii(b) == False: - continue - if self.enable_time == 'y' and self.next_line == True: - self.console.write_bytes(get_time_stamp() + ": ") - self.next_line = False - if self.enable_savelog == 'y': - self.log_file.write(get_time_stamp() + ": ") + def handle_serial_input(self, data, finalize_line=False): + sp = data.split(b'\n') + if self._last_line_part != b"": + # add unprocessed part from previous "data" to the first line + sp[0] = self._last_line_part + sp[0] + self._last_line_part = b"" + if sp[-1] != b"": + # last part is not a full line + self._last_line_part = sp.pop() + for line in sp: + if line != b"": + if self._serial_check_exit and line == self.exit_key.encode('latin-1'): + raise SerialStopException() + if self._force_line_print or self._line_matcher.match(line.decode(errors="ignore")): + self._print(line + b'\n') + self.handle_possible_pc_address_in_line(line) + self.check_gdbstub_trigger(line) + self._force_line_print = False + # Now we have the last part (incomplete line) in _last_line_part. By + # default we don't touch it and just wait for the arrival of the rest + # of the line. But after some time when we didn't received it we need + # to make a decision. + if self._last_line_part != b"": + if self._force_line_print or (finalize_line and self._line_matcher.match(self._last_line_part.decode(errors="ignore"))): + self._force_line_print = True + self._print(self._last_line_part) + self.handle_possible_pc_address_in_line(self._last_line_part) + self.check_gdbstub_trigger(self._last_line_part) + # It is possible that the incomplete line cuts in half the PC + # address. A small buffer is kept and will be used the next time + # handle_possible_pc_address_in_line is invoked to avoid this problem. + # MATCH_PCADDR matches 10 character long addresses. Therefore, we + # keep the last 9 characters. + self._pc_address_buffer = self._last_line_part[-9:] + # GDB sequence can be cut in half also. GDB sequence is 7 + # characters long, therefore, we save the last 6 characters. + self._gdb_buffer = self._last_line_part[-6:] + self._last_line_part = b"" + # else: keeping _last_line_part and it will be processed the next time + # handle_serial_input is invoked - self.console.write_bytes(b) - - if self.enable_savelog == 'y': - self.log_file.write(b) - - if b == b'\n': # end of line - self.handle_serial_input_line(self._read_line.strip()) - self._read_line = b"" - self.next_line = True - else: - self._read_line += b - self.check_gdbstub_trigger(b) - - def handle_serial_input_line(self, line): - for m in re.finditer(MATCH_PCADDR, line): + def handle_possible_pc_address_in_line(self, line): + line = self._pc_address_buffer + line + self._pc_address_buffer = b"" + for m in re.finditer(MATCH_PCADDR, line.decode(errors="ignore")): self.lookup_pc_address(m.group()) def handle_menu_key(self, c): if c == self.exit_key or c == self.menu_key: # send verbatim self.serial.write(codecs.encode(c)) - elif c in [ CTRL_H, 'h', 'H', '?' ]: + elif c in [CTRL_H, 'h', 'H', '?']: red_print(self.get_help_text()) elif c == CTRL_R: # Reset device via RTS self.serial.setRTS(True) time.sleep(0.2) self.serial.setRTS(False) + self.output_enable(True) elif c == CTRL_F: # Recompile & upload self.run_make("flash") - elif c == CTRL_A: # Recompile & upload app only + elif c in [CTRL_A, 'a', 'A']: # Recompile & upload app only + # "CTRL-A" cannot be captured with the default settings of the Windows command line, therefore, "A" can be used + # instead self.run_make("app-flash") + elif c == CTRL_Y: # Toggle output display + self.output_toggle() + elif c == CTRL_L: # Toggle saving output into file + self.toggle_logging() + elif c == CTRL_P: + yellow_print("Pause app (enter bootloader mode), press Ctrl-T Ctrl-R to restart") + # to fast trigger pause without press menu key + self.serial.setDTR(False) # IO0=HIGH + self.serial.setRTS(True) # EN=LOW, chip in reset + time.sleep(1.3) # timeouts taken from esptool.py, includes esp32r0 workaround. defaults: 0.1 + self.serial.setDTR(True) # IO0=LOW + self.serial.setRTS(False) # EN=HIGH, chip out of reset + time.sleep(0.45) # timeouts taken from esptool.py, includes esp32r0 workaround. defaults: 0.05 + self.serial.setDTR(False) # IO0=HIGH, done else: red_print('--- unknown menu character {} --'.format(key_description(c))) @@ -383,19 +506,23 @@ class Monitor(object): --- {exit:8} Exit program --- {menu:8} Menu escape key, followed by: --- Menu keys: ---- {menu:7} Send the menu character itself to remote ---- {exit:7} Send the exit character itself to remote ---- {reset:7} Reset target board via RTS line ---- {make:7} Run 'make flash' to build & flash ---- {appmake:7} Run 'make app-flash to build & flash app +--- {menu:14} Send the menu character itself to remote +--- {exit:14} Send the exit character itself to remote +--- {reset:14} Reset target board via RTS line +--- {makecmd:14} Build & flash project +--- {appmake:14} Build & flash app only +--- {output:14} Toggle output display +--- {log:14} Toggle saving output into file +--- {pause:14} Reset target into bootloader to pause app via RTS line """.format(version=__version__, exit=key_description(self.exit_key), menu=key_description(self.menu_key), reset=key_description(CTRL_R), - make=key_description(CTRL_F), - appmake=key_description(CTRL_A), - - ) + makecmd=key_description(CTRL_F), + appmake=key_description(CTRL_A) + ' (or A)', + output=key_description(CTRL_Y), + log=key_description(CTRL_L), + pause=key_description(CTRL_P)) def __enter__(self): """ Use 'with self' to temporarily disable monitoring behaviour """ @@ -413,8 +540,8 @@ class Monitor(object): red_print(""" --- {} --- Press {} to exit monitor. ---- Press {} to run 'make flash'. ---- Press {} to run 'make app-flash'. +--- Press {} to build & flash project. +--- Press {} to build & flash app. --- Press any other key to resume monitor (resets target).""".format(reason, key_description(self.exit_key), key_description(CTRL_F), @@ -426,36 +553,44 @@ class Monitor(object): self.console.cleanup() if k == self.exit_key: self.event_queue.put((TAG_KEY, k)) - elif k in [ CTRL_F, CTRL_A ]: + elif k in [CTRL_F, CTRL_A]: self.event_queue.put((TAG_KEY, self.menu_key)) self.event_queue.put((TAG_KEY, k)) def run_make(self, target): with self: - yellow_print("Running make %s..." % target) - p = subprocess.Popen([self.make, - target ]) + if isinstance(self.make, list): + popen_args = self.make + [target] + else: + popen_args = [self.make, target] + yellow_print("Running %s..." % " ".join(popen_args)) + p = subprocess.Popen(popen_args) try: p.wait() except KeyboardInterrupt: p.wait() if p.returncode != 0: self.prompt_next_action("Build failed") + else: + self.output_enable(True) def lookup_pc_address(self, pc_addr): - translation = subprocess.check_output( - ["%saddr2line" % self.toolchain_prefix, - "-pfiaC", "-e", self.elf_file, pc_addr], - cwd=".") - if not "?? ??:0" in translation: - yellow_print(translation) + cmd = ["%saddr2line" % self.toolchain_prefix, + "-pfiaC", "-e", self.elf_file, pc_addr] + try: + translation = subprocess.check_output(cmd, cwd=".") + if b"?? ??:0" not in translation: + self._print(translation.decode(), console_printer=yellow_print) + except OSError as e: + red_print("%s: %s" % (" ".join(cmd), e)) - def check_gdbstub_trigger(self, c): - self._gdb_buffer = self._gdb_buffer[-6:] + c # keep the last 7 characters seen - m = re.match(b"\\$(T..)#(..)", self._gdb_buffer) # look for a gdb "reason" for a break + def check_gdbstub_trigger(self, line): + line = self._gdb_buffer + line + self._gdb_buffer = b"" + m = re.search(b"\\$(T..)#(..)", line) # look for a gdb "reason" for a break if m is not None: try: - chsum = sum(ord(p) for p in m.group(1)) & 0xFF + chsum = sum(ord(bytes([p])) for p in m.group(1)) & 0xFF calc_chsum = int(m.group(2), 16) except ValueError: return # payload wasn't valid hex digits @@ -464,20 +599,84 @@ class Monitor(object): else: red_print("Malformed gdb message... calculated checksum %02x received %02x" % (chsum, calc_chsum)) - def run_gdb(self): with self: # disable console control sys.stderr.write(ANSI_NORMAL) try: - subprocess.call(["%sgdb" % self.toolchain_prefix, - "-ex", "set serial baud %d" % self.serial.baudrate, - "-ex", "target remote %s" % self.serial.port, - "-ex", "interrupt", # monitor has already parsed the first 'reason' command, need a second - self.elf_file], cwd=".") + cmd = ["%sgdb" % self.toolchain_prefix, + "-ex", "set serial baud %d" % self.serial.baudrate, + "-ex", "target remote %s" % self.serial.port, + "-ex", "interrupt", # monitor has already parsed the first 'reason' command, need a second + self.elf_file] + process = subprocess.Popen(cmd, cwd=".") + process.wait() + except OSError as e: + red_print("%s: %s" % (" ".join(cmd), e)) except KeyboardInterrupt: pass # happens on Windows, maybe other OSes + finally: + try: + # on Linux, maybe other OSes, gdb sometimes seems to be alive even after wait() returns... + process.terminate() + except Exception: + pass + try: + # also on Linux, maybe other OSes, gdb sometimes exits uncleanly and breaks the tty mode + subprocess.call(["stty", "sane"]) + except Exception: + pass # don't care if there's no stty, we tried... self.prompt_next_action("gdb exited") + def output_enable(self, enable): + self._output_enabled = enable + + def output_toggle(self): + self._output_enabled = not self._output_enabled + yellow_print("\nToggle output display: {}, Type Ctrl-T Ctrl-Y to show/disable output again.".format(self._output_enabled)) + + def toggle_logging(self): + if self._log_file: + self.stop_logging() + else: + self.start_logging() + + def start_logging(self): + if not self._log_file: + try: + name = "log.{}.{}.txt".format(os.path.splitext(os.path.basename(self.elf_file))[0], + datetime.datetime.now().strftime('%Y%m%d%H%M%S')) + self._log_file = open(name, "wb+") + yellow_print("\nLogging is enabled into file {}".format(name)) + except Exception as e: + red_print("\nLog file {} cannot be created: {}".format(name, e)) + + def stop_logging(self): + if self._log_file: + try: + name = self._log_file.name + self._log_file.close() + yellow_print("\nLogging is disabled and file {} has been closed".format(name)) + except Exception as e: + red_print("\nLog file cannot be closed: {}".format(e)) + finally: + self._log_file = None + + def _print(self, string, console_printer=None): + if console_printer is None: + console_printer = self.console.write_bytes + if self._output_enabled: + console_printer(string) + if self._log_file: + try: + if isinstance(string, type(u'')): + string = string.encode() + self._log_file.write(string) + except Exception as e: + red_print("\nCannot write to file: {}".format(e)) + # don't fill-up the screen with the previous errors (probably consequent prints would fail also) + self.stop_logging() + + def main(): parser = argparse.ArgumentParser("idf_monitor - a serial output monitor for esp-idf") @@ -515,16 +714,9 @@ def main(): type=argparse.FileType('rb')) parser.add_argument( - '--enable-time', - help='enable console timestamp', - default=False - ) - - parser.add_argument( - '--enable-savelog', - help='enable console log', - default=False - ) + '--print_filter', + help="Filtering string", + default=DEFAULT_PRINT_FILTER) args = parser.parse_args() @@ -551,7 +743,7 @@ def main(): except KeyError: pass # not running a make jobserver - monitor = Monitor(serial_instance, args.elf_file.name, args.make, args.toolchain_prefix, args.eol, args.enable_time, args.enable_savelog) + monitor = Monitor(serial_instance, args.elf_file.name, args.print_filter, args.make, args.toolchain_prefix, args.eol) yellow_print('--- idf_monitor on {p.name} {p.baudrate} ---'.format( p=serial_instance)) @@ -560,9 +752,12 @@ def main(): key_description(monitor.menu_key), key_description(monitor.menu_key), key_description(CTRL_H))) + if args.print_filter != DEFAULT_PRINT_FILTER: + yellow_print('--- Print filter: {} ---'.format(args.print_filter)) monitor.main_loop() + if os.name == 'nt': # Windows console stuff @@ -577,7 +772,7 @@ if os.name == 'nt': RE_ANSI_COLOR = re.compile(b'\033\\[([01]);3([0-7])m') # list mapping the 8 ANSI colors (the indexes) to Windows Console colors - ANSI_TO_WINDOWS_COLOR = [ 0, 4, 2, 6, 1, 5, 3, 7 ] + ANSI_TO_WINDOWS_COLOR = [0, 4, 2, 6, 1, 5, 3, 7] GetStdHandle = ctypes.windll.kernel32.GetStdHandle SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute @@ -593,19 +788,39 @@ if os.name == 'nt': least-bad working solution, as winpty doesn't support any "passthrough" mode for raw output. """ - def __init__(self, output): + def __init__(self, output=None, decode_output=False): self.output = output + self.decode_output = decode_output self.handle = GetStdHandle(STD_ERROR_HANDLE if self.output == sys.stderr else STD_OUTPUT_HANDLE) self.matched = b'' + def _output_write(self, data): + try: + if self.decode_output: + self.output.write(data.decode()) + else: + self.output.write(data) + except IOError: + # Windows 10 bug since the Fall Creators Update, sometimes writing to console randomly throws + # an exception (however, the character is still written to the screen) + # Ref https://github.com/espressif/esp-idf/issues/1136 + pass + def write(self, data): + if isinstance(data, bytes): + data = bytearray(data) + else: + data = bytearray(data, 'utf-8') for b in data: - l = len(self.matched) - if b == '\033': # ESC + b = bytes([b]) + length = len(self.matched) + if b == b'\033': # ESC self.matched = b - elif (l == 1 and b == '[') or (1 < l < 7): + elif (length == 1 and b == b'[') or (1 < length < 7): self.matched += b - if self.matched == ANSI_NORMAL: # reset console + if self.matched == ANSI_NORMAL.encode('latin-1'): # reset console + # Flush is required only with Python3 - switching color before it is printed would mess up the console + self.flush() SetConsoleTextAttribute(self.handle, FOREGROUND_GREY) self.matched = b'' elif len(self.matched) == 7: # could be an ANSI sequence @@ -614,22 +829,18 @@ if os.name == 'nt': color = ANSI_TO_WINDOWS_COLOR[int(m.group(2))] if m.group(1) == b'1': color |= FOREGROUND_INTENSITY + # Flush is required only with Python3 - switching color before it is printed would mess up the console + self.flush() SetConsoleTextAttribute(self.handle, color) else: - self.output.write(self.matched) # not an ANSI color code, display verbatim + self._output_write(self.matched) # not an ANSI color code, display verbatim self.matched = b'' else: - try: - self.output.write(b) - except IOError: - # Windows 10 bug since the Fall Creators Update, sometimes writing to console randomly fails - # (but always succeeds the second time, it seems.) Ref https://github.com/espressif/esp-idf/issues/1136 - self.output.write(b) + self._output_write(b) self.matched = b'' def flush(self): self.output.flush() - if __name__ == "__main__": main() diff --git a/tools/idf_size.py b/tools/idf_size.py new file mode 100755 index 00000000..e89312dd --- /dev/null +++ b/tools/idf_size.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# +# esp-idf alternative to "size" to print ELF file sizes, also analyzes +# the linker map file to dump higher resolution details. +# +# Includes information which is not shown in "xtensa-esp32-elf-size", +# or easy to parse from "xtensa-esp32-elf-objdump" or raw map files. +# +# Copyright 2017-2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +import argparse +import collections +import json +import os.path +import re +import sys + +DEFAULT_TOOLCHAIN_PREFIX = "xtensa-esp32-elf-" + +CHIP_SIZES = { + "esp32": { + "total_iram": 0x20000, + "total_irom": 0x330000, + "total_drom": 0x800000, + # total dram is determined from objdump output + } +} + + +def _json_dump(obj): + """ Pretty-print JSON object to stdout """ + json.dump(obj, sys.stdout, indent=4) + print('\n') + + +def scan_to_header(f, header_line): + """ Scan forward in a file until you reach 'header_line', then return """ + for line in f: + if line.strip() == header_line: + return + raise RuntimeError("Didn't find line '%s' in file" % header_line) + + +def load_map_data(map_file): + memory_config = load_memory_config(map_file) + sections = load_sections(map_file) + return memory_config, sections + + +def load_memory_config(map_file): + """ Memory Configuration section is the total size of each output section """ + result = {} + scan_to_header(map_file, "Memory Configuration") + RE_MEMORY_SECTION = r"(?P[^ ]+) +0x(?P[\da-f]+) +0x(?P[\da-f]+)" + for line in map_file: + m = re.match(RE_MEMORY_SECTION, line) + if m is None: + if len(result) == 0: + continue # whitespace or a header, before the content we want + else: + return result # we're at the end of the Memory Configuration + section = { + "name": m.group("name"), + "origin": int(m.group("origin"), 16), + "length": int(m.group("length"), 16), + } + if section["name"] != "*default*": + result[section["name"]] = section + raise RuntimeError("End of file while scanning memory configuration?") + + +def load_sections(map_file): + """ Load section size information from the MAP file. + + Returns a dict of 'sections', where each key is a section name and the value + is a dict with details about this section, including a "sources" key which holds a list of source file line + information for each symbol linked into the section. + """ + scan_to_header(map_file, "Linker script and memory map") + sections = {} + section = None + sym_backup = None + for line in map_file: + # output section header, ie '.iram0.text 0x0000000040080400 0x129a5' + RE_SECTION_HEADER = r"(?P[^ ]+) +0x(?P
[\da-f]+) +0x(?P[\da-f]+)$" + m = re.match(RE_SECTION_HEADER, line) + if m is not None: # start of a new section + section = { + "name": m.group("name"), + "address": int(m.group("address"), 16), + "size": int(m.group("size"), 16), + "sources": [], + } + sections[section["name"]] = section + continue + + # source file line, ie + # 0x0000000040080400 0xa4 /home/gus/esp/32/idf/examples/get-started/hello_world/build/esp32/libesp32.a(cpu_start.o) + RE_SOURCE_LINE = r"\s*(?P\S*).* +0x(?P
[\da-f]+) +0x(?P[\da-f]+) (?P.+\.a)\((?P.+\.ob?j?)\)" + + m = re.match(RE_SOURCE_LINE, line, re.M) + if not m: + # cmake build system links some object files directly, not part of any archive + RE_SOURCE_LINE = r"\s*(?P\S*).* +0x(?P
[\da-f]+) +0x(?P[\da-f]+) (?P.+\.ob?j?)" + m = re.match(RE_SOURCE_LINE, line) + if section is not None and m is not None: # input source file details=ma,e + sym_name = m.group("sym_name") if len(m.group("sym_name")) > 0 else sym_backup + try: + archive = m.group("archive") + except IndexError: + archive = "(exe)" + + source = { + "size": int(m.group("size"), 16), + "address": int(m.group("address"), 16), + "archive": os.path.basename(archive), + "object_file": os.path.basename(m.group("object_file")), + "sym_name": sym_name, + } + source["file"] = "%s:%s" % (source["archive"], source["object_file"]) + section["sources"] += [source] + + # In some cases the section name appears on the previous line, back it up in here + RE_SYMBOL_ONLY_LINE = r"^ (?P\S*)$" + m = re.match(RE_SYMBOL_ONLY_LINE, line) + if section is not None and m is not None: + sym_backup = m.group("sym_name") + + return sections + + +def sizes_by_key(sections, key): + """ Takes a dict of sections (from load_sections) and returns + a dict keyed by 'key' with aggregate output size information. + + Key can be either "archive" (for per-archive data) or "file" (for per-file data) in the result. + """ + result = {} + for section in sections.values(): + for s in section["sources"]: + if not s[key] in result: + result[s[key]] = {} + archive = result[s[key]] + if not section["name"] in archive: + archive[section["name"]] = 0 + archive[section["name"]] += s["size"] + return result + + +def main(): + parser = argparse.ArgumentParser("idf_size - a tool to print IDF elf file sizes") + + parser.add_argument( + '--toolchain-prefix', + help="Triplet prefix to add before objdump executable", + default=DEFAULT_TOOLCHAIN_PREFIX) + + parser.add_argument( + '--json', + help="Output results as JSON", + action="store_true") + + parser.add_argument( + 'map_file', help='MAP file produced by linker', + type=argparse.FileType('r')) + + parser.add_argument( + '--archives', help='Print per-archive sizes', action='store_true') + + parser.add_argument( + '--archive_details', help='Print detailed symbols per archive') + + parser.add_argument( + '--files', help='Print per-file sizes', action='store_true') + + args = parser.parse_args() + + memory_config, sections = load_map_data(args.map_file) + if not args.json or not (args.archives or args.files or args.archive_details): + print_summary(memory_config, sections, args.json) + + if args.archives: + print_detailed_sizes(sections, "archive", "Archive File", args.json) + if args.files: + print_detailed_sizes(sections, "file", "Object File", args.json) + if args.archive_details: + print_archive_symbols(sections, args.archive_details, args.json) + + +def print_summary(memory_config, sections, as_json=False): + def get_size(section): + try: + return sections[section]["size"] + except KeyError: + return 0 + + # if linker script changes, these need to change + total_iram = memory_config["iram0_0_seg"]["length"] + total_dram = memory_config["dram0_0_seg"]["length"] + used_data = get_size(".dram0.data") + used_bss = get_size(".dram0.bss") + used_dram = used_data + used_bss + try: + used_dram_ratio = used_dram / total_dram + except ZeroDivisionError: + used_dram_ratio = float('nan') + used_iram = sum(get_size(s) for s in sections if s.startswith(".iram0")) + try: + used_iram_ratio = used_iram / total_iram + except ZeroDivisionError: + used_iram_ratio = float('nan') + flash_code = get_size(".flash.text") + flash_rodata = get_size(".flash.rodata") + total_size = used_data + used_iram + flash_code + flash_rodata + + if as_json: + _json_dump(collections.OrderedDict([ + ("dram_data", used_data), + ("dram_bss", used_bss), + ("used_dram", used_dram), + ("available_dram", total_dram - used_dram), + ("used_dram_ratio", used_dram_ratio), + ("used_iram", used_iram), + ("available_iram", total_iram - used_iram), + ("used_iram_ratio", used_iram_ratio), + ("flash_code", flash_code), + ("flash_rodata", flash_rodata), + ("total_size", total_size) + ])) + else: + print("Total sizes:") + print(" DRAM .data size: %7d bytes" % used_data) + print(" DRAM .bss size: %7d bytes" % used_bss) + print("Used static DRAM: %7d bytes (%7d available, %.1f%% used)" % + (used_dram, total_dram - used_dram, 100.0 * used_dram_ratio)) + print("Used static IRAM: %7d bytes (%7d available, %.1f%% used)" % + (used_iram, total_iram - used_iram, 100.0 * used_iram_ratio)) + print(" Flash code: %7d bytes" % flash_code) + print(" Flash rodata: %7d bytes" % flash_rodata) + print("Total image size:~%7d bytes (.bin may be padded larger)" % (total_size)) + + +def print_detailed_sizes(sections, key, header, as_json=False): + sizes = sizes_by_key(sections, key) + + result = {} + for k in sizes: + v = sizes[k] + result[k] = collections.OrderedDict() + result[k]["data"] = v.get(".dram0.data", 0) + result[k]["bss"] = v.get(".dram0.bss", 0) + result[k]["iram"] = sum(t for (s,t) in v.items() if s.startswith(".iram0")) + result[k]["flash_text"] = v.get(".flash.text", 0) + result[k]["flash_rodata"] = v.get(".flash.rodata", 0) + result[k]["total"] = sum(result[k].values()) + + def return_total_size(elem): + val = elem[1] + return val["total"] + + def return_header(elem): + return elem[0] + s = sorted(list(result.items()), key=return_header) + + # do a secondary sort in order to have consistent order (for diff-ing the output) + s = sorted(s, key=return_total_size, reverse=True) + + if as_json: + _json_dump(collections.OrderedDict(s)) + else: + print("Per-%s contributions to ELF file:" % key) + headings = (header, + "DRAM .data", + "& .bss", + "IRAM", + "Flash code", + "& rodata", + "Total") + header_format = "%24s %10d %6d %6d %10d %8d %7d" + print(header_format.replace("d", "s") % headings) + + for k,v in s: + if ":" in k: # print subheadings for key of format archive:file + sh,k = k.split(":") + print(header_format % (k[:24], + v["data"], + v["bss"], + v["iram"], + v["flash_text"], + v["flash_rodata"], + v["total"])) + + +def print_archive_symbols(sections, archive, as_json=False): + interested_sections = [".dram0.data", ".dram0.bss", ".iram0.text", ".iram0.vectors", ".flash.text", ".flash.rodata"] + result = {} + for t in interested_sections: + result[t] = {} + for section in sections.values(): + section_name = section["name"] + if section_name not in interested_sections: + continue + for s in section["sources"]: + if archive != s["archive"]: + continue + s["sym_name"] = re.sub("(.text.|.literal.|.data.|.bss.|.rodata.)", "", s["sym_name"]) + result[section_name][s["sym_name"]] = result[section_name].get(s["sym_name"], 0) + s["size"] + + # build a new ordered dict of each section, where each entry is an ordereddict of symbols to sizes + section_symbols = collections.OrderedDict() + for t in interested_sections: + s = sorted(list(result[t].items()), key=lambda k_v: k_v[0]) + # do a secondary sort in order to have consistent order (for diff-ing the output) + s = sorted(s, key=lambda k_v: k_v[1], reverse=True) + section_symbols[t] = collections.OrderedDict(s) + + if as_json: + _json_dump(section_symbols) + else: + print("Symbols within the archive: %s (Not all symbols may be reported)" % (archive)) + for t,s in section_symbols.items(): + section_total = 0 + print("\nSymbols from section:", t) + for key, val in s.items(): + print(("%s(%d)" % (key.replace(t + ".", ""), val)), end=' ') + section_total += val + print("\nSection total:",section_total) + + +if __name__ == "__main__": + main() diff --git a/tools/idf_tools.py b/tools/idf_tools.py new file mode 100755 index 00000000..7cdd1823 --- /dev/null +++ b/tools/idf_tools.py @@ -0,0 +1,1250 @@ +#!/usr/bin/env python +# coding=utf-8 +# +# This script helps installing tools required to use the ESP-IDF, and updating PATH +# to use the installed tools. It can also create a Python virtual environment, +# and install Python requirements into it. +# It does not install OS dependencies. It does install tools such as the Xtensa +# GCC toolchain and ESP32 ULP coprocessor toolchain. +# +# By default, downloaded tools will be installed under $HOME/.espressif directory +# (%USERPROFILE%/.espressif on Windows). This path can be modified by setting +# IDF_TOOLS_PATH variable prior to running this tool. +# +# Users do not need to interact with this script directly. In IDF root directory, +# install.sh (.bat) and export.sh (.bat) scripts are provided to invoke this script. +# +# Usage: +# +# * To install the tools, run `idf_tools.py install`. +# +# * To install the Python environment, run `idf_tools.py install-python-env`. +# +# * To start using the tools, run `eval "$(idf_tools.py export)"` — this will update +# the PATH to point to the installed tools and set up other environment variables +# needed by the tools. +# +### +# +# Copyright 2019 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import subprocess +import sys +import argparse +import re +import platform +import hashlib +import tarfile +import zipfile +import errno +import shutil +import functools +from collections import OrderedDict, namedtuple + +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve + + +TOOLS_FILE = 'tools/tools.json' +TOOLS_SCHEMA_FILE = 'tools/tools_schema.json' +TOOLS_FILE_NEW = 'tools/tools.new.json' +TOOLS_FILE_VERSION = 1 +IDF_TOOLS_PATH_DEFAULT = os.path.join('~', '.espressif') +UNKNOWN_VERSION = 'unknown' +SUBST_TOOL_PATH_REGEX = re.compile(r'\${TOOL_PATH}') +VERSION_REGEX_REPLACE_DEFAULT = r'\1' +IDF_MAINTAINER = os.environ.get('IDF_MAINTAINER') or False +TODO_MESSAGE = 'TODO' +DOWNLOAD_RETRY_COUNT = 3 +URL_PREFIX_MAP_SEPARATOR = ',' +IDF_TOOLS_INSTALL_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD') +IDF_TOOLS_EXPORT_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD') + +PYTHON_PLATFORM = platform.system() + '-' + platform.machine() + +# Identifiers used in tools.json for different platforms. +PLATFORM_WIN32 = 'win32' +PLATFORM_WIN64 = 'win64' +PLATFORM_MACOS = 'macos' +PLATFORM_LINUX32 = 'linux-i686' +PLATFORM_LINUX64 = 'linux-amd64' +PLATFORM_LINUX_ARM32 = 'linux-armel' +PLATFORM_LINUX_ARM64 = 'linux-arm64' + + +# Mappings from various other names these platforms are known as, to the identifiers above. +# This includes strings produced from "platform.system() + '-' + platform.machine()", see PYTHON_PLATFORM +# definition above. +# This list also includes various strings used in release archives of xtensa-esp32-elf-gcc, OpenOCD, etc. +PLATFORM_FROM_NAME = { + # Windows + PLATFORM_WIN32: PLATFORM_WIN32, + 'Windows-i686': PLATFORM_WIN32, + 'Windows-x86': PLATFORM_WIN32, + PLATFORM_WIN64: PLATFORM_WIN64, + 'Windows-x86_64': PLATFORM_WIN64, + 'Windows-AMD64': PLATFORM_WIN64, + # macOS + PLATFORM_MACOS: PLATFORM_MACOS, + 'osx': PLATFORM_MACOS, + 'darwin': PLATFORM_MACOS, + 'Darwin-x86_64': PLATFORM_MACOS, + # Linux + PLATFORM_LINUX64: PLATFORM_LINUX64, + 'linux64': PLATFORM_LINUX64, + 'Linux-x86_64': PLATFORM_LINUX64, + PLATFORM_LINUX32: PLATFORM_LINUX32, + 'linux32': PLATFORM_LINUX32, + 'Linux-i686': PLATFORM_LINUX32, + PLATFORM_LINUX_ARM32: PLATFORM_LINUX_ARM32, + 'Linux-arm': PLATFORM_LINUX_ARM32, + 'Linux-armv7l': PLATFORM_LINUX_ARM32, + PLATFORM_LINUX_ARM64: PLATFORM_LINUX_ARM64, + 'Linux-arm64': PLATFORM_LINUX_ARM64, + 'Linux-aarch64': PLATFORM_LINUX_ARM64, + 'Linux-armv8l': PLATFORM_LINUX_ARM64, +} + +UNKNOWN_PLATFORM = 'unknown' +CURRENT_PLATFORM = PLATFORM_FROM_NAME.get(PYTHON_PLATFORM, UNKNOWN_PLATFORM) + +EXPORT_SHELL = 'shell' +EXPORT_KEY_VALUE = 'key-value' + + +global_quiet = False +global_non_interactive = False +global_idf_path = None +global_idf_tools_path = None +global_tools_json = None + + +def fatal(text, *args): + if not global_quiet: + sys.stderr.write('ERROR: ' + text + '\n', *args) + + +def warn(text, *args): + if not global_quiet: + sys.stderr.write('WARNING: ' + text + '\n', *args) + + +def info(text, f=None, *args): + if not global_quiet: + if f is None: + f = sys.stdout + f.write(text + '\n', *args) + + +def run_cmd_check_output(cmd, input_text=None, extra_paths=None): + # If extra_paths is given, locate the executable in one of these directories. + # Note: it would seem logical to add extra_paths to env[PATH], instead, and let OS do the job of finding the + # executable for us. However this does not work on Windows: https://bugs.python.org/issue8557. + if extra_paths: + found = False + extensions = [''] + if sys.platform == 'win32': + extensions.append('.exe') + for path in extra_paths: + for ext in extensions: + fullpath = os.path.join(path, cmd[0] + ext) + if os.path.exists(fullpath): + cmd[0] = fullpath + found = True + break + if found: + break + + try: + if input_text: + input_text = input_text.encode() + result = subprocess.run(cmd, capture_output=True, check=True, input=input_text) + return result.stdout + result.stderr + except (AttributeError, TypeError): + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate(input_text) + if p.returncode != 0: + try: + raise subprocess.CalledProcessError(p.returncode, cmd, stdout, stderr) + except TypeError: + raise subprocess.CalledProcessError(p.returncode, cmd, stdout) + return stdout + stderr + + +def to_shell_specific_paths(paths_list): + if sys.platform == 'win32': + paths_list = [p.replace('/', os.path.sep) if os.path.sep in p else p for p in paths_list] + + if 'MSYSTEM' in os.environ: + paths_msys = run_cmd_check_output(['cygpath', '-u', '-f', '-'], + input_text='\n'.join(paths_list)) + paths_list = paths_msys.decode().strip().split('\n') + + return paths_list + + +def get_env_for_extra_paths(extra_paths): + """ + Return a copy of environment variables dict, prepending paths listed in extra_paths + to the PATH environment variable. + """ + env_arg = os.environ.copy() + new_path = os.pathsep.join(extra_paths) + os.pathsep + env_arg['PATH'] + if sys.version_info.major == 2: + env_arg['PATH'] = new_path.encode('utf8') + else: + env_arg['PATH'] = new_path + return env_arg + + +def get_file_size_sha256(filename, block_size=65536): + sha256 = hashlib.sha256() + size = 0 + with open(filename, 'rb') as f: + for block in iter(lambda: f.read(block_size), b''): + sha256.update(block) + size += len(block) + return size, sha256.hexdigest() + + +def report_progress(count, block_size, total_size): + percent = int(count * block_size * 100 / total_size) + percent = min(100, percent) + sys.stdout.write("\r%d%%" % percent) + sys.stdout.flush() + + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno != errno.EEXIST or not os.path.isdir(path): + raise + + +def unpack(filename, destination): + info('Extracting {0} to {1}'.format(filename, destination)) + if filename.endswith('tar.gz'): + archive_obj = tarfile.open(filename, 'r:gz') + elif filename.endswith('zip'): + archive_obj = zipfile.ZipFile(filename) + else: + raise NotImplementedError('Unsupported archive type') + if sys.version_info.major == 2: + # This is a workaround for the issue that unicode destination is not handled: + # https://bugs.python.org/issue17153 + destination = str(destination) + archive_obj.extractall(destination) + + +def strip_container_dirs(path, levels): + assert levels > 0 + # move the original directory out of the way (add a .tmp suffix) + tmp_path = path + '.tmp' + if os.path.exists(tmp_path): + shutil.rmtree(tmp_path) + os.rename(path, tmp_path) + os.mkdir(path) + base_path = tmp_path + # walk given number of levels down + for level in range(levels): + contents = os.listdir(base_path) + if len(contents) > 1: + raise RuntimeError('at level {}, expected 1 entry, got {}'.format(level, contents)) + base_path = os.path.join(base_path, contents[0]) + if not os.path.isdir(base_path): + raise RuntimeError('at level {}, {} is not a directory'.format(level, contents[0])) + # get the list of directories/files to move + contents = os.listdir(base_path) + for name in contents: + move_from = os.path.join(base_path, name) + move_to = os.path.join(path, name) + os.rename(move_from, move_to) + shutil.rmtree(tmp_path) + + +class ToolNotFound(RuntimeError): + pass + + +class ToolExecError(RuntimeError): + pass + + +class DownloadError(RuntimeError): + pass + + +class IDFToolDownload(object): + def __init__(self, platform_name, url, size, sha256): + self.platform_name = platform_name + self.url = url + self.size = size + self.sha256 = sha256 + self.platform_name = platform_name + + +@functools.total_ordering +class IDFToolVersion(object): + STATUS_RECOMMENDED = 'recommended' + STATUS_SUPPORTED = 'supported' + STATUS_DEPRECATED = 'deprecated' + + STATUS_VALUES = [STATUS_RECOMMENDED, STATUS_SUPPORTED, STATUS_DEPRECATED] + + def __init__(self, version, status): + self.version = version + self.status = status + self.downloads = OrderedDict() + self.latest = False + + def __lt__(self, other): + if self.status != other.status: + return self.status > other.status + else: + assert not (self.status == IDFToolVersion.STATUS_RECOMMENDED + and other.status == IDFToolVersion.STATUS_RECOMMENDED) + return self.version < other.version + + def __eq__(self, other): + return self.status == other.status and self.version == other.version + + def add_download(self, platform_name, url, size, sha256): + self.downloads[platform_name] = IDFToolDownload(platform_name, url, size, sha256) + + def get_download_for_platform(self, platform_name): + if platform_name in PLATFORM_FROM_NAME.keys(): + platform_name = PLATFORM_FROM_NAME[platform_name] + if platform_name in self.downloads.keys(): + return self.downloads[platform_name] + if 'any' in self.downloads.keys(): + return self.downloads['any'] + return None + + def compatible_with_platform(self, platform_name=PYTHON_PLATFORM): + return self.get_download_for_platform(platform_name) is not None + + +OPTIONS_LIST = ['version_cmd', + 'version_regex', + 'version_regex_replace', + 'export_paths', + 'export_vars', + 'install', + 'info_url', + 'license', + 'strip_container_dirs'] + +IDFToolOptions = namedtuple('IDFToolOptions', OPTIONS_LIST) + + +class IDFTool(object): + # possible values of 'install' field + INSTALL_ALWAYS = 'always' + INSTALL_ON_REQUEST = 'on_request' + INSTALL_NEVER = 'never' + + def __init__(self, name, description, install, info_url, license, version_cmd, version_regex, version_regex_replace=None, + strip_container_dirs=0): + self.name = name + self.description = description + self.versions = OrderedDict() + self.version_in_path = None + self.versions_installed = [] + if version_regex_replace is None: + version_regex_replace = VERSION_REGEX_REPLACE_DEFAULT + self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace, + [], OrderedDict(), install, info_url, license, strip_container_dirs) + self.platform_overrides = [] + self._update_current_options() + + def _update_current_options(self): + self._current_options = IDFToolOptions(*self.options) + for override in self.platform_overrides: + if CURRENT_PLATFORM not in override['platforms']: + continue + override_dict = override.copy() + del override_dict['platforms'] + self._current_options = self._current_options._replace(**override_dict) + + def add_version(self, version): + assert(type(version) is IDFToolVersion) + self.versions[version.version] = version + + def get_path(self): + return os.path.join(global_idf_tools_path, 'tools', self.name) + + def get_path_for_version(self, version): + assert(version in self.versions) + return os.path.join(self.get_path(), version) + + def get_export_paths(self, version): + tool_path = self.get_path_for_version(version) + return [os.path.join(tool_path, *p) for p in self._current_options.export_paths] + + def get_export_vars(self, version): + """ + Get the dictionary of environment variables to be exported, for the given version. + Expands: + - ${TOOL_PATH} => the actual path where the version is installed + """ + result = {} + for k, v in self._current_options.export_vars.items(): + replace_path = self.get_path_for_version(version).replace('\\', '\\\\') + v_repl = re.sub(SUBST_TOOL_PATH_REGEX, replace_path, v) + if v_repl != v: + v_repl = to_shell_specific_paths([v_repl])[0] + result[k] = v_repl + return result + + def check_version(self, extra_paths=None): + """ + Execute the tool, optionally prepending extra_paths to PATH, + extract the version string and return it as a result. + Raises ToolNotFound if the tool is not found (not present in the paths). + Raises ToolExecError if the tool returns with a non-zero exit code. + Returns 'unknown' if tool returns something from which version string + can not be extracted. + """ + cmd = self._current_options.version_cmd + try: + version_cmd_result = run_cmd_check_output(cmd, None, extra_paths) + except OSError: + # tool is not on the path + raise ToolNotFound('Tool {} not found'.format(self.name)) + except subprocess.CalledProcessError as e: + raise ToolExecError('Command {} has returned non-zero exit code ({})\n'.format( + ' '.join(self._current_options.version_cmd), e.returncode)) + + in_str = version_cmd_result.decode() + match = re.search(self._current_options.version_regex, in_str) + if not match: + return UNKNOWN_VERSION + return re.sub(self._current_options.version_regex, self._current_options.version_regex_replace, match.group(0)) + + def get_install_type(self): + return self._current_options.install + + def get_recommended_version(self): + recommended_versions = [k for k, v in self.versions.items() + if v.status == IDFToolVersion.STATUS_RECOMMENDED + and v.compatible_with_platform()] + assert len(recommended_versions) <= 1 + if recommended_versions: + return recommended_versions[0] + return None + + def get_preferred_installed_version(self): + recommended_versions = [k for k in self.versions_installed + if self.versions[k].status == IDFToolVersion.STATUS_RECOMMENDED + and self.versions[k].compatible_with_platform()] + assert len(recommended_versions) <= 1 + if recommended_versions: + return recommended_versions[0] + return None + + def find_installed_versions(self): + """ + Checks whether the tool can be found in PATH and in global_idf_tools_path. + Writes results to self.version_in_path and self.versions_installed. + """ + # First check if the tool is in system PATH + try: + ver_str = self.check_version() + except ToolNotFound: + # not in PATH + pass + except ToolExecError: + warn('tool {} found in path, but failed to run'.format(self.name)) + else: + self.version_in_path = ver_str + + # Now check all the versions installed in global_idf_tools_path + self.versions_installed = [] + for version, version_obj in self.versions.items(): + if not version_obj.compatible_with_platform(): + continue + tool_path = self.get_path_for_version(version) + if not os.path.exists(tool_path): + # version not installed + continue + try: + ver_str = self.check_version(self.get_export_paths(version)) + except ToolNotFound: + warn('directory for tool {} version {} is present, but tool was not found'.format( + self.name, version)) + except ToolExecError: + warn('tool {} version {} is installed, but the tool failed to run'.format( + self.name, version)) + else: + if ver_str != version: + warn('tool {} version {} is installed, but has reported version {}'.format( + self.name, version, ver_str)) + else: + self.versions_installed.append(version) + + def download(self, version): + assert(version in self.versions) + download_obj = self.versions[version].get_download_for_platform(PYTHON_PLATFORM) + if not download_obj: + fatal('No packages for tool {} platform {}!'.format(self.name, PYTHON_PLATFORM)) + raise DownloadError() + + url = download_obj.url + archive_name = os.path.basename(url) + local_path = os.path.join(global_idf_tools_path, 'dist', archive_name) + mkdir_p(os.path.dirname(local_path)) + + if os.path.isfile(local_path): + if not self.check_download_file(download_obj, local_path): + warn('removing downloaded file {0} and downloading again'.format(archive_name)) + os.unlink(local_path) + else: + info('file {0} is already downloaded'.format(archive_name)) + return + + downloaded = False + for retry in range(DOWNLOAD_RETRY_COUNT): + local_temp_path = local_path + '.tmp' + info('Downloading {} to {}'.format(archive_name, local_temp_path)) + urlretrieve(url, local_temp_path, report_progress if not global_non_interactive else None) + sys.stdout.write("\rDone\n") + sys.stdout.flush() + if not self.check_download_file(download_obj, local_temp_path): + warn('Failed to download file {}'.format(local_temp_path)) + continue + os.rename(local_temp_path, local_path) + downloaded = True + break + if not downloaded: + fatal('Failed to download, and retry count has expired') + raise DownloadError() + + def install(self, version): + # Currently this is called after calling 'download' method, so here are a few asserts + # for the conditions which should be true once that method is done. + assert (version in self.versions) + download_obj = self.versions[version].get_download_for_platform(PYTHON_PLATFORM) + assert (download_obj is not None) + archive_name = os.path.basename(download_obj.url) + archive_path = os.path.join(global_idf_tools_path, 'dist', archive_name) + assert (os.path.isfile(archive_path)) + dest_dir = self.get_path_for_version(version) + if os.path.exists(dest_dir): + warn('destination path already exists, removing') + shutil.rmtree(dest_dir) + mkdir_p(dest_dir) + unpack(archive_path, dest_dir) + if self._current_options.strip_container_dirs: + strip_container_dirs(dest_dir, self._current_options.strip_container_dirs) + + @staticmethod + def check_download_file(download_obj, local_path): + expected_sha256 = download_obj.sha256 + expected_size = download_obj.size + file_size, file_sha256 = get_file_size_sha256(local_path) + if file_size != expected_size: + warn('file size mismatch for {}, expected {}, got {}'.format(local_path, expected_size, file_size)) + return False + if file_sha256 != expected_sha256: + warn('hash mismatch for {}, expected {}, got {}'.format(local_path, expected_sha256, file_sha256)) + return False + return True + + @classmethod + def from_json(cls, tool_dict): + # json.load will return 'str' types in Python 3 and 'unicode' in Python 2 + expected_str_type = type(u'') + + # Validate json fields + tool_name = tool_dict.get('name') + if type(tool_name) is not expected_str_type: + raise RuntimeError('tool_name is not a string') + + description = tool_dict.get('description') + if type(description) is not expected_str_type: + raise RuntimeError('description is not a string') + + version_cmd = tool_dict.get('version_cmd') + if type(version_cmd) is not list: + raise RuntimeError('version_cmd for tool %s is not a list of strings' % tool_name) + + version_regex = tool_dict.get('version_regex') + if type(version_regex) is not expected_str_type or not version_regex: + raise RuntimeError('version_regex for tool %s is not a non-empty string' % tool_name) + + version_regex_replace = tool_dict.get('version_regex_replace') + if version_regex_replace and type(version_regex_replace) is not expected_str_type: + raise RuntimeError('version_regex_replace for tool %s is not a string' % tool_name) + + export_paths = tool_dict.get('export_paths') + if type(export_paths) is not list: + raise RuntimeError('export_paths for tool %s is not a list' % tool_name) + + export_vars = tool_dict.get('export_vars', {}) + if type(export_vars) is not dict: + raise RuntimeError('export_vars for tool %s is not a mapping' % tool_name) + + versions = tool_dict.get('versions') + if type(versions) is not list: + raise RuntimeError('versions for tool %s is not an array' % tool_name) + + install = tool_dict.get('install', False) + if type(install) is not expected_str_type: + raise RuntimeError('install for tool %s is not a string' % tool_name) + + info_url = tool_dict.get('info_url', False) + if type(info_url) is not expected_str_type: + raise RuntimeError('info_url for tool %s is not a string' % tool_name) + + license = tool_dict.get('license', False) + if type(license) is not expected_str_type: + raise RuntimeError('license for tool %s is not a string' % tool_name) + + strip_container_dirs = tool_dict.get('strip_container_dirs', 0) + if strip_container_dirs and type(strip_container_dirs) is not int: + raise RuntimeError('strip_container_dirs for tool %s is not an int' % tool_name) + + overrides_list = tool_dict.get('platform_overrides', []) + if type(overrides_list) is not list: + raise RuntimeError('platform_overrides for tool %s is not a list' % tool_name) + + # Create the object + tool_obj = cls(tool_name, description, install, info_url, license, + version_cmd, version_regex, version_regex_replace, + strip_container_dirs) + + for path in export_paths: + tool_obj.options.export_paths.append(path) + + for name, value in export_vars.items(): + tool_obj.options.export_vars[name] = value + + for index, override in enumerate(overrides_list): + platforms_list = override.get('platforms') + if type(platforms_list) is not list: + raise RuntimeError('platforms for override %d of tool %s is not a list' % (index, tool_name)) + + install = override.get('install') + if install is not None and type(install) is not expected_str_type: + raise RuntimeError('install for override %d of tool %s is not a string' % (index, tool_name)) + + version_cmd = override.get('version_cmd') + if version_cmd is not None and type(version_cmd) is not list: + raise RuntimeError('version_cmd for override %d of tool %s is not a list of strings' % + (index, tool_name)) + + version_regex = override.get('version_regex') + if version_regex is not None and (type(version_regex) is not expected_str_type or not version_regex): + raise RuntimeError('version_regex for override %d of tool %s is not a non-empty string' % + (index, tool_name)) + + version_regex_replace = override.get('version_regex_replace') + if version_regex_replace is not None and type(version_regex_replace) is not expected_str_type: + raise RuntimeError('version_regex_replace for override %d of tool %s is not a string' % + (index, tool_name)) + + export_paths = override.get('export_paths') + if export_paths is not None and type(export_paths) is not list: + raise RuntimeError('export_paths for override %d of tool %s is not a list' % (index, tool_name)) + + export_vars = override.get('export_vars') + if export_vars is not None and type(export_vars) is not dict: + raise RuntimeError('export_vars for override %d of tool %s is not a mapping' % (index, tool_name)) + tool_obj.platform_overrides.append(override) + + recommended_versions = {} + for version_dict in versions: + version = version_dict.get('name') + if type(version) is not expected_str_type: + raise RuntimeError('version name for tool {} is not a string'.format(tool_name)) + + version_status = version_dict.get('status') + if type(version_status) is not expected_str_type and version_status not in IDFToolVersion.STATUS_VALUES: + raise RuntimeError('tool {} version {} status is not one of {}', tool_name, version, + IDFToolVersion.STATUS_VALUES) + + version_obj = IDFToolVersion(version, version_status) + for platform_id, platform_dict in version_dict.items(): + if platform_id in ['name', 'status']: + continue + if platform_id not in PLATFORM_FROM_NAME.keys(): + raise RuntimeError('invalid platform %s for tool %s version %s' % + (platform_id, tool_name, version)) + + version_obj.add_download(platform_id, + platform_dict['url'], platform_dict['size'], platform_dict['sha256']) + + if version_status == IDFToolVersion.STATUS_RECOMMENDED: + if platform_id not in recommended_versions: + recommended_versions[platform_id] = [] + recommended_versions[platform_id].append(version) + + tool_obj.add_version(version_obj) + for platform_id, version_list in recommended_versions.items(): + if len(version_list) > 1: + raise RuntimeError('tool {} for platform {} has {} recommended versions'.format( + tool_name, platform_id, len(recommended_versions))) + if install != IDFTool.INSTALL_NEVER and len(recommended_versions) == 0: + raise RuntimeError('required/optional tool {} for platform {} has no recommended versions'.format( + tool_name, platform_id)) + + tool_obj._update_current_options() + return tool_obj + + def to_json(self): + versions_array = [] + for version, version_obj in self.versions.items(): + version_json = { + 'name': version, + 'status': version_obj.status + } + for platform_id, download in version_obj.downloads.items(): + version_json[platform_id] = { + 'url': download.url, + 'size': download.size, + 'sha256': download.sha256 + } + versions_array.append(version_json) + overrides_array = self.platform_overrides + + tool_json = { + 'name': self.name, + 'description': self.description, + 'export_paths': self.options.export_paths, + 'export_vars': self.options.export_vars, + 'install': self.options.install, + 'info_url': self.options.info_url, + 'license': self.options.license, + 'version_cmd': self.options.version_cmd, + 'version_regex': self.options.version_regex, + 'versions': versions_array, + } + if self.options.version_regex_replace != VERSION_REGEX_REPLACE_DEFAULT: + tool_json['version_regex_replace'] = self.options.version_regex_replace + if overrides_array: + tool_json['platform_overrides'] = overrides_array + if self.options.strip_container_dirs: + tool_json['strip_container_dirs'] = self.options.strip_container_dirs + return tool_json + + +def load_tools_info(): + """ + Load tools metadata from tools.json, return a dictionary: tool name - tool info + """ + tool_versions_file_name = global_tools_json + + with open(tool_versions_file_name, 'r') as f: + tools_info = json.load(f) + + return parse_tools_info_json(tools_info) + + +def parse_tools_info_json(tools_info): + """ + Parse and validate the dictionary obtained by loading the tools.json file. + Returns a dictionary of tools (key: tool name, value: IDFTool object). + """ + if tools_info['version'] != TOOLS_FILE_VERSION: + raise RuntimeError('Invalid version') + + tools_dict = OrderedDict() + + tools_array = tools_info.get('tools') + if type(tools_array) is not list: + raise RuntimeError('tools property is missing or not an array') + + for tool_dict in tools_array: + tool = IDFTool.from_json(tool_dict) + tools_dict[tool.name] = tool + + return tools_dict + + +def dump_tools_json(tools_info): + tools_array = [] + for tool_name, tool_obj in tools_info.items(): + tool_json = tool_obj.to_json() + tools_array.append(tool_json) + file_json = {'version': TOOLS_FILE_VERSION, 'tools': tools_array} + return json.dumps(file_json, indent=2, separators=(',', ': '), sort_keys=True) + + +def get_python_env_path(): + python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor) + + version_file_path = os.path.join(global_idf_path, 'version.txt') + if os.path.exists(version_file_path): + with open(version_file_path, "r") as version_file: + idf_version_str = version_file.read() + else: + idf_version_str = subprocess.check_output(['git', '-C', global_idf_path, 'describe', '--tags'], cwd=global_idf_path, env=os.environ).decode() + match = re.match(r'^v([0-9]+\.[0-9]+).*', idf_version_str) + idf_version = match.group(1) + + idf_python_env_path = os.path.join(global_idf_tools_path, 'python_env', + 'idf{}_py{}_env'.format(idf_version, python_ver_major_minor)) + + if sys.platform == 'win32': + subdir = 'Scripts' + python_exe = 'python.exe' + else: + subdir = 'bin' + python_exe = 'python' + + idf_python_export_path = os.path.join(idf_python_env_path, subdir) + virtualenv_python = os.path.join(idf_python_export_path, python_exe) + + return idf_python_env_path, idf_python_export_path, virtualenv_python + + +def action_list(args): + tools_info = load_tools_info() + for name, tool in tools_info.items(): + if tool.get_install_type() == IDFTool.INSTALL_NEVER: + continue + optional_str = ' (optional)' if tool.get_install_type() == IDFTool.INSTALL_ON_REQUEST else '' + info('* {}: {}{}'.format(name, tool.description, optional_str)) + tool.find_installed_versions() + versions_for_platform = {k: v for k, v in tool.versions.items() if v.compatible_with_platform()} + if not versions_for_platform: + info(' (no versions compatible with platform {})'.format(PYTHON_PLATFORM)) + continue + versions_sorted = sorted(versions_for_platform.keys(), key=tool.versions.get, reverse=True) + for version in versions_sorted: + version_obj = tool.versions[version] + info(' - {} ({}{})'.format(version, version_obj.status, + ', installed' if version in tool.versions_installed else '')) + + +def action_check(args): + tools_info = load_tools_info() + not_found_list = [] + info('Checking for installed tools...') + for name, tool in tools_info.items(): + if tool.get_install_type() == IDFTool.INSTALL_NEVER: + continue + tool_found_somewhere = False + info('Checking tool %s' % name) + tool.find_installed_versions() + if tool.version_in_path: + info(' version found in PATH: %s' % tool.version_in_path) + tool_found_somewhere = True + else: + info(' no version found in PATH') + + for version in tool.versions_installed: + info(' version installed in tools directory: %s' % version) + tool_found_somewhere = True + if not tool_found_somewhere and tool.get_install_type() == IDFTool.INSTALL_ALWAYS: + not_found_list.append(name) + if not_found_list: + fatal('The following required tools were not found: ' + ' '.join(not_found_list)) + raise SystemExit(1) + + +def action_export(args): + tools_info = load_tools_info() + all_tools_found = True + export_vars = {} + paths_to_export = [] + for name, tool in tools_info.items(): + if tool.get_install_type() == IDFTool.INSTALL_NEVER: + continue + tool.find_installed_versions() + + if tool.version_in_path: + if tool.version_in_path not in tool.versions: + # unsupported version + if args.prefer_system: + warn('using an unsupported version of tool {} found in PATH: {}'.format( + tool.name, tool.version_in_path)) + continue + else: + # unsupported version in path + pass + else: + # supported/deprecated version in PATH, use it + version_obj = tool.versions[tool.version_in_path] + if version_obj.status == IDFToolVersion.STATUS_SUPPORTED: + info('Using a supported version of tool {} found in PATH: {}.'.format(name, tool.version_in_path), + f=sys.stderr) + info('However the recommended version is {}.'.format(tool.get_recommended_version()), + f=sys.stderr) + elif version_obj.status == IDFToolVersion.STATUS_DEPRECATED: + warn('using a deprecated version of tool {} found in PATH: {}'.format(name, tool.version_in_path)) + continue + + self_restart_cmd = '{} {}{}'.format(sys.executable, __file__, + (' --tools-json ' + args.tools_json) if args.tools_json else '') + self_restart_cmd = to_shell_specific_paths([self_restart_cmd])[0] + + if IDF_TOOLS_EXPORT_CMD: + prefer_system_hint = '' + else: + prefer_system_hint = ' To use it, run \'{} export --prefer-system\''.format(self_restart_cmd) + + if IDF_TOOLS_INSTALL_CMD: + install_cmd = to_shell_specific_paths([IDF_TOOLS_INSTALL_CMD])[0] + else: + install_cmd = self_restart_cmd + ' install' + + if not tool.versions_installed: + if tool.get_install_type() == IDFTool.INSTALL_ALWAYS: + all_tools_found = False + fatal('tool {} has no installed versions. Please run \'{}\' to install it.'.format( + tool.name, install_cmd)) + if tool.version_in_path and tool.version_in_path not in tool.versions: + info('An unsupported version of tool {} was found in PATH: {}. '.format(name, tool.version_in_path) + + prefer_system_hint, f=sys.stderr) + continue + else: + # tool is optional, and does not have versions installed + # use whatever is available in PATH + continue + + if tool.version_in_path and tool.version_in_path not in tool.versions: + info('Not using an unsupported version of tool {} found in PATH: {}.'.format( + tool.name, tool.version_in_path) + prefer_system_hint, f=sys.stderr) + + version_to_use = tool.get_preferred_installed_version() + export_paths = tool.get_export_paths(version_to_use) + if export_paths: + paths_to_export += export_paths + tool_export_vars = tool.get_export_vars(version_to_use) + for k, v in tool_export_vars.items(): + old_v = os.environ.get(k) + if old_v is None or old_v != v: + export_vars[k] = v + + current_path = os.getenv('PATH') + idf_python_env_path, idf_python_export_path, virtualenv_python = get_python_env_path() + if os.path.exists(virtualenv_python): + idf_python_env_path = to_shell_specific_paths([idf_python_env_path])[0] + if os.getenv('IDF_PYTHON_ENV_PATH') != idf_python_env_path: + export_vars['IDF_PYTHON_ENV_PATH'] = to_shell_specific_paths([idf_python_env_path])[0] + if idf_python_export_path not in current_path: + paths_to_export.append(idf_python_export_path) + + idf_tools_dir = os.path.join(global_idf_path, 'tools') + idf_tools_dir = to_shell_specific_paths([idf_tools_dir])[0] + if idf_tools_dir not in current_path: + paths_to_export.append(idf_tools_dir) + + if sys.platform == 'win32' and 'MSYSTEM' not in os.environ: + old_path = '%PATH%' + path_sep = ';' + else: + old_path = '$PATH' + # can't trust os.pathsep here, since for Windows Python started from MSYS shell, + # os.pathsep will be ';' + path_sep = ':' + + if args.format == EXPORT_SHELL: + if sys.platform == 'win32' and 'MSYSTEM' not in os.environ: + export_format = 'SET "{}={}"' + export_sep = '\n' + else: + export_format = 'export {}="{}"' + export_sep = ';' + elif args.format == EXPORT_KEY_VALUE: + export_format = '{}={}' + export_sep = '\n' + else: + raise NotImplementedError('unsupported export format {}'.format(args.format)) + + if paths_to_export: + export_vars['PATH'] = path_sep.join(to_shell_specific_paths(paths_to_export) + [old_path]) + + export_statements = export_sep.join([export_format.format(k, v) for k, v in export_vars.items()]) + + if export_statements: + print(export_statements) + + if not all_tools_found: + raise SystemExit(1) + + +def apply_mirror_prefix_map(args, tool_obj, tool_version): + """Rewrite URL for given tool_obj, given tool_version, and current platform, + if --mirror-prefix-map flag or IDF_MIRROR_PREFIX_MAP environment variable is given. + """ + mirror_prefix_map = None + mirror_prefix_map_env = os.getenv('IDF_MIRROR_PREFIX_MAP') + if mirror_prefix_map_env: + mirror_prefix_map = mirror_prefix_map_env.split(';') + if IDF_MAINTAINER and args.mirror_prefix_map: + if mirror_prefix_map: + warn('Both IDF_MIRROR_PREFIX_MAP environment variable and --mirror-prefix-map flag are specified, ' + + 'will use the value from the command line.') + mirror_prefix_map = args.mirror_prefix_map + download_obj = tool_obj.versions[tool_version].get_download_for_platform(PYTHON_PLATFORM) + if mirror_prefix_map and download_obj: + for item in mirror_prefix_map: + if URL_PREFIX_MAP_SEPARATOR not in item: + warn('invalid mirror-prefix-map item (missing \'{}\') {}'.format(URL_PREFIX_MAP_SEPARATOR, item)) + continue + search, replace = item.split(URL_PREFIX_MAP_SEPARATOR, 1) + old_url = download_obj.url + new_url = re.sub(search, replace, old_url) + if new_url != old_url: + info('Changed download URL: {} => {}'.format(old_url, new_url)) + download_obj.url = new_url + break + + +def action_install(args): + tools_info = load_tools_info() + tools_spec = args.tools + if not tools_spec: + tools_spec = [k for k, v in tools_info.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS] + info('Installing tools: {}'.format(', '.join(tools_spec))) + elif 'all' in tools_spec: + tools_spec = [k for k, v in tools_info.items() if v.get_install_type() != IDFTool.INSTALL_NEVER] + info('Installing tools: {}'.format(', '.join(tools_spec))) + + for tool_spec in tools_spec: + if '@' not in tool_spec: + tool_name = tool_spec + tool_version = None + else: + tool_name, tool_version = tool_spec.split('@', 1) + if tool_name not in tools_info: + fatal('unknown tool name: {}'.format(tool_name)) + raise SystemExit(1) + tool_obj = tools_info[tool_name] + if tool_version is not None and tool_version not in tool_obj.versions: + fatal('unknown version for tool {}: {}'.format(tool_name, tool_version)) + raise SystemExit(1) + if tool_version is None: + tool_version = tool_obj.get_recommended_version() + assert tool_version is not None + tool_obj.find_installed_versions() + tool_spec = '{}@{}'.format(tool_name, tool_version) + if tool_version in tool_obj.versions_installed: + info('Skipping {} (already installed)'.format(tool_spec)) + continue + + info('Installing {}'.format(tool_spec)) + apply_mirror_prefix_map(args, tool_obj, tool_version) + + tool_obj.download(tool_version) + tool_obj.install(tool_version) + + +def action_install_python_env(args): + idf_python_env_path, _, virtualenv_python = get_python_env_path() + + if args.reinstall and os.path.exists(idf_python_env_path): + warn('Removing the existing Python environment in {}'.format(idf_python_env_path)) + shutil.rmtree(idf_python_env_path) + + if not os.path.exists(virtualenv_python): + info('Creating a new Python environment in {}'.format(idf_python_env_path)) + + try: + import virtualenv # noqa: F401 + except ImportError: + info('Installing virtualenv') + subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'virtualenv'], + stdout=sys.stdout, stderr=sys.stderr) + + subprocess.check_call([sys.executable, '-m', 'virtualenv', '--no-site-packages', idf_python_env_path], + stdout=sys.stdout, stderr=sys.stderr) + run_args = [virtualenv_python, '-m', 'pip', 'install', '--no-warn-script-location'] + requirements_txt = os.path.join(global_idf_path, 'requirements.txt') + run_args += ['-r', requirements_txt] + if args.extra_wheels_dir: + run_args += ['--find-links', args.extra_wheels_dir] + info('Installing Python packages from {}'.format(requirements_txt)) + subprocess.check_call(run_args, stdout=sys.stdout, stderr=sys.stderr) + + +def action_add_version(args): + tools_info = load_tools_info() + tool_name = args.tool + tool_obj = tools_info.get(tool_name) + if not tool_obj: + info('Creating new tool entry for {}'.format(tool_name)) + tool_obj = IDFTool(tool_name, TODO_MESSAGE, IDFTool.INSTALL_ALWAYS, + TODO_MESSAGE, TODO_MESSAGE, [TODO_MESSAGE], TODO_MESSAGE) + tools_info[tool_name] = tool_obj + version = args.version + version_obj = tool_obj.versions.get(version) + if version not in tool_obj.versions: + info('Creating new version {}'.format(version)) + version_obj = IDFToolVersion(version, IDFToolVersion.STATUS_SUPPORTED) + tool_obj.versions[version] = version_obj + url_prefix = args.url_prefix or 'https://%s/' % TODO_MESSAGE + for file_path in args.files: + file_name = os.path.basename(file_path) + # Guess which platform this file is for + found_platform = None + for platform_alias, platform_id in PLATFORM_FROM_NAME.items(): + if platform_alias in file_name: + found_platform = platform_id + break + if found_platform is None: + info('Could not guess platform for file {}'.format(file_name)) + found_platform = TODO_MESSAGE + # Get file size and calculate the SHA256 + file_size, file_sha256 = get_file_size_sha256(file_path) + url = url_prefix + file_name + info('Adding download for platform {}'.format(found_platform)) + info(' size: {}'.format(file_size)) + info(' SHA256: {}'.format(file_sha256)) + info(' URL: {}'.format(url)) + version_obj.add_download(found_platform, url, file_size, file_sha256) + json_str = dump_tools_json(tools_info) + if not args.output: + args.output = os.path.join(global_idf_path, TOOLS_FILE_NEW) + with open(args.output, 'w') as f: + f.write(json_str) + f.write('\n') + info('Wrote output to {}'.format(args.output)) + + +def action_rewrite(args): + tools_info = load_tools_info() + json_str = dump_tools_json(tools_info) + if not args.output: + args.output = os.path.join(global_idf_path, TOOLS_FILE_NEW) + with open(args.output, 'w') as f: + f.write(json_str) + f.write('\n') + info('Wrote output to {}'.format(args.output)) + + +def action_validate(args): + try: + import jsonschema + except ImportError: + fatal('You need to install jsonschema package to use validate command') + raise SystemExit(1) + + with open(os.path.join(global_idf_path, TOOLS_FILE), 'r') as tools_file: + tools_json = json.load(tools_file) + + with open(os.path.join(global_idf_path, TOOLS_SCHEMA_FILE), 'r') as schema_file: + schema_json = json.load(schema_file) + jsonschema.validate(tools_json, schema_json) + # on failure, this will raise an exception with a fairly verbose diagnostic message + + +def main(argv): + parser = argparse.ArgumentParser() + + parser.add_argument('--quiet', help='Don\'t output diagnostic messages to stdout/stderr', action='store_true') + parser.add_argument('--non-interactive', help='Don\'t output interactive messages and questions', action='store_true') + parser.add_argument('--tools-json', help='Path to the tools.json file to use') + parser.add_argument('--idf-path', help='ESP-IDF path to use') + + subparsers = parser.add_subparsers(dest='action') + subparsers.add_parser('list', help='List tools and versions available') + subparsers.add_parser('check', help='Print summary of tools installed or found in PATH') + export = subparsers.add_parser('export', help='Output command for setting tool paths, suitable for shell') + export.add_argument('--format', choices=[EXPORT_SHELL, EXPORT_KEY_VALUE], default=EXPORT_SHELL, + help='Format of the output: shell (suitable for printing into shell), ' + + 'or key-value (suitable for parsing by other tools') + export.add_argument('--prefer-system', help='Normally, if the tool is already present in PATH, ' + + 'but has an unsupported version, a version from the tools directory ' + + 'will be used instead. If this flag is given, the version in PATH ' + + 'will be used.', action='store_true') + install = subparsers.add_parser('install', help='Download and install tools into the tools directory') + if IDF_MAINTAINER: + install.add_argument('--mirror-prefix-map', nargs='*', + help='Pattern to rewrite download URLs, with source and replacement separated by comma.' + + ' E.g. http://foo.com,http://test.foo.com') + install.add_argument('tools', nargs='*', help='Tools to install. ' + + 'To install a specific version use tool_name@version syntax.' + + 'Use \'all\' to install all tools, including the optional ones.') + + install_python_env = subparsers.add_parser('install-python-env', + help='Create Python virtual environment and install the ' + + 'required Python packages') + install_python_env.add_argument('--reinstall', help='Discard the previously installed environment', + action='store_true') + install_python_env.add_argument('--extra-wheels-dir', help='Additional directories with wheels ' + + 'to use during installation') + + if IDF_MAINTAINER: + add_version = subparsers.add_parser('add-version', help='Add or update download info for a version') + add_version.add_argument('--output', help='Save new tools.json into this file') + add_version.add_argument('--tool', help='Tool name to set add a version for', required=True) + add_version.add_argument('--version', help='Version identifier', required=True) + add_version.add_argument('--url-prefix', help='String to prepend to file names to obtain download URLs') + add_version.add_argument('files', help='File names of the download artifacts', nargs='*') + + rewrite = subparsers.add_parser('rewrite', help='Load tools.json, validate, and save the result back into JSON') + rewrite.add_argument('--output', help='Save new tools.json into this file') + + subparsers.add_parser('validate', help='Validate tools.json against schema file') + + args = parser.parse_args(argv) + + if args.action is None: + parser.print_help() + parser.exit(1) + + if args.quiet: + global global_quiet + global_quiet = True + + if args.non_interactive: + global global_non_interactive + global_non_interactive = True + + global global_idf_path + global_idf_path = os.environ.get('IDF_PATH') + if args.idf_path: + global_idf_path = args.idf_path + if not global_idf_path: + global_idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "..")) + + global global_idf_tools_path + global_idf_tools_path = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(IDF_TOOLS_PATH_DEFAULT) + + if sys.version_info.major == 2: + try: + global_idf_tools_path.decode('ascii') + except UnicodeDecodeError: + fatal('IDF_TOOLS_PATH contains non-ASCII characters: {}'.format(global_idf_tools_path) + + '\nThis is not supported yet with Python 2. ' + + 'Please set IDF_TOOLS_PATH to a directory with an ASCII name, or switch to Python 3.') + raise SystemExit(1) + + if CURRENT_PLATFORM == UNKNOWN_PLATFORM: + fatal('Platform {} appears to be unsupported'.format(PYTHON_PLATFORM)) + raise SystemExit(1) + + global global_tools_json + if args.tools_json: + global_tools_json = args.tools_json + else: + global_tools_json = os.path.join(global_idf_path, TOOLS_FILE) + + action_func_name = 'action_' + args.action.replace('-', '_') + action_func = globals()[action_func_name] + + action_func(args) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/kconfig/.gitignore b/tools/kconfig/.gitignore index 9e8c2433..30d8d457 100644 --- a/tools/kconfig/.gitignore +++ b/tools/kconfig/.gitignore @@ -11,6 +11,7 @@ zconf.hash.c gconf.glade.h *.pot *.mo +*.o # # configuration programs diff --git a/tools/kconfig/Makefile b/tools/kconfig/Makefile index 106ffb6b..b91cbc28 100644 --- a/tools/kconfig/Makefile +++ b/tools/kconfig/Makefile @@ -167,6 +167,8 @@ check-lxdialog := $(SRCDIR)/lxdialog/check-lxdialog.sh CFLAGS += $(shell $(CONFIG_SHELL) $(check-lxdialog) -ccflags) \ -DLOCALE -MMD +CFLAGS += -I "." -I "${SRCDIR}" + %.o: $(SRCDIR)/%.c $(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@ @@ -192,13 +194,13 @@ lxdialog/%.o: $(SRCDIR)/lxdialog/%.c lxdialog := lxdialog/checklist.o lxdialog/util.o lxdialog/inputbox.o lxdialog += lxdialog/textbox.o lxdialog/yesno.o lxdialog/menubox.o -conf-objs := conf.o zconf.tab.o -mconf-objs := mconf.o zconf.tab.o $(lxdialog) -nconf-objs := nconf.o zconf.tab.o nconf.gui.o -kxgettext-objs := kxgettext.o zconf.tab.o +conf-objs := conf.o zconf.tab.o expand_env.o +mconf-objs := mconf.o zconf.tab.o $(lxdialog) expand_env.o +nconf-objs := nconf.o zconf.tab.o nconf.gui.o expand_env.o +kxgettext-objs := kxgettext.o zconf.tab.o expand_env.o qconf-cxxobjs := qconf.o -qconf-objs := zconf.tab.o -gconf-objs := gconf.o zconf.tab.o +qconf-objs := zconf.tab.o expand_env.o +gconf-objs := gconf.o zconf.tab.o expand_env.o hostprogs-y := conf-idf nconf mconf-idf kxgettext qconf gconf diff --git a/tools/kconfig/confdata.c b/tools/kconfig/confdata.c index 2a8d5b54..70f6ab7f 100644 --- a/tools/kconfig/confdata.c +++ b/tools/kconfig/confdata.c @@ -831,119 +831,6 @@ next: return 0; } -static int conf_split_config(void) -{ - const char *name; - char path[PATH_MAX+1]; - char *s, *d, c; - struct symbol *sym; - struct stat sb; - int res, i, fd; - - name = conf_get_autoconfig_name(); - conf_read_simple(name, S_DEF_AUTO); - sym_calc_value(modules_sym); - - if (chdir("include/config")) - return 1; - - res = 0; - for_all_symbols(i, sym) { - sym_calc_value(sym); - if ((sym->flags & SYMBOL_AUTO) || !sym->name) - continue; - if (sym->flags & SYMBOL_WRITE) { - if (sym->flags & SYMBOL_DEF_AUTO) { - /* - * symbol has old and new value, - * so compare them... - */ - switch (sym->type) { - case S_BOOLEAN: - case S_TRISTATE: - if (sym_get_tristate_value(sym) == - sym->def[S_DEF_AUTO].tri) - continue; - break; - case S_STRING: - case S_HEX: - case S_INT: - if (!strcmp(sym_get_string_value(sym), - sym->def[S_DEF_AUTO].val)) - continue; - break; - default: - break; - } - } else { - /* - * If there is no old value, only 'no' (unset) - * is allowed as new value. - */ - switch (sym->type) { - case S_BOOLEAN: - case S_TRISTATE: - if (sym_get_tristate_value(sym) == no) - continue; - break; - default: - break; - } - } - } else if (!(sym->flags & SYMBOL_DEF_AUTO)) - /* There is neither an old nor a new value. */ - continue; - /* else - * There is an old value, but no new value ('no' (unset) - * isn't saved in auto.conf, so the old value is always - * different from 'no'). - */ - - /* Replace all '_' and append ".h" */ - s = sym->name; - d = path; - while ((c = *s++)) { - c = tolower(c); - *d++ = (c == '_') ? '/' : c; - } - strcpy(d, ".h"); - - /* Assume directory path already exists. */ - fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd == -1) { - if (errno != ENOENT) { - res = 1; - break; - } - /* - * Create directory components, - * unless they exist already. - */ - d = path; - while ((d = strchr(d, '/'))) { - *d = 0; - if (stat(path, &sb) && mkdir(path, 0755)) { - res = 1; - goto out; - } - *d++ = '/'; - } - /* Try it again. */ - fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd == -1) { - res = 1; - break; - } - } - close(fd); - } -out: - if (chdir("../..")) - return 1; - - return res; -} - int conf_write_autoconf(void) { struct symbol *sym; @@ -955,9 +842,6 @@ int conf_write_autoconf(void) file_write_dep("include/config/auto.conf.cmd"); - if (conf_split_config()) - return 1; - out = fopen(".tmpconfig", "w"); if (!out) return 1; diff --git a/tools/kconfig/expand_env.c b/tools/kconfig/expand_env.c new file mode 100644 index 00000000..185280e0 --- /dev/null +++ b/tools/kconfig/expand_env.c @@ -0,0 +1,88 @@ +#include +#include +#include +#include +#include +#include + +#include "expand_env.h" + +static bool allowed_env_var_name(char c) +{ + return c != '\0' && + !isblank(c) && + !iscntrl(c) && + c != '/' && + c != '\\' && + c != '=' && + c != '$'; +} + +#define MAX_LEN (128 * 1024) /* Longest a result can expand to */ + +/* Very basic expansion that looks for variable references like $NAME and expands them + * + */ +char *expand_environment(const char *input, const char *src_name, int src_line_no) +{ + char *result = malloc(MAX_LEN); + + char *out = result; + const char *in = input; + + while (*in != '\0') { + // check for buffer overflow + if (out >= result + MAX_LEN - 1) { + goto too_long; + } + + if (*in != '$') { + // not part of an environment variable name, copy directly + *out++ = *in++; + continue; + } + + // *in points to start of an environment variable reference + in++; + const char *env_start = in; + while (allowed_env_var_name(*in)) { // scan to the end of the name + in++; + } + size_t env_len = in - env_start; + + // make a buffer to hold the environment variable name + // + // strndup is not available on mingw32, apparently. + char *env_name = calloc(1, env_len + 1); + assert(env_name != NULL); + strncpy(env_name, env_start, env_len); + + const char *value = getenv(env_name); + if (value == NULL || strlen(value) == 0) { + printf("%s:%d: undefined environment variable \"%s\"\n", + src_name, src_line_no, env_name); + exit(1); + } + free(env_name); + if (out + strlen(value) >= result + MAX_LEN - 1) { + goto too_long; + } + strcpy(out, value); // append the value to the result (range checked in previous statement) + out += strlen(value); + } + + *out = '\0'; // null terminate the result string + + return result; + +too_long: + printf("%s:%d: Expansion is longer than %d bytes\n", + src_name, src_line_no, MAX_LEN); + free(result); + exit(1); +} + +void free_expanded(char *expanded) +{ + free(expanded); +} diff --git a/tools/kconfig/expand_env.h b/tools/kconfig/expand_env.h new file mode 100644 index 00000000..4404523a --- /dev/null +++ b/tools/kconfig/expand_env.h @@ -0,0 +1,13 @@ +#pragma once + +/* Expand any $ENV type environment variables in 'input', + return a newly allocated buffer with the result. + + Buffer should be freed after use. + + This is very basic expansion, doesn't do escaping or anything else. +*/ +char *expand_environment(const char *input, const char *src_name, int src_line_no); + +/* Free a buffer allocated by expand_environment */ +void free_expanded(char *expanded); diff --git a/tools/kconfig/lxdialog/check-lxdialog.sh b/tools/kconfig/lxdialog/check-lxdialog.sh index 9e981459..e9daa627 100755 --- a/tools/kconfig/lxdialog/check-lxdialog.sh +++ b/tools/kconfig/lxdialog/check-lxdialog.sh @@ -7,6 +7,10 @@ ldflags() if [ $(uname -s) == "Darwin" ]; then #OSX seems to need ncurses too echo -n "-lncurses " + elif [ $(uname -s) == "FreeBSD" ]; then + # On FreeBSD, the linker needs to know to search port libs for + # libintl + echo -n "-L/usr/local/lib -lintl " fi pkg-config --libs ncursesw 2>/dev/null && exit pkg-config --libs ncurses 2>/dev/null && exit @@ -44,6 +48,10 @@ ccflags() if [ $(uname -s) == "Darwin" ]; then #OSX doesn't have libintl echo -n "-DKBUILD_NO_NLS -Wno-format-security " + elif [ $(uname -s) == "FreeBSD" ]; then + # FreeBSD gettext port has libintl.h, but the compiler needs + # to know to search port includes + echo -n "-I/usr/local/include " fi } diff --git a/tools/kconfig/zconf.l b/tools/kconfig/zconf.l index 8fd75491..b8cc8939 100644 --- a/tools/kconfig/zconf.l +++ b/tools/kconfig/zconf.l @@ -13,9 +13,9 @@ #include #include #include -#include #include "lkc.h" +#include "expand_env.h" #define START_STRSIZE 16 @@ -151,7 +151,7 @@ n [A-Za-z0-9_-] return T_WORD; } #.* /* comment */ - \\\n current_file->lineno++; + \\\r?\n current_file->lineno++; [[:blank:]]+ . warn_ignored_character(*yytext); <> { @@ -348,19 +348,33 @@ void zconf_nextfile(const char *name) current_file = file; } -void zconf_nextfiles(const char *wildcard) +void zconf_nextfiles(const char *expression) { - wordexp_t p; - char **w; - int i; + /* Expand environment variables in 'expression' */ + char* str = expand_environment(expression, zconf_curname(), zconf_lineno()); - wordexp(wildcard, &p, 0); - w = p.we_wordv; + /* zconf_nextfile() processes files in LIFO order, so to keep the + files in the order provided we need to process the list backwards + */ + if (str != NULL && strlen(str)) { + char* pos = str + strlen(str); // start at null terminator - for (i = p.we_wordc - 1; i >= 0; i--) - zconf_nextfile(w[i]); + while (pos != str) { + pos--; + if(*pos == ' ') { + *pos = '\0'; // split buffer into multiple c-strings + if (strlen(pos + 1)) { + zconf_nextfile(pos + 1); + } + } + } - wordfree(&p); + if (strlen(str)) { // re-check as first character may have been a space + zconf_nextfile(str); + } + } + + free_expanded(str); } static void zconf_endfile(void) diff --git a/tools/kconfig_new/README.md b/tools/kconfig_new/README.md new file mode 100644 index 00000000..b03ffd32 --- /dev/null +++ b/tools/kconfig_new/README.md @@ -0,0 +1,100 @@ +# kconfig_new + +kconfig_new is the kconfig support used by the CMake-based build system. + +It uses a fork of [kconfiglib](https://github.com/ulfalizer/Kconfiglib) which adds a few small features (newer upstream kconfiglib also has the support we need, we just haven't updated yet). See comments at top of kconfiglib.py for details + +## confserver.py + +confserver.py is a small Python program intended to support IDEs and other clients who want to allow editing sdkconfig, without needing to reproduce all of the kconfig logic in a particular program. + +After launching confserver.py (which can be done via `idf.py confserver` command or `confserver` build target in ninja/make), the confserver communicates via JSON sent/received on stdout. Out-of-band errors are logged via stderr. + +### Configuration Structure + +During cmake run, the CMake-based build system produces a number of metadata files including `build/config/kconfig_menus.json`, which is a JSON representation of all the menu items in the project configuration and their structure. + +This format is currently undocumented, however running CMake with an IDF project will give an indication of the format. The format is expected to be stable. + +### Initial Process + +After initializing, the server will print "Server running, waiting for requests on stdin..." on stderr. + +Then it will print a JSON dictionary on stdout, representing the initial state of sdkconfig: + +``` +{ + "version": 2, + "ranges": { + "TEST_CONDITIONAL_RANGES": [0, 10] }, + "visible": { "TEST_CONDITIONAL_RANGES": true, + "CHOICE_A": true, + "test-config-submenu": true }, + "values": { "TEST_CONDITIONAL_RANGES": 1, + "CHOICE_A": true }, +} +``` + +(Note: actual output is not pretty-printed and will print on a single line. Order of dictionary keys is undefined.) + +* "version" key is the protocol version in use. +* "ranges" holds is a dictionary for any config symbol which has a valid integer range. The array value has two values for min/max. +* "visible" holds a dictionary showing initial visibility status of config symbols (identified by the config symbol name) and menus (which don't represent a symbol but are represented as an id 'slug'). Both these names (symbol name and menu slug) correspond to the 'id' key in kconfig_menus.json. +* "values" holds a dictionary showing initial values of all config symbols. Invisible symbols are not included here. + +### Interaction + +Interaction consists of the client sending JSON dictionary "requests" to the server one at a time. The server will respond to each request with a JSON dictionary response. Interaction is done when the client closes stdout (at this point the server will exit). + +Requests look like: + +``` +{ "version": 2, + "set": { "TEST_CHILD_STR": "New value", + "TEST_BOOL": true } +} +``` + +Note: Requests don't need to be pretty-printed, they just need to be valid JSON. + +The `version` key *must* be present in the request and must match a protocol version supported by confserver. + +The `set` key is optional. If present, its value must be a dictionary of new values to set on kconfig symbols. + +Additional optional keys: + +* `load`: If this key is set, sdkconfig file will be reloaded from filesystem before any values are set applied. The value of this key can be a filename, in which case configuration will be loaded from this file. If the value of this key is `null`, configuration will be loaded from the last used file. + +* `save`: If this key is set, sdkconfig file will be saved after any values are set. Similar to `load`, the value of this key can be a filename to save to a particular file, or `null` to reuse the last used file. + +After a request is processed, a response is printed to stdout similar to this: + +``` +{ "version": 2, + "ranges": {}, + "visible": { "test-config-submenu": false}, + "values": { "SUBMENU_TRIGGER": false } +} +``` + +* `version` is the protocol version used by the server. +* `ranges` contains any changed ranges, where the new range of the config symbol has changed (due to some other configuration change or because a new sdkconfig has been loaded). +* `visible` contains any visibility changes, where the visible config symbols have changed. +* `values` contains any value changes, where a config symbol value has changed. This may be due to an explicit change (ie the client `set` this value), or a change caused by some other change in the config system. Note that a change which is set by the client may not be reflected exactly the same in the response, due to restrictions on allowed values which are enforced by the config server. Invalid changes are ignored by the config server. + +### Error Responses + +In some cases, a request may lead to an error message. In this case, the error message is printed to stderr but an array of errors is also returned in the `error` key of the response: + +``` +{ "version": 777, + "error": [ "Unsupported request version 777. Server supports versions 1-2" ] +} +``` + +These error messages are intended to be human readable, not machine parseable. + +### Protocol Version Changes + +* V2: Added the `visible` key to the response. Invisible items are no longer represented as having value null. +* V2: `load` now sends changes compared to values before the load, not the whole list of config items. diff --git a/tools/kconfig_new/confgen.py b/tools/kconfig_new/confgen.py index 8255592f..e58f55ef 100755 --- a/tools/kconfig_new/confgen.py +++ b/tools/kconfig_new/confgen.py @@ -22,21 +22,155 @@ # limitations under the License. from __future__ import print_function import argparse -import sys +import fnmatch +import json import os import os.path +import re +import sys import tempfile -import json import gen_kconfig_doc import kconfiglib -import pprint __version__ = "0.1" -if not "IDF_CMAKE" in os.environ: +if "IDF_CMAKE" not in os.environ: os.environ["IDF_CMAKE"] = "" + +class DeprecatedOptions(object): + _REN_FILE = 'sdkconfig.rename' + _DEP_OP_BEGIN = '# Deprecated options for backward compatibility' + _DEP_OP_END = '# End of deprecated options' + _RE_DEP_OP_BEGIN = re.compile(_DEP_OP_BEGIN) + _RE_DEP_OP_END = re.compile(_DEP_OP_END) + + def __init__(self, config_prefix, path_rename_files, ignore_dirs=()): + self.config_prefix = config_prefix + # r_dic maps deprecated options to new options; rev_r_dic maps in the opposite direction + self.r_dic, self.rev_r_dic = self._parse_replacements(path_rename_files, ignore_dirs) + + # note the '=' at the end of regex for not getting partial match of configs + self._RE_CONFIG = re.compile(r'{}(\w+)='.format(self.config_prefix)) + + def _parse_replacements(self, repl_dir, ignore_dirs): + rep_dic = {} + rev_rep_dic = {} + + def remove_config_prefix(string): + if string.startswith(self.config_prefix): + return string[len(self.config_prefix):] + raise RuntimeError('Error in {} (line {}): Config {} is not prefixed with {}' + ''.format(rep_path, line_number, string, self.config_prefix)) + + for root, dirnames, filenames in os.walk(repl_dir): + for filename in fnmatch.filter(filenames, self._REN_FILE): + rep_path = os.path.join(root, filename) + + if rep_path.startswith(ignore_dirs): + print('Ignoring: {}'.format(rep_path)) + continue + + with open(rep_path) as f_rep: + for line_number, line in enumerate(f_rep, start=1): + sp_line = line.split() + if len(sp_line) == 0 or sp_line[0].startswith('#'): + # empty line or comment + continue + if len(sp_line) != 2 or not all(x.startswith(self.config_prefix) for x in sp_line): + raise RuntimeError('Syntax error in {} (line {})'.format(rep_path, line_number)) + if sp_line[0] in rep_dic: + raise RuntimeError('Error in {} (line {}): Replacement {} exist for {} and new ' + 'replacement {} is defined'.format(rep_path, line_number, + rep_dic[sp_line[0]], sp_line[0], + sp_line[1])) + + (dep_opt, new_opt) = (remove_config_prefix(x) for x in sp_line) + rep_dic[dep_opt] = new_opt + rev_rep_dic[new_opt] = dep_opt + return rep_dic, rev_rep_dic + + def get_deprecated_option(self, new_option): + return self.rev_r_dic.get(new_option, None) + + def get_new_option(self, deprecated_option): + return self.r_dic.get(deprecated_option, None) + + def replace(self, sdkconfig_in, sdkconfig_out): + replace_enabled = True + with open(sdkconfig_in, 'r') as f_in, open(sdkconfig_out, 'w') as f_out: + for line_num, line in enumerate(f_in, start=1): + if self._RE_DEP_OP_BEGIN.search(line): + replace_enabled = False + elif self._RE_DEP_OP_END.search(line): + replace_enabled = True + elif replace_enabled: + m = self._RE_CONFIG.search(line) + if m and m.group(1) in self.r_dic: + depr_opt = self.config_prefix + m.group(1) + new_opt = self.config_prefix + self.r_dic[m.group(1)] + line = line.replace(depr_opt, new_opt) + print('{}:{} {} was replaced with {}'.format(sdkconfig_in, line_num, depr_opt, new_opt)) + f_out.write(line) + + def append_doc(self, config, path_output): + + def option_was_written(opt): + return any(gen_kconfig_doc.node_should_write(node) for node in config.syms[opt].nodes) + + if len(self.r_dic) > 0: + with open(path_output, 'a') as f_o: + header = 'Deprecated options and their replacements' + f_o.write('.. _configuration-deprecated-options:\n\n{}\n{}\n\n'.format(header, '-' * len(header))) + for dep_opt in sorted(self.r_dic): + new_opt = self.r_dic[dep_opt] + if new_opt not in config.syms or (config.syms[new_opt].choice is None and option_was_written(new_opt)): + # everything except config for a choice (no link reference for those in the docs) + f_o.write('- {}{} (:ref:`{}{}`)\n'.format(config.config_prefix, dep_opt, + config.config_prefix, new_opt)) + + if new_opt in config.named_choices: + # here are printed config options which were filtered out + syms = config.named_choices[new_opt].syms + for sym in syms: + if sym.name in self.rev_r_dic: + # only if the symbol has been renamed + dep_name = self.rev_r_dic[sym.name] + + # config options doesn't have references + f_o.write(' - {}{}\n'.format(config.config_prefix, dep_name)) + + def append_config(self, config, path_output): + tmp_list = [] + + def append_config_node_process(node): + item = node.item + if isinstance(item, kconfiglib.Symbol) and item.env_var is None: + if item.name in self.rev_r_dic: + c_string = item.config_string + if c_string: + tmp_list.append(c_string.replace(self.config_prefix + item.name, + self.config_prefix + self.rev_r_dic[item.name])) + + config.walk_menu(append_config_node_process) + + if len(tmp_list) > 0: + with open(path_output, 'a') as f_o: + f_o.write('\n{}\n'.format(self._DEP_OP_BEGIN)) + f_o.writelines(tmp_list) + f_o.write('{}\n'.format(self._DEP_OP_END)) + + def append_header(self, config, path_output): + if len(self.r_dic) > 0: + with open(path_output, 'a') as f_o: + f_o.write('\n/* List of deprecated options */\n') + for dep_opt in sorted(self.r_dic): + new_opt = self.r_dic[dep_opt] + f_o.write('#ifdef {}{}\n#define {}{} {}{}\n#endif\n\n'.format(self.config_prefix, new_opt, + self.config_prefix, dep_opt, self.config_prefix, new_opt)) + + def main(): parser = argparse.ArgumentParser(description='confgen.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0])) @@ -46,9 +180,11 @@ def main(): default=None) parser.add_argument('--defaults', - help='Optional project defaults file, used if --config file doesn\'t exist', + help='Optional project defaults file, used if --config file doesn\'t exist. ' + 'Multiple files can be specified using multiple --defaults arguments.', nargs='?', - default=None) + default=[], + action='append') parser.add_argument('--kconfig', help='KConfig file with config item definitions', @@ -62,43 +198,70 @@ def main(): parser.add_argument('--env', action='append', default=[], help='Environment to set when evaluating the config file', metavar='NAME=VAL') + parser.add_argument('--env-file', type=argparse.FileType('r'), + help='Optional file to load environment variables from. Contents ' + 'should be a JSON object where each key/value pair is a variable.') + args = parser.parse_args() for fmt, filename in args.output: - if not fmt in OUTPUT_FORMATS.keys(): + if fmt not in OUTPUT_FORMATS.keys(): print("Format '%s' not recognised. Known formats: %s" % (fmt, OUTPUT_FORMATS.keys())) sys.exit(1) try: - args.env = [ (name,value) for (name,value) in ( e.split("=",1) for e in args.env) ] + args.env = [(name,value) for (name,value) in (e.split("=",1) for e in args.env)] except ValueError: - print("--env arguments must each contain =. To unset an environment variable, use 'ENV='") - sys.exit(1) + print("--env arguments must each contain =. To unset an environment variable, use 'ENV='") + sys.exit(1) for name, value in args.env: os.environ[name] = value + if args.env_file is not None: + env = json.load(args.env_file) + os.environ.update(env) + config = kconfiglib.Kconfig(args.kconfig) config.disable_redun_warnings() config.disable_override_warnings() - if args.defaults is not None: + if len(args.defaults) > 0: # always load defaults first, so any items which are not defined in that config # will have the default defined in the defaults file - if not os.path.exists(args.defaults): - raise RuntimeError("Defaults file not found: %s" % args.defaults) - config.load_config(args.defaults) + for name in args.defaults: + print("Loading defaults file %s..." % name) + if not os.path.exists(name): + raise RuntimeError("Defaults file not found: %s" % name) + config.load_config(name, replace=False) + + # don't collect rename options from examples because those are separate projects and no need to "stay compatible" + # with example projects + deprecated_options = DeprecatedOptions(config.config_prefix, path_rename_files=os.environ["IDF_PATH"], + ignore_dirs=(os.path.join(os.environ["IDF_PATH"], 'examples'))) # If config file previously exists, load it if args.config and os.path.exists(args.config): - config.load_config(args.config, replace=False) + # ... but replace deprecated options before that + with tempfile.NamedTemporaryFile(prefix="confgen_tmp", delete=False) as f: + temp_file = f.name + try: + deprecated_options.replace(sdkconfig_in=args.config, sdkconfig_out=temp_file) + config.load_config(temp_file, replace=False) + update_if_changed(temp_file, args.config) + finally: + try: + os.remove(temp_file) + except OSError: + pass # Output the files specified in the arguments for output_type, filename in args.output: - temp_file = tempfile.mktemp(prefix="confgen_tmp") + with tempfile.NamedTemporaryFile(prefix="confgen_tmp", delete=False) as f: + temp_file = f.name try: output_function = OUTPUT_FORMATS[output_type] - output_function(config, temp_file) + output_function(deprecated_options, config, temp_file) update_if_changed(temp_file, filename) finally: try: @@ -107,15 +270,56 @@ def main(): pass -def write_config(config, filename): +def write_config(deprecated_options, config, filename): CONFIG_HEADING = """# # Automatically generated file. DO NOT EDIT. # Espressif IoT Development Framework (ESP-IDF) Project Configuration # """ config.write_config(filename, header=CONFIG_HEADING) + deprecated_options.append_config(config, filename) -def write_header(config, filename): + +def write_makefile(deprecated_options, config, filename): + CONFIG_HEADING = """# +# Automatically generated file. DO NOT EDIT. +# Espressif IoT Development Framework (ESP-IDF) Project Makefile Configuration +# +""" + with open(filename, "w") as f: + tmp_dep_lines = [] + f.write(CONFIG_HEADING) + + def get_makefile_config_string(name, value, orig_type): + if orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE): + return "{}{}={}\n".format(config.config_prefix, name, '' if value == 'n' else value) + elif orig_type in (kconfiglib.INT, kconfiglib.HEX): + return "{}{}={}\n".format(config.config_prefix, name, value) + elif orig_type == kconfiglib.STRING: + return '{}{}="{}"\n'.format(config.config_prefix, name, kconfiglib.escape(value)) + else: + raise RuntimeError('{}{}: unknown type {}'.format(config.config_prefix, name, orig_type)) + + def write_makefile_node(node): + item = node.item + if isinstance(item, kconfiglib.Symbol) and item.env_var is None: + # item.config_string cannot be used because it ignores hidden config items + val = item.str_value + f.write(get_makefile_config_string(item.name, val, item.orig_type)) + + dep_opt = deprecated_options.get_deprecated_option(item.name) + if dep_opt: + # the same string but with the deprecated name + tmp_dep_lines.append(get_makefile_config_string(dep_opt, val, item.orig_type)) + + config.walk_menu(write_makefile_node, True) + + if len(tmp_dep_lines) > 0: + f.write('\n# List of deprecated options\n') + f.writelines(tmp_dep_lines) + + +def write_header(deprecated_options, config, filename): CONFIG_HEADING = """/* * Automatically generated file. DO NOT EDIT. * Espressif IoT Development Framework (ESP-IDF) Configuration Header @@ -123,9 +327,12 @@ def write_header(config, filename): #pragma once """ config.write_autoconf(filename, header=CONFIG_HEADING) + deprecated_options.append_header(config, filename) -def write_cmake(config, filename): + +def write_cmake(deprecated_options, config, filename): with open(filename, "w") as f: + tmp_dep_list = [] write = f.write prefix = config.config_prefix @@ -134,31 +341,46 @@ def write_cmake(config, filename): # Espressif IoT Development Framework (ESP-IDF) Configuration cmake include file # """) + + configs_list = list() + def write_node(node): sym = node.item if not isinstance(sym, kconfiglib.Symbol): return - # Note: str_value calculates _write_to_conf, due to - # internal magic in kconfiglib... - val = sym.str_value - if sym._write_to_conf: + if sym.config_string: + val = sym.str_value if sym.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and val == "n": val = "" # write unset values as empty variables write("set({}{} \"{}\")\n".format( prefix, sym.name, val)) + + configs_list.append(prefix + sym.name) + dep_opt = deprecated_options.get_deprecated_option(sym.name) + if dep_opt: + tmp_dep_list.append("set({}{} \"{}\")\n".format(prefix, dep_opt, val)) + configs_list.append(prefix + dep_opt) + config.walk_menu(write_node) + write("set(CONFIGS_LIST {})".format(";".join(configs_list))) + + if len(tmp_dep_list) > 0: + write('\n# List of deprecated options for backward compatibility\n') + f.writelines(tmp_dep_list) + def get_json_values(config): config_dict = {} + def write_node(node): sym = node.item if not isinstance(sym, kconfiglib.Symbol): return - val = sym.str_value # this calculates _write_to_conf, due to kconfiglib magic - if sym._write_to_conf: - if sym.type in [ kconfiglib.BOOL, kconfiglib.TRISTATE ]: + if sym.config_string: + val = sym.str_value + if sym.type in [kconfiglib.BOOL, kconfiglib.TRISTATE]: val = (val != "n") elif sym.type == kconfiglib.HEX: val = int(val, 16) @@ -168,12 +390,39 @@ def get_json_values(config): config.walk_menu(write_node) return config_dict -def write_json(config, filename): + +def write_json(deprecated_options, config, filename): config_dict = get_json_values(config) with open(filename, "w") as f: json.dump(config_dict, f, indent=4, sort_keys=True) -def write_json_menus(config, filename): + +def get_menu_node_id(node): + """ Given a menu node, return a unique id + which can be used to identify it in the menu structure + + Will either be the config symbol name, or a menu identifier + 'slug' + + """ + try: + if not isinstance(node.item, kconfiglib.Choice): + return node.item.name + except AttributeError: + pass + + result = [] + while node.parent is not None: + slug = re.sub(r'\W+', '-', node.prompt[0]).lower() + result.append(slug) + node = node.parent + + result = "-".join(reversed(result)) + return result + + +def write_json_menus(deprecated_options, config, filename): + existing_ids = set() result = [] # root level items node_lookup = {} # lookup from MenuNode to an item in result @@ -181,7 +430,7 @@ def write_json_menus(config, filename): try: json_parent = node_lookup[node.parent]["children"] except KeyError: - assert not node.parent in node_lookup # if fails, we have a parent node with no "children" entity (ie a bug) + assert node.parent not in node_lookup # if fails, we have a parent node with no "children" entity (ie a bug) json_parent = result # root level node # node.kconfig.y means node has no dependency, @@ -197,11 +446,11 @@ def write_json_menus(config, filename): new_json = None if node.item == kconfiglib.MENU or is_menuconfig: - new_json = { "type" : "menu", - "title" : node.prompt[0], - "depends_on": depends, - "children": [] - } + new_json = {"type": "menu", + "title": node.prompt[0], + "depends_on": depends, + "children": [], + } if is_menuconfig: sym = node.item new_json["name"] = sym.name @@ -224,15 +473,17 @@ def write_json_menus(config, filename): # should have one condition which is true for min_range, max_range, cond_expr in sym.ranges: if kconfiglib.expr_value(cond_expr): - greatest_range = [int(min_range.str_value), int(max_range.str_value)] + base = 16 if sym.type == kconfiglib.HEX else 10 + greatest_range = [int(min_range.str_value, base), int(max_range.str_value, base)] + break new_json = { - "type" : kconfiglib.TYPE_TO_STR[sym.type], - "name" : sym.name, + "type": kconfiglib.TYPE_TO_STR[sym.type], + "name": sym.name, "title": node.prompt[0] if node.prompt else None, - "depends_on" : depends, + "depends_on": depends, "help": node.help, - "range" : greatest_range, + "range": greatest_range, "children": [], } elif isinstance(node.item, kconfiglib.Choice): @@ -241,12 +492,18 @@ def write_json_menus(config, filename): "type": "choice", "title": node.prompt[0], "name": choice.name, - "depends_on" : depends, + "depends_on": depends, "help": node.help, "children": [] } if new_json: + node_id = get_menu_node_id(node) + if node_id in existing_ids: + raise RuntimeError("Config file contains two items with the same id: %s (%s). " + + "Please rename one of these items to avoid ambiguity." % (node_id, node.prompt[0])) + new_json["id"] = node_id + json_parent.append(new_json) node_lookup[node] = new_json @@ -254,6 +511,12 @@ def write_json_menus(config, filename): with open(filename, "w") as f: f.write(json.dumps(result, sort_keys=True, indent=4)) + +def write_docs(deprecated_options, config, filename): + gen_kconfig_doc.write_docs(config, filename) + deprecated_options.append_doc(config, filename) + + def update_if_changed(source, destination): with open(source, "r") as f: source_contents = f.read() @@ -268,14 +531,15 @@ def update_if_changed(source, destination): f.write(source_contents) -OUTPUT_FORMATS = { - "config" : write_config, - "header" : write_header, - "cmake" : write_cmake, - "docs" : gen_kconfig_doc.write_docs, - "json" : write_json, - "json_menus" : write_json_menus, - } +OUTPUT_FORMATS = {"config": write_config, + "makefile": write_makefile, # only used with make in order to generate auto.conf + "header": write_header, + "cmake": write_cmake, + "docs": write_docs, + "json": write_json, + "json_menus": write_json_menus, + } + class FatalError(RuntimeError): """ @@ -283,6 +547,7 @@ class FatalError(RuntimeError): """ pass + if __name__ == '__main__': try: main() diff --git a/tools/kconfig_new/config.env.in b/tools/kconfig_new/config.env.in new file mode 100644 index 00000000..d685c85d --- /dev/null +++ b/tools/kconfig_new/config.env.in @@ -0,0 +1,7 @@ +{ + "COMPONENT_KCONFIGS": "${kconfigs}", + "COMPONENT_KCONFIGS_PROJBUILD": "${kconfig_projbuilds}", + "IDF_CMAKE": "y", + "IDF_TARGET": "${idf_target}", + "IDF_PATH": "${idf_path}" +} diff --git a/tools/kconfig_new/confserver.py b/tools/kconfig_new/confserver.py index cf63a598..914a2dd7 100755 --- a/tools/kconfig_new/confserver.py +++ b/tools/kconfig_new/confserver.py @@ -5,19 +5,25 @@ # from __future__ import print_function import argparse +import confgen import json import kconfiglib import os import sys -import confgen +import tempfile from confgen import FatalError, __version__ +# Min/Max supported protocol versions +MIN_PROTOCOL_VERSION = 1 +MAX_PROTOCOL_VERSION = 2 + + def main(): parser = argparse.ArgumentParser(description='confserver.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0])) parser.add_argument('--config', - help='Project configuration settings', - required=True) + help='Project configuration settings', + required=True) parser.add_argument('--kconfig', help='KConfig file with config item definitions', @@ -26,41 +32,90 @@ def main(): parser.add_argument('--env', action='append', default=[], help='Environment to set when evaluating the config file', metavar='NAME=VAL') + parser.add_argument('--env-file', type=argparse.FileType('r'), + help='Optional file to load environment variables from. Contents ' + 'should be a JSON object where each key/value pair is a variable.') + + parser.add_argument('--version', help='Set protocol version to use on initial status', + type=int, default=MAX_PROTOCOL_VERSION) + args = parser.parse_args() + if args.version < MIN_PROTOCOL_VERSION: + print("Version %d is older than minimum supported protocol version %d. Client is much older than ESP-IDF version?" % + (args.version, MIN_PROTOCOL_VERSION)) + + if args.version > MAX_PROTOCOL_VERSION: + print("Version %d is newer than maximum supported protocol version %d. Client is newer than ESP-IDF version?" % + (args.version, MAX_PROTOCOL_VERSION)) + try: - args.env = [ (name,value) for (name,value) in ( e.split("=",1) for e in args.env) ] + args.env = [(name,value) for (name,value) in (e.split("=",1) for e in args.env)] except ValueError: - print("--env arguments must each contain =. To unset an environment variable, use 'ENV='") - sys.exit(1) + print("--env arguments must each contain =. To unset an environment variable, use 'ENV='") + sys.exit(1) for name, value in args.env: os.environ[name] = value - print("Server running, waiting for requests on stdin...", file=sys.stderr) + if args.env_file is not None: + env = json.load(args.env_file) + os.environ.update(env) + run_server(args.kconfig, args.config) -def run_server(kconfig, sdkconfig): +def run_server(kconfig, sdkconfig, default_version=MAX_PROTOCOL_VERSION): config = kconfiglib.Kconfig(kconfig) + deprecated_options = confgen.DeprecatedOptions(config.config_prefix, path_rename_files=os.environ["IDF_PATH"]) + with tempfile.NamedTemporaryFile(mode='w+b') as f_o: + with open(sdkconfig, mode='rb') as f_i: + f_o.write(f_i.read()) + f_o.flush() + f_o.seek(0) + deprecated_options.replace(sdkconfig_in=f_o.name, sdkconfig_out=sdkconfig) config.load_config(sdkconfig) + print("Server running, waiting for requests on stdin...", file=sys.stderr) + config_dict = confgen.get_json_values(config) ranges_dict = get_ranges(config) - json.dump({"version": 1, "values" : config_dict, "ranges" : ranges_dict}, sys.stdout) + visible_dict = get_visible(config) + + if default_version == 1: + # V1: no 'visibility' key, send value None for any invisible item + values_dict = dict((k, v if visible_dict[k] else False) for (k,v) in config_dict.items()) + json.dump({"version": 1, "values": values_dict, "ranges": ranges_dict}, sys.stdout) + else: + # V2 onwards: separate visibility from version + json.dump({"version": default_version, "values": config_dict, "ranges": ranges_dict, "visible": visible_dict}, sys.stdout) print("\n") + sys.stdout.flush() while True: line = sys.stdin.readline() if not line: break - req = json.loads(line) + try: + req = json.loads(line) + except ValueError as e: # json module throws JSONDecodeError (sublcass of ValueError) on Py3 but ValueError on Py2 + response = {"version": default_version, "error": ["JSON formatting error: %s" % e]} + json.dump(response, sys.stdout) + print("\n") + sys.stdout.flush() + continue before = confgen.get_json_values(config) before_ranges = get_ranges(config) + before_visible = get_visible(config) - if "load" in req: # if we're loading a different sdkconfig, response should have all items in it - before = {} - before_ranges = {} + if "load" in req: # load a new sdkconfig + + if req.get("version", default_version) == 1: + # for V1 protocol, send all items when loading new sdkconfig. + # (V2+ will only send changes, same as when setting an item) + before = {} + before_ranges = {} + before_visible = {} # if no new filename is supplied, use existing sdkconfig path, otherwise update the path if req["load"] is None: @@ -74,27 +129,41 @@ def run_server(kconfig, sdkconfig): else: sdkconfig = req["save"] - error = handle_request(config, req) + error = handle_request(deprecated_options, config, req) after = confgen.get_json_values(config) after_ranges = get_ranges(config) + after_visible = get_visible(config) values_diff = diff(before, after) ranges_diff = diff(before_ranges, after_ranges) - response = {"version" : 1, "values" : values_diff, "ranges" : ranges_diff} + visible_diff = diff(before_visible, after_visible) + if req["version"] == 1: + # V1 response, invisible items have value None + for k in (k for (k,v) in visible_diff.items() if not v): + values_diff[k] = None + response = {"version": 1, "values": values_diff, "ranges": ranges_diff} + else: + # V2+ response, separate visibility values + response = {"version": req["version"], "values": values_diff, "ranges": ranges_diff, "visible": visible_diff} if error: for e in error: print("Error: %s" % e, file=sys.stderr) response["error"] = error json.dump(response, sys.stdout) print("\n") + sys.stdout.flush() -def handle_request(config, req): - if not "version" in req: - return [ "All requests must have a 'version'" ] - if int(req["version"]) != 1: - return [ "Only version 1 requests supported" ] +def handle_request(deprecated_options, config, req): + if "version" not in req: + return ["All requests must have a 'version'"] + + if req["version"] < MIN_PROTOCOL_VERSION or req["version"] > MAX_PROTOCOL_VERSION: + return ["Unsupported request version %d. Server supports versions %d-%d" % ( + req["version"], + MIN_PROTOCOL_VERSION, + MAX_PROTOCOL_VERSION)] error = [] @@ -103,7 +172,7 @@ def handle_request(config, req): try: config.load_config(req["load"]) except Exception as e: - error += [ "Failed to load from %s: %s" % (req["load"], e) ] + error += ["Failed to load from %s: %s" % (req["load"], e)] if "set" in req: handle_set(config, error, req["set"]) @@ -111,18 +180,19 @@ def handle_request(config, req): if "save" in req: try: print("Saving config to %s..." % req["save"], file=sys.stderr) - confgen.write_config(config, req["save"]) + confgen.write_config(deprecated_options, config, req["save"]) except Exception as e: - error += [ "Failed to save to %s: %s" % (req["save"], e) ] + error += ["Failed to save to %s: %s" % (req["save"], e)] return error + def handle_set(config, error, to_set): - missing = [ k for k in to_set if not k in config.syms ] + missing = [k for k in to_set if k not in config.syms] if missing: error.append("The following config symbol(s) were not found: %s" % (", ".join(missing))) # replace name keys with the full config symbol for each key: - to_set = dict((config.syms[k],v) for (k,v) in to_set.items() if not k in missing) + to_set = dict((config.syms[k],v) for (k,v) in to_set.items() if k not in missing) # Work through the list of values to set, noting that # some may not be immediately applicable (maybe they depend @@ -130,14 +200,14 @@ def handle_set(config, error, to_set): # knowing if any value is unsettable until then end while len(to_set): - set_pass = [ (k,v) for (k,v) in to_set.items() if k.visibility ] + set_pass = [(k,v) for (k,v) in to_set.items() if k.visibility] if not set_pass: break # no visible keys left for (sym,val) in set_pass: if sym.type in (kconfiglib.BOOL, kconfiglib.TRISTATE): - if val == True: + if val is True: sym.set_value(2) - elif val == False: + elif val is False: sym.set_value(0) else: error.append("Boolean symbol %s only accepts true/false values" % sym.name) @@ -150,20 +220,18 @@ def handle_set(config, error, to_set): error.append("The following config symbol(s) were not visible so were not updated: %s" % (", ".join(s.name for s in to_set))) - def diff(before, after): """ - Return a dictionary with the difference between 'before' and 'after' (either with the new value if changed, - or None as the value if a key in 'before' is missing in 'after' + Return a dictionary with the difference between 'before' and 'after', + for items which are present in 'after' dictionary """ diff = dict((k,v) for (k,v) in after.items() if before.get(k, None) != v) - hidden = dict((k,None) for k in before if k not in after) - diff.update(hidden) return diff def get_ranges(config): ranges_dict = {} + def handle_node(node): sym = node.item if not isinstance(sym, kconfiglib.Symbol): @@ -176,10 +244,38 @@ def get_ranges(config): return ranges_dict +def get_visible(config): + """ + Return a dict mapping node IDs (config names or menu node IDs) to True/False for their visibility + """ + result = {} + menus = [] + + # when walking the menu the first time, only + # record whether the config symbols are visible + # and make a list of menu nodes (that are not symbols) + def handle_node(node): + sym = node.item + try: + visible = (sym.visibility != 0) + result[node] = visible + except AttributeError: + menus.append(node) + config.walk_menu(handle_node) + + # now, figure out visibility for each menu. A menu is visible if any of its children are visible + for m in reversed(menus): # reverse to start at leaf nodes + result[m] = any(v for (n,v) in result.items() if n.parent == m) + + # return a dict mapping the node ID to its visibility. + result = dict((confgen.get_menu_node_id(n),v) for (n,v) in result.items()) + + return result + + if __name__ == '__main__': try: main() except FatalError as e: print("A fatal error occurred: %s" % e, file=sys.stderr) sys.exit(2) - diff --git a/tools/kconfig_new/gen_kconfig_doc.py b/tools/kconfig_new/gen_kconfig_doc.py index af73c886..6d6d7b1f 100644 --- a/tools/kconfig_new/gen_kconfig_doc.py +++ b/tools/kconfig_new/gen_kconfig_doc.py @@ -21,7 +21,6 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function -import os import re import kconfiglib @@ -33,7 +32,8 @@ HEADING_SYMBOLS = '#*=-^"+' # Keep the heading level in sync with api-reference/kconfig.rst INITIAL_HEADING_LEVEL = 3 -MAX_HEADING_LEVEL = len(HEADING_SYMBOLS)-1 +MAX_HEADING_LEVEL = len(HEADING_SYMBOLS) - 1 + def write_docs(config, filename): """ Note: writing .rst documentation ignores the current value @@ -42,22 +42,25 @@ def write_docs(config, filename): with open(filename, "w") as f: config.walk_menu(lambda node: write_menu_item(f, node)) + def node_is_menu(node): try: return node.item == kconfiglib.MENU or node.is_menuconfig except AttributeError: return False # not all MenuNodes have is_menuconfig for some reason + def get_breadcrumbs(node): # this is a bit wasteful as it recalculates each time, but still... result = [] node = node.parent while node.parent: if node.prompt: - result = [ ":ref:`%s`" % get_link_anchor(node) ] + result + result = [":ref:`%s`" % get_link_anchor(node)] + result node = node.parent return " > ".join(result) + def get_link_anchor(node): try: return "CONFIG_%s" % node.item.name @@ -68,11 +71,12 @@ def get_link_anchor(node): result = [] while node.parent: if node.prompt: - result = [ re.sub(r"[^a-zA-z0-9]+", "-", node.prompt[0]) ] + result + result = [re.sub(r"[^a-zA-z0-9]+", "-", node.prompt[0])] + result node = node.parent result = "-".join(result).lower() return result + def get_heading_level(node): result = INITIAL_HEADING_LEVEL node = node.parent @@ -83,15 +87,19 @@ def get_heading_level(node): node = node.parent return result + def format_rest_text(text, indent): # Format an indented text block for use with ReST text = indent + text.replace('\n', '\n' + indent) # Escape some characters which are inline formatting in ReST text = text.replace("*", "\\*") text = text.replace("_", "\\_") + # replace absolute links to documentation by relative ones + text = re.sub(r'https://docs.espressif.com/projects/esp-idf/\w+/\w+/(.+)\.html', r':doc:`../\1`', text) text += '\n' return text + def node_should_write(node): if not node.prompt: return False # Don't do anything for invisible menu items @@ -101,6 +109,7 @@ def node_should_write(node): return True + def write_menu_item(f, node): if not node_should_write(node): return @@ -112,7 +121,7 @@ def write_menu_item(f, node): is_menu = node_is_menu(node) - ## Heading + # Heading if name: title = 'CONFIG_%s' % name else: @@ -167,6 +176,6 @@ def write_menu_item(f, node): child = child.next f.write('\n') + if __name__ == '__main__': print("Run this via 'confgen.py --output doc FILENAME'") - diff --git a/tools/kconfig_new/test/Kconfig b/tools/kconfig_new/test/Kconfig index 95822674..8529fd52 100644 --- a/tools/kconfig_new/test/Kconfig +++ b/tools/kconfig_new/test/Kconfig @@ -1,44 +1,78 @@ menu "Test config" -config TEST_BOOL - bool "Test boolean" - default n + config TEST_BOOL + bool "Test boolean" + default n -config TEST_CHILD_BOOL - bool "Test boolean" - depends on TEST_BOOL - default y + config TEST_CHILD_BOOL + bool "Test boolean" + depends on TEST_BOOL + default y -config TEST_CHILD_STR - string "Test str" - depends on TEST_BOOL - default "OHAI!" + config TEST_CHILD_STR + string "Test str" + depends on TEST_BOOL + default "OHAI!" -choice TEST_CHOICE - prompt "Some choice" - default CHOICE_A + choice TEST_CHOICE + prompt "Some choice" + default CHOICE_A -config CHOICE_A - bool "A" + config CHOICE_A + bool "A" -config CHOICE_B - bool "B" + config CHOICE_B + bool "B" -endchoice + endchoice -config DEPENDS_ON_CHOICE - string "Depends on choice" - default "Depends on A" if CHOICE_A - default "Depends on B" if CHOICE_B - default "WAT" + config DEPENDS_ON_CHOICE + string "Depends on choice" + default "Depends on A" if CHOICE_A + default "Depends on B" if CHOICE_B + default "WAT" -config SOME_UNRELATED_THING - bool "Some unrelated thing" + config SOME_UNRELATED_THING + bool "Some unrelated thing" -config TEST_CONDITIONAL_RANGES - int "Something with a range" - range 0 100 if TEST_BOOL - range 0 10 - default 1 + config TEST_CONDITIONAL_RANGES + int "Something with a range" + range 0 100 if TEST_BOOL + range 0 10 + default 1 -endmenu + config TEST_CONDITIONAL_HEX_RANGES + hex "Something with a hex range" + range 0x00 0xaf if TEST_BOOL + range 0x10 0xaf + default 0xa0 + + config SUBMENU_TRIGGER + bool "I enable/disable some submenu items" + default y + + menu "Submenu" + + config SUBMENU_ITEM_A + int "I am a submenu item" + depends on SUBMENU_TRIGGER + default 77 + + config SUBMENU_ITEM_B + bool "I am also submenu item" + depends on SUBMENU_TRIGGER + + endmenu # Submenu + + menuconfig SUBMENU_CONFIG + bool "Submenuconfig" + default y + help + I am a submenu which is also a config item. + + config SUBMENU_CONFIG_ITEM + bool "Depends on submenuconfig" + depends on SUBMENU_CONFIG + default y + +endmenu # Test config diff --git a/tools/kconfig_new/test/README.md b/tools/kconfig_new/test/README.md new file mode 100644 index 00000000..ca7416e3 --- /dev/null +++ b/tools/kconfig_new/test/README.md @@ -0,0 +1,32 @@ +# KConfig Tests + +## confserver.py tests + +Install pexpect (`pip install pexpect`). + +Then run the tests manually like this: + +``` +./test_confserver.py --logfile tests.log +``` + +If a weird error message comes up from the test, check the log file (`tests.log`) which has the full interaction session (input and output) from confserver.py - sometimes the test suite misinterprets some JSON-like content in a Python error message as JSON content. + +Note: confserver.py prints its error messages on stderr, to avoid overlap with JSON content on stdout. However pexpect uses a pty (virtual terminal) which can't distinguish stderr and stdout. + +Test cases apply to `KConfig` config schema. Cases are listed in `testcases.txt` and are each of this form: + +``` +* Set TEST_BOOL, showing child items +> { "TEST_BOOL" : true } +< { "values" : { "TEST_BOOL" : true, "TEST_CHILD_STR" : "OHAI!", "TEST_CHILD_BOOL" : true }, "ranges": {"TEST_CONDITIONAL_RANGES": [0, 100]}, "visible": {"TEST_CHILD_BOOL" : true, "TEST_CHILD_STR" : true} } + +``` + +* First line (`*`) is description +* Second line (`>`) is changes to send +* Third line (`<`) is response to expect back +* (Blank line between cases) + +Test cases are run in sequence, so any test case depends on the state changes caused by all items above it. + diff --git a/tools/kconfig_new/test/test_confserver.py b/tools/kconfig_new/test/test_confserver.py index 222cc454..ee9cf434 100755 --- a/tools/kconfig_new/test/test_confserver.py +++ b/tools/kconfig_new/test/test_confserver.py @@ -1,25 +1,19 @@ #!/usr/bin/env python from __future__ import print_function import os -import sys -import threading -import time import json import argparse -import shutil import tempfile import pexpect -sys.path.append("..") -import confserver +# Each protocol version to be tested needs a 'testcases_vX.txt' file +PROTOCOL_VERSIONS = [1, 2] -def create_server_thread(*args): - t = threading.Thread() -def parse_testcases(): - with open("testcases.txt", "r") as f: - cases = [ l for l in f.readlines() if len(l.strip()) > 0 ] +def parse_testcases(version): + with open("testcases_v%d.txt" % version, "r") as f: + cases = [l for l in f.readlines() if len(l.strip()) > 0] # Each 3 lines in the file should be formatted as: # * Description of the test change # * JSON "changes" to send to the server @@ -29,19 +23,20 @@ def parse_testcases(): for i in range(0, len(cases), 3): desc = cases[i] - send = cases[i+1] - expect = cases[i+2] + send = cases[i + 1] + expect = cases[i + 2] if not desc.startswith("* "): - raise RuntimeError("Unexpected description at line %d: '%s'" % (i+1, desc)) + raise RuntimeError("Unexpected description at line %d: '%s'" % (i + 1, desc)) if not send.startswith("> "): - raise RuntimeError("Unexpected send at line %d: '%s'" % (i+2, send)) + raise RuntimeError("Unexpected send at line %d: '%s'" % (i + 2, send)) if not expect.startswith("< "): - raise RuntimeError("Unexpected expect at line %d: '%s'" % (i+3, expect)) + raise RuntimeError("Unexpected expect at line %d: '%s'" % (i + 3, expect)) desc = desc[2:] send = json.loads(send[2:]) expect = json.loads(expect[2:]) yield (desc, send, expect) + def main(): parser = argparse.ArgumentParser() parser.add_argument('--logfile', type=argparse.FileType('w'), help='Optional session log of the interactions with confserver.py') @@ -56,53 +51,19 @@ def main(): cmdline = "../confserver.py --kconfig Kconfig --config %s" % temp_sdkconfig_path print("Running: %s" % cmdline) - p = pexpect.spawn(cmdline, timeout=0.5) - p.logfile = args.logfile - p.setecho(False) - - def expect_json(): - # run p.expect() to expect a json object back, and return it as parsed JSON - p.expect("{.+}\r\n") - return json.loads(p.match.group(0).strip().decode()) + p = pexpect.spawn(cmdline, timeout=30, logfile=args.logfile, echo=False, use_poll=True, maxread=1) p.expect("Server running.+\r\n") - initial = expect_json() + initial = expect_json(p) print("Initial: %s" % initial) - cases = parse_testcases() - for (desc, send, expected) in cases: - print(desc) - req = { "version" : "1", "set" : send } - req = json.dumps(req) - print("Sending: %s" % (req)) - p.send("%s\n" % req) - readback = expect_json() - print("Read back: %s" % (json.dumps(readback))) - if readback.get("version", None) != 1: - raise RuntimeError('Expected {"version" : 1} in response') - for expect_key in expected.keys(): - read_vals = readback[expect_key] - exp_vals = expected[expect_key] - if read_vals != exp_vals: - expect_diff = dict((k,v) for (k,v) in exp_vals.items() if not k in read_vals or v != read_vals[k]) - raise RuntimeError("Test failed! Was expecting %s: %s" % (expect_key, json.dumps(expect_diff))) - print("OK") + for version in PROTOCOL_VERSIONS: + test_protocol_version(p, version) - print("Testing load/save...") - before = os.stat(temp_sdkconfig_path).st_mtime - p.send("%s\n" % json.dumps({ "version" : "1", "save" : temp_sdkconfig_path })) - save_result = expect_json() - print("Save result: %s" % (json.dumps(save_result))) - assert len(save_result["values"]) == 0 - assert len(save_result["ranges"]) == 0 - after = os.stat(temp_sdkconfig_path).st_mtime - assert after > before + test_load_save(p, temp_sdkconfig_path) + + test_invalid_json(p) - p.send("%s\n" % json.dumps({ "version" : "1", "load" : temp_sdkconfig_path })) - load_result = expect_json() - print("Load result: %s" % (json.dumps(load_result))) - assert len(load_result["values"]) > 0 # loading same file should return all config items - assert len(load_result["ranges"]) > 0 print("Done. All passed.") finally: @@ -111,6 +72,92 @@ def main(): except OSError: pass + +def expect_json(p): + # run p.expect() to expect a json object back, and return it as parsed JSON + p.expect("{.+}\r\n") + result = p.match.group(0).strip().decode() + print("Read raw data from server: %s" % result) + return json.loads(result) + + +def send_request(p, req): + req = json.dumps(req) + print("Sending: %s" % (req)) + p.send("%s\n" % req) + readback = expect_json(p) + print("Read back: %s" % (json.dumps(readback))) + return readback + + +def test_protocol_version(p, version): + print("*****") + print("Testing version %d..." % version) + + # reload the config from the sdkconfig file + req = {"version": version, "load": None} + readback = send_request(p, req) + print("Reset response: %s" % (json.dumps(readback))) + + # run through each test case + cases = parse_testcases(version) + for (desc, send, expected) in cases: + print(desc) + req = {"version": version, "set": send} + readback = send_request(p, req) + if readback.get("version", None) != version: + raise RuntimeError('Expected {"version" : %d} in response' % version) + for expect_key in expected.keys(): + read_vals = readback[expect_key] + exp_vals = expected[expect_key] + if read_vals != exp_vals: + expect_diff = dict((k,v) for (k,v) in exp_vals.items() if k not in read_vals or v != read_vals[k]) + raise RuntimeError("Test failed! Was expecting %s: %s" % (expect_key, json.dumps(expect_diff))) + print("OK") + print("Version %d OK" % version) + + +def test_load_save(p, temp_sdkconfig_path): + print("Testing load/save...") + before = os.stat(temp_sdkconfig_path).st_mtime + save_result = send_request(p, {"version": 2, "save": temp_sdkconfig_path}) + print("Save result: %s" % (json.dumps(save_result))) + assert "error" not in save_result + assert len(save_result["values"]) == 0 # nothing changes when we save + assert len(save_result["ranges"]) == 0 + after = os.stat(temp_sdkconfig_path).st_mtime + assert after > before # something got written to disk + + # Do a V2 load + load_result = send_request(p, {"version": 2, "load": temp_sdkconfig_path}) + print("V2 Load result: %s" % (json.dumps(load_result))) + assert "error" not in load_result + assert len(load_result["values"]) == 0 # in V2, loading same file should return no config items + assert len(load_result["ranges"]) == 0 + + # Do a V1 load + load_result = send_request(p, {"version": 1, "load": temp_sdkconfig_path}) + print("V1 Load result: %s" % (json.dumps(load_result))) + assert "error" not in load_result + assert len(load_result["values"]) > 0 # in V1, loading same file should return all config items + assert len(load_result["ranges"]) > 0 + + +def test_invalid_json(p): + print("Testing invalid JSON formatting...") + + bad_escaping = r'{ "version" : 2, "load" : "c:\some\path\not\escaped\as\json" }' + p.send("%s\n" % bad_escaping) + readback = expect_json(p) + print(readback) + assert "json" in readback["error"][0].lower() + + not_really_json = 'Hello world!!' + p.send("%s\n" % not_really_json) + readback = expect_json(p) + print(readback) + assert "json" in readback["error"][0].lower() + + if __name__ == "__main__": main() - diff --git a/tools/kconfig_new/test/testcases.txt b/tools/kconfig_new/test/testcases_v1.txt similarity index 91% rename from tools/kconfig_new/test/testcases.txt rename to tools/kconfig_new/test/testcases_v1.txt index 4563d492..0f68536c 100644 --- a/tools/kconfig_new/test/testcases.txt +++ b/tools/kconfig_new/test/testcases_v1.txt @@ -1,6 +1,6 @@ * Set TEST_BOOL, showing child items > { "TEST_BOOL" : true } -< { "values" : { "TEST_BOOL" : true, "TEST_CHILD_STR" : "OHAI!", "TEST_CHILD_BOOL" : true }, "ranges": {"TEST_CONDITIONAL_RANGES": [0, 100]} } +< { "values" : { "TEST_BOOL" : true, "TEST_CHILD_STR" : "OHAI!", "TEST_CHILD_BOOL" : true }, "ranges": {"TEST_CONDITIONAL_RANGES": [0, 100], "TEST_CONDITIONAL_HEX_RANGES": [0, 175]} } * Set TEST_CHILD_STR > { "TEST_CHILD_STR" : "Other value" } @@ -8,7 +8,7 @@ * Clear TEST_BOOL, hiding child items > { "TEST_BOOL" : false } -< { "values" : { "TEST_BOOL" : false, "TEST_CHILD_STR" : null, "TEST_CHILD_BOOL" : null }, "ranges": {"TEST_CONDITIONAL_RANGES": [0, 10]} } +< { "values" : { "TEST_BOOL" : false, "TEST_CHILD_STR" : null, "TEST_CHILD_BOOL" : null }, "ranges": {"TEST_CONDITIONAL_RANGES": [0, 10], "TEST_CONDITIONAL_HEX_RANGES": [16, 175]} } * Set TEST_CHILD_BOOL, invalid as parent is disabled > { "TEST_CHILD_BOOL" : false } diff --git a/tools/kconfig_new/test/testcases_v2.txt b/tools/kconfig_new/test/testcases_v2.txt new file mode 100644 index 00000000..7a735fd1 --- /dev/null +++ b/tools/kconfig_new/test/testcases_v2.txt @@ -0,0 +1,47 @@ +* Set TEST_BOOL, showing child items +> { "TEST_BOOL" : true } +< { "values" : { "TEST_BOOL" : true, "TEST_CHILD_STR" : "OHAI!", "TEST_CHILD_BOOL" : true }, "ranges": {"TEST_CONDITIONAL_RANGES": [0, 100], "TEST_CONDITIONAL_HEX_RANGES": [0, 175]}, "visible": {"TEST_CHILD_BOOL" : true, "TEST_CHILD_STR" : true} } + +* Set TEST_CHILD_STR +> { "TEST_CHILD_STR" : "Other value" } +< { "values" : { "TEST_CHILD_STR" : "Other value" } } + +* Clear TEST_BOOL, hiding child items +> { "TEST_BOOL" : false } +< { "values" : { "TEST_BOOL" : false }, "ranges": {"TEST_CONDITIONAL_RANGES": [0, 10], "TEST_CONDITIONAL_HEX_RANGES": [16, 175]}, "visible": { "TEST_CHILD_BOOL" : false, "TEST_CHILD_STR" : false } } + +* Set TEST_CHILD_BOOL, invalid as parent is disabled +> { "TEST_CHILD_BOOL" : false } +< { "values" : { } } + +* Set TEST_BOOL & TEST_CHILD_STR together +> { "TEST_BOOL" : true, "TEST_CHILD_STR" : "New value" } +< { "values" : { "TEST_BOOL" : true, "TEST_CHILD_STR" : "New value", "TEST_CHILD_BOOL" : true } } + +* Set choice +> { "CHOICE_B" : true } +< { "values" : { "CHOICE_B" : true, "CHOICE_A" : false, "DEPENDS_ON_CHOICE" : "Depends on B" } } + +* Set string which depends on choice B +> { "DEPENDS_ON_CHOICE" : "oh, really?" } +< { "values" : { "DEPENDS_ON_CHOICE" : "oh, really?" } } + +* Try setting boolean values to invalid types +> { "CHOICE_A" : 11, "TEST_BOOL" : "false" } +< { "values" : { } } + +* Disabling all items in a submenu causes all sub-items to have visible:False +> { "SUBMENU_TRIGGER": false } +< { "values" : { "SUBMENU_TRIGGER": false}, "visible": { "test-config-submenu" : false, "SUBMENU_ITEM_A": false, "SUBMENU_ITEM_B": false} } + +* Re-enabling submenu causes that menu to be visible again, and refreshes sub-items +> { "SUBMENU_TRIGGER": true } +< { "values" : { "SUBMENU_TRIGGER": true}, "visible": {"test-config-submenu": true, "SUBMENU_ITEM_A": true, "SUBMENU_ITEM_B": true}, "values": {"SUBMENU_TRIGGER": true, "SUBMENU_ITEM_A": 77, "SUBMENU_ITEM_B": false } } + +* Disabling submenuconfig item hides its children +> { "SUBMENU_CONFIG": false } +< { "values" : { "SUBMENU_CONFIG": false }, "visible": { "SUBMENU_CONFIG_ITEM": false } } + +* Enabling submenuconfig item re-shows its children +> { "SUBMENU_CONFIG": true } +< { "values" : { "SUBMENU_CONFIG_ITEM": true, "SUBMENU_CONFIG" : true }, "visible": { "SUBMENU_CONFIG_ITEM": true } } diff --git a/tools/make_cacert.py b/tools/make_cacert.py deleted file mode 100644 index e7faf4d5..00000000 --- a/tools/make_cacert.py +++ /dev/null @@ -1,41 +0,0 @@ -import os - - -class Cert(object): - def __init__(self, name, buff): - self.name = name - self.len = len(buff) - self.buff = buff - pass - - def __str__(self): - out_str = ['\0']*32 - for i in range(len(self.name)): - out_str[i] = self.name[i] - out_str = "".join(out_str) - out_str += str(chr(self.len & 0xFF)) - out_str += str(chr((self.len & 0xFF00) >> 8)) - out_str += self.buff - return out_str - pass - - -def main(): - cert_list = [] - file_list = os.listdir(os.getcwd()) - cert_file_list = [] - for _file in file_list: - if _file.endswith(".cer"): - cert_file_list.append(_file) - print cert_file_list - for cert_file in cert_file_list: - with open(cert_file, 'rb') as f: - buff = f.read() - cert_list.append(Cert(cert_file, buff)) - with open('esp_ca_cert.bin', 'wb+') as f: - for _cert in cert_list: - f.write("%s" % _cert) - pass -if __name__ == '__main__': - main() - diff --git a/tools/make_cert.py b/tools/make_cert.py deleted file mode 100644 index 9a5b057b..00000000 --- a/tools/make_cert.py +++ /dev/null @@ -1,53 +0,0 @@ -import os - - -class Cert(object): - def __init__(self, name, buff): - self.name = name - self.len = len(buff) - self.buff = buff - pass - - def __str__(self): - out_str = ['\0']*32 - for i in range(len(self.name)): - out_str[i] = self.name[i] - out_str = "".join(out_str) - out_str += str(chr(self.len & 0xFF)) - out_str += str(chr((self.len & 0xFF00) >> 8)) - out_str += self.buff - return out_str - pass - - -def main(): - cert_list = [] - file_list = os.listdir(os.getcwd()) - cert_file_list = [] - for _file in file_list: - pos = _file.find(".key_1024") - if pos != -1: - cert_file_list.append(_file[:pos]) - - pos = _file.find(".cer") - if pos!= -1: - cert_file_list.append(_file[:pos]) - - for cert_file in cert_file_list: - if cert_file == 'private_key': - with open(cert_file+".key_1024", 'rb') as f: - buff = f.read() - cert_list.append(Cert(cert_file, buff)) - - if cert_file == 'certificate': - with open(cert_file+".cer", 'rb') as f: - buff = f.read() - cert_list.append(Cert(cert_file, buff)) - - with open('esp_cert_private_key.bin', 'wb+') as f: - for _cert in cert_list: - f.write("%s" % _cert) - pass -if __name__ == '__main__': - main() - diff --git a/tools/makefile.sh b/tools/makefile.sh deleted file mode 100755 index df37d512..00000000 --- a/tools/makefile.sh +++ /dev/null @@ -1,92 +0,0 @@ -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of the axTLS project nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -# -# Generate the certificates and keys for testing. -# - -PROJECT_NAME="TLS Project" - -# Generate the openssl configuration files. -cat > ca_cert.conf << EOF -[ req ] -distinguished_name = req_distinguished_name -prompt = no - -[ req_distinguished_name ] - O = $PROJECT_NAME Dodgy Certificate Authority -EOF - -cat > certs.conf << EOF -[ req ] -distinguished_name = req_distinguished_name -prompt = no - -[ req_distinguished_name ] - O = $PROJECT_NAME - CN = 127.0.0.1 -EOF - -cat > device_cert.conf << EOF -[ req ] -distinguished_name = req_distinguished_name -prompt = no - -[ req_distinguished_name ] - O = $PROJECT_NAME Device Certificate -EOF - -# private key generation -openssl genrsa -out TLS.ca_key.pem 1024 -openssl genrsa -out TLS.key_1024.pem 1024 - -# convert private keys into DER format -openssl rsa -in TLS.key_1024.pem -out TLS.key_1024 -outform DER - -# cert requests -openssl req -out TLS.ca_x509.req -key TLS.ca_key.pem -new \ - -config ./ca_cert.conf -openssl req -out TLS.x509_1024.req -key TLS.key_1024.pem -new \ - -config ./certs.conf - -# generate the actual certs. -openssl x509 -req -in TLS.ca_x509.req -out TLS.ca_x509.pem \ - -sha1 -days 5000 -signkey TLS.ca_key.pem -openssl x509 -req -in TLS.x509_1024.req -out TLS.x509_1024.pem \ - -sha1 -CAcreateserial -days 5000 \ - -CA TLS.ca_x509.pem -CAkey TLS.ca_key.pem - -# some cleanup -rm TLS*.req -rm *.conf - -openssl x509 -in TLS.ca_x509.pem -outform DER -out TLS.ca_x509.cer -openssl x509 -in TLS.x509_1024.pem -outform DER -out TLS.x509_1024.cer - -# -# Generate the certificates and keys for encrypt. -# - -# set default cert for use in the client -xxd -i TLS.x509_1024.cer | sed -e \ - "s/TLS_x509_1024_cer/default_certificate/" > cert.h -# set default key for use in the server -xxd -i TLS.key_1024 | sed -e \ - "s/TLS_key_1024/default_private_key/" > private_key.h diff --git a/tools/pack_fw.py b/tools/pack_fw.py old mode 100644 new mode 100755 diff --git a/tools/set-submodules-to-github.sh b/tools/set-submodules-to-github.sh new file mode 100755 index 00000000..5495fb4a --- /dev/null +++ b/tools/set-submodules-to-github.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Explicitly switches the relative submodules locations on GitHub to the original public URLs +# +# '../../group/repo.git' to 'https://github.com/group/repo.git' +# +# This can be useful for non-GitHub forks to automate getting of right submodules sources. +# + +# +# It makes sense to do +# +# git submodule deinit --force . +# git submodule init +# +# before running this, and +# +# git submodule update --recursive +# +# after that. These were not included over this script deliberately, to use the script flexibly +# + +set -o errexit +set -o pipefail +set -o nounset + +DEBUG_SHELL=${DEBUG_SHELL:-"0"} +[ "${DEBUG_SHELL}" = "1" ] && set -x + +### '../../' relative locations + +for LINE in $(git config -f .gitmodules --list | grep "\.url=../../[^.]") +do + SUBPATH=$(echo "${LINE}" | sed "s|^submodule\.\([^.]*\)\.url.*$|\1|") + LOCATION=$(echo "${LINE}" | sed 's|.*\.url=\.\./\.\./\(.*\)$|\1|') + SUBURL="https://github.com/$LOCATION" + + git config submodule."${SUBPATH}".url "${SUBURL}" +done + +git config --get-regexp '^submodule\..*\.url$' diff --git a/tools/test_check_kconfigs.py b/tools/test_check_kconfigs.py new file mode 100755 index 00000000..a38f16ab --- /dev/null +++ b/tools/test_check_kconfigs.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from check_kconfigs import LineRuleChecker +from check_kconfigs import SourceChecker +from check_kconfigs import InputError +from check_kconfigs import IndentAndNameChecker +from check_kconfigs import CONFIG_NAME_MAX_LENGTH + + +class ApplyLine(object): + def apply_line(self, string): + self.checker.process_line(string + '\n', 0) + + def expect_error(self, string, expect, cleanup=None): + try: + with self.assertRaises(InputError) as cm: + self.apply_line(string) + if expect: + self.assertEqual(cm.exception.suggested_line, expect + '\n') + finally: + if cleanup: + # cleanup of the previous failure + self.apply_line(cleanup) + + def expt_success(self, string): + self.apply_line(string) + + +class TestLineRuleChecker(unittest.TestCase, ApplyLine): + def setUp(self): + self.checker = LineRuleChecker('Kconfig') + + def tearDown(self): + pass + + def test_tabulators(self): + self.expect_error('\ttest', expect=' test') + self.expect_error('\t test', expect=' test') + self.expect_error(' \ttest', expect=' test') + self.expect_error(' \t test', expect=' test') + self.expt_success(' test') + self.expt_success('test') + + def test_trailing_whitespaces(self): + self.expect_error(' ', expect='') + self.expect_error(' ', expect='') + self.expect_error('test ', expect='test') + self.expt_success('test') + self.expt_success('') + + def test_line_length(self): + self.expect_error('x' * 120, expect=None) + self.expt_success('x' * 119) + self.expt_success('') + + def test_backslashes(self): + self.expect_error('test \\', expect=None) + + +class TestSourceChecker(unittest.TestCase, ApplyLine): + def setUp(self): + self.checker = SourceChecker('Kconfig') + + def tearDown(self): + pass + + def test_source_file_name(self): + self.expect_error('source "Kconfig.test"', expect='source "Kconfig.in"') + self.expect_error('source "/tmp/Kconfig.test"', expect='source "/tmp/Kconfig.in"') + self.expect_error('source "Kconfig"', expect='source "Kconfig.in"') + self.expt_success('source "Kconfig.in"') + self.expt_success('source "/tmp/Kconfig.in"') + self.expect_error('source"Kconfig.in"', expect='source "Kconfig.in"') + self.expt_success('source "/tmp/Kconfig.in" # comment') + + +class TestIndentAndNameChecker(unittest.TestCase, ApplyLine): + def setUp(self): + self.checker = IndentAndNameChecker('Kconfig') + self.checker.min_prefix_length = 4 + + def tearDown(self): + self.checker.__exit__('Kconfig', None, None) + + +class TestIndent(TestIndentAndNameChecker): + def setUp(self): + super(TestIndent, self).setUp() + self.checker.min_prefix_length = 0 # prefixes are ignored in this test case + + def test_indent_characters(self): + self.expt_success('menu "test"') + self.expect_error(' test', expect=' test') + self.expect_error(' test', expect=' test') + self.expect_error(' test', expect=' test') + self.expect_error(' test', expect=' test') + self.expt_success(' test') + self.expt_success(' test2') + self.expt_success(' config') + self.expect_error(' default', expect=' default') + self.expt_success(' help') + self.expect_error(' text', expect=' text') + self.expt_success(' help text') + self.expt_success(' menu') + self.expt_success(' endmenu') + self.expect_error(' choice', expect=' choice', cleanup=' endchoice') + self.expect_error(' choice', expect=' choice', cleanup=' endchoice') + self.expt_success(' choice') + self.expt_success(' endchoice') + self.expt_success(' config') + self.expt_success('endmenu') + + def test_help_content(self): + self.expt_success('menu "test"') + self.expt_success(' config') + self.expt_success(' help') + self.expt_success(' description') + self.expt_success(' config keyword in the help') + self.expt_success(' menu keyword in the help') + self.expt_success(' menuconfig keyword in the help') + self.expt_success(' endmenu keyword in the help') + self.expt_success(' endmenu keyword in the help') + self.expt_success('') # newline in help + self.expt_success(' endmenu keyword in the help') + self.expect_error(' menu "real menu with wrong indent"', + expect=' menu "real menu with wrong indent"', cleanup=' endmenu') + self.expt_success('endmenu') + + def test_mainmenu(self): + self.expt_success('mainmenu "test"') + self.expect_error('test', expect=' test') + self.expt_success(' not_a_keyword') + self.expt_success(' config') + self.expt_success(' menuconfig') + self.expect_error('test', expect=' test') + self.expect_error(' test', expect=' test') + self.expt_success(' menu') + self.expt_success(' endmenu') + + def test_ifendif(self): + self.expt_success('menu "test"') + self.expt_success(' config') + self.expt_success(' help') + self.expect_error(' if', expect=' if', cleanup=' endif') + self.expt_success(' if') + self.expect_error(' config', expect=' config') + self.expt_success(' config') + self.expt_success(' help') + self.expt_success(' endif') + self.expt_success(' config') + self.expt_success('endmenu') + + def test_config_without_menu(self): + self.expt_success('menuconfig') + self.expt_success(' help') + self.expt_success(' text') + self.expt_success('') + self.expt_success(' text') + self.expt_success('config') + self.expt_success(' help') + + def test_source_after_config(self): + self.expt_success('menuconfig') + self.expt_success(' help') + self.expt_success(' text') + self.expect_error(' source', expect='source') + self.expt_success('source "Kconfig.in"') + + def test_comment_after_config(self): + self.expt_success('menuconfig') + self.expt_success(' # comment') + self.expt_success(' help') + self.expt_success(' text') + self.expect_error('# comment', expect=' # comment') + self.expt_success(' # second not realcomment"') + + +class TestName(TestIndentAndNameChecker): + def setUp(self): + super(TestName, self).setUp() + self.checker.min_prefix_length = 0 # prefixes are ignored in this test case + + def test_name_length(self): + max_length = CONFIG_NAME_MAX_LENGTH + too_long = max_length + 1 + self.expt_success('menu "test"') + self.expt_success(' config ABC') + self.expt_success(' config ' + ('X' * max_length)) + self.expect_error(' config ' + ('X' * too_long), expect=None) + self.expt_success(' menuconfig ' + ('X' * max_length)) + self.expect_error(' menuconfig ' + ('X' * too_long), expect=None) + self.expt_success(' choice ' + ('X' * max_length)) + self.expect_error(' choice ' + ('X' * too_long), expect=None) + self.expt_success('endmenu') + + +class TestPrefix(TestIndentAndNameChecker): + def test_prefix_len(self): + self.expt_success('menu "test"') + self.expt_success(' config ABC_1') + self.expt_success(' config ABC_2') + self.expt_success(' config ABC_DEBUG') + self.expt_success(' config ABC_ANOTHER') + self.expt_success('endmenu') + self.expt_success('menu "test2"') + self.expt_success(' config A') + self.expt_success(' config B') + self.expect_error('endmenu', expect=None) + + def test_choices(self): + self.expt_success('menu "test"') + self.expt_success(' choice ASSERTION_LEVEL') + self.expt_success(' config ASSERTION_DEBUG') + self.expt_success(' config ASSERTION_RELEASE') + self.expt_success(' menuconfig ASSERTION_XY') + self.expt_success(' endchoice') + self.expt_success(' choice DEBUG') + self.expt_success(' config DE_1') + self.expt_success(' config DE_2') + self.expect_error(' endchoice', expect=None) + self.expect_error('endmenu', expect=None) + + def test_nested_menu(self): + self.expt_success('menu "test"') + self.expt_success(' config DOESNT_MATTER') + self.expt_success(' menu "inner menu"') + self.expt_success(' config MENUOP_1') + self.expt_success(' config MENUOP_2') + self.expt_success(' config MENUOP_3') + self.expt_success(' endmenu') + self.expt_success('endmenu') + + def test_nested_ifendif(self): + self.expt_success('menu "test"') + self.expt_success(' config MENUOP_1') + self.expt_success(' if MENUOP_1') + self.expt_success(' config MENUOP_2') + self.expt_success(' endif') + self.expt_success('endmenu') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/toolchain_versions.mk b/tools/toolchain_versions.mk index b18d4eff..5f468815 100644 --- a/tools/toolchain_versions.mk +++ b/tools/toolchain_versions.mk @@ -2,5 +2,5 @@ SUPPORTED_TOOLCHAIN_COMMIT_DESC = crosstool-ng-1.22.0-92-g8facf4c SUPPORTED_TOOLCHAIN_GCC_VERSIONS = 5.2.0 CURRENT_TOOLCHAIN_COMMIT_DESC = crosstool-ng-1.22.0-92-g8facf4c -CURRENT_TOOLCHAIN_COMMIT_DESC_SHORT = 1.22.0-92-g8facf4c +CURRENT_TOOLCHAIN_COMMIT_DESC_SHORT = crosstool-ng-1.22.0-92-g8facf4c CURRENT_TOOLCHAIN_GCC_VERSION = 5.2.0 diff --git a/tools/tools.json b/tools/tools.json new file mode 100644 index 00000000..5bbed7e8 --- /dev/null +++ b/tools/tools.json @@ -0,0 +1,391 @@ +{ + "tools": [ + { + "description": "Toolchain for Xtensa (ESP32) based on GCC", + "export_paths": [ + [ + "xtensa-esp32-elf", + "bin" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/espressif/crosstool-NG", + "install": "always", + "license": "GPL-3.0-with-GCC-exception", + "name": "xtensa-esp32-elf", + "version_cmd": [ + "xtensa-esp32-elf-gcc", + "--version" + ], + "version_regex": "\\(crosstool-NG\\s+(?:crosstool-ng-)?([0-9a-z\\.\\-]+)\\)\\s*([0-9\\.]+)", + "version_regex_replace": "\\1-\\2", + "versions": [ + { + "linux-amd64": { + "sha256": "39db59b13f25e83e53c55f56979dbfce77b7f23126ad79de833509ad902d3f0a", + "size": 63025996, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_2_0-esp32-2019r1-linux-amd64.tar.gz" + }, + "linux-armel": { + "sha256": "4ffd19839fcb241af3111da7c419448b80be3bd844da570e95f8f3d5a7eccf79", + "size": 61164309, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_2_0-esp32-2019r1-linux-armel.tar.gz" + }, + "linux-i686": { + "sha256": "85c02a4310bb97ac46e6f943b0de10e9e9572596c7d33d09b6f93f8bace3b784", + "size": 65016647, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_2_0-esp32-2019r1-linux-i686.tar.gz" + }, + "macos": { + "sha256": "adb256394c948ca424ec6ef1d9bee91baa99a304d8ace8e6701303da952eb007", + "size": 69674700, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_2_0-esp32-2019r1-macos.tar.gz" + }, + "name": "esp32-2019r1-8.2.0", + "status": "recommended", + "win32": { + "sha256": "ff00dbb02287219a61873c3b2649a50b94e80c82e607c336383f2838abbefbde", + "size": 73245169, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_2_0-esp32-2019r1-win32.zip" + }, + "win64": { + "sha256": "ff00dbb02287219a61873c3b2649a50b94e80c82e607c336383f2838abbefbde", + "size": 73245169, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_2_0-esp32-2019r1-win32.zip" + } + } + ] + }, + { + "description": "Toolchain for ESP32 ULP coprocessor", + "export_paths": [ + [ + "esp32ulp-elf-binutils", + "bin" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/espressif/binutils-esp32ulp", + "install": "always", + "license": "GPL-2.0-or-later", + "name": "esp32ulp-elf", + "version_cmd": [ + "esp32ulp-elf-as", + "--version" + ], + "version_regex": "\\(GNU Binutils\\)\\s+([0-9a-z\\.\\-]+)", + "versions": [ + { + "linux-amd64": { + "sha256": "c1bbcd65e1e30c7312a50344c8dbc70c2941580a79aa8f8abbce8e0e90c79566", + "size": 8246604, + "url": "https://dl.espressif.com/dl/binutils-esp32ulp-linux64-2.28.51-esp32ulp-20180809.tar.gz" + }, + "macos": { + "sha256": "c92937d85cc9a90eb6c6099ce767ca021108c18c94e34bd7b1fa0cde168f94a0", + "size": 5726662, + "url": "https://dl.espressif.com/dl/binutils-esp32ulp-macos-2.28.51-esp32ulp-20180809.tar.gz" + }, + "name": "2.28.51.20170517", + "status": "recommended", + "win32": { + "sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd", + "size": 12231559, + "url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip" + }, + "win64": { + "sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd", + "size": 12231559, + "url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip" + } + } + ] + }, + { + "description": "CMake build system", + "export_paths": [ + [ + "bin" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/Kitware/CMake", + "install": "on_request", + "license": "BSD-3-Clause", + "name": "cmake", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win32", + "win64" + ] + }, + { + "export_paths": [ + [ + "CMake.app", + "Contents", + "bin" + ] + ], + "platforms": [ + "macos" + ] + } + ], + "strip_container_dirs": 1, + "version_cmd": [ + "cmake", + "--version" + ], + "version_regex": "cmake version ([0-9.]+)", + "versions": [ + { + "linux-amd64": { + "sha256": "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76", + "size": 38405896, + "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz" + }, + "macos": { + "sha256": "fef537614d73fda848f6168273b6c7ba45f850484533361e7bc50ac1d315f780", + "size": 32062124, + "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Darwin-x86_64.tar.gz" + }, + "name": "3.13.4", + "status": "recommended", + "win32": { + "sha256": "28daf772f55d817a13ef14e25af2a5569f8326dac66a6aa3cc5208cf1f8e943f", + "size": 26385104, + "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win32-x86.zip" + }, + "win64": { + "sha256": "bcd477d49e4a9400b41213d53450b474beaedb264631693c958ef9affa8e5623", + "size": 29696565, + "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win64-x64.zip" + } + } + ] + }, + { + "description": "OpenOCD for ESP32", + "export_paths": [ + [ + "openocd-esp32", + "bin" + ] + ], + "export_vars": { + "OPENOCD_SCRIPTS": "${TOOL_PATH}/openocd-esp32/share/openocd/scripts" + }, + "info_url": "https://github.com/espressif/openocd-esp32", + "install": "always", + "license": "GPL-2.0-only", + "name": "openocd-esp32", + "version_cmd": [ + "openocd", + "--version" + ], + "version_regex": "Open On-Chip Debugger\\s+([a-z0-9.-]+)\\s+", + "versions": [ + { + "linux-amd64": { + "sha256": "e5b5579edffde090e426b4995b346e281843bf84394f8e68c8e41bd1e4c576bd", + "size": 1681596, + "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-linux64-0.10.0-esp32-20190313.tar.gz" + }, + "macos": { + "sha256": "09504eea5aa92646a117f16573c95b34e04b4010791a2f8fefcd2bd8c430f081", + "size": 1760536, + "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-macos-0.10.0-esp32-20190313.tar.gz" + }, + "name": "v0.10.0-esp32-20190313", + "status": "recommended", + "win32": { + "sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2", + "size": 2098513, + "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip" + }, + "win64": { + "sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2", + "size": 2098513, + "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip" + } + } + ] + }, + { + "description": "menuconfig tool", + "export_paths": [ + [ + "" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/espressif/kconfig-frontends", + "install": "never", + "license": "GPL-2.0-only", + "name": "mconf", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win32", + "win64" + ] + } + ], + "strip_container_dirs": 1, + "version_cmd": [ + "mconf-idf", + "-v" + ], + "version_regex": "mconf-idf version mconf-([a-z0-9.-]+)-win32", + "versions": [ + { + "name": "v4.6.0.0-idf-20190628", + "status": "recommended", + "win32": { + "sha256": "1b8f17f48740ab669c13bd89136e8cc92efe0cd29872f0d6c44148902a2dc40c", + "size": 826114, + "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190628/mconf-v4.6.0.0-idf-20190628-win32.zip" + }, + "win64": { + "sha256": "1b8f17f48740ab669c13bd89136e8cc92efe0cd29872f0d6c44148902a2dc40c", + "size": 826114, + "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190628/mconf-v4.6.0.0-idf-20190628-win32.zip" + } + } + ] + }, + { + "description": "Ninja build system", + "export_paths": [ + [ + "" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/ninja-build/ninja", + "install": "on_request", + "license": "Apache-2.0", + "name": "ninja", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win32", + "win64" + ] + } + ], + "version_cmd": [ + "ninja", + "--version" + ], + "version_regex": "([0-9.]+)", + "versions": [ + { + "linux-amd64": { + "sha256": "978fd9e26c2db8d33392c6daef50e9edac0a3db6680710a9f9ad47e01f3e49b7", + "size": 85276, + "url": "https://dl.espressif.com/dl/ninja-1.9.0-linux64.tar.gz" + }, + "macos": { + "sha256": "9504cd1783ef3c242d06330a50d54dc8f838b605f5fc3e892c47254929f7350c", + "size": 91457, + "url": "https://dl.espressif.com/dl/ninja-1.9.0-osx.tar.gz" + }, + "name": "1.9.0", + "status": "recommended", + "win64": { + "sha256": "2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9", + "size": 254497, + "url": "https://dl.espressif.com/dl/ninja-1.9.0-win64.zip" + } + } + ] + }, + { + "description": "IDF wrapper tool for Windows", + "export_paths": [ + [ + "" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/espressif/esp-idf/tree/master/tools/windows/idf_exe", + "install": "never", + "license": "Apache-2.0", + "name": "idf-exe", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win32", + "win64" + ] + } + ], + "version_cmd": [ + "idf.py.exe", + "-v" + ], + "version_regex": "([0-9.]+)", + "versions": [ + { + "name": "1.0.1", + "status": "recommended", + "win32": { + "sha256": "53eb6aaaf034cc7ed1a97d5c577afa0f99815b7793905e9408e74012d357d04a", + "size": 11297, + "url": "https://dl.espressif.com/dl/idf-exe-v1.0.1.zip" + }, + "win64": { + "sha256": "53eb6aaaf034cc7ed1a97d5c577afa0f99815b7793905e9408e74012d357d04a", + "size": 11297, + "url": "https://dl.espressif.com/dl/idf-exe-v1.0.1.zip" + } + } + ] + }, + { + "description": "Ccache (compiler cache)", + "export_paths": [ + [ + "" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/ccache/ccache", + "install": "never", + "license": "GPL-3.0-or-later", + "name": "ccache", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win64" + ] + } + ], + "version_cmd": [ + "ccache.exe", + "--version" + ], + "version_regex": "ccache version ([0-9.]+)", + "versions": [ + { + "name": "3.7", + "status": "recommended", + "win64": { + "sha256": "37e833f3f354f1145503533e776c1bd44ec2e77ff8a2476a1d2039b0b10c78d6", + "size": 142401, + "url": "https://dl.espressif.com/dl/ccache-3.7-w64.zip" + } + } + ] + } + ], + "version": 1 +} diff --git a/tools/tools_schema.json b/tools/tools_schema.json new file mode 100644 index 00000000..6f112154 --- /dev/null +++ b/tools/tools_schema.json @@ -0,0 +1,234 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/espressif/esp-idf/blob/master/tools/tools-schema.json", + "type": "object", + "properties": { + "version": { + "type": "integer", + "description": "Metadata file version" + }, + "tools": { + "type": "array", + "description": "List of tools", + "items": { + "$ref": "#/definitions/toolInfo" + } + } + }, + "required": [ + "version", + "tools" + ], + "definitions": { + "toolInfo": { + "type": "object", + "description": "Information about one tool", + "properties": { + "name" : { + "description": "Tool name (used as a directory name)", + "type": "string" + }, + "description" : { + "description": "A short (one sentence) description of the tool.", + "type": "string" + }, + "export_paths": { + "$ref": "#/definitions/exportPaths" + }, + "export_vars": { + "$ref": "#/definitions/envVars", + "description": "Some variable expansions are done on the values. 1) ${TOOL_PATH} is replaced with the directory where the tool is installed." + }, + "info_url": { + "description": "URL of the page with information about the tool", + "type": "string" + }, + "install": { + "$ref": "#/definitions/installRequirementInfo", + "description": "If 'always', the tool will be installed by default. If 'on_request', tool will be installed when specifically requested. If 'never', tool will not be considered for installation." + }, + "license": { + "description": "License name. Use SPDX license identifier if it exists, short name of the license otherwise.", + "type": "string" + }, + "version_cmd": { + "$ref": "#/definitions/arrayOfStrings", + "description": "Command to be executed (along with any extra arguments). The executable be present in one of the export_paths." + }, + "version_regex": { + "description": "Regex which is to be applied to version_cmd output to extract the version. By default, the version will be the first capture group of the expression. If version_regex_replace is specified, version will be obtained by doing a substitution using version_regex_replace instead.", + "$ref": "#/definitions/regex" + }, + "version_regex_replace": { + "description": "If given, this will be used as substitute expression for the regex defined in version_regex, to obtain the version string. Not specifying this is equivalent to setting it to '\\1' (i.e. return the first capture group).", + "type": "string" + }, + "strip_container_dirs": { + "type": "integer", + "description": "If specified, this number of top directory levels will removed when extracting. E.g. if strip_container_dirs=2, archive path a/b/c/d.txt will be extracted as c/d.txt" + }, + "versions": { + "type": "array", + "description": "List of versions", + "items": { + "$ref": "#/definitions/versionInfo" + } + }, + "platform_overrides": { + "type": "array", + "description": "List of platform-specific overrides", + "items": { + "$ref": "#/definitions/platformOverrideInfo" + } + } + }, + "required": [ + "description", + "export_paths", + "version_cmd", + "version_regex", + "versions", + "install", + "info_url", + "license" + ] + }, + "arrayOfStrings": { + "description": "Array of strings. Used to represent paths (split into components) and command lines (split into arguments)", + "type": "array", + "items": { + "type": "string" + } + }, + "exportPaths": { + "description": "Array of paths to be exported (added to PATH). Each item in the array is relative to the directory where the tool will be installed.", + "type": "array", + "items": { + "$ref": "#/definitions/arrayOfStrings" + } + }, + "envVars": { + "description": "Collection of environment variables. Keys and values are the environment variable names and values, respectively.", + "type": "object", + "patternProperties": { + "^([A-Z_0-9]+)+$": { + "type": "string" + } + }, + "additionalProperties": false + }, + "regex": { + "description": "A regular expression", + "type": "string" + }, + "versionInfo": { + "type": "object", + "properties": { + "name" : { + "description": "Version name (used as a directory name)", + "type": "string" + }, + "status": { + "description": "Determines whether the version is recommended/supported/deprecated", + "type": "string", + "enum": ["recommended", "supported", "deprecated"] + }, + "linux-i686": { + "$ref": "#/definitions/platformDownloadInfo" + }, + "linux-amd64": { + "$ref": "#/definitions/platformDownloadInfo" + }, + "linux-armel": { + "$ref": "#/definitions/platformDownloadInfo" + }, + "linux-arm64": { + "$ref": "#/definitions/platformDownloadInfo" + }, + "macos": { + "$ref": "#/definitions/platformDownloadInfo" + }, + "win32": { + "$ref": "#/definitions/platformDownloadInfo" + }, + "win64": { + "$ref": "#/definitions/platformDownloadInfo" + }, + "any": { + "$ref": "#/definitions/platformDownloadInfo" + } + } + }, + "platformDownloadInfo": { + "description": "Information about download artifact for one platform", + "type": "object", + "properties": { + "sha256": { + "type": "string", + "description": "SHA256 sum of the file" + }, + "size": { + "type": "integer", + "description": "Size of the file, in bytes" + }, + "url": { + "type": "string", + "description": "Download URL" + } + }, + "required": [ + "sha256", + "url", + "size" + ] + }, + "installRequirementInfo": { + "description": "If 'always', the tool will be installed by default. If 'on_request', tool will be installed when specifically requested. If 'never', tool will not be considered for installation.", + "type": "string", + "enum": ["always", "on_request", "never"] + }, + "platformOverrideInfo": { + "description": "Platform-specific values which override the defaults", + "type": "object", + "properties": { + "platforms": { + "description": "List of platforms to which this override applies", + "type": "array", + "items": { + "type": "string", + "enum": ["linux-i686", "linux-amd64", "linux-armel", "linux-arm64", "macos", "win32", "win64"] + } + }, + "export_paths": { + "description": "Platform-specific replacement for toolInfo/export_paths", + "$ref": "#/definitions/exportPaths" + }, + "export_vars": { + "description": "Platform-specific replacement for toolInfo/export_vars", + "$ref": "#/definitions/envVars" + }, + "install": { + "description": "Platform-specific replacement for toolInfo/install", + "$ref": "#/definitions/installRequirementInfo" + }, + "version_cmd": { + "description": "Platform-specific replacement for toolInfo/version_cmd", + "$ref": "#/definitions/arrayOfStrings" + }, + "version_regex": { + "description": "Platform-specific replacement for toolInfo/version_regex", + "$ref": "#/definitions/regex" + }, + "version_regex_replace": { + "description": "Platform-specific replacement for toolInfo/version_regex_replace", + "type": "string" + }, + "strip_container_dirs": { + "type": "string", + "description": "Platform-specific replacement for toolInfo/strip_container_dirs" + } + }, + "required": ["platforms"] + } + } +} diff --git a/tools/windows/eclipse_make.py b/tools/windows/eclipse_make.py index c0b037d6..76f510c0 100644 --- a/tools/windows/eclipse_make.py +++ b/tools/windows/eclipse_make.py @@ -3,11 +3,16 @@ # Wrapper to run make and preprocess any paths in the output from MSYS Unix-style paths # to Windows paths, for Eclipse from __future__ import print_function, division -import sys, subprocess, os.path, re +import sys +import subprocess +import os.path +import re UNIX_PATH_RE = re.compile(r'(/[^ \'"]+)+') paths = {} + + def check_path(path): try: return paths[path] @@ -24,13 +29,15 @@ def check_path(path): paths[path] = winpath return winpath + def main(): print("Running make in '%s'" % check_path(os.getcwd())) make = subprocess.Popen(["make"] + sys.argv[1:] + ["BATCH_BUILD=1"], stdout=subprocess.PIPE) for line in iter(make.stdout.readline, ''): - line = re.sub(UNIX_PATH_RE, lambda m: check_path(m.group(0)), line) - print(line.rstrip()) + line = re.sub(UNIX_PATH_RE, lambda m: check_path(m.group(0)), line) + print(line.rstrip()) sys.exit(make.wait()) + if __name__ == "__main__": main() diff --git a/tools/windows/idf_exe/CMakeLists.txt b/tools/windows/idf_exe/CMakeLists.txt new file mode 100644 index 00000000..7693412e --- /dev/null +++ b/tools/windows/idf_exe/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.5) +project(idfexe) + +set(VERSION 1.0.1) +set(ARCHIVE_NAME idf-exe-v${VERSION}.zip) + +add_executable(idf idf_main.c) +target_compile_definitions(idf PRIVATE -DVERSION=\"${VERSION}\") +set_target_properties(idf PROPERTIES C_STANDARD 99) +target_link_libraries(idf "-lshlwapi") + +if(CMAKE_BUILD_TYPE STREQUAL Release) + add_custom_command(TARGET idf + POST_BUILD + COMMAND ${CMAKE_STRIP} idf.exe) +endif() + +add_custom_target(dist ALL DEPENDS idf) + +add_custom_command( + TARGET dist + POST_BUILD + COMMAND ${CMAKE_COMMAND} ARGS -E copy "${CMAKE_CURRENT_BINARY_DIR}/idf.exe" "${CMAKE_CURRENT_BINARY_DIR}/idf.py.exe" + COMMAND ${CMAKE_COMMAND} ARGS -E tar cfv ${ARCHIVE_NAME} --format=zip + "${CMAKE_CURRENT_BINARY_DIR}/idf.py.exe" + ) diff --git a/tools/windows/idf_exe/README.md b/tools/windows/idf_exe/README.md new file mode 100644 index 00000000..0a0b5438 --- /dev/null +++ b/tools/windows/idf_exe/README.md @@ -0,0 +1,34 @@ +# IDF wrapper tool (idf.py.exe) + +This tools helps invoke idf.py in Windows CMD shell. + +In Windows CMD shell, python scripts can be executed directly (by typing their name) if `.py` extension is associated with Python. The issue with such association is that it is incompatible with virtual environments. That is, if `.py` extension is associated with a global (or user-specific) copy of `python.exe`, then the virtual environment will not be used when running the script. [Python Launcher](https://www.python.org/dev/peps/pep-0397/) solves this issue, but it is installed by default only with Python 3.6 and newer. In addition to that, the user may choose not to install Python Launcher (for example, to keep `.py` files associated with an editor). + +Hence, `idf.py.exe` is introduced. It is a simple program which forwards the command line arguments to `idf.py`. That is, + +``` +idf.py args... +``` +has the same effect as: + +``` +python.exe %IDF_PATH%\tools\idf.py args... +``` + +`python.exe` location is determined using the default search rules, which include searching the directories in `%PATH%`. Standard I/O streams are forwarded between `idf.py.exe` and `python.exe` processes. The exit code of `idf.py.exe` is the same as the exit code of `python.exe` process. + +For compatibility with `idf_tools.py`, a flag to obtain the version of `idf.py.exe` is provided: `idf.py.exe -v` or `idf.py.exe --version`. Note that this flag only works when `idf.py.exe` is started by the full name (with `.exe` extension). Running `idf.py -v` results in same behavior as `python idf.py -v`, that is `-v` argument is propagated to the Python script. + +## Building + +On Linux/Mac, install mingw-w64 toolchain (`i686-w64-mingw32-gcc`). Then build `idf.py.exe` using CMake: + +``` +mkdir -p build +cd build +cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release .. +cmake --build . +``` + +On Windows, it is also possible to build using Visual Studio, with CMake support installed. + diff --git a/tools/windows/idf_exe/idf_main.c b/tools/windows/idf_exe/idf_main.c new file mode 100644 index 00000000..3a8e6dcf --- /dev/null +++ b/tools/windows/idf_exe/idf_main.c @@ -0,0 +1,111 @@ +// Copyright 2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at", +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +#include +#include +#include +#include + +#define LINESIZE 1024 + +static void fail(LPCSTR message, ...) __attribute__((noreturn)); + +static void fail(LPCSTR message, ...) +{ + DWORD written; + char msg[LINESIZE]; + va_list args = NULL; + va_start(args, message); + StringCchVPrintfA(msg, sizeof(msg), message, args); + WriteFile(GetStdHandle(STD_ERROR_HANDLE), message, lstrlen(msg), &written, NULL); + ExitProcess(1); +} + +int main(int argc, LPTSTR argv[]) +{ + /* Print the version of this wrapper tool, but only if invoked as "idf.exe". + * "idf -v" will invoke idf.py as expected. + */ + + LPCTSTR cmdname = PathFindFileName(argv[0]); + int cmdname_length = strlen(cmdname); + + if (argc == 2 && + cmdname_length > 4 && + StrCmp(cmdname + cmdname_length - 4, TEXT(".exe")) == 0 && + (StrCmp(argv[1], TEXT("--version")) == 0 || + StrCmp(argv[1], TEXT("-v")) == 0)) { + LPCSTR msg = VERSION "\n"; + DWORD written; + WriteFile(GetStdHandle(STD_OUTPUT_HANDLE), msg, lstrlen(msg), &written, NULL); + return 0; + } + + LPCTSTR idfpy_script_name = TEXT("idf.py"); + + /* Get IDF_PATH */ + TCHAR idf_path[LINESIZE] = {}; + if (GetEnvironmentVariable(TEXT("IDF_PATH"), idf_path, sizeof(idf_path)) == 0) { + DWORD err = GetLastError(); + if (err == ERROR_ENVVAR_NOT_FOUND) { + fail("IDF_PATH environment variable needs to be set to use this tool\n"); + } else { + fail("Unknown error (%u)\n", err); + } + } + + /* Prepare the command line: python.exe %IDF_PATH%\\tools\idf.py */ + TCHAR cmdline[LINESIZE] = {}; + StringCchCat(cmdline, sizeof(cmdline), TEXT("python.exe ")); + StringCchCat(cmdline, sizeof(cmdline), idf_path); + StringCchCat(cmdline, sizeof(cmdline), TEXT("\\tools\\")); + StringCchCat(cmdline, sizeof(cmdline), idfpy_script_name); + StringCchCat(cmdline, sizeof(cmdline), TEXT(" ")); + for (int i = 1; i < argc; ++i) { + StringCchCat(cmdline, sizeof(cmdline), argv[i]); + StringCchCat(cmdline, sizeof(cmdline), TEXT(" ")); + } + + SetEnvironmentVariable(TEXT("IDF_PY_PROGRAM_NAME"), idfpy_script_name); + + /* Reuse the standard streams of this process */ + STARTUPINFO start_info = { + .cb = sizeof(STARTUPINFO), + .hStdError = GetStdHandle(STD_ERROR_HANDLE), + .hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE), + .hStdInput = GetStdHandle(STD_INPUT_HANDLE), + .dwFlags = STARTF_USESTDHANDLES + }; + + /* Run the child process */ + PROCESS_INFORMATION child_process; + if (!CreateProcess(NULL, cmdline, NULL, NULL, TRUE, 0, NULL, NULL, &start_info, &child_process)) { + DWORD err = GetLastError(); + if (err == ERROR_FILE_NOT_FOUND) { + fail("Can not find Python\n"); + } else { + fail("Unknown error (%u)\n", err); + } + } + + /* Wait for it to complete */ + WaitForSingleObject(child_process.hProcess, INFINITE); + + /* Return with the exit code of the child process */ + DWORD exitcode; + if (!GetExitCodeProcess(child_process.hProcess, &exitcode)) { + fail("Couldn't get the exit code (%u)\n", GetLastError()); + } + return exitcode; +} diff --git a/tools/windows/idf_exe/toolchain-i686-w64-mingw32.cmake b/tools/windows/idf_exe/toolchain-i686-w64-mingw32.cmake new file mode 100644 index 00000000..8e9acb4a --- /dev/null +++ b/tools/windows/idf_exe/toolchain-i686-w64-mingw32.cmake @@ -0,0 +1,7 @@ +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR x86) +set(CMAKE_C_COMPILER i686-w64-mingw32-gcc) +set(CMAKE_CXX_COMPILER i686-w64-mingw32-g++) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/tools/windows/tool_setup/.gitignore b/tools/windows/tool_setup/.gitignore new file mode 100644 index 00000000..620ec0e2 --- /dev/null +++ b/tools/windows/tool_setup/.gitignore @@ -0,0 +1,6 @@ +Output +cmdlinerunner/build +dist +unzip +keys +idf_versions.txt diff --git a/tools/windows/tool_setup/README.md b/tools/windows/tool_setup/README.md new file mode 100644 index 00000000..f9e71e9b --- /dev/null +++ b/tools/windows/tool_setup/README.md @@ -0,0 +1,39 @@ +# ESP-IDF Tools Installer for Windows + +This directory contains source files required to build the tools installer for Windows. + +The installer is built using [Inno Setup](http://www.jrsoftware.org/isinfo.php). At the time of writing, the installer can be built with Inno Setup version 6.0.2. + +The main source file of the installer is `idf_tools_setup.iss`. PascalScript code is split into multiple `*.iss.inc` files. + +Some functionality of the installer depends on additional programs: + +* [Inno Download Plugin](https://bitbucket.org/mitrich_k/inno-download-plugin) — used to download additional files during the installation. + +* [7-zip](https://www.7-zip.org) — used to extract downloaded IDF archives. + +* [cmdlinerunner](cmdlinerunner/cmdlinerunner.c) — a helper DLL used to run external command line programs from the installer, capture live console output, and get the exit code. + +## Steps required to build the installer + +* Build cmdlinerunner DLL. + - On Linux/Mac, install mingw-w64 toolchain (`i686-w64-mingw32-gcc`). Then build the DLL using CMake: + ``` + mkdir -p cmdlinerunner/build + cd cmdlinerunner/build + cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . + ``` + This will produce `cmdlinerunner.dll` in the build directory. + - On Windows, it is possible to build using Visual Studio, with CMake support installed. By default, VS produces build artifacts in some hard to find directory. You can adjust this in CmakeSettings.json file generated by VS. + +* Download 7zip.exe [("standalone console version")](https://www.7-zip.org/download.html) and put it into `unzip` directory (to get `unzip/7za.exe`). + +* Download [idf_versions.txt](https://dl.espressif.com/dl/esp-idf/idf_versions.txt) and place it into the current directory. The installer will use it as a fallback, if it can not download idf_versions.txt at run time. + +* Create the `dist` directory and populate it with the tools which should be bundled with the installer. At the moment the easiest way to obtain it is to use `install.sh`/`install.bat` in IDF, and then copy the contents of `$HOME/.espressif/dist` directory. If the directory is empty, the installer should still work, and the tools will be downloaded during the installation. + +* Build the installer using Inno Setup Compiler: `ISCC.exe idf_tools_setup.iss`. + +* Obtain the signing keys, then sign `Output/esp-idf-tools-setup-unsigned.exe`. + diff --git a/tools/windows/tool_setup/build_installer.sh b/tools/windows/tool_setup/build_installer.sh index 8733629b..270fa8ef 100755 --- a/tools/windows/tool_setup/build_installer.sh +++ b/tools/windows/tool_setup/build_installer.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Setup script to build Windows tool installer with Inno Setup # @@ -10,19 +10,58 @@ # - Runs ISCC under wine to compile the installer itself set -e -cd `dirname $0` -pushd dl -wget --continue "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip" -wget --continue "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20180319/mconf-v4.6.0.0-idf-20180319-win32.zip" -wget --continue "https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip" -popd +if [ -z "${KEYPASSWORD}" ]; then + echo "KEYPASSWORD should be set" + exit 1 +fi -rm -rf input/* -pushd input -unzip ../dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip -unzip ../dl/mconf-v4.6.0.0-idf-20180319-win32.zip -unzip ../dl/ninja-win.zip -popd +if [ "$1" != "--no-download" ]; then + + mkdir -p dl input + + cd `dirname $0` + pushd dl + wget --continue "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip" + wget --continue "https://github.com/espressif/binutils-esp32ulp/releases/download/v2.28.51-esp32ulp-20180809/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip" + wget --continue "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20180920/openocd-esp32-win32-0.10.0-esp32-20180920.zip" + wget --continue "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20180525/mconf-v4.6.0.0-idf-20180525-win32.zip" + wget --continue "https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip" + popd + + rm -rf input/* + pushd input + unzip ../dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip + unzip ../dl/mconf-v4.6.0.0-idf-20180525-win32.zip + unzip ../dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip + unzip ../dl/openocd-esp32-win32-0.10.0-esp32-20180920.zip + unzip ../dl/ninja-win.zip + popd +fi wine "C:\Program Files\Inno Setup 5\ISCC.exe" "`winepath -w ./idf_tool_setup.iss`" +# sign the installer with osslsigncode, parsing the version number out of the +# installer config + +VERSION=`grep "^AppVersion=" idf_tool_setup.iss | cut -d'=' -f2` + +echo "Signing installer..." + +# Note: The cert chain passed to -certs needs to contain the intermediate +# cert(s) as well, appended after the code signing cert, or Windows may see +# it as "Unknown Publisher" +# +# See https://stackoverflow.com/a/52637050 for full details +# +umask 770 # for the process substitution FIFO + +osslsigncode -certs ./keys/certchain.pem -key ./keys/key.pem \ + -readpass <(echo "$KEYPASSWORD") \ + -in Output/esp-idf-tools-setup-unsigned.exe \ + -out Output/esp-idf-tools-setup-${VERSION}.exe \ + -h sha256 \ + -n "Espressif Systems (Shanghai) Pte. Ltd." \ + -i "https://www.espressif.com/" \ + -ts http://timestamp.digicert.com + +chmod 644 Output/esp-idf-tools-setup-${VERSION}.exe # make up for the umask diff --git a/tools/windows/tool_setup/choice_page.iss.inc b/tools/windows/tool_setup/choice_page.iss.inc new file mode 100644 index 00000000..8291edc2 --- /dev/null +++ b/tools/windows/tool_setup/choice_page.iss.inc @@ -0,0 +1,247 @@ +var + ChoicePagePrepare: array of TNotifyEvent; + ChoicePageSelectionChange: array of TNotifyEvent; + ChoicePageValidate: array of TWizardPageButtonEvent; + ChoicePageMaxTag: Integer; + ChoicePages: array of TInputOptionWizardPage; + +procedure ChoicePageOnClickCheck(Sender: TObject); +var + ListBox: TNewCheckListBox; + Id: Integer; +begin + ListBox := TNewCheckListBox(Sender); + Id := Integer(ListBox.Tag); + ChoicePageSelectionChange[Id](ChoicePages[Id]); +end; + +function ChoicePageGetInput(Page: TInputOptionWizardPage): TNewEdit; +begin + Result := TNewEdit(Page.FindComponent('ChoicePageInput')); +end; + +function ChoicePageGetLabel(Page: TInputOptionWizardPage): TNewStaticText; +begin + Result := TNewStaticText(Page.FindComponent('ChoicePageLabel')); +end; + +function ChoicePageGetButton(Page: TInputOptionWizardPage): TNewButton; +begin + Result := TNewButton(Page.FindComponent('ChoicePageBrowseButton')); +end; + +procedure ChoicePageSetEditLabel(Page: TInputOptionWizardPage; Caption: String); +var + InputLabel: TNewStaticText; +begin + InputLabel := ChoicePageGetLabel(Page); + InputLabel.Caption := Caption; +end; + +function ChoicePageGetInputText(Page: TInputOptionWizardPage): String; +begin + Result := ChoicePageGetInput(Page).Text; +end; + +procedure ChoicePageSetInputText(Page: TInputOptionWizardPage; Text: String); +begin + ChoicePageGetInput(Page).Text := Text; +end; + +procedure ChoicePageSetInputEnabled(Page: TInputOptionWizardPage; Enabled: Boolean); +begin + ChoicePageGetLabel(Page).Enabled := Enabled; + ChoicePageGetInput(Page).Enabled := Enabled; + ChoicePageGetButton(Page).Enabled := Enabled; +end; + + +procedure ChoicePageOnBrowseButtonClick(Sender: TObject); +var + Button: TNewButton; + Page: TInputOptionWizardPage; + InputLabel: TNewStaticText; + Input: TNewEdit; + Dir: String; +begin + Button := TNewButton(Sender); + Page := TInputOptionWizardPage(Button.Owner); + Input := ChoicePageGetInput(Page); + InputLabel := ChoicePageGetLabel(Page); + Dir := Input.Text; + if BrowseForFolder(InputLabel.Caption, Dir, True) then + begin + Input.Text := Dir; + end; +end; + + +procedure ChoicePageOnCurPageChanged(CurPageID: Integer); +var + i: Integer; +begin + for i := 1 to ChoicePageMaxTag do + begin + if ChoicePages[i].ID = CurPageID then + begin + ChoicePagePrepare[i](ChoicePages[i]); + break; + end; + end; +end; + + +function ChoicePageOnNextButtonClick(CurPageID: Integer): Boolean; +var + i: Integer; +begin + Result := True; + for i := 1 to ChoicePageMaxTag do + begin + if ChoicePages[i].ID = CurPageID then + begin + Result := ChoicePageValidate[i](ChoicePages[i]); + break; + end; + end; +end; + + +procedure InitChoicePages(); +begin + ChoicePages := [ ]; + ChoicePagePrepare := [ ]; + ChoicePageSelectionChange := [ ]; + ChoicePageValidate := [ ]; +end; + +function FindLinkInText(Text: String): String; +var + Tmp: String; + LinkStartPos, LinkEndPos: Integer; +begin + Result := ''; + Tmp := Text; + LinkStartPos := Pos('https://', Tmp); + if LinkStartPos = 0 then exit; + Delete(Tmp, 1, LinkStartPos - 1); + + { Try to find the end of the link } + LinkEndPos := 0 + if LinkEndPos = 0 then LinkEndPos := Pos(' ', Tmp); + if LinkEndPos = 0 then LinkEndPos := Pos(',', Tmp); + if LinkEndPos = 0 then LinkEndPos := Pos('.', Tmp); + if LinkEndPos = 0 then LinkEndPos := Length(Tmp); + Delete(Text, LinkEndPos, Length(Tmp)); + + Log('Found link in "' + Text + '": "' + Tmp + '"'); + Result := Tmp; +end; + +procedure OnStaticTextClick(Sender: TObject); +var + StaticText: TNewStaticText; + Link: String; + Err: Integer; +begin + StaticText := TNewStaticText(Sender); + Link := FindLinkInText(StaticText.Caption); + if Link = '' then + exit; + + ShellExec('open', Link, '', '', SW_SHOWNORMAL, ewNoWait, Err); +end; + +procedure MakeStaticTextClickable(StaticText: TNewStaticText); +begin + if FindLinkInText(StaticText.Caption) = '' then + exit; + + StaticText.OnClick := @OnStaticTextClick; + StaticText.Cursor := crHand; +end; + +function ChoicePageCreate( + const AfterID: Integer; + const Caption, Description, SubCaption, EditCaption: String; + HasDirectoryChooser: Boolean; + Prepare: TNotifyEvent; + SelectionChange: TNotifyEvent; + Validate: TWizardPageButtonEvent): TInputOptionWizardPage; +var + VSpace, Y : Integer; + ChoicePage: TInputOptionWizardPage; + InputLabel: TNewStaticText; + Input: TNewEdit; + Button: TNewButton; + +begin + ChoicePageMaxTag := ChoicePageMaxTag + 1; + VSpace := ScaleY(8); + ChoicePage := CreateInputOptionPage(AfterID, Caption, + Description, SubCaption, True, True); + + MakeStaticTextClickable(ChoicePage.SubCaptionLabel); + + ChoicePage.Tag := ChoicePageMaxTag; + ChoicePage.CheckListBox.OnClickCheck := @ChoicePageOnClickCheck; + ChoicePage.CheckListBox.Tag := ChoicePageMaxTag; + + if HasDirectoryChooser then + begin + ChoicePage.CheckListBox.Anchors := [ akLeft, akTop, akRight ]; + ChoicePage.CheckListBox.Height := ChoicePage.CheckListBox.Height - ScaleY(60); + Y := ChoicePage.CheckListBox.Top + ChoicePage.CheckListBox.Height + VSpace; + + InputLabel := TNewStaticText.Create(ChoicePage); + with InputLabel do + begin + Top := Y; + Anchors := [akTop, akLeft, akRight]; + Caption := EditCaption; + AutoSize := True; + Parent := ChoicePage.Surface; + Name := 'ChoicePageLabel'; + end; + MakeStaticTextClickable(InputLabel); + Y := Y + InputLabel.Height + VSpace; + + Input := TNewEdit.Create(ChoicePage); + with Input do + begin + Top := Y; + Anchors := [akTop, akLeft, akRight]; + Parent := ChoicePage.Surface; + Name := 'ChoicePageInput'; + Text := ''; + end; + + Button := TNewButton.Create(ChoicePage); + with Button do + begin + Anchors := [akTop, akRight]; + Parent := ChoicePage.Surface; + Width := WizardForm.NextButton.Width; + Height := WizardForm.NextButton.Height; + Top := Y - (Height - Input.Height) / 2; + Left := ChoicePage.SurfaceWidth - Button.Width; + Name := 'ChoicePageBrowseButton'; + Caption := SetupMessage(msgButtonWizardBrowse); + OnClick := @ChoicePageOnBrowseButtonClick; + end; + + Input.Width := Button.Left - ScaleX(8); + end; + + SetArrayLength(ChoicePages, ChoicePageMaxTag+1); + SetArrayLength(ChoicePagePrepare, ChoicePageMaxTag+1); + SetArrayLength(ChoicePageSelectionChange, ChoicePageMaxTag+1); + SetArrayLength(ChoicePageValidate, ChoicePageMaxTag+1); + + ChoicePages[ChoicePageMaxTag] := ChoicePage; + ChoicePagePrepare[ChoicePageMaxTag] := Prepare; + ChoicePageSelectionChange[ChoicePageMaxTag] := SelectionChange; + ChoicePageValidate[ChoicePageMaxTag] := Validate; + + Result := ChoicePage; +end; diff --git a/tools/windows/tool_setup/cmdline_page.iss.inc b/tools/windows/tool_setup/cmdline_page.iss.inc new file mode 100644 index 00000000..fa5fdc63 --- /dev/null +++ b/tools/windows/tool_setup/cmdline_page.iss.inc @@ -0,0 +1,154 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Progress & log page for command line tools ------------------------------ } + +var + CmdlineInstallCancel: Boolean; + +{ ------------------------------ Splitting strings into lines and adding them to TStrings ------------------------------ } + +procedure StringsAddLine(Dest: TStrings; Line: String; var ReplaceLastLine: Boolean); +begin + if ReplaceLastLine then + begin + Dest.Strings[Dest.Count - 1] := Line; + ReplaceLastLine := False; + end else begin + Dest.Add(Line); + end; +end; + +procedure StrSplitAppendToList(Text: String; Dest: TStrings; var LastLine: String); +var + pCR, pLF, Len: Integer; + Tmp: String; + ReplaceLastLine: Boolean; +begin + if Length(LastLine) > 0 then + begin + ReplaceLastLine := True; + Text := LastLine + Text; + end; + repeat + Len := Length(Text); + pLF := Pos(#10, Text); + pCR := Pos(#13, Text); + if (pLF > 0) and ((pCR = 0) or (pLF < pCR) or (pLF = pCR + 1)) then + begin + if pLF < pCR then + Tmp := Copy(Text, 1, pLF - 1) + else + Tmp := Copy(Text, 1, pLF - 2); + StringsAddLine(Dest, Tmp, ReplaceLastLine); + Text := Copy(Text, pLF + 1, Len) + end else begin + if (pCR = Len) or (pCR = 0) then + begin + break; + end; + Text := Copy(Text, pCR + 1, Len) + end; + until (pLF = 0) and (pCR = 0); + + LastLine := Text; + if pCR = Len then + begin + Text := Copy(Text, 1, pCR - 1); + end; + if Length(LastLine) > 0 then + begin + StringsAddLine(Dest, Text, ReplaceLastLine); + end; + +end; + +{ ------------------------------ The actual command line install page ------------------------------ } + +procedure OnCmdlineInstallCancel(Sender: TObject); +begin + CmdlineInstallCancel := True; +end; + +function DoCmdlineInstall(caption, description, command: String): Boolean; +var + CmdlineInstallPage: TOutputProgressWizardPage; + Res: Integer; + Handle: Longword; + ExitCode: Integer; + LogTextAnsi: AnsiString; + LogText, LeftOver: String; + Memo: TNewMemo; + PrevCancelButtonOnClick: TNotifyEvent; + +begin + CmdlineInstallPage := CreateOutputProgressPage('', '') + CmdlineInstallPage.Caption := caption; + CmdlineInstallPage.Description := description; + + Memo := TNewMemo.Create(CmdlineInstallPage); + Memo.Top := CmdlineInstallPage.ProgressBar.Top + CmdlineInstallPage.ProgressBar.Height + ScaleY(8); + Memo.Width := CmdlineInstallPage.SurfaceWidth; + Memo.Height := ScaleY(120); + Memo.ScrollBars := ssVertical; + Memo.Parent := CmdlineInstallPage.Surface; + Memo.Lines.Clear(); + + CmdlineInstallPage.Show(); + + try + WizardForm.CancelButton.Visible := True; + WizardForm.CancelButton.Enabled := True; + PrevCancelButtonOnClick := WizardForm.CancelButton.OnClick; + WizardForm.CancelButton.OnClick := @OnCmdlineInstallCancel; + + CmdlineInstallPage.SetProgress(0, 100); + CmdlineInstallPage.ProgressBar.Style := npbstMarquee; + + ExitCode := -1; + Memo.Lines.Append('Running command: ' + command); + Handle := ProcStart(command, ExpandConstant('{tmp}')) + if Handle = 0 then + begin + Log('ProcStart failed'); + ExitCode := -2; + end; + while (ExitCode = -1) and not CmdlineInstallCancel do + begin + ExitCode := ProcGetExitCode(Handle); + SetLength(LogTextAnsi, 4096); + Res := ProcGetOutput(Handle, LogTextAnsi, 4096) + if Res > 0 then + begin + SetLength(LogTextAnsi, Res); + LogText := LeftOver + String(LogTextAnsi); + StrSplitAppendToList(LogText, Memo.Lines, LeftOver); + end; + CmdlineInstallPage.SetProgress(0, 100); + Sleep(10); + end; + ProcEnd(Handle); + finally + Log('Done, exit code=' + IntToStr(ExitCode)); + Log('--------'); + Log(Memo.Lines.Text); + Log('--------'); + if CmdlineInstallCancel then + begin + MsgBox('Installation has been cancelled.', mbError, MB_OK); + Result := False; + end else if ExitCode <> 0 then + begin + MsgBox('Installation has failed with exit code ' + IntToStr(ExitCode), mbError, MB_OK); + Result := False; + end else begin + Result := True; + end; + CmdlineInstallPage.Hide; + CmdlineInstallPage.Free; + WizardForm.CancelButton.OnClick := PrevCancelButtonOnClick; + end; + if not Result then + RaiseException('Installation has failed at step: ' + caption); +end; + diff --git a/tools/windows/tool_setup/cmdlinerunner/CMakeLists.txt b/tools/windows/tool_setup/cmdlinerunner/CMakeLists.txt new file mode 100644 index 00000000..1f4368a3 --- /dev/null +++ b/tools/windows/tool_setup/cmdlinerunner/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.5) +project(cmdlinerunner) +set(CMAKE_EXE_LINKER_FLAGS " -static") +add_library(cmdlinerunner SHARED cmdlinerunner.c) +target_compile_definitions(cmdlinerunner PUBLIC UNICODE _UNICODE) +set_target_properties(cmdlinerunner PROPERTIES PREFIX "") +set_target_properties(cmdlinerunner PROPERTIES C_STANDARD 99) +target_link_libraries(cmdlinerunner "-static-libgcc") diff --git a/tools/windows/tool_setup/cmdlinerunner/cmdlinerunner.c b/tools/windows/tool_setup/cmdlinerunner/cmdlinerunner.c new file mode 100644 index 00000000..0688ea43 --- /dev/null +++ b/tools/windows/tool_setup/cmdlinerunner/cmdlinerunner.c @@ -0,0 +1,194 @@ +// Copyright 2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at", +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +#define CMDLINERUNNER_EXPORTS + +#include +#include +#include +#include "cmdlinerunner.h" + +#define LINESIZE 1024 + +#ifdef WITH_DEBUG +#include +#define DEBUGV(...) do { fprintf(stderr, __VA_ARG__); } while(0) +#else +#define DEBUGV(...) +#endif + +struct proc_instance_s { + PROCESS_INFORMATION child_process; + HANDLE pipe_server_handle; + HANDLE pipe_client_handle; +}; + +#ifdef WITH_DEBUG +static void print_last_error() +{ + DWORD dw; + TCHAR errmsg[LINESIZE]; + dw = GetLastError(); + + FormatMessage( FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, dw, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + errmsg, sizeof(errmsg) - 1, NULL ); + DEBUGV("error %d: %s\n", dw, errmsg); +} +#define PRINT_LAST_ERROR() print_last_error() +#else +#define PRINT_LAST_ERROR() +#endif + +static proc_instance_t *proc_instance_allocate() +{ + return (proc_instance_t*) HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, sizeof(proc_instance_t)); +} + +static void proc_instance_free(proc_instance_t *instance) +{ + if (instance->pipe_server_handle) { + CloseHandle(instance->pipe_server_handle); + } + if (instance->pipe_client_handle) { + CloseHandle(instance->pipe_client_handle); + } + if (instance->child_process.hProcess) { + TerminateProcess(instance->child_process.hProcess, 1); + CloseHandle(instance->child_process.hProcess); + CloseHandle(instance->child_process.hThread); + } + HeapFree(GetProcessHeap(), 0, instance); +} + +void proc_end(proc_instance_t *inst) +{ + if (inst == NULL) { + return; + } + proc_instance_free(inst); +} + +CMDLINERUNNER_API proc_instance_t * proc_start(LPCTSTR cmdline, LPCTSTR workdir) +{ + proc_instance_t *inst = proc_instance_allocate(); + if (inst == NULL) { + return NULL; + } + + SECURITY_ATTRIBUTES sec_attr = { + .nLength = sizeof(SECURITY_ATTRIBUTES), + .bInheritHandle = TRUE, + .lpSecurityDescriptor = NULL + }; + + LPCTSTR pipename = TEXT("\\\\.\\pipe\\cmdlinerunner_pipe"); + + inst->pipe_server_handle = CreateNamedPipe(pipename, PIPE_ACCESS_DUPLEX, + PIPE_TYPE_BYTE | PIPE_WAIT, 1, 1024 * 16, 1024 * 16, + NMPWAIT_WAIT_FOREVER, &sec_attr); + if (inst->pipe_server_handle == INVALID_HANDLE_VALUE) { + DEBUGV("inst->pipe_server_handle == INVALID_HANDLE_VALUE\n"); + goto error; + } + + inst->pipe_client_handle = CreateFile(pipename, GENERIC_WRITE | GENERIC_READ, + 0, &sec_attr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (inst->pipe_client_handle == INVALID_HANDLE_VALUE) { + DEBUGV("inst->pipe_client_handle == INVALID_HANDLE_VALUE\n"); + goto error; + } + + DWORD new_mode = PIPE_READMODE_BYTE | PIPE_NOWAIT; + if (!SetNamedPipeHandleState(inst->pipe_server_handle, &new_mode, NULL, + NULL)) { + DEBUGV("SetNamedPipeHandleState failed\n"); + goto error; + } + + if (!SetHandleInformation(inst->pipe_server_handle, HANDLE_FLAG_INHERIT, 0)) { + DEBUGV("SetHandleInformation failed\n"); + goto error; + } + + if (!SetHandleInformation(inst->pipe_client_handle, HANDLE_FLAG_INHERIT, + HANDLE_FLAG_INHERIT)) { + DEBUGV("SetHandleInformation failed\n"); + goto error; + } + + STARTUPINFO siStartInfo = { + .cb = sizeof(STARTUPINFO), + .hStdError = inst->pipe_client_handle, + .hStdOutput = inst->pipe_client_handle, + .hStdInput = inst->pipe_client_handle, + .dwFlags = STARTF_USESTDHANDLES + }; + + size_t workdir_len = 0; + StringCbLength(workdir, STRSAFE_MAX_CCH * sizeof(TCHAR), &workdir_len); + if (workdir_len == 0) { + workdir = NULL; + } + + TCHAR cmdline_tmp[LINESIZE]; + StringCbCopy(cmdline_tmp, sizeof(cmdline_tmp), cmdline); + if (!CreateProcess(NULL, cmdline_tmp, + NULL, NULL, TRUE, CREATE_NO_WINDOW, NULL, workdir, &siStartInfo, + &inst->child_process)) { + DEBUGV("CreateProcess failed\n"); + goto error; + } + return inst; + +error: + PRINT_LAST_ERROR(); + proc_instance_free(inst); + return NULL; +} + +int proc_get_exit_code(proc_instance_t *inst) +{ + DWORD result; + if (!GetExitCodeProcess(inst->child_process.hProcess, &result)) { + return -2; + } + if (result == STILL_ACTIVE) { + return -1; + } + return (int) result; +} + +DWORD proc_get_output(proc_instance_t *inst, LPSTR dest, DWORD sz) +{ + DWORD read_bytes; + BOOL res = ReadFile(inst->pipe_server_handle, dest, + sz - 1, &read_bytes, NULL); + if (!res) { + if (GetLastError() == ERROR_NO_DATA) { + return 0; + } else { + PRINT_LAST_ERROR(); + return 0; + } + } + dest[read_bytes] = 0; + return read_bytes; +} + +BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved ) +{ + return TRUE; +} + diff --git a/tools/windows/tool_setup/cmdlinerunner/cmdlinerunner.h b/tools/windows/tool_setup/cmdlinerunner/cmdlinerunner.h new file mode 100644 index 00000000..bdfdf2dc --- /dev/null +++ b/tools/windows/tool_setup/cmdlinerunner/cmdlinerunner.h @@ -0,0 +1,32 @@ +// Copyright 2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at", +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +#pragma once + +#include + +struct proc_instance_s; +typedef struct proc_instance_s proc_instance_t; + +#ifdef CMDLINERUNNER_EXPORTS +#define CMDLINERUNNER_API __declspec(dllexport) +#else +#define CMDLINERUNNER_API __declspec(dllimport) +#endif + +CMDLINERUNNER_API proc_instance_t * proc_start(LPCTSTR cmdline, LPCTSTR workdir); +CMDLINERUNNER_API int proc_get_exit_code(proc_instance_t *inst); +CMDLINERUNNER_API DWORD proc_get_output(proc_instance_t *inst, LPSTR dest, DWORD sz); +CMDLINERUNNER_API void proc_end(proc_instance_t *inst); +CMDLINERUNNER_API BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved ); diff --git a/tools/windows/tool_setup/cmdlinerunner/toolchain-i686-w64-mingw32.cmake b/tools/windows/tool_setup/cmdlinerunner/toolchain-i686-w64-mingw32.cmake new file mode 100644 index 00000000..8e9acb4a --- /dev/null +++ b/tools/windows/tool_setup/cmdlinerunner/toolchain-i686-w64-mingw32.cmake @@ -0,0 +1,7 @@ +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR x86) +set(CMAKE_C_COMPILER i686-w64-mingw32-gcc) +set(CMAKE_CXX_COMPILER i686-w64-mingw32-g++) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/tools/windows/tool_setup/git_find_installed.iss.inc b/tools/windows/tool_setup/git_find_installed.iss.inc new file mode 100644 index 00000000..328294cd --- /dev/null +++ b/tools/windows/tool_setup/git_find_installed.iss.inc @@ -0,0 +1,98 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Find installed copies of Git ------------------------------ } + +var + InstalledGitVersions: TStringList; + InstalledGitDisplayNames: TStringList; + InstalledGitExecutables: TStringList; + + +procedure GitVersionAdd(Version, DisplayName, Executable: String); +begin + Log('Adding Git version=' + Version + ' name='+DisplayName+' executable='+Executable); + InstalledGitVersions.Append(Version); + InstalledGitDisplayNames.Append(DisplayName); + InstalledGitExecutables.Append(Executable); +end; + +function GetVersionOfGitExe(Path: String; var Version: String; var ErrStr: String): Boolean; +var + VersionOutputFile: String; + Args: String; + GitVersionAnsi: AnsiString; + GitVersion: String; + GitVersionPrefix: String; + Err: Integer; +begin + VersionOutputFile := ExpandConstant('{tmp}\gitver.txt'); + + DeleteFile(VersionOutputFile); + Args := '/C "' + Path + '" --version >gitver.txt'; + Log('Running ' + Args); + if not ShellExec('', 'cmd.exe', Args, + ExpandConstant('{tmp}'), SW_HIDE, ewWaitUntilTerminated, Err) then + begin + ErrStr := 'Failed to get git version, error=' + IntToStr(err); + Log(ErrStr); + Result := False; + exit; + end; + + LoadStringFromFile(VersionOutputFile, GitVersionAnsi); + GitVersion := Trim(String(GitVersionAnsi)); + GitVersionPrefix := 'git version '; + if Pos(GitVersionPrefix, GitVersion) <> 1 then + begin + ErrStr := 'Unexpected git version format: ' + GitVersion; + Log(ErrStr); + Result := False; + exit; + end; + + Delete(GitVersion, 1, Length(GitVersionPrefix)); + Version := GitVersion; + Result := True; +end; + +procedure FindGitInPath(); +var + Args: String; + GitListFile: String; + GitPaths: TArrayOfString; + GitVersion: String; + ErrStr: String; + Err: Integer; + i: Integer; +begin + GitListFile := ExpandConstant('{tmp}\gitlist.txt'); + Args := '/C where git.exe >"' + GitListFile + '"'; + if not ShellExec('', 'cmd.exe', Args, + '', SW_HIDE, ewWaitUntilTerminated, Err) then + begin + Log('Failed to find git using "where", error='+IntToStr(Err)); + exit; + end; + + LoadStringsFromFile(GitListFile, GitPaths); + + for i:= 0 to GetArrayLength(GitPaths) - 1 do + begin + Log('Git path: ' + GitPaths[i]); + if not GetVersionOfGitExe(GitPaths[i], GitVersion, ErrStr) then + continue; + + Log('Git version: ' + GitVersion); + GitVersionAdd(GitVersion, GitVersion, GitPaths[i]); + end; +end; + +procedure FindInstalledGitVersions(); +begin + InstalledGitVersions := TStringList.Create(); + InstalledGitDisplayNames := TStringList.Create(); + InstalledGitExecutables := TStringList.Create(); + + FindGitInPath(); +end; diff --git a/tools/windows/tool_setup/git_page.iss.inc b/tools/windows/tool_setup/git_page.iss.inc new file mode 100644 index 00000000..b9c1ef7d --- /dev/null +++ b/tools/windows/tool_setup/git_page.iss.inc @@ -0,0 +1,194 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Page to select Git ------------------------------ } + +#include "git_find_installed.iss.inc" + +var + GitPage: TInputOptionWizardPage; + GitPath, GitExecutablePath, GitVersion: String; + GitUseExisting: Boolean; + GitSelectionInstallIndex: Integer; + GitSelectionCustomPathIndex: Integer; + +function GetGitPath(Unused: String): String; +begin + Result := GitPath; +end; + +function GitInstallRequired(): Boolean; +begin + Result := not GitUseExisting; +end; + +function GitVersionSupported(Version: String): Boolean; +var + Major, Minor: Integer; +begin + Result := False; + if not VersionExtractMajorMinor(Version, Major, Minor) then + begin + Log('GitVersionSupported: Could not parse version=' + Version); + exit; + end; + + { Need at least git 2.12 for 'git clone --reference' to work with submodules } + if (Major = 2) and (Minor >= 12) then Result := True; + if (Major > 2) then Result := True; +end; + +procedure GitCustomPathUpdateEnabled(); +var + Enable: Boolean; +begin + if GitPage.SelectedValueIndex = GitSelectionCustomPathIndex then + Enable := True; + + ChoicePageSetInputEnabled(GitPage, Enable); +end; + +procedure OnGitPagePrepare(Sender: TObject); +var + Page: TInputOptionWizardPage; + FullName: String; + i, Index, FirstEnabledIndex: Integer; + OfferToInstall: Boolean; + VersionToInstall: String; + VersionSupported: Boolean; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnGitPagePrepare'); + if Page.CheckListBox.Items.Count > 0 then + exit; + + FindInstalledGitVersions(); + + VersionToInstall := '{#GitVersion}'; + OfferToInstall := True; + FirstEnabledIndex := -1; + + for i := 0 to InstalledGitVersions.Count - 1 do + begin + VersionSupported := GitVersionSupported(InstalledGitVersions[i]); + FullName := InstalledGitDisplayNames.Strings[i]; + if not VersionSupported then + begin + FullName := FullName + ' (unsupported)'; + end; + FullName := FullName + #13#10 + InstalledGitExecutables.Strings[i]; + Index := Page.Add(FullName); + if not VersionSupported then + begin + Page.CheckListBox.ItemEnabled[Index] := False; + end else begin + if FirstEnabledIndex < 0 then FirstEnabledIndex := Index; + end; + if InstalledGitVersions[i] = VersionToInstall then + begin + OfferToInstall := False; + end; + end; + + if OfferToInstall then + begin + Index := Page.Add('Install Git ' + VersionToInstall); + if FirstEnabledIndex < 0 then FirstEnabledIndex := Index; + GitSelectionInstallIndex := Index; + end; + + Index := Page.Add('Custom git.exe location'); + if FirstEnabledIndex < 0 then FirstEnabledIndex := Index; + GitSelectionCustomPathIndex := Index; + + Page.SelectedValueIndex := FirstEnabledIndex; + GitCustomPathUpdateEnabled(); +end; + +procedure OnGitSelectionChange(Sender: TObject); +var + Page: TInputOptionWizardPage; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnGitSelectionChange index=' + IntToStr(Page.SelectedValueIndex)); + GitCustomPathUpdateEnabled(); +end; + +function OnGitPageValidate(Sender: TWizardPage): Boolean; +var + Page: TInputOptionWizardPage; + Version, ErrStr: String; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnGitPageValidate index=' + IntToStr(Page.SelectedValueIndex)); + if Page.SelectedValueIndex = GitSelectionInstallIndex then + begin + GitUseExisting := False; + GitExecutablePath := ''; + GitPath := ''; + GitVersion := '{#GitVersion}'; + Result := True; + end else if Page.SelectedValueIndex = GitSelectionCustomPathIndex then + begin + GitPath := ChoicePageGetInputText(Page); + GitExecutablePath := GitPath + '\git.exe'; + if not FileExists(GitExecutablePath) then + begin + MsgBox('Can not find git.exe in ' + GitPath, mbError, MB_OK); + Result := False; + exit; + end; + + if not GetVersionOfGitExe(GitExecutablePath, Version, ErrStr) then + begin + MsgBox('Can not determine version of git.exe.' + #13#10 + + 'Please check that this copy of git works from cmd.exe.', mbError, MB_OK); + Result := False; + exit; + end; + Log('Version of ' + GitExecutablePath + ' is ' + Version); + if not GitVersionSupported(Version) then + begin + MsgBox('Selected git version (' + Version + ') is not supported.', mbError, MB_OK); + Result := False; + exit; + end; + Log('Version of git is supported'); + GitUseExisting := True; + GitVersion := Version; + end else begin + GitUseExisting := True; + GitExecutablePath := InstalledGitExecutables[Page.SelectedValueIndex]; + GitPath := ExtractFilePath(GitExecutablePath); + GitVersion := InstalledGitVersions[Page.SelectedValueIndex]; + Result := True; + end; +end; + +procedure GitExecutablePathUpdateAfterInstall(); +var + GitInstallPath: String; +begin + GitInstallPath := GetInstallPath('SOFTWARE\GitForWindows', 'InstallPath'); + if GitInstallPath = '' then + begin + Log('Failed to find Git install path'); + exit; + end; + GitPath := GitInstallPath + '\cmd'; + GitExecutablePath := GitPath + '\git.exe'; +end; + + +procedure CreateGitPage(); +begin + GitPage := ChoicePageCreate( + wpLicense, + 'Git choice', 'Please choose Git version', + 'Available Git versions', + 'Enter custom location of git.exe', + True, + @OnGitPagePrepare, + @OnGitSelectionChange, + @OnGitPageValidate); +end; diff --git a/tools/windows/tool_setup/idf_cmd_init.bat b/tools/windows/tool_setup/idf_cmd_init.bat new file mode 100644 index 00000000..853dcf4d --- /dev/null +++ b/tools/windows/tool_setup/idf_cmd_init.bat @@ -0,0 +1,117 @@ +@echo off + +:: This script is called from a shortcut (cmd.exe /k export_fallback.bat), with +:: the working directory set to an ESP-IDF directory. +:: Its purpose is to support using the "IDF Tools Directory" method of +:: installation for ESP-IDF versions older than IDF v4.0. +:: It does the same thing as "export.bat" in IDF v4.0. + +set IDF_PATH=%CD% +if not exist "%IDF_PATH%\tools\idf.py" ( + echo This script must be invoked from ESP-IDF directory. + goto :end +) + +if "%~2"=="" ( + echo Usage: idf_cmd_init.bat ^ ^ + echo This script must be invoked from ESP-IDF directory. + goto :end +) + +set IDF_PYTHON_DIR=%1 +set IDF_GIT_DIR=%2 + +:: Strip quoutes +set "IDF_PYTHON_DIR=%IDF_PYTHON_DIR:"=%" +set "IDF_GIT_DIR=%IDF_GIT_DIR:"=%" + +:: Clear PYTHONPATH as it may contain libraries of other Python versions +if not "%PYTHONPATH%"=="" ( + echo Clearing PYTHONPATH, was set to %PYTHONPATH% + set PYTHONPATH= +) + +:: Add Python and Git paths to PATH +set "PATH=%IDF_PYTHON_DIR%;%IDF_GIT_DIR%;%PATH%" +echo Using Python in %IDF_PYTHON_DIR% +python.exe --version +echo Using Git in %IDF_GIT_DIR% +git.exe --version + +:: Check if this is a recent enough copy of ESP-IDF. +:: If so, use export.bat provided there. +:: Note: no "call", will not return into this batch file. +if exist "%IDF_PATH%\export.bat" %IDF_PATH%\export.bat + +echo IDF version does not include export.bat. Using the fallback version. + +if exist "%IDF_PATH%\tools\tools.json" ( + set "IDF_TOOLS_JSON_PATH=%IDF_PATH%\tools\tools.json" +) else ( + echo IDF version does not include tools\tools.json. Using the fallback version. + set "IDF_TOOLS_JSON_PATH=%~dp0%tools_fallback.json" +) + +if exist "%IDF_PATH%\tools\idf_tools.py" ( + set "IDF_TOOLS_PY_PATH=%IDF_PATH%\tools\idf_tools.py" +) else ( + echo IDF version does not include tools\idf_tools.py. Using the fallback version. + set "IDF_TOOLS_PY_PATH=%~dp0%idf_tools_fallback.py" +) + +echo. +echo Setting IDF_PATH: %IDF_PATH% +echo. + +set "OLD_PATH=%PATH%" +echo Adding ESP-IDF tools to PATH... +:: Export tool paths and environment variables. +:: It is possible to do this without a temporary file (running idf_tools.py from for /r command), +:: but that way it is impossible to get the exit code of idf_tools.py. +set "IDF_TOOLS_EXPORTS_FILE=%TEMP%\idf_export_vars.tmp" +python.exe %IDF_TOOLS_PY_PATH% --tools-json %IDF_TOOLS_JSON_PATH% export --format key-value >"%IDF_TOOLS_EXPORTS_FILE%" +if %errorlevel% neq 0 goto :end + +for /f "usebackq tokens=1,2 eol=# delims==" %%a in ("%IDF_TOOLS_EXPORTS_FILE%") do ( + call set "%%a=%%b" + ) + +:: This removes OLD_PATH substring from PATH, leaving only the paths which have been added, +:: and prints semicolon-delimited components of the path on separate lines +call set PATH_ADDITIONS=%%PATH:%OLD_PATH%=%% +if "%PATH_ADDITIONS%"=="" call :print_nothing_added +if not "%PATH_ADDITIONS%"=="" echo %PATH_ADDITIONS:;=&echo. % + +echo Checking if Python packages are up to date... +python.exe %IDF_PATH%\tools\check_python_dependencies.py +if %errorlevel% neq 0 goto :end + +echo. +echo Done! You can now compile ESP-IDF projects. +echo Go to the project directory and run: +echo. +echo idf.py build +echo. + +goto :end + +:print_nothing_added + echo No directories added to PATH: + echo. + echo %PATH% + echo. + goto :eof + +:end + +:: Clean up +if not "%IDF_TOOLS_EXPORTS_FILE%"=="" ( + del "%IDF_TOOLS_EXPORTS_FILE%" 1>nul 2>nul +) +set IDF_TOOLS_EXPORTS_FILE= +set IDF_PYTHON_DIR= +set IDF_GIT_DIR= +set IDF_TOOLS_PY_PATH= +set IDF_TOOLS_JSON_PATH= +set OLD_PATH= +set PATH_ADDITIONS= diff --git a/tools/windows/tool_setup/idf_download_page.iss.inc b/tools/windows/tool_setup/idf_download_page.iss.inc new file mode 100644 index 00000000..3636123e --- /dev/null +++ b/tools/windows/tool_setup/idf_download_page.iss.inc @@ -0,0 +1,142 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Page to select the version of ESP-IDF to download ------------------------------ } + +var + IDFDownloadPage: TInputOptionWizardPage; + IDFDownloadAvailableVersions: TArrayOfString; + IDFDownloadPath, IDFDownloadVersion: String; + +function GetSuggestedIDFDirectory(): String; +var +BaseName: String; +RepeatIndex: Integer; +begin + { Start with Desktop\esp-idf name and if it already exists, + keep trying with Desktop\esp-idf-N for N=2 and above. } + BaseName := ExpandConstant('{userdesktop}\esp-idf'); + Result := BaseName; + RepeatIndex := 1; + while DirExists(Result) do + begin + RepeatIndex := RepeatIndex + 1; + Result := BaseName + '-' + IntToStr(RepeatIndex); + end; +end; + +function GetIDFVersionDescription(Version: String): String; +begin + if WildCardMatch(Version, 'v*-beta*') then + Result := 'beta version' + else if WildCardMatch(Version, 'v*-rc*') then + Result := 'pre-release version' + else if WildCardMatch(Version, 'v*') then + Result := 'release version' + else if WildCardMatch(Version, 'release/v*') then + Result := 'release branch' + else if WildCardMatch(Version, 'master') then + Result := 'development branch' + else + Result := ''; +end; + +procedure DownloadIDFVersionsList(); +var + Url: String; + VersionFile: String; +begin + Url := '{#IDFVersionsURL}'; + VersionFile := ExpandConstant('{tmp}\idf_versions.txt'); + if idpDownloadFile(Url, VersionFile) then + begin + Log('Downloaded ' + Url + ' to ' + VersionFile); + end else begin + Log('Download of ' + Url + ' failed, using a fallback versions list'); + ExtractTemporaryFile('idf_versions.txt'); + end; +end; + +procedure OnIDFDownloadPagePrepare(Sender: TObject); +var + Page: TInputOptionWizardPage; + VersionFile: String; + i: Integer; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnIDFDownloadPagePrepare'); + if Page.CheckListBox.Items.Count > 0 then + exit; + + DownloadIDFVersionsList(); + + VersionFile := ExpandConstant('{tmp}\idf_versions.txt'); + if not LoadStringsFromFile(VersionFile, IDFDownloadAvailableVersions) then + begin + Log('Failed to load versions from ' + VersionFile); + exit; + end; + + Log('Versions count: ' + IntToStr(GetArrayLength(IDFDownloadAvailableVersions))) + for i := 0 to GetArrayLength(IDFDownloadAvailableVersions) - 1 do + begin + Log('Version ' + IntToStr(i) + ': ' + IDFDownloadAvailableVersions[i]); + Page.Add(IDFDownloadAvailableVersions[i] + ' (' + + GetIDFVersionDescription(IDFDownloadAvailableVersions[i]) + ')'); + end; + Page.SelectedValueIndex := 0; + + ChoicePageSetInputText(Page, GetSuggestedIDFDirectory()); +end; + +procedure OnIDFDownloadSelectionChange(Sender: TObject); +var + Page: TInputOptionWizardPage; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnIDFDownloadSelectionChange index=' + IntToStr(Page.SelectedValueIndex)); +end; + +function OnIDFDownloadPageValidate(Sender: TWizardPage): Boolean; +var + Page: TInputOptionWizardPage; + IDFPath: String; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnIDFDownloadPageValidate index=' + IntToStr(Page.SelectedValueIndex)); + + IDFPath := ChoicePageGetInputText(Page); + if DirExists(IDFPath) and not DirIsEmpty(IDFPath) then + begin + MsgBox('Directory already exists and is not empty:' + #13#10 + + IDFPath + #13#10 + 'Please choose a different directory.', mbError, MB_OK); + Result := False; + exit; + end; + + IDFDownloadPath := IDFPath; + IDFDownloadVersion := IDFDownloadAvailableVersions[Page.SelectedValueIndex]; + Result := True; +end; + + +function ShouldSkipIDFDownloadPage(PageID: Integer): Boolean; +begin + if (PageID = IDFDownloadPage.ID) and not IDFDownloadRequired() then + Result := True; +end; + + +procedure CreateIDFDownloadPage(); +begin + IDFDownloadPage := ChoicePageCreate( + IDFPage.ID, + 'Download ESP-IDF', 'Please choose ESP-IDF version to download', + 'For more information about ESP-IDF versions, see' + #13#10 + + 'https://docs.espressif.com/projects/esp-idf/en/latest/versions.html', + 'Choose a directory to download ESP-IDF to', + True, + @OnIDFDownloadPagePrepare, + @OnIDFDownloadSelectionChange, + @OnIDFDownloadPageValidate); +end; diff --git a/tools/windows/tool_setup/idf_page.iss.inc b/tools/windows/tool_setup/idf_page.iss.inc new file mode 100644 index 00000000..87cc5c5d --- /dev/null +++ b/tools/windows/tool_setup/idf_page.iss.inc @@ -0,0 +1,111 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Page to select whether to download ESP-IDF, or use an existing copy ------------------------------ } + +var + IDFPage: TInputOptionWizardPage; + IDFSelectionDownloadIndex: Integer; + IDFSelectionCustomPathIndex: Integer; + IDFUseExisting: Boolean; + IDFExistingPath: String; + +function IDFDownloadRequired(): Boolean; +begin + Result := not IDFUseExisting; +end; + +procedure IDFPageUpdateInput(); +var + Enable: Boolean; +begin + if IDFPage.SelectedValueIndex = IDFSelectionCustomPathIndex then + Enable := True; + + ChoicePageSetInputEnabled(IDFPage, Enable); +end; + +procedure OnIDFPagePrepare(Sender: TObject); +var + Page: TInputOptionWizardPage; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnIDFPagePrepare'); + if Page.CheckListBox.Items.Count > 0 then + exit; + + IDFSelectionDownloadIndex := Page.Add('Download ESP-IDF') + IDFSelectionCustomPathIndex := Page.Add('Use an existing ESP-IDF directory'); + + Page.SelectedValueIndex := 0; + IDFPageUpdateInput(); +end; + +procedure OnIDFSelectionChange(Sender: TObject); +var + Page: TInputOptionWizardPage; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnIDFSelectionChange index=' + IntToStr(Page.SelectedValueIndex)); + IDFPageUpdateInput(); +end; + +function OnIDFPageValidate(Sender: TWizardPage): Boolean; +var + Page: TInputOptionWizardPage; + NotSupportedMsg, IDFPath, IDFPyPath, RequirementsPath: String; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnIDFPageValidate index=' + IntToStr(Page.SelectedValueIndex)); + + if Page.SelectedValueIndex = IDFSelectionDownloadIndex then + begin + IDFUseExisting := False; + Result := True; + end else begin + IDFUseExisting := True; + Result := False; + NotSupportedMsg := 'The selected version of ESP-IDF is not supported:' + #13#10; + IDFPath := ChoicePageGetInputText(Page); + + if not DirExists(IDFPath) then + begin + MsgBox('Directory doesn''t exist: ' + IDFPath + #13#10 + + 'Please choose an existing ESP-IDF directory', mbError, MB_OK); + exit; + end; + + IDFPyPath := IDFPath + '\tools\idf.py'; + if not FileExists(IDFPyPath) then + begin + MsgBox(NotSupportedMsg + + 'Can not find idf.py in ' + IDFPath + '\tools', mbError, MB_OK); + exit; + end; + + RequirementsPath := IDFPath + '\requirements.txt'; + if not FileExists(RequirementsPath) then + begin + MsgBox(NotSupportedMsg + + 'Can not find requirements.txt in ' + IDFPath, mbError, MB_OK); + exit; + end; + + IDFExistingPath := IDFPath; + Result := True; + end; +end; + + +procedure CreateIDFPage(); +begin + IDFPage := ChoicePageCreate( + wpLicense, + 'Download or use ESP-IDF', 'Please choose ESP-IDF version to download, or use an existing ESP-IDF copy', + 'Available ESP-IDF versions', + 'Choose existing ESP-IDF directory', + True, + @OnIDFPagePrepare, + @OnIDFSelectionChange, + @OnIDFPageValidate); +end; diff --git a/tools/windows/tool_setup/idf_setup.iss.inc b/tools/windows/tool_setup/idf_setup.iss.inc new file mode 100644 index 00000000..30c626a8 --- /dev/null +++ b/tools/windows/tool_setup/idf_setup.iss.inc @@ -0,0 +1,255 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Downloading ESP-IDF ------------------------------ } + +var + IDFZIPFileVersion, IDFZIPFileName: String; + +function GetIDFPath(Unused: String): String; +begin + if IDFUseExisting then + Result := IDFExistingPath + else + Result := IDFDownloadPath; +end; + +function GetIDFZIPFileVersion(Version: String): String; +var + ReleaseVerPart: String; + i: Integer; + Found: Boolean; +begin + if WildCardMatch(Version, 'v*') or WildCardMatch(Version, 'v*-rc*') then + Result := Version + else if Version = 'master' then + Result := '' + else if WildCardMatch(Version, 'release/v*') then + begin + ReleaseVerPart := Version; + Log('ReleaseVerPart=' + ReleaseVerPart) + Delete(ReleaseVerPart, 1, Length('release/')); + Log('ReleaseVerPart=' + ReleaseVerPart) + Found := False; + for i := 0 to GetArrayLength(IDFDownloadAvailableVersions) - 1 do + begin + if Pos(ReleaseVerPart, IDFDownloadAvailableVersions[i]) = 1 then + begin + Result := IDFDownloadAvailableVersions[i]; + Found := True; + break; + end; + end; + if not Found then + Result := ''; + end; + Log('GetIDFZIPFileVersion(' + Version + ')=' + Result); +end; + +procedure IDFAddDownload(); +var + Url, MirrorUrl: String; +begin + IDFZIPFileVersion := GetIDFZIPFileVersion(IDFDownloadVersion); + if IDFZIPFileVersion <> '' then + begin + Url := 'https://github.com/espressif/esp-idf/releases/download/' + IDFZIPFileVersion + '/esp-idf-' + IDFZIPFileVersion + '.zip'; + MirrorUrl := 'https://dl.espressif.com/dl/esp-idf/releases/esp-idf-' + IDFZIPFileVersion + '.zip'; + IDFZIPFileName := ExpandConstant('{app}\releases\esp-idf-' + IDFZIPFileVersion + '.zip') + if not FileExists(IDFZIPFileName) then + begin + ForceDirectories(ExpandConstant('{app}\releases')) + Log('Adding download: ' + Url + ', mirror: ' + MirrorUrl + ', destination: ' + IDFZIPFileName); + idpAddFile(Url, IDFZIPFileName); + idpAddMirror(Url, MirrorUrl); + end else begin + Log(IDFZIPFileName + ' already exists') + end; + end; +end; + +procedure RemoveAlternatesFile(Path: String); +begin + Log('Removing ' + Path); + DeleteFile(Path); +end; + +{ + Replacement of the '--dissociate' flag of 'git clone', to support older versions of Git. + '--reference' is supported for submodules since git 2.12, but '--dissociate' only from 2.18. +} +procedure GitRepoDissociate(Path: String); +var + CmdLine: String; +begin + CmdLine := GitExecutablePath + ' -C ' + Path + ' repack -d -a' + DoCmdlineInstall('Finishing ESP-IDF installation', 'Re-packing the repository', CmdLine); + CmdLine := GitExecutablePath + ' -C ' + Path + ' submodule foreach git repack -d -a' + DoCmdlineInstall('Finishing ESP-IDF installation', 'Re-packing the submodules', CmdLine); + + FindFileRecusive(Path + '\.git', 'alternates', @RemoveAlternatesFile); +end; + +{ Run git reset --hard in the repo and in the submodules, to fix the newlines. } +procedure GitRepoFixNewlines(Path: String); +var + CmdLine: String; +begin + CmdLine := GitExecutablePath + ' -C ' + Path + ' reset --hard'; + Log('Resetting the repository: ' + CmdLine); + DoCmdlineInstall('Finishing ESP-IDF installation', 'Updating newlines', CmdLine); + + Log('Resetting the submodules: ' + CmdLine); + CmdLine := GitExecutablePath + ' -C ' + Path + ' submodule foreach git reset --hard'; + DoCmdlineInstall('Finishing ESP-IDF installation', 'Updating newlines in submodules', CmdLine); +end; + +{ + There are 3 possible ways how an ESP-IDF copy can be obtained: + - Download the .zip archive with submodules included, extract to destination directory, + then do 'git reset --hard' and 'git submodule foreach git reset --hard' to correct for + possibly different newlines. This is done for release versions. + - Do a git clone of the Github repository into the destination directory. + This is done for the master branch. + - Download the .zip archive of a "close enough" release version, extract into a temporary + directory. Then do a git clone of the Github repository, using the temporary directory + as a '--reference'. This is done for other versions (such as release branches). +} + +procedure IDFDownload(); +var + CmdLine: String; + IDFTempPath: String; + IDFPath: String; + NeedToClone: Boolean; + Res: Boolean; + +begin + IDFPath := IDFDownloadPath; + { If there is a release archive to download, IDFZIPFileName and IDFZIPFileVersion will be set. + See GetIDFZIPFileVersion function. + } + + if IDFZIPFileName <> '' then + begin + if IDFZIPFileVersion <> IDFDownloadVersion then + begin + { The version of .zip file downloaded is not the same as the version the user has requested. + Will use 'git clone --reference' to obtain the correct version, using the contents + of the .zip file as reference. + } + NeedToClone := True; + end; + + ExtractTemporaryFile('7za.exe') + CmdLine := ExpandConstant('{tmp}\7za.exe x -o' + ExpandConstant('{tmp}') + ' -r -aoa ' + IDFZIPFileName); + IDFTempPath := ExpandConstant('{tmp}\esp-idf-') + IDFZIPFileVersion; + Log('Extracting ESP-IDF reference repository: ' + CmdLine); + Log('Reference repository path: ' + IDFTempPath); + DoCmdlineInstall('Extracting ESP-IDF', 'Setting up reference repository', CmdLine); + end else begin + { IDFZIPFileName is not set, meaning that we will rely on 'git clone'. } + NeedToClone := True; + Log('Not .zip release archive. Will do full clone.'); + end; + + if NeedToClone then + begin + CmdLine := GitExecutablePath + ' clone --recursive --progress -b ' + IDFDownloadVersion; + + if IDFTempPath <> '' then + CmdLine := CmdLine + ' --reference ' + IDFTempPath; + + CmdLine := CmdLine + ' https://github.com/espressif/esp-idf.git ' + IDFPath; + Log('Cloning IDF: ' + CmdLine); + DoCmdlineInstall('Downloading ESP-IDF', 'Using git to clone ESP-IDF repository', CmdLine); + + if IDFTempPath <> '' then + GitRepoDissociate(IDFPath); + + end else begin + Log('Moving ' + IDFTempPath + ' to ' + IDFPath); + if DirExists(IDFPath) then + begin + if not DirIsEmpty(IDFPath) then + begin + MsgBox('Destination directory exists and is not empty: ' + IDFPath, mbError, MB_OK); + RaiseException('Failed to copy ESP-IDF') + end; + + Res := RemoveDir(IDFPath); + if not Res then + begin + MsgBox('Failed to remove destination directory: ' + IDFPath, mbError, MB_OK); + RaiseException('Failed to copy ESP-IDF') + end; + end; + Res := RenameFile(IDFTempPath, IDFPath); + if not Res then + begin + MsgBox('Failed to copy ESP-IDF to the destination directory: ' + IDFPath, mbError, MB_OK); + RaiseException('Failed to copy ESP-IDF'); + end; + GitRepoFixNewlines(IDFPath); + end; +end; + +{ ------------------------------ IDF Tools setup, Python environment setup ------------------------------ } + +procedure IDFToolsSetup(); +var + CmdLine: String; + IDFPath: String; + IDFToolsPyPath: String; + IDFToolsPyCmd: String; +begin + IDFPath := GetIDFPath(''); + IDFToolsPyPath := IDFPath + '\tools\idf_tools.py'; + if FileExists(IDFToolsPyPath) then + begin + Log('idf_tools.py exists in IDF directory'); + IDFToolsPyCmd := PythonExecutablePath + ' ' + IDFToolsPyPath; + end else begin + Log('idf_tools.py does not exist in IDF directory, using a fallback version'); + IDFToolsPyCmd := ExpandConstant(PythonExecutablePath + + ' {app}\idf_tools_fallback.py' + + ' --idf-path ' + IDFPath + + ' --tools {app}\tools_fallback.json'); + end; + + Log('idf_tools.py command: ' + IDFToolsPyCmd); + CmdLine := IDFToolsPyCmd + ' install'; + Log('Installing tools:' + CmdLine); + DoCmdlineInstall('Installing ESP-IDF tools', '', CmdLine); + + CmdLine := IDFToolsPyCmd + ' install-python-env'; + Log('Installing Python environment:' + CmdLine); + DoCmdlineInstall('Installing Python environment', '', CmdLine); +end; + +{ ------------------------------ Start menu shortcut ------------------------------ } + +procedure CreateIDFCommandPromptShortcut(); +var + Destination: String; + Description: String; + Command: String; +begin + ForceDirectories(ExpandConstant('{group}')); + Destination := ExpandConstant('{group}\{#IDFCmdExeShortcutFile}'); + Description := '{#IDFCmdExeShortcutDescription}'; + Command := ExpandConstant('/k {app}\idf_cmd_init.bat "') + PythonPath + '" "' + GitPath + '"'; + Log('CreateShellLink Destination=' + Destination + ' Description=' + Description + ' Command=' + Command) + try + CreateShellLink( + Destination, + Description, + 'cmd.exe', + Command, + GetIDFPath(''), + '', 0, SW_SHOWNORMAL); + except + MsgBox('Failed to create the Start menu shortcut: ' + Destination, mbError, MB_OK); + RaiseException('Failed to create the shortcut'); + end; +end; diff --git a/tools/windows/tool_setup/idf_tool_setup.iss b/tools/windows/tool_setup/idf_tool_setup.iss index 70e60d2a..7fd8b2f7 100644 --- a/tools/windows/tool_setup/idf_tool_setup.iss +++ b/tools/windows/tool_setup/idf_tool_setup.iss @@ -1,225 +1,95 @@ +; Copyright 2019 Espressif Systems (Shanghai) PTE LTD +; SPDX-License-Identifier: Apache-2.0 + +#pragma include __INCLUDE__ + ";" + ReadReg(HKLM, "Software\Mitrich Software\Inno Download Plugin", "InstallDir") #include -[Setup] -AppName=ESP-IDF Tools -AppVersion=1.0 -OutputBaseFilename=esp-idf-tools-setup-1.0 +#define MyAppName "ESP-IDF Tools" +#define MyAppVersion "2.0" +#define MyAppPublisher "Espressif Systems (Shanghai) Co. Ltd." +#define MyAppURL "https://github.com/espressif/esp-idf" -DefaultDirName={pf}\Espressif\ESP-IDF Tools -DefaultGroupName=ESP-IDF Tools -Compression=lzma2 +#define PythonVersion "3.7" +#define PythonInstallerName "python-3.7.3-amd64.exe" +#define PythonInstallerDownloadURL "https://www.python.org/ftp/python/3.7.3/python-3.7.3-amd64.exe" + +#define GitVersion "2.21.0" +#define GitInstallerName "Git-2.21.0-64-bit.exe" +#define GitInstallerDownloadURL "https://github.com/git-for-windows/git/releases/download/v2.21.0.windows.1/Git-2.21.0-64-bit.exe" + +#define IDFVersionsURL "https://dl.espressif.com/dl/esp-idf/idf_versions.txt" + +#define IDFCmdExeShortcutDescription "Open ESP-IDF Command Prompt (cmd.exe)" +#define IDFCmdExeShortcutFile "ESP-IDF Command Prompt (cmd.exe).lnk" + +[Setup] +; NOTE: The value of AppId uniquely identifies this application. +; Do not use the same AppId value in installers for other applications. +; (To generate a new GUID, click Tools | Generate GUID inside the IDE.) +AppId={{9E068D99-5C4B-4E5F-96A3-B17CF291E6BD} +AppName={#MyAppName} +AppVersion={#MyAppVersion} +AppVerName={#MyAppName} {#MyAppVersion} +AppPublisher={#MyAppPublisher} +AppPublisherURL={#MyAppURL} +AppSupportURL={#MyAppURL} +AppUpdatesURL={#MyAppURL} +DefaultDirName={%USERPROFILE}\.espressif +DirExistsWarning=no +DefaultGroupName=ESP-IDF +DisableProgramGroupPage=yes +OutputBaseFilename=esp-idf-tools-setup-unsigned +Compression=lzma SolidCompression=yes -ChangesEnvironment=yes -; Note: the rest of the installer setup is written to work cleanly on win32 also, *however* -; Ninja doesn't ship a 32-bit binary so there's no way yet to install on win32 :( -; See https://github.com/ninja-build/ninja/issues/1339 ArchitecturesAllowed=x64 ArchitecturesInstallIn64BitMode=x64 +LicenseFile=license.txt +PrivilegesRequired=lowest +SetupLogging=yes +ChangesEnvironment=yes +WizardStyle=modern -[Types] -Name: "full"; Description: "Default installation" -Name: "custom"; Description: "Custom installation"; Flags: iscustom +[Languages] +Name: "english"; MessagesFile: "compiler:Default.isl" -[Components] -Name: toolchain; Description: ESP32 Xtensa GCC Cross-Toolchain; Types: full custom; -Name: mconf; Description: ESP-IDF console menuconfig tool; Types: full custom; -Name: ninja; Description: Install Ninja build v1.8.2; Types: full custom - -[Tasks] -; Should installer prepend to Path (does this by default) -Name: addpath; Description: "Add tools to Path"; GroupDescription: "Add to Path:"; -Name: addpath\allusers; Description: "For all users"; GroupDescription: "Add to Path:"; Flags: exclusive -Name: addpath\user; Description: "For the current user only"; GroupDescription: "Add to Path:"; Flags: exclusive unchecked - -; External installation tasks -; -; Note: The Check conditions here auto-select 32-bit or 64-bit installers, as needed -; The tasks won't appear if CMake/Python27 already appear to be installed on this system -Name: cmake32; Description: Download and Run CMake 3.11.1 Installer; GroupDescription: "Other Required Tools:"; Check: not IsWin64 and not CMakeInstalled -Name: cmake64; Description: Download and Run CMake 3.11.1 Installer; GroupDescription: "Other Required Tools:"; Check: IsWin64 and not CMakeInstalled -Name: python32; Description: Download and Run Python 2.7.14 Installer and install pyserial; GroupDescription: "Other Required Tools:"; Check: not IsWin64 and not Python27Installed -Name: python64; Description: Download and Run Python 2.7.14 Installer and install pyserial; GroupDescription: "Other Required Tools:"; Check: IsWin64 and not Python27Installed +[Dirs] +Name: "{app}\dist" [Files] -Components: toolchain; Source: "input\xtensa-esp8266-elf\*"; DestDir: "{app}\toolchain\"; Flags: recursesubdirs; -Components: mconf; Source: "input\mconf-v4.6.0.0-idf-20180319-win32\*"; DestDir: "{app}\mconf\"; -Components: ninja; Source: "input\ninja.exe"; DestDir: "{app}"; +Source: "cmdlinerunner\build\cmdlinerunner.dll"; Flags: dontcopy +Source: "unzip\7za.exe"; Flags: dontcopy +Source: "idf_versions.txt"; Flags: dontcopy +Source: "..\..\idf_tools.py"; DestDir: "{app}"; DestName: "idf_tools_fallback.py" +; Note: this tools.json matches the requirements of IDF v3.x versions. +Source: "tools_fallback.json"; DestDir: "{app}"; DestName: "tools_fallback.json" +Source: "idf_cmd_init.bat"; DestDir: "{app}" +Source: "dist\*"; DestDir: "{app}\dist" + +[UninstallDelete] +Type: filesandordirs; Name: "{app}\dist" +Type: filesandordirs; Name: "{app}\releases" +Type: filesandordirs; Name: "{app}\tools" +Type: filesandordirs; Name: "{app}\python_env" [Run] -Tasks: cmake32 cmake64; Filename: "msiexec.exe"; Parameters: "/i ""{tmp}\cmake.msi"" /qb! {code:GetCMakeInstallerArgs}"; StatusMsg: Running CMake installer...; -Tasks: python32 python64; Filename: "msiexec.exe"; Parameters: "/i ""{tmp}\python.msi"" /qb! {code:GetPythonInstallerArgs} REBOOT=Supress"; StatusMsg: Running Python installer...; -Tasks: python32 python64; Filename: "C:\Python27\Scripts\pip.exe"; Parameters: "install pyserial"; StatusMsg: Installing pyserial...; +Filename: "{app}\dist\{#PythonInstallerName}"; Parameters: "/passive PrependPath=1 InstallLauncherAllUsers=0 Include_dev=0 Include_tcltk=0 Include_launcher=0 Include_test=0 Include_doc=0"; Description: "Installing Python"; Check: PythonInstallRequired +Filename: "{app}\dist\{#GitInstallerName}"; Parameters: "/silent /tasks="""" /norestart"; Description: "Installing Git"; Check: GitInstallRequired +Filename: "{group}\{#IDFCmdExeShortcutFile}"; Flags: postinstall shellexec; Description: "Run ESP-IDF Command Prompt (cmd.exe)"; Check: InstallationSuccessful [Registry] -; Prepend various entries to Path in the registry. Can either be HKLM (all users) or HKCU (single user only) - -; ninja path (in root app directory) -Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; \ - ValueType: expandsz; ValueName: "Path"; ValueData: "{app};{olddata}"; Check: not IsInPath('{app}'); \ - Components: ninja; Tasks: addpath\allusers -Root: HKCU; Subkey: "Environment"; \ - ValueType: expandsz; ValueName: "Path"; ValueData: "{app};{olddata}"; Check: not IsInPath('{app}'); \ - Components: ninja; Tasks: addpath\user - -; mconf path -Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; \ - ValueType: expandsz; ValueName: "Path"; ValueData: "{app}\mconf;{olddata}"; Check: not IsInPath('{app}\mconf'); \ - Components: mconf; Tasks: addpath\allusers -Root: HKCU; Subkey: "Environment"; \ - ValueType: expandsz; ValueName: "Path"; ValueData: "{app}\mconf;{olddata}"; Check: not IsInPath('{app}\mconf'); \ - Components: mconf; Tasks: addpath\user - -; toolchain path -Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; \ - ValueType: expandsz; ValueName: "Path"; ValueData: "{app}\toolchain\bin;{olddata}"; Check: not IsInPath('{app}\toolchain\bin'); \ - Components: toolchain; Tasks: addpath\allusers -Root: HKCU; Subkey: "Environment"; \ - ValueType: expandsz; ValueName: "Path"; ValueData: "{app}\toolchain\bin;{olddata}"; Check: not IsInPath('{app}\toolchain\bin'); \ - Components: toolchain; Tasks: addpath\user - +Root: HKCU; Subkey: "Environment"; ValueType: string; ValueName: "IDF_TOOLS_PATH"; \ + ValueData: "{app}"; Flags: preservestringtype createvalueifdoesntexist; [Code] -procedure InitializeWizard; -begin - idpDownloadAfter(wpReady); -end; - -procedure CurPageChanged(CurPageID: Integer); -begin - { When the Ready page is being displayed, initialise downloads based on which Tasks are selected } - if CurPageID=wpReady then - begin - if IsTaskSelected('python32') then - begin - idpAddFile('https://www.python.org/ftp/python/2.7.14/python-2.7.14.msi', ExpandConstant('{tmp}\python.msi')); - end; - if IsTaskSelected('python64') then - begin - idpAddFile('https://www.python.org/ftp/python/2.7.14/python-2.7.14.amd64.msi', ExpandConstant('{tmp}\python.msi')); - end; - if IsTaskSelected('cmake32') then - begin - idpAddFile('https://cmake.org/files/v3.11/cmake-3.11.1-win32-x86.msi', ExpandConstant('{tmp}\cmake.msi')); - end; - if IsTaskSelected('cmake64') then - begin - idpAddFile('https://cmake.org/files/v3.11/cmake-3.11.1-win64-x64.msi', ExpandConstant('{tmp}\cmake.msi')); - end; - end; -end; - -{ Utility to search in HKLM for an installation path. Looks in both 64-bit & 32-bit registry. } -function GetInstallPath(key, valuename : String) : Variant; -var - value : string; -begin - Result := Null; - if RegQueryStringValue(HKEY_LOCAL_MACHINE, key, valuename, value) then - begin - Result := value; - end - else - begin - { This is 32-bit setup running on 64-bit Windows, but ESP-IDF can use 64-bit tools also } - if IsWin64 and RegQueryStringValue(HKLM64, key, valuename, value) then - begin - Result := value; - end; - end; -end; - -{ Return the path of the CMake install, if there is one } -function CMakeInstallPath() : Variant; -begin - Result := GetInstallPath('SOFTWARE\Kitware\CMake', 'InstallDir'); -end; - -{ Return 'True' if CMake is installed } -function CMakeInstalled() : Boolean; -begin - Result := not VarIsNull(CMakeInstallPath()); -end; - -{ Return the path where Python 2.7 is installed, if there is one } -function Python27InstallPath() : Variant; -begin - Result := GetInstallPath('SOFTWARE\Python\PythonCore\2.7\InstallPath', ''); -end; - -{ Return True if Python 2.7 is installed } -function Python27Installed() : Boolean; -begin - Result := not VarIsNull(Python27InstallPath()); -end; - -{ Return arguments to pass to CMake installer, ie should it add CMake to the Path } -function GetCMakeInstallerArgs(Param : String) : String; -begin - if IsTaskSelected('addpath\allusers') then - begin - Result := 'ADD_CMAKE_TO_PATH=System'; - end - else if IsTaskSelected('addpath\user') then - begin - Result := 'ADD_CMAKE_TO_PATH=User'; - end - else begin - Result := 'ADD_CMAKE_TO_PATH=None'; - end; -end; - -{ Return arguments to pass to the Python installer, - ie should it install for all users and should it prepend to the Path } -function GetPythonInstallerArgs(Param : String) : String; -begin - { Note: The Python 2.7 installer appears to always add PATH to - system environment variables, regardless of ALLUSERS setting. - - This appears to be fixed in the Python 3.x installers (which use WiX) } - if IsTaskSelected('addpath') then - begin - Result := 'ADDLOCAL=ALL '; - end - else begin - Result := '' - end; - if IsTaskSelected('addpath\allusers') then - begin - Result := Result + 'ALLUSERS=1'; - end - else begin - Result := Result + 'ALLUSERS='; - end; -end; -{ Return True if the param is already set in the Path - (user or system, depending on which Task is chosen) - - Adapted from https://stackoverflow.com/a/3431379 -} -function IsInPath(Param: string): boolean; -var - OrigPath: string; - RootKey : Integer; - SubKey : String; -begin - if IsTaskSelected('addpath\allusers') then - begin - RootKey := HKEY_LOCAL_MACHINE; - SubKey := 'SYSTEM\CurrentControlSet\Control\Session Manager\Environment'; - end - else begin - RootKey := HKEY_CURRENT_USER; - SubKey := 'Environment'; - end; - - if not RegQueryStringValue(RootKey, SubKey, 'Path', OrigPath) - then begin - Result := False; - end - else begin - { look for the path with leading and trailing semicolon } - Result := Pos(';' + Param + ';', ';' + OrigPath + ';') > 0; - end; -end; +#include "utils.iss.inc" +#include "choice_page.iss.inc" +#include "cmdline_page.iss.inc" +#include "idf_page.iss.inc" +#include "git_page.iss.inc" +#include "python_page.iss.inc" +#include "idf_download_page.iss.inc" +#include "idf_setup.iss.inc" +#include "summary.iss.inc" +#include "main.iss.inc" diff --git a/tools/windows/tool_setup/license.txt b/tools/windows/tool_setup/license.txt new file mode 100644 index 00000000..73ef0a6c --- /dev/null +++ b/tools/windows/tool_setup/license.txt @@ -0,0 +1,357 @@ +This installer incorporates the following software programs licensed under the terms of GNU General Public License Version 2 + +- GNU Compiler Collection (GCC) +- GNU development tools ("binutils") +- GNU Debugger ("gdb") +- OpenOCD +- KConfig Frontends + +Text of this license is included below. + +Source code for these programs can be obtained from the following URLs: + +- https://github.com/espressif/crosstool-NG +- https://github.com/espressif/binutils-esp32ulp +- https://github.com/espressif/openocd-esp32 +- https://github.com/espressif/kconfig-frontends + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/tools/windows/tool_setup/main.iss.inc b/tools/windows/tool_setup/main.iss.inc new file mode 100644 index 00000000..857f4487 --- /dev/null +++ b/tools/windows/tool_setup/main.iss.inc @@ -0,0 +1,121 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Custom steps before the main installation flow ------------------------------ } + +var + SetupAborted: Boolean; + +function InstallationSuccessful(): Boolean; +begin + Result := not SetupAborted; +end; + + +procedure InitializeDownloader(); +begin + idpDownloadAfter(wpReady); +end; + + +function PreInstallSteps(CurPageID: Integer): Boolean; +var + DestPath: String; +begin + Result := True; + if CurPageID <> wpReady then + exit; + + ForceDirectories(ExpandConstant('{app}\dist')); + + if not PythonUseExisting then + begin + DestPath := ExpandConstant('{app}\dist\{#PythonInstallerName}'); + if FileExists(DestPath) then + begin + Log('Python installer already downloaded: ' + DestPath); + end else begin + idpAddFile('{#PythonInstallerDownloadURL}', DestPath); + end; + end; + + if not GitUseExisting then + begin + DestPath := ExpandConstant('{app}\dist\{#GitInstallerName}'); + if FileExists(DestPath) then + begin + Log('Git installer already downloaded: ' + DestPath); + end else begin + idpAddFile('{#GitInstallerDownloadURL}', DestPath); + end; + end; + + if not IDFUseExisting then + begin + IDFAddDownload(); + end; +end; + +{ ------------------------------ Custom steps after the main installation flow ------------------------------ } + +procedure AddPythonGitToPath(); +var + EnvPath: String; + PythonLibPath: String; +begin + EnvPath := GetEnv('PATH'); + + if not PythonUseExisting then + PythonExecutablePathUpdateAfterInstall(); + + if not GitUseExisting then + GitExecutablePathUpdateAfterInstall(); + + EnvPath := PythonPath + ';' + GitPath + ';' + EnvPath; + Log('Setting PATH for this process: ' + EnvPath); + SetEnvironmentVariable('PATH', EnvPath); + + { Log and clear PYTHONPATH variable, as it might point to libraries of another Python version} + PythonLibPath := GetEnv('PYTHONPATH') + Log('PYTHONPATH=' + PythonLibPath) + SetEnvironmentVariable('PYTHONPATH', '') +end; + + +procedure PostInstallSteps(CurStep: TSetupStep); +var + Err: Integer; +begin + if CurStep <> ssPostInstall then + exit; + + try + AddPythonGitToPath(); + + if not IDFUseExisting then + IDFDownload(); + + IDFToolsSetup(); + + CreateIDFCommandPromptShortcut(); + except + SetupAborted := True; + if MsgBox('Installation log has been created, it may contain more information about the problem.' + #13#10 + + 'Display the installation log now?', mbConfirmation, MB_YESNO or MB_DEFBUTTON1) = IDYES then + begin + ShellExec('', 'notepad.exe', ExpandConstant('{log}'), ExpandConstant('{tmp}'), SW_SHOW, ewNoWait, Err); + end; + Abort(); + end; +end; + + +function SkipFinishedPage(PageID: Integer): Boolean; +begin + Result := False; + + if PageID = wpFinished then + begin + Result := SetupAborted; + end; +end; diff --git a/tools/windows/tool_setup/python_find_installed.iss.inc b/tools/windows/tool_setup/python_find_installed.iss.inc new file mode 100644 index 00000000..4485030d --- /dev/null +++ b/tools/windows/tool_setup/python_find_installed.iss.inc @@ -0,0 +1,113 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Find installed Python interpreters in Windows Registry (see PEP 514) ------------------------------ } + +var + InstalledPythonVersions: TStringList; + InstalledPythonDisplayNames: TStringList; + InstalledPythonExecutables: TStringList; + +procedure PythonVersionAdd(Version, DisplayName, Executable: String); +begin + Log('Adding Python version=' + Version + ' name='+DisplayName+' executable='+Executable); + InstalledPythonVersions.Append(Version); + InstalledPythonDisplayNames.Append(DisplayName); + InstalledPythonExecutables.Append(Executable); +end; + +function GetPythonVersionInfoFromKey(RootKey: Integer; SubKeyName, CompanyName, TagName: String; + var Version: String; + var DisplayName: String; + var ExecutablePath: String): Boolean; +var + TagKey, InstallPathKey, DefaultPath: String; +begin + TagKey := SubKeyName + '\' + CompanyName + '\' + TagName; + InstallPathKey := TagKey + '\InstallPath'; + + if not RegQueryStringValue(RootKey, InstallPathKey, '', DefaultPath) then + begin + Log('No (Default) key, skipping'); + Result := False; + exit; + end; + + if not RegQueryStringValue(RootKey, InstallPathKey, 'ExecutablePath', ExecutablePath) then + begin + Log('No ExecutablePath, using the default'); + ExecutablePath := DefaultPath + '\python.exe'; + end; + + if not RegQueryStringValue(RootKey, TagKey, 'SysVersion', Version) then + begin + if CompanyName = 'PythonCore' then + begin + Version := TagName; + Delete(Version, 4, Length(Version)); + end else begin + Log('Can not determine SysVersion'); + Result := False; + exit; + end; + end; + + if not RegQueryStringValue(RootKey, TagKey, 'DisplayName', DisplayName) then + begin + DisplayName := 'Python ' + Version; + end; + + Result := True; +end; + +procedure FindPythonVersionsFromKey(RootKey: Integer; SubKeyName: String); +var + CompanyNames: TArrayOfString; + CompanyName, CompanySubKey, TagName, TagSubKey: String; + ExecutablePath, DisplayName, Version: String; + TagNames: TArrayOfString; + CompanyId, TagId: Integer; +begin + if not RegGetSubkeyNames(RootKey, SubKeyName, CompanyNames) then + begin + Log('Nothing found in ' + IntToStr(RootKey) + '\' + SubKeyName); + Exit; + end; + + for CompanyId := 0 to GetArrayLength(CompanyNames) - 1 do + begin + CompanyName := CompanyNames[CompanyId]; + + if CompanyName = 'PyLauncher' then + continue; + + CompanySubKey := SubKeyName + '\' + CompanyName; + Log('In ' + IntToStr(RootKey) + '\' + CompanySubKey); + + if not RegGetSubkeyNames(RootKey, CompanySubKey, TagNames) then + continue; + + for TagId := 0 to GetArrayLength(TagNames) - 1 do + begin + TagName := TagNames[TagId]; + TagSubKey := CompanySubKey + '\' + TagName; + Log('In ' + IntToStr(RootKey) + '\' + TagSubKey); + + if not GetPythonVersionInfoFromKey(RootKey, SubKeyName, CompanyName, TagName, Version, DisplayName, ExecutablePath) then + continue; + + PythonVersionAdd(Version, DisplayName, ExecutablePath); + end; + end; +end; + +procedure FindInstalledPythonVersions(); +begin + InstalledPythonVersions := TStringList.Create(); + InstalledPythonDisplayNames := TStringList.Create(); + InstalledPythonExecutables := TStringList.Create(); + + FindPythonVersionsFromKey(HKEY_CURRENT_USER, 'Software\Python'); + FindPythonVersionsFromKey(HKEY_LOCAL_MACHINE, 'Software\Python'); + FindPythonVersionsFromKey(HKEY_LOCAL_MACHINE, 'Software\Wow6432Node\Python'); +end; diff --git a/tools/windows/tool_setup/python_page.iss.inc b/tools/windows/tool_setup/python_page.iss.inc new file mode 100644 index 00000000..a0325fc7 --- /dev/null +++ b/tools/windows/tool_setup/python_page.iss.inc @@ -0,0 +1,149 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Page to select Python interpreter ------------------------------ } + +#include "python_find_installed.iss.inc" + +var + PythonPage: TInputOptionWizardPage; + PythonVersion, PythonPath, PythonExecutablePath: String; + PythonUseExisting: Boolean; + + +function GetPythonPath(Unused: String): String; +begin + Result := PythonPath; +end; + +function PythonInstallRequired(): Boolean; +begin + Result := not PythonUseExisting; +end; + +function PythonVersionSupported(Version: String): Boolean; +var + Major, Minor: Integer; +begin + Result := False; + if not VersionExtractMajorMinor(Version, Major, Minor) then + begin + Log('PythonVersionSupported: Could not parse version=' + Version); + exit; + end; + + if (Major = 2) and (Minor = 7) then Result := True; + if (Major = 3) and (Minor >= 5) then Result := True; +end; + +procedure OnPythonPagePrepare(Sender: TObject); +var + Page: TInputOptionWizardPage; + FullName: String; + i, Index, FirstEnabledIndex: Integer; + OfferToInstall: Boolean; + VersionToInstall: String; + VersionSupported: Boolean; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnPythonPagePrepare'); + if Page.CheckListBox.Items.Count > 0 then + exit; + + FindInstalledPythonVersions(); + + VersionToInstall := '{#PythonVersion}'; + OfferToInstall := True; + FirstEnabledIndex := -1; + + for i := 0 to InstalledPythonVersions.Count - 1 do + begin + VersionSupported := PythonVersionSupported(InstalledPythonVersions[i]); + FullName := InstalledPythonDisplayNames.Strings[i]; + if not VersionSupported then + begin + FullName := FullName + ' (unsupported)'; + end; + FullName := FullName + #13#10 + InstalledPythonExecutables.Strings[i]; + Index := Page.Add(FullName); + if not VersionSupported then + begin + Page.CheckListBox.ItemEnabled[Index] := False; + end else begin + if FirstEnabledIndex < 0 then FirstEnabledIndex := Index; + end; + if InstalledPythonVersions[i] = VersionToInstall then + begin + OfferToInstall := False; + end; + end; + + if OfferToInstall then + begin + Index := Page.Add('Install Python ' + VersionToInstall); + if FirstEnabledIndex < 0 then FirstEnabledIndex := Index; + end; + + Page.SelectedValueIndex := FirstEnabledIndex; +end; + +procedure OnPythonSelectionChange(Sender: TObject); +var + Page: TInputOptionWizardPage; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnPythonSelectionChange index=' + IntToStr(Page.SelectedValueIndex)); +end; + +function OnPythonPageValidate(Sender: TWizardPage): Boolean; +var + Page: TInputOptionWizardPage; +begin + Page := TInputOptionWizardPage(Sender); + Log('OnPythonPageValidate index=' + IntToStr(Page.SelectedValueIndex)); + if Page.SelectedValueIndex < InstalledPythonExecutables.Count then + begin + PythonUseExisting := True; + PythonExecutablePath := InstalledPythonExecutables[Page.SelectedValueIndex]; + PythonPath := ExtractFilePath(PythonExecutablePath); + PythonVersion := InstalledPythonVersions[Page.SelectedValueIndex]; + end else begin + PythonUseExisting := False; + PythonExecutablePath := ''; + PythonPath := ''; + PythonVersion := '{#PythonVersion}'; + end; + Log('OnPythonPageValidate: PythonPath='+PythonPath+' PythonExecutablePath='+PythonExecutablePath); + Result := True; +end; + +procedure PythonExecutablePathUpdateAfterInstall(); +var + Version, DisplayName, ExecutablePath: String; +begin + if not GetPythonVersionInfoFromKey( + HKEY_CURRENT_USER, 'Software\Python', 'PythonCore', '{#PythonVersion}', + Version, DisplayName, ExecutablePath) then + begin + Log('Failed to find ExecutablePath for the installed copy of Python'); + exit; + end; + Log('Found ExecutablePath for ' + DisplayName + ': ' + ExecutablePath); + PythonExecutablePath := ExecutablePath; + PythonPath := ExtractFilePath(PythonExecutablePath); + Log('PythonExecutablePathUpdateAfterInstall: PythonPath='+PythonPath+' PythonExecutablePath='+PythonExecutablePath); +end; + + +procedure CreatePythonPage(); +begin + PythonPage := ChoicePageCreate( + wpLicense, + 'Python choice', 'Please choose Python version', + 'Available Python versions', + '', + False, + @OnPythonPagePrepare, + @OnPythonSelectionChange, + @OnPythonPageValidate); +end; diff --git a/tools/windows/tool_setup/summary.iss.inc b/tools/windows/tool_setup/summary.iss.inc new file mode 100644 index 00000000..6a592360 --- /dev/null +++ b/tools/windows/tool_setup/summary.iss.inc @@ -0,0 +1,40 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Installation summary page ------------------------------ } + +function UpdateReadyMemo(Space, NewLine, MemoUserInfoInfo, MemoDirInfo, + MemoTypeInfo, MemoComponentsInfo, MemoGroupInfo, MemoTasksInfo: String): String; +begin + Result := '' + + if PythonUseExisting then + begin + Result := Result + 'Using Python ' + PythonVersion + ':' + NewLine + + Space + PythonExecutablePath + NewLine + NewLine; + end else begin + Result := Result + 'Will download and install Python ' + PythonVersion + NewLine + NewLine; + end; + + if GitUseExisting then + begin + Result := Result + 'Using Git ' + GitVersion + ':' + NewLine + + Space + GitExecutablePath + NewLine + NewLine; + end else begin + Result := Result + 'Will download and install Git for Windows ' + GitVersion + NewLine + NewLine; + end; + + if IDFUseExisting then + begin + Result := Result + 'Using existing ESP-IDF copy: ' + NewLine + + Space + IDFExistingPath + NewLine + NewLine; + end else begin + Result := Result + 'Will download ESP-IDF ' + IDFDownloadVersion + ' into:' + NewLine + + Space + IDFDownloadPath + NewLine + NewLine; + end; + + Result := Result + 'IDF tools directory (IDF_TOOLS_PATH):' + NewLine + + Space + ExpandConstant('{app}') + NewLine + NewLine; + + Log('Summary message: ' + NewLine + Result); +end; diff --git a/tools/windows/tool_setup/tools_fallback.json b/tools/windows/tool_setup/tools_fallback.json new file mode 100644 index 00000000..2422f2f4 --- /dev/null +++ b/tools/windows/tool_setup/tools_fallback.json @@ -0,0 +1,390 @@ +{ + "tools": [ + { + "description": "Toolchain for Xtensa (ESP32) based on GCC", + "export_paths": [ + [ + "xtensa-esp32-elf", + "bin" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/espressif/crosstool-NG", + "install": "always", + "license": "GPL-3.0-with-GCC-exception", + "name": "xtensa-esp32-elf", + "version_cmd": [ + "xtensa-esp32-elf-gcc", + "--version" + ], + "version_regex": "\\(crosstool-NG\\s+(?:crosstool-ng-)?([0-9a-z\\.\\-]+)\\)\\s*([0-9\\.]+)", + "version_regex_replace": "\\1-\\2", + "versions": [ + { + "name": "1.22.0-80-g6c4433a5-5.2.0", + "status": "recommended", + "win32": { + "sha256": "f217fccbeaaa8c92db239036e0d6202458de4488b954a3a38f35ac2ec48058a4", + "size": 125719261, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip" + }, + "win64": { + "sha256": "f217fccbeaaa8c92db239036e0d6202458de4488b954a3a38f35ac2ec48058a4", + "size": 125719261, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip" + } + }, + { + "linux-amd64": { + "sha256": "3fe96c151d46c1d4e5edc6ed690851b8e53634041114bad04729bc16b0445156", + "size": 44219107, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-linux64-1.22.0-80-g6c4433a-5.2.0.tar.gz" + }, + "linux-i686": { + "sha256": "b4055695ffc2dfc0bcb6dafdc2572a6e01151c4179ef5fa972b3fcb2183eb155", + "size": 45566336, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-linux32-1.22.0-80-g6c4433a-5.2.0.tar.gz" + }, + "macos": { + "sha256": "a4307a97945d2f2f2745f415fbe80d727750e19f91f9a1e7e2f8a6065652f9da", + "size": 46517409, + "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-osx-1.22.0-80-g6c4433a-5.2.0.tar.gz" + }, + "name": "1.22.0-80-g6c4433a-5.2.0", + "status": "recommended" + } + ] + }, + { + "description": "Toolchain for ESP32 ULP coprocessor", + "export_paths": [ + [ + "esp32ulp-elf-binutils", + "bin" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/espressif/binutils-esp32ulp", + "install": "always", + "license": "GPL-2.0-or-later", + "name": "esp32ulp-elf", + "version_cmd": [ + "esp32ulp-elf-as", + "--version" + ], + "version_regex": "\\(GNU Binutils\\)\\s+([0-9a-z\\.\\-]+)", + "versions": [ + { + "linux-amd64": { + "sha256": "c1bbcd65e1e30c7312a50344c8dbc70c2941580a79aa8f8abbce8e0e90c79566", + "size": 8246604, + "url": "https://dl.espressif.com/dl/binutils-esp32ulp-linux64-2.28.51-esp32ulp-20180809.tar.gz" + }, + "macos": { + "sha256": "c92937d85cc9a90eb6c6099ce767ca021108c18c94e34bd7b1fa0cde168f94a0", + "size": 5726662, + "url": "https://dl.espressif.com/dl/binutils-esp32ulp-macos-2.28.51-esp32ulp-20180809.tar.gz" + }, + "name": "2.28.51.20170517", + "status": "recommended", + "win32": { + "sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd", + "size": 12231559, + "url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip" + }, + "win64": { + "sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd", + "size": 12231559, + "url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip" + } + } + ] + }, + { + "description": "CMake build system", + "export_paths": [ + [ + "bin" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/Kitware/CMake", + "install": "on_request", + "license": "BSD-3-Clause", + "name": "cmake", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win32", + "win64" + ] + }, + { + "export_paths": [ + [ + "CMake.app", + "Contents", + "bin" + ] + ], + "platforms": [ + "macos" + ] + } + ], + "strip_container_dirs": 1, + "version_cmd": [ + "cmake", + "--version" + ], + "version_regex": "cmake version ([0-9.]+)", + "versions": [ + { + "linux-amd64": { + "sha256": "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76", + "size": 38405896, + "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz" + }, + "macos": { + "sha256": "fef537614d73fda848f6168273b6c7ba45f850484533361e7bc50ac1d315f780", + "size": 32062124, + "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Darwin-x86_64.tar.gz" + }, + "name": "3.13.4", + "status": "recommended", + "win32": { + "sha256": "28daf772f55d817a13ef14e25af2a5569f8326dac66a6aa3cc5208cf1f8e943f", + "size": 26385104, + "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win32-x86.zip" + }, + "win64": { + "sha256": "bcd477d49e4a9400b41213d53450b474beaedb264631693c958ef9affa8e5623", + "size": 29696565, + "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win64-x64.zip" + } + } + ] + }, + { + "description": "OpenOCD for ESP32", + "export_paths": [ + [ + "openocd-esp32", + "bin" + ] + ], + "export_vars": { + "OPENOCD_SCRIPTS": "${TOOL_PATH}/openocd-esp32/share/openocd/scripts" + }, + "info_url": "https://github.com/espressif/openocd-esp32", + "install": "always", + "license": "GPL-2.0-only", + "name": "openocd-esp32", + "version_cmd": [ + "openocd", + "--version" + ], + "version_regex": "Open On-Chip Debugger\\s+([a-z0-9.-]+)\\s+", + "versions": [ + { + "linux-amd64": { + "sha256": "e5b5579edffde090e426b4995b346e281843bf84394f8e68c8e41bd1e4c576bd", + "size": 1681596, + "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-linux64-0.10.0-esp32-20190313.tar.gz" + }, + "macos": { + "sha256": "09504eea5aa92646a117f16573c95b34e04b4010791a2f8fefcd2bd8c430f081", + "size": 1760536, + "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-macos-0.10.0-esp32-20190313.tar.gz" + }, + "name": "v0.10.0-esp32-20190313", + "status": "recommended", + "win32": { + "sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2", + "size": 2098513, + "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip" + }, + "win64": { + "sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2", + "size": 2098513, + "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip" + } + } + ] + }, + { + "description": "menuconfig tool", + "export_paths": [ + [ + "" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/espressif/kconfig-frontends", + "install": "never", + "license": "GPL-2.0-only", + "name": "mconf", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win32", + "win64" + ] + } + ], + "strip_container_dirs": 1, + "version_cmd": [ + "mconf-idf", + "-v" + ], + "version_regex": "mconf-idf version mconf-([a-z0-9.-]+)-win32", + "versions": [ + { + "name": "v4.6.0.0-idf-20190628", + "status": "recommended", + "win32": { + "sha256": "1b8f17f48740ab669c13bd89136e8cc92efe0cd29872f0d6c44148902a2dc40c", + "size": 826114, + "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190628/mconf-v4.6.0.0-idf-20190628-win32.zip" + }, + "win64": { + "sha256": "1b8f17f48740ab669c13bd89136e8cc92efe0cd29872f0d6c44148902a2dc40c", + "size": 826114, + "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190628/mconf-v4.6.0.0-idf-20190628-win32.zip" + } + } + ] + }, + { + "description": "Ninja build system", + "export_paths": [ + [ + "" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/ninja-build/ninja", + "install": "on_request", + "license": "Apache-2.0", + "name": "ninja", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win32", + "win64" + ] + } + ], + "version_cmd": [ + "ninja", + "--version" + ], + "version_regex": "([0-9.]+)", + "versions": [ + { + "linux-amd64": { + "sha256": "978fd9e26c2db8d33392c6daef50e9edac0a3db6680710a9f9ad47e01f3e49b7", + "size": 85276, + "url": "https://dl.espressif.com/dl/ninja-1.9.0-linux64.tar.gz" + }, + "macos": { + "sha256": "9504cd1783ef3c242d06330a50d54dc8f838b605f5fc3e892c47254929f7350c", + "size": 91457, + "url": "https://dl.espressif.com/dl/ninja-1.9.0-osx.tar.gz" + }, + "name": "1.9.0", + "status": "recommended", + "win64": { + "sha256": "2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9", + "size": 254497, + "url": "https://dl.espressif.com/dl/ninja-1.9.0-win64.zip" + } + } + ] + }, + { + "description": "IDF wrapper tool for Windows", + "export_paths": [ + [ + "" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/espressif/esp-idf/tree/master/tools/windows/idf_exe", + "install": "never", + "license": "Apache-2.0", + "name": "idf-exe", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win32", + "win64" + ] + } + ], + "version_cmd": [ + "idf.py.exe", + "-v" + ], + "version_regex": "([0-9.]+)", + "versions": [ + { + "name": "1.0.1", + "status": "recommended", + "win32": { + "sha256": "53eb6aaaf034cc7ed1a97d5c577afa0f99815b7793905e9408e74012d357d04a", + "size": 11297, + "url": "https://dl.espressif.com/dl/idf-exe-v1.0.1.zip" + }, + "win64": { + "sha256": "53eb6aaaf034cc7ed1a97d5c577afa0f99815b7793905e9408e74012d357d04a", + "size": 11297, + "url": "https://dl.espressif.com/dl/idf-exe-v1.0.1.zip" + } + } + ] + }, + { + "description": "Ccache (compiler cache)", + "export_paths": [ + [ + "" + ] + ], + "export_vars": {}, + "info_url": "https://github.com/ccache/ccache", + "install": "never", + "license": "GPL-3.0-or-later", + "name": "ccache", + "platform_overrides": [ + { + "install": "always", + "platforms": [ + "win64" + ] + } + ], + "version_cmd": [ + "ccache.exe", + "--version" + ], + "version_regex": "ccache version ([0-9.]+)", + "versions": [ + { + "name": "3.7", + "status": "recommended", + "win64": { + "sha256": "37e833f3f354f1145503533e776c1bd44ec2e77ff8a2476a1d2039b0b10c78d6", + "size": 142401, + "url": "https://dl.espressif.com/dl/ccache-3.7-w64.zip" + } + } + ] + } + ], + "version": 1 +} diff --git a/tools/windows/tool_setup/utils.iss.inc b/tools/windows/tool_setup/utils.iss.inc new file mode 100644 index 00000000..a93f6ad4 --- /dev/null +++ b/tools/windows/tool_setup/utils.iss.inc @@ -0,0 +1,157 @@ +{ Copyright 2019 Espressif Systems (Shanghai) PTE LTD + SPDX-License-Identifier: Apache-2.0 } + +{ ------------------------------ Helper functions from libcmdlinerunner.dll ------------------------------ } + +function ProcStart(cmdline, workdir: string): Longword; + external 'proc_start@files:cmdlinerunner.dll cdecl'; + +function ProcGetExitCode(inst: Longword): DWORD; + external 'proc_get_exit_code@files:cmdlinerunner.dll cdecl'; + +function ProcGetOutput(inst: Longword; dest: PAnsiChar; sz: DWORD): DWORD; + external 'proc_get_output@files:cmdlinerunner.dll cdecl'; + +procedure ProcEnd(inst: Longword); + external 'proc_end@files:cmdlinerunner.dll cdecl'; + +{ ------------------------------ WinAPI functions ------------------------------ } + +#ifdef UNICODE + #define AW "W" +#else + #define AW "A" +#endif + +function SetEnvironmentVariable(lpName: string; lpValue: string): BOOL; + external 'SetEnvironmentVariable{#AW}@kernel32.dll stdcall'; + +{ ------------------------------ Functions to query the registry ------------------------------ } + +{ Utility to search in HKLM and HKCU for an installation path. Looks in both 64-bit & 32-bit registry. } +function GetInstallPath(key, valuename : String) : String; +var + value: String; +begin + Result := ''; + if RegQueryStringValue(HKEY_LOCAL_MACHINE, key, valuename, value) then + begin + Result := value; + exit; + end; + + if RegQueryStringValue(HKEY_CURRENT_USER, key, valuename, value) then + begin + Result := value; + exit; + end; + + { This is 32-bit setup running on 64-bit Windows, but ESP-IDF can use 64-bit tools also } + if IsWin64 and RegQueryStringValue(HKLM64, key, valuename, value) then + begin + Result := value; + exit; + end; + + if IsWin64 and RegQueryStringValue(HKCU64, key, valuename, value) then + begin + Result := value; + exit; + end; +end; + +{ ------------------------------ Function to exit from the installer ------------------------------ } + +procedure AbortInstallation(Message: String); +begin + MsgBox(Message, mbError, MB_OK); + Abort(); +end; + +{ ------------------------------ File system related functions ------------------------------ } + +function DirIsEmpty(DirName: String): Boolean; +var + FindRec: TFindRec; +begin + Result := True; + if FindFirst(DirName+'\*', FindRec) then begin + try + repeat + if (FindRec.Name <> '.') and (FindRec.Name <> '..') then begin + Result := False; + break; + end; + until not FindNext(FindRec); + finally + FindClose(FindRec); + end; + end; +end; + +type + TFindFileCallback = procedure(Filename: String); + +procedure FindFileRecusive(Directory: string; FileName: string; Callback: TFindFileCallback); +var + FindRec: TFindRec; + FilePath: string; +begin + if FindFirst(Directory + '\*', FindRec) then + begin + try + repeat + if (FindRec.Name = '.') or (FindRec.Name = '..') then + continue; + + FilePath := Directory + '\' + FindRec.Name; + if FindRec.Attributes and FILE_ATTRIBUTE_DIRECTORY <> 0 then + begin + FindFileRecusive(FilePath, FileName, Callback); + end else if CompareText(FindRec.Name, FileName) = 0 then + begin + Callback(FilePath); + end; + until not FindNext(FindRec); + finally + FindClose(FindRec); + end; + end; +end; + +{ ------------------------------ Version related functions ------------------------------ } + +function VersionExtractMajorMinor(Version: String; var Major: Integer; var Minor: Integer): Boolean; +var + Delim: Integer; + MajorStr, MinorStr: String; + OrigVersion, ExpectedPrefix: String; +begin + Result := False; + OrigVersion := Version; + Delim := Pos('.', Version); + if Delim = 0 then exit; + + MajorStr := Version; + Delete(MajorStr, Delim, Length(MajorStr)); + Delete(Version, 1, Delim); + Major := StrToInt(MajorStr); + + Delim := Pos('.', Version); + if Delim = 0 then Delim := Length(MinorStr); + + MinorStr := Version; + Delete(MinorStr, Delim, Length(MinorStr)); + Delete(Version, 1, Delim); + Minor := StrToInt(MinorStr); + + { Sanity check } + ExpectedPrefix := IntToStr(Major) + '.' + IntToStr(Minor); + if Pos(ExpectedPrefix, OrigVersion) <> 1 then + begin + Log('VersionExtractMajorMinor: version=' + OrigVersion + ', expected=' + ExpectedPrefix); + exit; + end; + + Result := True; +end; diff --git a/tools/windows/windows_install_prerequisites.sh b/tools/windows/windows_install_prerequisites.sh index 94718570..99393108 100644 --- a/tools/windows/windows_install_prerequisites.sh +++ b/tools/windows/windows_install_prerequisites.sh @@ -1,11 +1,11 @@ #!/bin/bash # -# Setup script to configure an MSYS2 environment for ESP8266_RTOS_SDK. +# Setup script to configure an MSYS2 environment for ESP-IDF. # # Use of this script is optional, there is also a prebuilt MSYS2 environment available # which can be downloaded and used as-is. # -# See https://docs.espressif.com/projects/esp8266-rtos-sdk/en/latest/get-started/windows-setup.html for full details. +# See https://docs.espressif.com/projects/esp-idf/en/latest/get-started/windows-setup.html for full details. if [ "$OSTYPE" != "msys" ]; then echo "This setup script expects to be run from an MSYS2 environment on Windows." @@ -34,26 +34,28 @@ set -e pacman --noconfirm -Syu # This step may require the terminal to be closed and restarted pacman --noconfirm -S --needed gettext-devel gcc git make ncurses-devel flex bison gperf vim \ - mingw-w64-i686-python2-pip mingw-w64-i686-python2-cryptography unzip winpty tar + mingw-w64-i686-python2-pip mingw-w64-i686-python2-cryptography unzip winpty -if [ -n $IDF_PATH ]; then - python -m pip install -r $IDF_PATH/requirements.txt +# if IDF_PATH is set, install requirements now as well +if [ -n "$IDF_PATH" ]; then + python -m pip install -r "$IDF_PATH/requirements.txt" fi -# Automatically download precompiled toolchain, unpack at /opt/xtensa-lx106-elf/ -TOOLCHAIN_GZ=xtensa-lx106-elf-win32-1.22.0-92-g8facf4c-5.2.0.tar.gz -echo "Downloading precompiled toolchain ${TOOLCHAIN_GZ}..." +# Automatically download precompiled toolchain, unpack at /opt/xtensa-esp32-elf/ +TOOLCHAIN_ZIP=xtensa-esp32-elf-gcc8_2_0-esp32-2019r1-win32.zip +echo "Downloading precompiled toolchain ${TOOLCHAIN_ZIP}..." cd ~ -curl -LO --retry 10 http://dl.espressif.com/dl/${TOOLCHAIN_GZ} +curl -LO --retry 10 http://dl.espressif.com/dl/${TOOLCHAIN_ZIP} +mkdir -p /opt cd /opt -rm -rf /opt/xtensa-lx106-elf # for upgrades -tar -xf ~/${TOOLCHAIN_GZ} || echo "Uncompressing cross toolchain has some little error, you may ignore it if it can be used." -rm ~/${TOOLCHAIN_GZ} +rm -rf /opt/xtensa-esp32-elf # for upgrades +unzip ~/${TOOLCHAIN_ZIP} +rm ~/${TOOLCHAIN_ZIP} -cat > /etc/profile.d/esp8266_toolchain.sh << EOF -# This file was created by ESP8266_RTOS_SDK windows_install_prerequisites.sh +cat > /etc/profile.d/esp32_toolchain.sh << EOF +# This file was created by ESP-IDF windows_install_prerequisites.sh # and will be overwritten if that script is run again. -export PATH="\$PATH:/opt/xtensa-lx106-elf/bin" +export PATH="/opt/xtensa-esp32-elf/bin:\$PATH" EOF # clean up pacman package cache to save some disk space @@ -61,19 +63,19 @@ pacman --noconfirm -Scc cat << EOF ************************************************ -MSYS2 environment is now ready to use ESP8266_RTOS_SDK. +MSYS2 environment is now ready to use ESP-IDF. 1) Run 'source /etc/profile' to add the toolchain to your path in this terminal. This command produces no output. You only need to do this once, future terminals do this automatically when opened. -2) After ESP8266_RTOS_SDK is set up (see setup guide), edit the file +2) After ESP-IDF is set up (see setup guide), edit the file `cygpath -w /etc/profile` and add a line to set the variable IDF_PATH so it points to the IDF directory, ie: -export IDF_PATH=/c/path/to/ESP8266_RTOS_SDK/directory +export IDF_PATH=/c/path/to/esp-idf/directory ************************************************ EOF