feature(script): update compiling script for new make and cmake

components/app_update/CMakeLists.txt

@@ -1,17 +1,57 @@
-set(COMPONENT_SRCDIRS ".")
+idf_component_register(SRCS "esp_ota_ops.c"
-set(COMPONENT_ADD_INCLUDEDIRS "include")
+                    INCLUDE_DIRS "include"
+                    REQUIRES spi_flash partition_table bootloader_support
+                    PRIV_REQUIRES util)
 
-set(COMPONENT_REQUIRES "spi_flash" "partition_table")
+# esp_app_desc structure is added as an undefined symbol because otherwise the
-set(COMPONENT_PRIV_REQUIRES "bootloader_support" "util")
+# linker will ignore this structure as it has no other files depending on it.
+target_link_libraries(${COMPONENT_LIB} INTERFACE "-u esp_app_desc")
 
-register_component()
+# cut PROJECT_VER and PROJECT_NAME to required 32 characters.
+idf_build_get_property(project_ver PROJECT_VER)
+idf_build_get_property(project_name PROJECT_NAME)
+string(SUBSTRING "${project_ver}" 0 31 PROJECT_VER_CUT)
+string(SUBSTRING "${project_name}" 0 31 PROJECT_NAME_CUT)
 
-# Add custom target for generating empty otadata partition for flashing
+set_source_files_properties(
-if(${OTADATA_PARTITION_OFFSET})
+    SOURCE "esp_app_desc.c"
-    add_custom_command(OUTPUT "${PROJECT_BINARY_DIR}/${BLANK_OTADATA_FILE}"
+    PROPERTIES COMPILE_DEFINITIONS
-        COMMAND ${PYTHON} ${CMAKE_CURRENT_SOURCE_DIR}/gen_empty_partition.py
+    "PROJECT_VER=\"${PROJECT_VER_CUT}\"; PROJECT_NAME=\"${PROJECT_NAME_CUT}\"")
-        --size ${OTADATA_PARTITION_SIZE} "${PROJECT_BINARY_DIR}/${BLANK_OTADATA_FILE}")
 
-    add_custom_target(blank_ota_data ALL DEPENDS "${PROJECT_BINARY_DIR}/${BLANK_OTADATA_FILE}")
+if(NOT BOOTLOADER_BUILD)
-    add_dependencies(flash blank_ota_data)
+    partition_table_get_partition_info(otadata_offset "--partition-type data --partition-subtype ota" "offset")
+    partition_table_get_partition_info(otadata_size "--partition-type data --partition-subtype ota" "size")
+
+    # Add custom target for generating empty otadata partition for flashing
+    if(otadata_size AND otadata_offset)
+        idf_build_get_property(build_dir BUILD_DIR)
+        set(blank_otadata_file ${build_dir}/ota_data_initial.bin)
+
+        idf_build_get_property(idf_path IDF_PATH)
+        idf_build_get_property(python PYTHON)
+        add_custom_command(OUTPUT ${blank_otadata_file}
+            COMMAND ${python} ${idf_path}/components/partition_table/gen_empty_partition.py
+            ${otadata_size} ${blank_otadata_file})
+
+        add_custom_target(blank_ota_data ALL DEPENDS ${blank_otadata_file})
+        add_dependencies(flash blank_ota_data)
+
+        set(otatool_py ${python} ${COMPONENT_DIR}/otatool.py)
+
+        set(esptool_args --esptool-args before=${CONFIG_ESPTOOLPY_BEFORE} after=${CONFIG_ESPTOOLPY_AFTER})
+
+        add_custom_target(read_otadata DEPENDS "${PARTITION_CSV_PATH}"
+                        COMMAND ${otatool_py} ${esptool_args}
+                        --partition-table-file ${PARTITION_CSV_PATH} 
+                        --partition-table-offset ${PARTITION_TABLE_OFFSET}
+                        read_otadata)
+
+        add_custom_target(erase_otadata DEPENDS "${PARTITION_CSV_PATH}"
+                        COMMAND ${otatool_py} ${esptool_args}
+                        --partition-table-file ${PARTITION_CSV_PATH}
+                        --partition-table-offset ${PARTITION_TABLE_OFFSET}
+                        erase_otadata)
+
+        esptool_py_flash_project_args(otadata ${otadata_offset} "${blank_otadata_file}" FLASH_IN_PROJECT)
+    endif()
 endif()

components/app_update/Makefile.projbuild

@@ -1,60 +1,57 @@
 # Generate partition binary
 #
-.PHONY: dump_otadata erase_ota blank_ota_data
+.PHONY: blank_ota_data erase_otadata read_otadata
 
-GEN_EMPTY_PART := $(PYTHON) $(COMPONENT_PATH)/gen_empty_partition.py
+OTATOOL_PY := $(PYTHON) $(COMPONENT_PATH)/otatool.py
+PARTTOOL_PY := $(PYTHON) $(IDF_PATH)/components/partition_table/parttool.py
+
+# Generate blank partition file
 BLANK_OTA_DATA_FILE = $(BUILD_DIR_BASE)/ota_data_initial.bin
 
-PARTITION_TABLE_LEN := 0xC00
+# Copy PARTITION_TABLE_CSV_PATH definition here from $IDF_PATH/components/partition_table/Makefile.projbuild
-OTADATA_LEN := 0x2000
+# to avoid undefined variables warning for PARTITION_TABLE_CSV_PATH
+ifndef PARTITION_TABLE_CSV_PATH
+PARTITION_TABLE_ROOT := $(call dequote,$(if $(CONFIG_PARTITION_TABLE_CUSTOM),$(PROJECT_PATH),$(IDF_PATH)/components/partition_table))
+PARTITION_TABLE_CSV_PATH := $(call dequote,$(abspath $(PARTITION_TABLE_ROOT)/$(call dequote,$(CONFIG_PARTITION_TABLE_FILENAME))))
+endif
 
-PARTITION_TABLE_ONCHIP_BIN_PATH := $(call dequote,$(abspath $(BUILD_DIR_BASE)))
+$(BLANK_OTA_DATA_FILE): partition_table_get_info $(PARTITION_TABLE_CSV_PATH) | check_python_dependencies
-PARTITION_TABLE_ONCHIP_BIN_NAME := "onchip_partition.bin"
+	$(shell if [ "$(OTA_DATA_OFFSET)" != "" ] && [ "$(OTA_DATA_SIZE)" != "" ]; then \
-OTADATA_ONCHIP_BIN_NAME := "onchip_otadata.bin"
+				$(PYTHON) $(IDF_PATH)/components/partition_table/gen_empty_partition.py $(OTA_DATA_SIZE) $(BLANK_OTA_DATA_FILE); \
+			fi; )
+	$(eval BLANK_OTA_DATA_FILE = $(shell if [ "$(OTA_DATA_OFFSET)" != "" ] && [ "$(OTA_DATA_SIZE)" != "" ]; then \
+			echo $(BLANK_OTA_DATA_FILE); else echo " "; fi) )
 
-PARTITION_TABLE_ONCHIP_BIN := $(PARTITION_TABLE_ONCHIP_BIN_PATH)/$(call dequote,$(PARTITION_TABLE_ONCHIP_BIN_NAME)) 
+blank_ota_data: $(BLANK_OTA_DATA_FILE)
-OTADATA_ONCHIP_BIN := $(PARTITION_TABLE_ONCHIP_BIN_PATH)/$(call dequote,$(OTADATA_ONCHIP_BIN_NAME))
-
-PARTITION_TABLE_GET_BIN_CMD = $(ESPTOOLPY_SERIAL) read_flash $(PARTITION_TABLE_OFFSET) $(PARTITION_TABLE_LEN) $(PARTITION_TABLE_ONCHIP_BIN)
-OTADATA_GET_BIN_CMD = $(ESPTOOLPY_SERIAL) read_flash $(OTADATA_OFFSET) $(OTADATA_LEN) $(OTADATA_ONCHIP_BIN)
-
-GEN_OTADATA = $(IDF_PATH)/components/app_update/dump_otadata.py
-ERASE_OTADATA_CMD = $(ESPTOOLPY_SERIAL) erase_region $(OTADATA_OFFSET) $(OTADATA_LEN)
 
 # If there is no otadata partition, both OTA_DATA_OFFSET and BLANK_OTA_DATA_FILE 
 # expand to empty values.
 ESPTOOL_ALL_FLASH_ARGS += $(OTA_DATA_OFFSET) $(BLANK_OTA_DATA_FILE)
 
-$(PARTITION_TABLE_ONCHIP_BIN):
+ESPTOOL_ARGS := --esptool-args port=$(CONFIG_ESPTOOLPY_PORT) baud=$(CONFIG_ESPTOOLPY_BAUD) before=$(CONFIG_ESPTOOLPY_BEFORE) after=$(CONFIG_ESPTOOLPY_AFTER)
-	$(PARTITION_TABLE_GET_BIN_CMD)
 
-onchip_otadata_get_info: $(PARTITION_TABLE_ONCHIP_BIN) 
+erase_otadata: $(PARTITION_TABLE_CSV_PATH) partition_table_get_info | check_python_dependencies
-	$(eval OTADATA_OFFSET:=$(shell $(GET_PART_INFO) --type data --subtype ota --offset $(PARTITION_TABLE_ONCHIP_BIN)))
+	$(OTATOOL_PY) $(ESPTOOL_ARGS) --partition-table-file $(PARTITION_TABLE_CSV_PATH) \
-	@echo $(if $(OTADATA_OFFSET), $(shell export OTADATA_OFFSET), $(shell rm -f $(PARTITION_TABLE_ONCHIP_BIN));$(error "ERROR: ESP32 does not have otadata partition."))
+	--partition-table-offset $(PARTITION_TABLE_OFFSET) \
+	erase_otadata
 
-$(OTADATA_ONCHIP_BIN):
+read_otadata: $(PARTITION_TABLE_CSV_PATH) partition_table_get_info | check_python_dependencies
-	$(OTADATA_GET_BIN_CMD)
+	$(OTATOOL_PY) $(ESPTOOL_ARGS) --partition-table-file $(PARTITION_TABLE_CSV_PATH) \ 
+	--partition-table-offset $(partition_table_offset) \ 
+	read_otadata
 
-dump_otadata: onchip_otadata_get_info $(OTADATA_ONCHIP_BIN) $(PARTITION_TABLE_ONCHIP_BIN) 
+erase_ota: erase_otadata
-	@echo "otadata retrieved. Contents:"
+	@echo "WARNING: erase_ota is deprecated. Use erase_otadata instead."
-	@echo $(SEPARATOR)
-	$(GEN_OTADATA) $(OTADATA_ONCHIP_BIN)
-	@echo $(SEPARATOR)
-	rm -f $(PARTITION_TABLE_ONCHIP_BIN)
-	rm -f $(OTADATA_ONCHIP_BIN)
-
-$(BLANK_OTA_DATA_FILE): partition_table_get_info
-	$(GEN_EMPTY_PART) --size $(OTA_DATA_SIZE) $(BLANK_OTA_DATA_FILE)
-	$(eval BLANK_OTA_DATA_FILE = $(shell if [ $(OTA_DATA_SIZE) != 0 ]; then echo $(BLANK_OTA_DATA_FILE); else echo " "; fi) )
-
-blank_ota_data: $(BLANK_OTA_DATA_FILE)
-
-erase_ota: partition_table_get_info | check_python_dependencies
-	@echo $(if $(OTA_DATA_OFFSET), "Erase ota_data [addr=$(OTA_DATA_OFFSET) size=$(OTA_DATA_SIZE)] ...", $(error "ERROR: Partition table does not have ota_data partition."))
-	$(ESPTOOLPY_SERIAL) erase_region $(OTA_DATA_OFFSET) $(OTA_DATA_SIZE)
 
 all: blank_ota_data
 flash: blank_ota_data
+ifdef CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT
+encrypted-flash: blank_ota_data
+endif
+
+TMP_DEFINES := $(BUILD_DIR_BASE)/app_update/tmp_cppflags.txt
+export TMP_DEFINES
 
 clean:
 	rm -f $(BLANK_OTA_DATA_FILE)
+	rm -f $(TMP_DEFINES)

components/app_update/dump_otadata.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python
-#
-# gen_otadata prints info about the otadata partition.
-#
-# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from __future__ import print_function, division
-import argparse
-import os
-import re
-import struct
-import sys
-import hashlib
-import binascii
-
-__version__ = '1.0'
-
-quiet = False
-
-def status(msg):
-    """ Print status message to stderr """
-    if not quiet:
-        critical(msg)
-
-def critical(msg):
-    """ Print critical message to stderr """
-    if not quiet:
-        sys.stderr.write(msg)
-        sys.stderr.write('\n')
-
-def little_endian(buff, offset):
-    data = buff[offset:offset+4]
-    data.reverse()
-    data = ''.join(data)
-    return data
-               
-def main():
-    global quiet
-    parser = argparse.ArgumentParser(description='Prints otadata partition in human readable form.')
-
-    parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
-
-    search_type = parser.add_mutually_exclusive_group()
-
-    parser.add_argument('input', help='Path to binary file containing otadata partition to parse.',
-                        type=argparse.FileType('rb'))
-
-    args = parser.parse_args()
-
-    quiet = args.quiet
-
-    input = args.input.read()
-    
-    hex_input_0 = binascii.hexlify(input)
-    hex_input_0 = map(''.join, zip(*[iter(hex_input_0)]*2))
-    hex_input_1 = binascii.hexlify(input[4096:])
-    hex_input_1 = map(''.join, zip(*[iter(hex_input_1)]*2))
-    
-    print("\t%11s\t%8s    |\t%8s\t%8s" %("OTA_SEQ", "CRC", "OTA_SEQ", "CRC"))
-    print("Firmware:  0x%s  \t 0x%s |\t0x%s \t 0x%s" % (little_endian(hex_input_0, 0), little_endian(hex_input_0, 28), \
-                                                        little_endian(hex_input_1, 0), little_endian(hex_input_1, 28)))
-class InputError(RuntimeError):
-    def __init__(self, e):
-        super(InputError, self).__init__(e)
-
-class ValidationError(InputError):
-    def __init__(self, partition, message):
-        super(ValidationError, self).__init__(
-            "Partition %s invalid: %s" % (partition.name, message))
-
-if __name__ == '__main__':
-    try:
-        r = main()
-        sys.exit(r)
-    except InputError as e:
-        print(e, file=sys.stderr)
-        sys.exit(2)

components/app_update/otatool.py

@@ -0,0 +1,388 @@
+#!/usr/bin/env python
+#
+# otatool is used to perform ota-level operations - flashing ota partition
+# erasing ota partition and switching ota partition
+#
+# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function, division
+import argparse
+import os
+import sys
+import binascii
+import tempfile
+import collections
+import struct
+
+try:
+    from parttool import PartitionName, PartitionType, ParttoolTarget, PARTITION_TABLE_OFFSET
+except ImportError:
+    COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components"))
+    PARTTOOL_DIR = os.path.join(COMPONENTS_PATH, "partition_table")
+
+    sys.path.append(PARTTOOL_DIR)
+    from parttool import PartitionName, PartitionType, ParttoolTarget, PARTITION_TABLE_OFFSET
+
+__version__ = '2.0'
+
+SPI_FLASH_SEC_SIZE = 0x2000
+
+quiet = False
+
+
+def status(msg):
+    if not quiet:
+        print(msg)
+
+
+class OtatoolTarget():
+
+    OTADATA_PARTITION = PartitionType("data", "ota")
+
+    def __init__(self, port=None, baud=None, partition_table_offset=PARTITION_TABLE_OFFSET, partition_table_file=None,
+                 spi_flash_sec_size=SPI_FLASH_SEC_SIZE, esptool_args=[], esptool_write_args=[],
+                 esptool_read_args=[], esptool_erase_args=[]):
+        self.target = ParttoolTarget(port, baud, partition_table_offset, partition_table_file, esptool_args,
+                                     esptool_write_args, esptool_read_args, esptool_erase_args)
+        self.spi_flash_sec_size = spi_flash_sec_size
+
+        temp_file = tempfile.NamedTemporaryFile(delete=False)
+        temp_file.close()
+        try:
+            self.target.read_partition(OtatoolTarget.OTADATA_PARTITION, temp_file.name)
+            with open(temp_file.name, "rb") as f:
+                self.otadata = f.read()
+        except Exception:
+            self.otadata = None
+        finally:
+            os.unlink(temp_file.name)
+
+    def _check_otadata_partition(self):
+        if not self.otadata:
+            raise Exception("No otadata partition found")
+
+    def erase_otadata(self):
+        self._check_otadata_partition()
+        self.target.erase_partition(OtatoolTarget.OTADATA_PARTITION)
+
+    def _get_otadata_info(self):
+        info = []
+
+        otadata_info = collections.namedtuple("otadata_info", "seq crc")
+
+        for i in range(2):
+            start = i * (self.spi_flash_sec_size >> 1)
+
+            seq = bytearray(self.otadata[start:start + 4])
+            crc = bytearray(self.otadata[start + 28:start + 32])
+
+            seq = struct.unpack('>I', seq)
+            crc = struct.unpack('>I', crc)
+
+            info.append(otadata_info(seq[0], crc[0]))
+
+        return info
+
+    def _get_partition_id_from_ota_id(self, ota_id):
+        if isinstance(ota_id, int):
+            return PartitionType("app", "ota_" + str(ota_id))
+        else:
+            return PartitionName(ota_id)
+
+    def switch_ota_partition(self, ota_id):
+        self._check_otadata_partition()
+
+        sys.path.append(PARTTOOL_DIR)
+        import gen_esp32part as gen
+
+        def is_otadata_info_valid(status):
+            seq = status.seq % (1 << 32)
+            crc = hex(binascii.crc32(struct.pack("I", seq), 0xFFFFFFFF) % (1 << 32))
+            return seq < (int('0xFFFFFFFF', 16) % (1 << 32)) and status.crc == crc
+
+        partition_table = self.target.partition_table
+
+        ota_partitions = list()
+
+        for i in range(gen.NUM_PARTITION_SUBTYPE_APP_OTA):
+            ota_partition = filter(lambda p: p.subtype == (gen.MIN_PARTITION_SUBTYPE_APP_OTA + i), partition_table)
+
+            try:
+                ota_partitions.append(list(ota_partition)[0])
+            except IndexError:
+                break
+
+        ota_partitions = sorted(ota_partitions, key=lambda p: p.subtype)
+
+        if not ota_partitions:
+            raise Exception("No ota app partitions found")
+
+        # Look for the app partition to switch to
+        ota_partition_next = None
+
+        try:
+            if isinstance(ota_id, int):
+                ota_partition_next = filter(lambda p: p.subtype - gen.MIN_PARTITION_SUBTYPE_APP_OTA  == ota_id, ota_partitions)
+            else:
+                ota_partition_next = filter(lambda p: p.name == ota_id, ota_partitions)
+
+            ota_partition_next = list(ota_partition_next)[0]
+        except IndexError:
+            raise Exception("Partition to switch to not found")
+
+        otadata_info = self._get_otadata_info()
+
+        # Find the copy to base the computation for ota sequence number on
+        otadata_compute_base = -1
+
+        # Both are valid, take the max as computation base
+        if is_otadata_info_valid(otadata_info[0]) and is_otadata_info_valid(otadata_info[1]):
+            if otadata_info[0].seq >= otadata_info[1].seq:
+                otadata_compute_base = 0
+            else:
+                otadata_compute_base = 1
+        # Only one copy is valid, use that
+        elif is_otadata_info_valid(otadata_info[0]):
+            otadata_compute_base = 0
+        elif is_otadata_info_valid(otadata_info[1]):
+            otadata_compute_base = 1
+        # Both are invalid (could be initial state - all 0xFF's)
+        else:
+            pass
+
+        ota_seq_next = 0
+        ota_partitions_num = len(ota_partitions)
+
+        target_seq = (ota_partition_next.subtype & 0x0F) + 1
+
+        # Find the next ota sequence number
+        if otadata_compute_base == 0 or otadata_compute_base == 1:
+            base_seq = otadata_info[otadata_compute_base].seq % (1 << 32)
+
+            i = 0
+            while base_seq > target_seq % ota_partitions_num + i * ota_partitions_num:
+                i += 1
+
+            ota_seq_next = target_seq % ota_partitions_num + i * ota_partitions_num
+        else:
+            ota_seq_next = target_seq
+
+        # Create binary data from computed values
+        ota_seq_next = struct.pack("I", ota_seq_next)
+        ota_seq_crc_next = binascii.crc32(ota_seq_next, 0xFFFFFFFF) % (1 << 32)
+        ota_seq_crc_next = struct.pack("I", ota_seq_crc_next)
+
+        temp_file = tempfile.NamedTemporaryFile(delete=False)
+        temp_file.close()
+
+        try:
+            with open(temp_file.name, "wb") as otadata_next_file:
+                start = (1 if otadata_compute_base == 0 else 0) * (self.spi_flash_sec_size >> 1)
+
+                otadata_next_file.write(self.otadata)
+
+                otadata_next_file.seek(start)
+                otadata_next_file.write(ota_seq_next)
+
+                otadata_next_file.seek(start + 28)
+                otadata_next_file.write(ota_seq_crc_next)
+
+                otadata_next_file.flush()
+
+            self.target.write_partition(OtatoolTarget.OTADATA_PARTITION, temp_file.name)
+        finally:
+            os.unlink(temp_file.name)
+
+    def read_ota_partition(self, ota_id, output):
+        self.target.read_partition(self._get_partition_id_from_ota_id(ota_id), output)
+
+    def write_ota_partition(self, ota_id, input):
+        self.target.write_partition(self._get_partition_id_from_ota_id(ota_id), input)
+
+    def erase_ota_partition(self, ota_id):
+        self.target.erase_partition(self._get_partition_id_from_ota_id(ota_id))
+
+
+def _read_otadata(target):
+    target._check_otadata_partition()
+
+    otadata_info = target._get_otadata_info()
+
+    print("             {:8s} \t  {:8s} | \t  {:8s} \t   {:8s}".format("OTA_SEQ", "CRC", "OTA_SEQ", "CRC"))
+    print("Firmware:  0x{:8x} \t0x{:8x} | \t0x{:8x} \t 0x{:8x}".format(otadata_info[0].seq, otadata_info[0].crc,
+          otadata_info[1].seq, otadata_info[1].crc))
+
+
+def _erase_otadata(target):
+    target.erase_otadata()
+    status("Erased ota_data partition contents")
+
+
+def _switch_ota_partition(target, ota_id):
+    target.switch_ota_partition(ota_id)
+
+
+def _read_ota_partition(target, ota_id, output):
+    target.read_ota_partition(ota_id, output)
+    status("Read ota partition contents to file {}".format(output))
+
+
+def _write_ota_partition(target, ota_id, input):
+    target.write_ota_partition(ota_id, input)
+    status("Written contents of file {} to ota partition".format(input))
+
+
+def _erase_ota_partition(target, ota_id):
+    target.erase_ota_partition(ota_id)
+    status("Erased contents of ota partition")
+
+
+def main():
+    global quiet
+
+    parser = argparse.ArgumentParser("ESP-IDF OTA Partitions Tool")
+
+    parser.add_argument("--quiet", "-q", help="suppress stderr messages", action="store_true")
+    parser.add_argument("--esptool-args", help="additional main arguments for esptool", nargs="+")
+    parser.add_argument("--esptool-write-args", help="additional subcommand arguments for esptool write_flash", nargs="+")
+    parser.add_argument("--esptool-read-args", help="additional subcommand arguments for esptool read_flash", nargs="+")
+    parser.add_argument("--esptool-erase-args", help="additional subcommand arguments for esptool erase_region", nargs="+")
+
+    # There are two possible sources for the partition table: a device attached to the host
+    # or a partition table CSV/binary file. These sources are mutually exclusive.
+    parser.add_argument("--port", "-p", help="port where the device to read the partition table from is attached")
+
+    parser.add_argument("--baud", "-b", help="baudrate to use", type=int)
+
+    parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from",  type=str)
+
+    parser.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from; \
+                                                            overrides device attached to specified port as the partition table source when defined")
+
+    subparsers = parser.add_subparsers(dest="operation", help="run otatool -h for additional help")
+
+    spi_flash_sec_size = argparse.ArgumentParser(add_help=False)
+    spi_flash_sec_size.add_argument("--spi-flash-sec-size", help="value of SPI_FLASH_SEC_SIZE macro", type=str)
+
+    # Specify the supported operations
+    subparsers.add_parser("read_otadata", help="read otadata partition", parents=[spi_flash_sec_size])
+    subparsers.add_parser("erase_otadata", help="erase otadata partition")
+
+    slot_or_name_parser = argparse.ArgumentParser(add_help=False)
+    slot_or_name_parser_args = slot_or_name_parser.add_mutually_exclusive_group()
+    slot_or_name_parser_args.add_argument("--slot", help="slot number of the ota partition", type=int)
+    slot_or_name_parser_args.add_argument("--name", help="name of the ota partition")
+
+    subparsers.add_parser("switch_ota_partition", help="switch otadata partition", parents=[slot_or_name_parser, spi_flash_sec_size])
+
+    read_ota_partition_subparser = subparsers.add_parser("read_ota_partition", help="read contents of an ota partition", parents=[slot_or_name_parser])
+    read_ota_partition_subparser.add_argument("--output", help="file to write the contents of the ota partition to")
+
+    write_ota_partition_subparser = subparsers.add_parser("write_ota_partition", help="write contents to an ota partition", parents=[slot_or_name_parser])
+    write_ota_partition_subparser.add_argument("--input", help="file whose contents to write to the ota partition")
+
+    subparsers.add_parser("erase_ota_partition", help="erase contents of an ota partition", parents=[slot_or_name_parser])
+
+    args = parser.parse_args()
+
+    quiet = args.quiet
+
+    # No operation specified, display help and exit
+    if args.operation is None:
+        if not quiet:
+            parser.print_help()
+        sys.exit(1)
+
+    target_args = {}
+
+    if args.port:
+        target_args["port"] = args.port
+
+    if args.partition_table_file:
+        target_args["partition_table_file"] = args.partition_table_file
+
+    if args.partition_table_offset:
+        target_args["partition_table_offset"] = int(args.partition_table_offset, 0)
+
+    try:
+        if args.spi_flash_sec_size:
+            target_args["spi_flash_sec_size"] = int(args.spi_flash_sec_size, 0)
+    except AttributeError:
+        pass
+
+    if args.esptool_args:
+        target_args["esptool_args"] = args.esptool_args
+
+    if args.esptool_write_args:
+        target_args["esptool_write_args"] = args.esptool_write_args
+
+    if args.esptool_read_args:
+        target_args["esptool_read_args"] = args.esptool_read_args
+
+    if args.esptool_erase_args:
+        target_args["esptool_erase_args"] = args.esptool_erase_args
+
+    if args.baud:
+        target_args["baud"] = args.baud
+
+    target = OtatoolTarget(**target_args)
+
+    # Create the operation table and execute the operation
+    common_args = {'target':target}
+
+    ota_id = []
+
+    try:
+        if args.name is not None:
+            ota_id = ["name"]
+        else:
+            if args.slot is not None:
+                ota_id = ["slot"]
+    except AttributeError:
+        pass
+
+    otatool_ops = {
+        'read_otadata':(_read_otadata, []),
+        'erase_otadata':(_erase_otadata, []),
+        'switch_ota_partition':(_switch_ota_partition, ota_id),
+        'read_ota_partition':(_read_ota_partition, ["output"] + ota_id),
+        'write_ota_partition':(_write_ota_partition, ["input"] + ota_id),
+        'erase_ota_partition':(_erase_ota_partition, ota_id)
+    }
+
+    (op, op_args) = otatool_ops[args.operation]
+
+    for op_arg in op_args:
+        common_args.update({op_arg:vars(args)[op_arg]})
+
+    try:
+        common_args['ota_id'] = common_args.pop('name')
+    except KeyError:
+        try:
+            common_args['ota_id'] = common_args.pop('slot')
+        except KeyError:
+            pass
+
+    if quiet:
+        # If exceptions occur, suppress and exit quietly
+        try:
+            op(**common_args)
+        except Exception:
+            sys.exit(2)
+    else:
+        op(**common_args)
+
+
+if __name__ == '__main__':
+    main()

components/bootloader/CMakeLists.txt

@@ -1,7 +1,21 @@
-# bootloader component logic is all in project_include.cmake,
+idf_component_register(PRIV_REQUIRES partition_table)
-# and subproject/CMakeLists.txt.
-#
-# This file is only included so the build system finds the
-# component
 
+# Do not generate flash file when building bootloader or is in early expansion of the build
+if(BOOTLOADER_BUILD)
+    return()
+endif()
 
+# When secure boot is enabled, do not flash bootloader along with invocation of `idf.py flash`
+if(NOT CONFIG_SECURE_BOOT_ENABLED)
+    set(flash_bootloader FLASH_IN_PROJECT)
+endif()
+
+# Set values used in flash_bootloader_args.in and generate flash file
+# for bootloader
+esptool_py_flash_project_args(bootloader 0x0
+                            ${BOOTLOADER_BUILD_DIR}/bootloader.bin
+                            ${flash_bootloader}
+                            FLASH_FILE_TEMPLATE flash_bootloader_args.in)
+
+esptool_py_custom_target(bootloader-flash bootloader "bootloader")
+add_dependencies(bootloader partition_table)

components/bootloader/Kconfig.projbuild

@@ -55,7 +55,7 @@ config BOOTLOADER_SPI_WP_PIN
     int "SPI Flash WP Pin when customising pins via efuse (read help)"
     range 0 33
     default 7
-    depends on (FLASHMODE_QIO || FLASHMODE_QOUT) && IDF_TARGET_ESP32
+    depends on (ESPTOOLPY_FLASHMODE_QIO || ESPTOOLPY_FLASHMODE_QOUT) && IDF_TARGET_ESP32
     help
         This value is ignored unless flash mode is set to QIO or QOUT *and* the SPI flash pins have been
         overriden by setting the efuses SPI_PAD_CONFIG_xxx.

components/bootloader/flash_bootloader_args.in

@@ -1,4 +1,4 @@
 --flash_mode ${ESPFLASHMODE}
 --flash_size ${ESPFLASHSIZE}
 --flash_freq ${ESPFLASHFREQ}
-0x0000 bootloader/bootloader.bin
+${OFFSET} ${IMAGE}

components/bootloader/project_include.cmake

@@ -1,28 +1,122 @@
+set(BOOTLOADER_OFFSET 0x1000)
+
+# Do not generate flash file when building bootloader
 if(BOOTLOADER_BUILD)
-    return()  # don't keep recursing!
+    return()
 endif()
 
 # Glue to build the bootloader subproject binary as an external
 # cmake project under this one
 #
 #
-set(bootloader_build_dir "${CMAKE_BINARY_DIR}/bootloader")
+idf_build_get_property(build_dir BUILD_DIR)
+set(BOOTLOADER_BUILD_DIR "${build_dir}/bootloader")
 set(bootloader_binary_files
-    "${bootloader_build_dir}/bootloader.elf"
+    "${BOOTLOADER_BUILD_DIR}/bootloader.elf"
-    "${bootloader_build_dir}/bootloader.bin"
+    "${BOOTLOADER_BUILD_DIR}/bootloader.bin"
-    "${bootloader_build_dir}/bootloader.map"
+    "${BOOTLOADER_BUILD_DIR}/bootloader.map"
     )
 
+idf_build_get_property(project_dir PROJECT_DIR)
+
+# There are some additional processing when CONFIG_CONFIG_SECURE_SIGNED_APPS. This happens
+# when either CONFIG_SECURE_BOOT_ENABLED or SECURE_BOOT_BUILD_SIGNED_BINARIES.
+# For both cases, the user either sets binaries to be signed during build or not
+# using CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES.
+#
+# Regardless, pass the main project's keys (signing/verification) to the bootloader subproject
+# via config.
+if(CONFIG_SECURE_SIGNED_APPS)
+    add_custom_target(gen_secure_boot_keys)
+
+    if(CONFIG_SECURE_BOOT_ENABLED)
+        # Check that the configuration is sane
+        if((CONFIG_SECURE_BOOTLOADER_REFLASHABLE AND CONFIG_SECURE_BOOTLOADER_ONE_TIME_FLASH) OR
+            (NOT CONFIG_SECURE_BOOTLOADER_REFLASHABLE AND NOT CONFIG_SECURE_BOOTLOADER_ONE_TIME_FLASH))
+            fail_at_build_time(bootloader "Invalid bootloader target: bad sdkconfig?")
+        endif()
+
+        if(CONFIG_SECURE_BOOTLOADER_REFLASHABLE)
+            set(bootloader_binary_files
+                ${bootloader_binary_files}
+                "${BOOTLOADER_BUILD_DIR}/bootloader-reflash-digest.bin"
+                "${BOOTLOADER_BUILD_DIR}/secure-bootloader-key-192.bin"
+                "${BOOTLOADER_BUILD_DIR}/secure-bootloader-key-256.bin"
+                )
+        endif()
+    endif()
+
+    # Since keys are usually given relative to main project dir, get the absolute paths to the keys
+    # for use by the bootloader subproject. Replace the values in config with these absolute paths,
+    # so that bootloader subproject does not need to assume main project dir to obtain path to the keys.
+    if(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)
+        get_filename_component(secure_boot_signing_key
+            "${CONFIG_SECURE_BOOT_SIGNING_KEY}"
+            ABSOLUTE BASE_DIR "${project_dir}")
+
+        if(NOT EXISTS ${secure_boot_signing_key})
+            # If the signing key is not found, create a phony gen_secure_boot_signing_key target that
+            # fails the build. fail_at_build_time causes a cmake run next time
+            # (to pick up a new signing key if one exists, etc.)
+            fail_at_build_time(gen_secure_boot_signing_key
+                "Secure Boot Signing Key ${CONFIG_SECURE_BOOT_SIGNING_KEY} does not exist. Generate using:"
+                "\tespsecure.py generate_signing_key ${CONFIG_SECURE_BOOT_SIGNING_KEY}")
+        else()
+            add_custom_target(gen_secure_boot_signing_key)
+        endif()
+
+        set(SECURE_BOOT_SIGNING_KEY ${secure_boot_signing_key}) # needed by some other components
+        set(sign_key_arg "-DSECURE_BOOT_SIGNING_KEY=${secure_boot_signing_key}")
+
+        add_dependencies(gen_secure_boot_keys gen_secure_boot_signing_key)
+    else()
+
+        get_filename_component(secure_boot_verification_key
+            ${CONFIG_SECURE_BOOT_VERIFICATION_KEY}
+            ABSOLUTE BASE_DIR "${project_dir}")
+
+        if(NOT EXISTS ${secure_boot_verification_key})
+            # If the verification key is not found, create a phony gen_secure_boot_verification_key target that
+            # fails the build. fail_at_build_time causes a cmake run next time
+            # (to pick up a new verification key if one exists, etc.)
+            fail_at_build_time(gen_secure_boot_verification_key
+                "Secure Boot Verification Public Key ${CONFIG_SECURE_BOOT_VERIFICATION_KEY} does not exist."
+                "\tThis can be extracted from the private signing key."
+                "\tSee docs/security/secure-boot.rst for details.")
+        else()
+            add_custom_target(gen_secure_boot_verification_key)
+        endif()
+
+        set(ver_key_arg "-DSECURE_BOOT_VERIFICATION_KEY=${secure_boot_verification_key}")
+
+        add_dependencies(gen_secure_boot_keys gen_secure_boot_verification_key)
+    endif()
+endif()
+
+idf_build_get_property(idf_path IDF_PATH)
+idf_build_get_property(idf_target IDF_TARGET)
+idf_build_get_property(sdkconfig SDKCONFIG)
+
 externalproject_add(bootloader
-    # TODO: support overriding the bootloader in COMPONENT_PATHS
+    SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/subproject"
-    SOURCE_DIR "${IDF_PATH}/components/bootloader/subproject"
+    BINARY_DIR "${BOOTLOADER_BUILD_DIR}"
-    BINARY_DIR "${bootloader_build_dir}"
+    CMAKE_ARGS  -DSDKCONFIG=${sdkconfig} -DIDF_PATH=${idf_path} -DIDF_TARGET=${idf_target}
-    CMAKE_ARGS -DSDKCONFIG=${SDKCONFIG} -DIDF_PATH=${IDF_PATH}
+                -DPYTHON_DEPS_CHECKED=1
+                -DEXTRA_COMPONENT_DIRS=${CMAKE_CURRENT_LIST_DIR}
+                ${sign_key_arg} ${ver_key_arg}
+                # LEGACY_INCLUDE_COMMON_HEADERS has to be passed in via cache variable since
+                # the bootloader common component requirements depends on this and
+                # config variables are not available before project() call.
+                -DLEGACY_INCLUDE_COMMON_HEADERS=${CONFIG_LEGACY_INCLUDE_COMMON_HEADERS}
     INSTALL_COMMAND ""
     BUILD_ALWAYS 1  # no easy way around this...
     BUILD_BYPRODUCTS ${bootloader_binary_files}
     )
 
+if(CONFIG_SECURE_SIGNED_APPS)
+    add_dependencies(bootloader gen_secure_boot_keys)
+endif()
+
 # this is a hack due to an (annoying) shortcoming in cmake, it can't
 # extend the 'clean' target to the external project
 # see thread: https://cmake.org/pipermail/cmake/2016-December/064660.html

components/bootloader/subproject/CMakeLists.txt

@@ -10,18 +10,119 @@ if(NOT IDF_PATH)
         "in by the parent build process.")
 endif()
 
-set(COMPONENTS esptool_py bootloader bootloader_support spi_flash log esp8266 util partition_table main)
+if(NOT IDF_TARGET)
+    message(FATAL_ERROR "Bootloader subproject expects the IDF_TARGET variable to be passed "
+        "in by the parent build process.")
+endif()
+
+set(COMPONENTS bootloader esptool_py partition_table esp8266 bootloader_support log spi_flash main)
 set(BOOTLOADER_BUILD 1)
-add_definitions(-DBOOTLOADER_BUILD=1)
-
-set(COMPONENT_REQUIRES_COMMON log esp8266 spi_flash)
-
 include("${IDF_PATH}/tools/cmake/project.cmake")
+set(common_req log)
+if(LEGACY_INCLUDE_COMMON_HEADERS)
+    list(APPEND common_req soc)
+endif()
+idf_build_set_property(__COMPONENT_REQUIRES_COMMON "${common_req}")
+idf_build_set_property(__OUTPUT_SDKCONFIG 0)
 project(bootloader)
 
-target_linker_script(bootloader.elf
+idf_build_set_property(COMPILE_DEFINITIONS "-DBOOTLOADER_BUILD=1" APPEND)
-    "main/esp8266.bootloader.ld"
+idf_build_set_property(COMPILE_OPTIONS "-fno-stack-protector" APPEND)
-    "main/esp8266.bootloader.rom.ld")
+
-# Imported from esp8266 component
+string(REPLACE ";" " " espsecurepy "${ESPSECUREPY}")
-target_linker_script(bootloader.elf ${ESP8266_BOOTLOADER_LINKER_SCRIPTS})
+string(REPLACE ";" " " espefusepy "${ESPEFUSEPY}")
-target_link_libraries(bootloader.elf ${ESP8266_BOOTLOADER_LIBS})
+set(esptoolpy_write_flash "${ESPTOOLPY_WRITE_FLASH_STR}")
+
+if(CONFIG_SECURE_BOOTLOADER_REFLASHABLE)
+    if(CONFIG_SECURE_BOOTLOADER_KEY_ENCODING_192BIT)
+        set(key_digest_len 192)
+    else()
+        set(key_digest_len 256)
+    endif()
+
+    get_filename_component(bootloader_digest_bin
+        "bootloader-reflash-digest.bin"
+        ABSOLUTE BASE_DIR "${CMAKE_BINARY_DIR}")
+
+    get_filename_component(secure_bootloader_key
+        "secure-bootloader-key-${key_digest_len}.bin"
+        ABSOLUTE BASE_DIR "${CMAKE_BINARY_DIR}")
+
+    add_custom_command(OUTPUT "${secure_bootloader_key}"
+        COMMAND ${ESPSECUREPY} digest_private_key
+            --keylen "${key_digest_len}"
+            --keyfile "${SECURE_BOOT_SIGNING_KEY}"
+            "${secure_bootloader_key}"
+        VERBATIM)
+
+    if(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)
+        add_custom_target(gen_secure_bootloader_key ALL DEPENDS "${secure_bootloader_key}")
+    else()
+        if(NOT EXISTS "${secure_bootloader_key}")
+            message(FATAL_ERROR
+                "No pre-generated key for a reflashable secure bootloader is available, "
+                "due to signing configuration."
+                "\nTo generate one, you can use this command:"
+                "\n\t${espsecurepy} generate_flash_encryption_key ${secure_bootloader_key}"
+                "\nIf a signing key is present, then instead use:"
+                "\n\t${espsecurepy} digest_private_key "
+                "--keylen (192/256) --keyfile KEYFILE "
+                "${secure_bootloader_key}")
+        endif()
+        add_custom_target(gen_secure_bootloader_key)
+    endif()
+
+    add_custom_command(OUTPUT "${bootloader_digest_bin}"
+        COMMAND ${CMAKE_COMMAND} -E echo "DIGEST ${bootloader_digest_bin}"
+        COMMAND ${ESPSECUREPY} digest_secure_bootloader --keyfile "${secure_bootloader_key}"
+            -o "${bootloader_digest_bin}" "${CMAKE_BINARY_DIR}/bootloader.bin"
+        DEPENDS gen_secure_bootloader_key gen_project_binary
+        VERBATIM)
+
+    add_custom_target (gen_bootloader_digest_bin ALL DEPENDS "${bootloader_digest_bin}")
+endif()
+
+if(CONFIG_SECURE_BOOTLOADER_ONE_TIME_FLASH)
+    add_custom_command(TARGET bootloader.elf POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "=============================================================================="
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "Bootloader built. Secure boot enabled, so bootloader not flashed automatically."
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "One-time flash command is:"
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "\t${esptoolpy_write_flash} ${BOOTLOADER_OFFSET} ${CMAKE_BINARY_DIR}/bootloader.bin"
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "* IMPORTANT: After first boot, BOOTLOADER CANNOT BE RE-FLASHED on same device"
+        VERBATIM)
+elseif(CONFIG_SECURE_BOOTLOADER_REFLASHABLE)
+    add_custom_command(TARGET bootloader.elf POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "=============================================================================="
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "Bootloader built and secure digest generated."
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "Secure boot enabled, so bootloader not flashed automatically."
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "Burn secure boot key to efuse using:"
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "\t${espefusepy} burn_key secure_boot ${secure_bootloader_key}"
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "First time flash command is:"
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "\t${esptoolpy_write_flash} ${BOOTLOADER_OFFSET} ${CMAKE_BINARY_DIR}/bootloader.bin"
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "=============================================================================="
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "To reflash the bootloader after initial flash:"
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "\t${esptoolpy_write_flash} 0x0 ${bootloader_digest_bin}"
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "=============================================================================="
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "* After first boot, only re-flashes of this kind (with same key) will be accepted."
+        COMMAND ${CMAKE_COMMAND} -E echo
+            "* Not recommended to re-use the same secure boot keyfile on multiple production devices."
+        DEPENDS gen_secure_bootloader_key gen_bootloader_digest_bin
+        VERBATIM)
+endif()

components/bootloader/subproject/main/CMakeLists.txt

@@ -1,4 +1,11 @@
-set(COMPONENT_SRCS "bootloader_start.c")
+idf_component_register(SRCS "bootloader_start.c"
-set(COMPONENT_ADD_INCLUDEDIRS "")
+                    REQUIRES bootloader bootloader_support spi_flash)
-set(COMPONENT_REQUIRES "bootloader_support")
+
-register_component()
+idf_build_get_property(target IDF_TARGET)
+set(scripts "${target}.bootloader.ld"
+            "${target}.bootloader.rom.ld"
+            "${ESP8266_BOOTLOADER_LINKER_SCRIPTS}")
+
+target_linker_script(${COMPONENT_LIB} INTERFACE "${scripts}")
+
+target_link_libraries(${COMPONENT_LIB} INTERFACE "${ESP8266_BOOTLOADER_LIBS}")

components/bootloader/subproject/main/bootloader_start.c

@@ -20,8 +20,8 @@
 #include "bootloader_utility.h"
 #include "bootloader_common.h"
 #include "esp_image_format.h"
-#include "esp_log.h"
 #include "esp_spi_flash.h"
+#include "esp_log.h"
 
 static const char* TAG = "boot";
 

components/bootloader_support/src/bootloader_init.c

@@ -164,7 +164,7 @@ static esp_err_t bootloader_main()
     ESP_LOGI(TAG, "Enabling RNG early entropy source...");
     bootloader_random_enable();
 
-#if CONFIG_FLASHMODE_QIO || CONFIG_FLASHMODE_QOUT
+#if CONFIG_ESPTOOLPY_FLASHMODE_QIO || CONFIG_ESPTOOLPY_FLASHMODE_QOUT
     bootloader_enable_qio_mode();
 #endif
 
@@ -625,7 +625,7 @@ static esp_err_t bootloader_main()
 
     ESP_LOGI(TAG, "compile time " __TIME__ );
 
-#if defined(CONFIG_FLASHMODE_QIO) || defined(CONFIG_FLASHMODE_QOUT)
+#if defined(CONFIG_ESPTOOLPY_FLASHMODE_QIO) || defined(CONFIG_ESPTOOLPY_FLASHMODE_QOUT)
     fhdr.spi_mode = CONFIG_SPI_FLASH_MODE;
 #endif
 

components/bootloader_support/src/flash_qio_mode.c

@@ -208,7 +208,7 @@ static esp_err_t enable_qio_mode(read_status_fn_t read_status_fn,
     ESP_LOGD(TAG, "Enabling QIO mode...");
 
     esp_rom_spiflash_read_mode_t mode;
-#if CONFIG_FLASHMODE_QOUT
+#if CONFIG_ESPTOOLPY_FLASHMODE_QOUT
     mode = ESP_ROM_SPIFLASH_QOUT_MODE;
 #else
     mode = ESP_ROM_SPIFLASH_QIO_MODE;

components/cjson/CMakeLists.txt

@@ -6,5 +6,5 @@ set(COMPONENT_REQUIRES newlib)
 register_component()
 
 if(CONFIG_NEWLIB_LIBRARY_LEVEL_NORMAL)
-target_compile_definitions(${COMPONENT_NAME} PRIVATE -D CJSON_SPRINTF_FLOAT=1)
+target_compile_definitions(${COMPONENT_LIB} PRIVATE -DCJSON_SPRINTF_FLOAT=1)
 endif()

components/coap/CMakeLists.txt

@@ -26,7 +26,7 @@ register_component()
 # Needed for coap headers in public builds, also.
 #
 # TODO: find a way to move this to a port header
-target_compile_definitions(coap PUBLIC WITH_POSIX)
+target_compile_definitions(${COMPONENT_LIB} PUBLIC WITH_POSIX)
 
 set_source_files_properties(
     libcoap/src/debug.c
@@ -40,4 +40,4 @@ set_source_files_properties(
     PROPERTIES COMPILE_FLAGS
     -Wno-implicit-fallthrough)
 
-target_compile_definitions(${COMPONENT_NAME} PUBLIC -D WITH_POSIX)
+target_compile_definitions(${COMPONENT_LIB} PUBLIC -D WITH_POSIX)

components/esp8266/CMakeLists.txt

@@ -21,32 +21,72 @@ if(BOOTLOADER_BUILD)
 else()
     # Regular app build
 
-    set(COMPONENT_SRCDIRS "driver source")
+    set(srcs
-    set(COMPONENT_ADD_INCLUDEDIRS "include")
+        "source/chip_boot.c"
-    set(COMPONENT_PRIV_INCLUDEDIRS "include/driver")
+        "source/esp_err_to_name.c"
+        "source/esp_timer.c"
+        "source/esp_wifi_os_adapter.c"
+        "source/esp_wifi.c"
+        "source/ets_printf.c"
+        "source/event_default_handlers.c"
+        "source/event_loop.c"
+        "source/phy_init.c"
+        "source/reset_reason.c"
+        "source/startup.c"
+        "source/system_api.c"
+        "source/task_wdt.c"
+        "driver/adc.c"
+        "driver/gpio.c"
+        "driver/hw_timer.c"
+        "driver/i2c.c"
+        "driver/i2s.c"
+        "driver/pwm.c"
+        "driver/spi.c"
+        "driver/uart.c")
 
-    set(COMPONENT_REQUIRES newlib)
+    set(include_dirs "include" "include/driver")
-    # driver is a public requirement because esp_sleep.h uses gpio_num_t & touch_pad_t
-    # tcpip_adapter is a public requirement because esp_event.h uses tcpip_adapter types
-    set(COMPONENT_PRIV_REQUIRES "log" "nvs_flash" "spi_flash" "tcpip_adapter" "bootloader_support" "util" "esp_ringbuf")
 
-    register_component()
+    set(priv_requires "wpa_supplicant" "log" "spi_flash" "tcpip_adapter" "esp_ringbuf" "bootloader_support" "nvs_flash" "util")
+
+    idf_component_register(SRCS "${srcs}"
+                    INCLUDE_DIRS "${include_dirs}"
+                    REQUIRES "${requires}"
+                    PRIV_REQUIRES "${priv_requires}"
+                    REQUIRED_IDF_TARGETS esp8266)
+
+    target_link_libraries(${COMPONENT_LIB} PUBLIC "-L ${CMAKE_CURRENT_SOURCE_DIR}/lib" "-lstdc++")
 
-    target_link_libraries(esp8266 "-L ${CMAKE_CURRENT_SOURCE_DIR}/lib")
     if(NOT CONFIG_NO_BLOBS)
-        target_link_libraries(esp8266 gcc hal core net80211 phy pp smartconfig ssc wpa espnow wps)
+        set(blobs "gcc" "hal" "core" "net80211" "phy" "pp" "smartconfig" "ssc" "wpa" "espnow" "wps")
+        foreach(blob ${blobs})
+            add_library(${blob} STATIC IMPORTED)
+            set_property(TARGET ${blob} PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_SOURCE_DIR}/lib/lib${blob}.a)
+            target_link_libraries(${COMPONENT_LIB} PUBLIC ${blob})
+
+            foreach(_blob ${blobs})
+                if(NOT _blob STREQUAL ${blob})
+                    set_property(TARGET ${blob} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${_blob})
+                endif()
+            endforeach()
+
+            set_property(TARGET ${blob} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${COMPONENT_LIB})
+        endforeach()
     endif()
-    target_linker_script(esp8266 "${CMAKE_CURRENT_BINARY_DIR}/esp8266_out.ld" "${CMAKE_CURRENT_BINARY_DIR}/esp8266_common_out.ld")
+    target_linker_script(${COMPONENT_LIB} INTERFACE "${CMAKE_CURRENT_BINARY_DIR}/esp8266_out.ld")
+    target_linker_script(${COMPONENT_LIB} INTERFACE "${CMAKE_CURRENT_BINARY_DIR}/esp8266_common_out.ld")
 
-    target_linker_script(esp8266
+    target_linker_script(${COMPONENT_LIB} INTERFACE "ld/esp8266.rom.ld")
-        "ld/esp8266.rom.ld"
+    target_linker_script(${COMPONENT_LIB} INTERFACE "ld/esp8266.peripherals.ld")
-        "ld/esp8266.peripherals.ld"
-        )
 
-    target_link_libraries(esp8266 "-u call_user_start")
+    target_link_libraries(${COMPONENT_LIB} INTERFACE "-u call_user_start")
 
     # Preprocess esp8266.ld linker script to include configuration, becomes esp8266_out.ld
     set(LD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ld)
+
+    partition_table_get_partition_info(app_offset "--partition-boot-default" "offset")
+    partition_table_get_partition_info(app_size "--partition-boot-default" "size")
+    set(CFLAGS ${CFLAGS} -DAPP_OFFSET=${app_offset} -DAPP_SIZE=${app_size})
+
     add_custom_command(
         OUTPUT esp8266_out.ld
         COMMAND "${CMAKE_C_COMPILER}" -C -P -x c -E -o esp8266_out.ld ${CFLAGS} -I ${CONFIG_DIR} ${LD_DIR}/esp8266.ld
@@ -60,7 +100,7 @@ else()
         COMMENT "Generating section linker script..."
         VERBATIM)
     add_custom_target(esp8266_linker_script DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/esp8266_out.ld" "${CMAKE_CURRENT_BINARY_DIR}/esp8266_common_out.ld")
-    add_dependencies(esp8266 esp8266_linker_script)
+    add_dependencies(${COMPONENT_LIB} esp8266_linker_script)
 
     if(CONFIG_ESP8266_PHY_INIT_DATA_IN_PARTITION)
         set(PHY_INIT_DATA_BIN phy_init_data.bin)
@@ -82,15 +122,15 @@ else()
     endif()
 
     if(CONFIG_ESP_FILENAME_MACRO_NO_PATH)
-        target_compile_definitions(${COMPONENT_NAME} PUBLIC -D __ESP_FILE__=__FILE__)
+        target_compile_definitions(${COMPONENT_LIB} PUBLIC -D __ESP_FILE__=__FILE__)
     endif()
 
     if(CONFIG_ESP_FILENAME_MACRO_RAW)
-        target_compile_definitions(${COMPONENT_NAME} PUBLIC -D __ESP_FILE__=__FILE__)
+        target_compile_definitions(${COMPONENT_LIB} PUBLIC -D __ESP_FILE__=__FILE__)
     endif()
 
     if(CONFIG_ESP_FILENAME_MACRO_NULL)
-        target_compile_definitions(${COMPONENT_NAME} PUBLIC -D __ESP_FILE__="null")
+        target_compile_definitions(${COMPONENT_LIB} PUBLIC -D __ESP_FILE__="null")
     endif()
 
 endif()

components/esp8266/include/rom/crc.h

@@ -0,0 +1,4 @@
+
+#pragma once
+
+#include "esp_crc.h"

components/esptool_py/CMakeLists.txt

@@ -1,14 +1,81 @@
-register_config_only_component()
+idf_component_register(REQUIRES bootloader)
 
-# Generate pre-canned flasher args files suitable for passing to esptool.py
+if(NOT BOOTLOADER_BUILD)
-foreach(part project app bootloader partition_table)
+    string(REPLACE ";" " " ESPTOOLPY_FLASH_PROJECT_OPTIONS "${ESPTOOLPY_FLASH_OPTIONS}")
-    configure_file(
+    set(ESPTOOLPY_FLASH_PROJECT_OPTIONS
-        "${CMAKE_CURRENT_LIST_DIR}/flash_${part}_args.in"
+        "${ESPTOOLPY_FLASH_PROJECT_OPTIONS}"
-        "${CMAKE_BINARY_DIR}/flash_${part}_args"
         )
-endforeach()
 
-configure_file(
+    if(CONFIG_SECURE_BOOT_ENABLED)
-    "${CMAKE_CURRENT_LIST_DIR}/flasher_args.json.in"
+        set(ESPTOOLPY_FLASH_PROJECT_OPTIONS "")
-    "${CMAKE_BINARY_DIR}/flasher_args.json"
+    endif()
-    )
+
+    # FLASH_PROJECT_ARGS, FLASH_PROJECT_ARGS_JSON, FLASH_PROJECT_ARGS_ENTRY_JSON
+    # are used in the flasher args input files (flash_project_args.in, flasher_args.json.in)
+    idf_component_get_property(FLASH_PROJECT_ARGS ${COMPONENT_NAME} 
+                            FLASH_PROJECT_ARGS GENERATOR_EXPRESSION)
+    idf_component_get_property(FLASH_PROJECT_ARGS_JSON ${COMPONENT_NAME}  
+                            FLASH_PROJECT_ARGS_JSON GENERATOR_EXPRESSION)
+    idf_component_get_property(FLASH_PROJECT_ARGS_ENTRY_JSON ${COMPONENT_NAME} 
+                            FLASH_PROJECT_ARGS_ENTRY_JSON GENERATOR_EXPRESSION)
+
+    # Generate the flash project args and the flasher args json file using the accumulated values
+    # from esptool_py_flash_project_args calls. The file is first configured using configure_file() for all variable values,
+    # and then generated using file(GENERATE... for generator expressions.
+    configure_file(${COMPONENT_DIR}/flash_project_args.in
+                ${CMAKE_CURRENT_BINARY_DIR}/flash_project_args.in)
+
+    file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/flash_project_args.in2
+                INPUT ${CMAKE_CURRENT_BINARY_DIR}/flash_project_args.in)
+    file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/flash_project_args
+                INPUT ${CMAKE_CURRENT_BINARY_DIR}/flash_project_args.in2)
+
+    if(CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT)
+        configure_file(${COMPONENT_DIR}/flash_encrypted_project_args.in
+                    ${CMAKE_CURRENT_BINARY_DIR}/flash_encrypted_project_args.in)
+
+        file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/flash_encrypted_project_args.in2
+                    INPUT ${CMAKE_CURRENT_BINARY_DIR}/flash_encrypted_project_args.in)
+        file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/flash_encrypted_project_args
+                    INPUT ${CMAKE_CURRENT_BINARY_DIR}/flash_encrypted_project_args.in2)
+    endif()
+
+    configure_file(${COMPONENT_DIR}/flasher_args.json.in
+                ${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json.in)
+
+    file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json.in2
+                INPUT ${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json.in)
+    file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/flasher_args.json
+                INPUT ${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json.in2)
+
+    set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES
+        "${CMAKE_CURRENT_BINARY_DIR}/flash_project_args.in"
+        "${CMAKE_CURRENT_BINARY_DIR}/flash_project_args.in2"
+        "${CMAKE_BINARY_DIR}/flash_project_args"
+        "${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json.in"
+        "${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json.in2"
+        "${CMAKE_BINARY_DIR}/flasher_args.json")
+
+    idf_build_get_property(build_dir BUILD_DIR)
+    partition_table_get_partition_info(app_partition_offset "--partition-boot-default" "offset")
+    esptool_py_flash_project_args(app ${app_partition_offset} ${build_dir}/${PROJECT_BIN} FLASH_IN_PROJECT)
+
+    if(CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT)
+        file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/flash_encrypted_app_args.in "--encrypt ${app_partition_offset} ${PROJECT_BIN}")
+        esptool_py_flash_project_args(encrypted_app ${app_partition_offset} ${build_dir}/${PROJECT_BIN}
+                                    FLASH_FILE_TEMPLATE ${CMAKE_CURRENT_BINARY_DIR}/flash_encrypted_app_args.in)
+    endif()
+
+    add_dependencies(flash partition_table)
+
+    # If anti-rollback option is set then factory partition should not be in Partition Table.
+    # In this case, should be used the partition table with two ota app without the factory.
+    partition_table_get_partition_info(factory_offset "--partition-type app --partition-subtype factory" "offset")
+    if(CONFIG_BOOTLOADER_APP_ANTI_ROLLBACK AND factory_offset)
+        fail_at_build_time(check_table_contents
+            "ERROR: Anti-rollback option is enabled. Partition table should consist of two ota app without factory partition.")
+        add_dependencies(app check_table_contents)
+    endif()
+endif()
+

components/esptool_py/Kconfig.projbuild

@@ -48,20 +48,20 @@ config ESPTOOLPY_COMPRESSED
 
 choice FLASHMODE
 	prompt "Flash SPI mode"
-	default FLASHMODE_QIO
+	default ESPTOOLPY_FLASHMODE_QIO
 	help
 		Mode the flash chip is flashed in, as well as the default mode for the
 		binary to run in.
 
 		The esptool.py flashes firmware at DIO mode when user select "QIO", "QOUT" or "DIO" mode.
 
-config FLASHMODE_QIO
+config ESPTOOLPY_FLASHMODE_QIO
 	bool "QIO"
-config FLASHMODE_QOUT
+config ESPTOOLPY_FLASHMODE_QOUT
 	bool "QOUT"
-config FLASHMODE_DIO
+config ESPTOOLPY_FLASHMODE_DIO
 	bool "DIO"
-config FLASHMODE_DOUT
+config ESPTOOLPY_FLASHMODE_DOUT
 	bool "DOUT"
 endchoice
 
@@ -70,17 +70,17 @@ endchoice
 # itself to quad mode during initialisation
 config ESPTOOLPY_FLASHMODE
 	string
-	default "dio" if FLASHMODE_QIO
+	default "dio" if ESPTOOLPY_FLASHMODE_QIO
-	default "dio" if FLASHMODE_QOUT
+	default "dio" if ESPTOOLPY_FLASHMODE_QOUT
-	default "dio" if FLASHMODE_DIO
+	default "dio" if ESPTOOLPY_FLASHMODE_DIO
-	default "dout" if FLASHMODE_DOUT
+	default "dout" if ESPTOOLPY_FLASHMODE_DOUT
 
 config SPI_FLASH_MODE
 	hex
-	default 0x0 if FLASHMODE_QIO
+	default 0x0 if ESPTOOLPY_FLASHMODE_QIO
-	default 0x1 if FLASHMODE_QOUT
+	default 0x1 if ESPTOOLPY_FLASHMODE_QOUT
-	default 0x2 if FLASHMODE_DIO
+	default 0x2 if ESPTOOLPY_FLASHMODE_DIO
-	default 0x3 if FLASHMODE_DOUT	
+	default 0x3 if ESPTOOLPY_FLASHMODE_DOUT	
 
 choice ESPTOOLPY_FLASHFREQ
 	prompt "Flash SPI speed"
@@ -192,7 +192,7 @@ config ESPTOOLPY_AFTER
     default "soft_reset" if ESPTOOLPY_AFTER_SOFT_RESET
     default "no_reset" if ESPTOOLPY_AFTER_NORESET
 
-choice MONITOR_BAUD
+choice ESPTOOLPY_MONITOR_BAUD
     prompt "'make monitor' baud rate"
     default MONITOR_BAUD_74880B
     help
@@ -201,52 +201,38 @@ choice MONITOR_BAUD
 
         Can override by setting the MONITORBAUD environment variable.
 
-config MONITOR_BAUD_9600B
+config ESPTOOLPY_MONITOR_BAUD_9600B
     bool "9600 bps"
-config MONITOR_BAUD_57600B
+config ESPTOOLPY_MONITOR_BAUD_57600B
     bool "57600 bps"
-config MONITOR_BAUD_74880B
+config ESPTOOLPY_MONITOR_BAUD_74880B
     bool "74880 bps"
-config MONITOR_BAUD_115200B
+config ESPTOOLPY_MONITOR_BAUD_115200B
     bool "115200 bps"
-config MONITOR_BAUD_230400B
+config ESPTOOLPY_MONITOR_BAUD_230400B
     bool "230400 bps"
-config MONITOR_BAUD_921600B
+config ESPTOOLPY_MONITOR_BAUD_921600B
     bool "921600 bps"
-config MONITOR_BAUD_2MB
+config ESPTOOLPY_MONITOR_BAUD_2MB
     bool "2 Mbps"
-config MONITOR_BAUD_OTHER
+config ESPTOOLPY_MONITOR_BAUD_OTHER
     bool "Custom baud rate"
 
 endchoice
 
-config MONITOR_BAUD_OTHER_VAL
+config ESPTOOLPY_MONITOR_BAUD_OTHER_VAL
     int "Custom baud rate value" if MONITOR_BAUD_OTHER
     default 74880
 
-config MONITOR_BAUD
+config ESPTOOLPY_MONITOR_BAUD
     int
-    default 9600 if MONITOR_BAUD_9600B
+    default 9600 if ESPTOOLPY_MONITOR_BAUD_9600B
-    default 57600 if MONITOR_BAUD_57600B
+    default 57600 if ESPTOOLPY_MONITOR_BAUD_57600B
-    default 74880 if MONITOR_BAUD_74880B
+    default 74880 if ESPTOOLPY_MONITOR_BAUD_74880B
-    default 115200 if MONITOR_BAUD_115200B
+    default 115200 if ESPTOOLPY_MONITOR_BAUD_115200B
-    default 230400 if MONITOR_BAUD_230400B
+    default 230400 if ESPTOOLPY_MONITOR_BAUD_230400B
-    default 921600 if MONITOR_BAUD_921600B
+    default 921600 if ESPTOOLPY_MONITOR_BAUD_921600B
-    default 2000000 if MONITOR_BAUD_2MB
+    default 2000000 if ESPTOOLPY_MONITOR_BAUD_2MB
-    default MONITOR_BAUD_OTHER_VAL if MONITOR_BAUD_OTHER
+    default ESPTOOLPY_MONITOR_BAUD_OTHER_VAL if ESPTOOLPY_MONITOR_BAUD_OTHER
-
-config ESPTOOLPY_ENABLE_TIME
-	bool "Enable monitor time information"
-	default n
-	help
-		Enable this option, time string will be added at the head of serial input data line.
-
-config ESPTOOLPY_ENABLE_SAVELOG
-	bool "Enable console log save"
-	default n
-	help
-		Enable this option, all console output will auto-save to current 'log' directory which same level as the 'build` directory; 
-		if 'log' directory is not exist, will create it. 
-		All console output will be saved as a log filename, named PROJECT_NAME + timestamp.log, which PROJECT_NAME defined in your demo Makefile.
 
 endmenu

components/esptool_py/Makefile.projbuild

@@ -90,7 +90,7 @@ erase_flash:
 	@echo "Erasing entire flash..."
 	$(ESPTOOLPY_SERIAL) erase_flash
 
-MONITORBAUD ?= $(CONFIG_MONITOR_BAUD)
+MONITORBAUD ?= $(CONFIG_ESPTOOLPY_MONITOR_BAUD)
 
 MONITOR_PYTHON := $(PYTHON)
 
@@ -106,7 +106,7 @@ endif
 simple_monitor: $(call prereq_if_explicit,%flash)
 	$(MONITOR_PYTHON) -m serial.tools.miniterm --rts 0 --dtr 0 --raw $(ESPPORT) $(MONITORBAUD)
 
-MONITOR_OPTS := --baud $(MONITORBAUD) --port $(ESPPORT) --toolchain-prefix $(CONFIG_TOOLPREFIX) --make "$(MAKE)" --enable-time $(ENABLE_TIME) --enable-savelog $(ENABLE_SAVELOG)
+MONITOR_OPTS := --baud $(MONITORBAUD) --port $(ESPPORT) --toolchain-prefix $(CONFIG_SDK_TOOLPREFIX) --make "$(MAKE)"
 
 monitor: $(call prereq_if_explicit,%flash)
 	$(summary) MONITOR

components/esptool_py/flash_app_args.in

@@ -1 +0,0 @@
-${APP_PARTITION_OFFSET} ${PROJECT_NAME}.bin

components/esptool_py/flash_partition_table_args.in

@@ -1 +0,0 @@
-${PARTITION_TABLE_OFFSET} partition_table/partition-table.bin

components/esptool_py/flash_project_args.in

@@ -1,8 +1,3 @@
---flash_mode ${ESPFLASHMODE}
+${ESPTOOLPY_FLASH_PROJECT_OPTIONS}
---flash_size ${ESPFLASHSIZE}
+$<JOIN:${FLASH_PROJECT_ARGS},
---flash_freq ${ESPFLASHFREQ}
+>
-0x0000 bootloader/bootloader.bin
-${PARTITION_TABLE_OFFSET} partition_table/partition-table.bin
-${PHY_PARTITION_OFFSET} ${PHY_PARTITION_BIN_FILE}
-${OTADATA_PARTITION_OFFSET} ${BLANK_OTADATA_FILE}
-${APP_PARTITION_OFFSET} ${PROJECT_NAME}.bin

components/esptool_py/flasher_args.json.in

@@ -2,19 +2,19 @@
     "write_flash_args" : [ "--flash_mode", "${ESPFLASHMODE}",
                            "--flash_size", "${ESPFLASHSIZE}",
                            "--flash_freq", "${ESPFLASHFREQ}" ],
-    "flash_files" : {
+    "flash_settings" : {
-        "0x0000" : "bootloader/bootloader.bin",
+        "flash_mode": "${ESPFLASHMODE}",
-        "${PARTITION_TABLE_OFFSET}" : "partition_table/partition-table.bin",
+        "flash_size": "${ESPFLASHSIZE}",
-        "${PHY_PARTITION_OFFSET}" : "${PHY_PARTITION_BIN_FILE}",
+        "flash_freq": "${ESPFLASHFREQ}"
-        "${OTADATA_PARTITION_OFFSET}" : "${BLANK_OTADATA_FILE}",
-        "${APP_PARTITION_OFFSET}" : "${PROJECT_NAME}.bin"
     },
-    "bootloader" :      { "offset" : "0x1000",
+    "flash_files" : {
-                          "file" : "bootloader/bootloader.bin" },
+        $<JOIN:${FLASH_PROJECT_ARGS_JSON},,
-    "partition_table" : { "offset" : "${PARTITION_TABLE_OFFSET}",
+        >
-                          "file" : "partition_table/partition-table.bin" },
+    },
-    "otadata" : { "offset" : "${OTADATA_PARTITION_OFFSET}",
+    $<JOIN:${FLASH_PROJECT_ARGS_ENTRY_JSON},,
-                  "file" : "${BLANK_OTADATA_FILE}" },
+    >,
-    "app" :             { "offset" : "${APP_PARTITION_OFFSET}",
+    "extra_esptool_args" : {
-                          "file" : "${PROJECT_NAME}.bin" }
+        "after"  : "${ESPTOOLPY_AFTER}",
+        "before" : "${ESPTOOLPY_BEFORE}"
+    }
 }

components/esptool_py/project_include.cmake

@@ -1,61 +1,212 @@
 # Set some global esptool.py variables
 #
 # Many of these are read when generating flash_app_args & flash_project_args
-set(ESPCHIP "esp8266")
+idf_build_get_property(python PYTHON)
-set(ESPTOOLPY "${PYTHON}" "${CMAKE_CURRENT_LIST_DIR}/esptool/esptool.py" --chip "${ESPCHIP}")
+set(ESPTOOLPY ${python} "${CMAKE_CURRENT_LIST_DIR}/esptool/esptool.py" --chip esp8266)
-set(ESPSECUREPY "${PYTHON}" "${CMAKE_CURRENT_LIST_DIR}/esptool/espsecure.py")
+set(ESPSECUREPY ${python} "${CMAKE_CURRENT_LIST_DIR}/esptool/espsecure.py")
+set(ESPEFUSEPY ${python} "${CMAKE_CURRENT_LIST_DIR}/esptool/espefuse.py")
 
 set(ESPFLASHMODE ${CONFIG_ESPTOOLPY_FLASHMODE})
 set(ESPFLASHFREQ ${CONFIG_ESPTOOLPY_FLASHFREQ})
 set(ESPFLASHSIZE ${CONFIG_ESPTOOLPY_FLASHSIZE})
 
-set(ESPTOOLPY_SERIAL "${ESPTOOLPY}" --port "${ESPPORT}" --baud ${ESPBAUD})
+set(ESPTOOLPY_BEFORE "${CONFIG_ESPTOOLPY_BEFORE}")
+set(ESPTOOLPY_AFTER  "${CONFIG_ESPTOOLPY_AFTER}")
 
-set(ESPTOOLPY_ELF2IMAGE_FLASH_OPTIONS
+if(CONFIG_SECURE_BOOT_ENABLED OR CONFIG_SECURE_FLASH_ENC_ENABLED)
+    # If security enabled then override post flash option
+    set(ESPTOOLPY_AFTER "no_reset")
+endif()
+
+set(ESPTOOLPY_SERIAL "${ESPTOOLPY}"
+    --port "${ESPPORT}"
+    --baud ${ESPBAUD}
+    --before "${ESPTOOLPY_BEFORE}"
+    --after "${ESPTOOLPY_AFTER}"
+    )
+
+if(CONFIG_ESPTOOLPY_COMPRESSED)
+    set(ESPTOOLPY_COMPRESSED_OPT -z)
+else()
+    set(ESPTOOLPY_COMPRESSED_OPT -u)
+endif()
+
+set(ESPTOOLPY_FLASH_OPTIONS
     --flash_mode ${ESPFLASHMODE}
     --flash_freq ${ESPFLASHFREQ}
     --flash_size ${ESPFLASHSIZE}
     )
 
+# String for printing flash command
+string(REPLACE ";" " " ESPTOOLPY_WRITE_FLASH_STR
+    "${ESPTOOLPY} --port (PORT) --baud (BAUD) --before ${ESPTOOLPY_BEFORE} --after ${ESPTOOLPY_AFTER} "
+    "write_flash ${ESPTOOLPY_FLASH_OPTIONS} ${ESPTOOLPY_EXTRA_FLASH_OPTIONS} ${ESPTOOLPY_COMPRESSED_OPT}")
+
+if(CONFIG_SECURE_BOOT_ENABLED AND
+    NOT CONFIG_SECURE_BOOT_ALLOW_SHORT_APP_PARTITION
+    AND NOT BOOTLOADER_BUILD)
+    set(ESPTOOLPY_ELF2IMAGE_OPTIONS ${ESPTOOLPY_ELF2IMAGE_OPTIONS} --secure-pad)
+endif()
+
+if(BOOTLOADER_BUILD)
+    set(ESPTOOLPY_ELF2IMAGE_OPTIONS ${ESPTOOLPY_ELF2IMAGE_OPTIONS})
+else()
+    set(ESPTOOLPY_ELF2IMAGE_OPTIONS ${ESPTOOLPY_ELF2IMAGE_OPTIONS} "--version=3")
+endif()
+
+
 if(CONFIG_ESPTOOLPY_FLASHSIZE_DETECT)
     # Set ESPFLASHSIZE to 'detect' *after* elf2image options are generated,
     # as elf2image can't have 'detect' as an option...
     set(ESPFLASHSIZE detect)
 endif()
 
-# Set variables if the PHY data partition is in the flash
+idf_build_get_property(build_dir BUILD_DIR)
-if(CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION)
+
-    set(PHY_PARTITION_OFFSET   ${CONFIG_PHY_DATA_OFFSET})
+idf_build_get_property(elf_name EXECUTABLE_NAME GENERATOR_EXPRESSION)
-    set(PHY_PARTITION_BIN_FILE "esp32/phy_init_data.bin")
+idf_build_get_property(elf EXECUTABLE GENERATOR_EXPRESSION)
+
+if(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES AND NOT BOOTLOADER_BUILD)
+    set(unsigned_project_binary "${elf_name}-unsigned.bin")
+else()
+    set(unsigned_project_binary "${elf_name}.bin")
 endif()
 
-if(BOOTLOADER_BUILD)
+set(PROJECT_BIN "${elf_name}.bin")
-set(ESPTOOL_ELF2IMAGE_OPTIONS "")
-else()
-set(ESPTOOL_ELF2IMAGE_OPTIONS "--version=3")
-endif()
 
 #
 # Add 'app.bin' target - generates with elf2image
 #
-add_custom_command(OUTPUT "${PROJECT_NAME}.bin"
+add_custom_command(OUTPUT "${build_dir}/.bin_timestamp"
-    COMMAND ${ESPTOOLPY} elf2image ${ESPTOOLPY_ELF2IMAGE_FLASH_OPTIONS} ${ESPTOOL_ELF2IMAGE_OPTIONS} -o "${PROJECT_NAME}.bin" "${PROJECT_NAME}.elf"
+    COMMAND ${ESPTOOLPY} elf2image ${ESPTOOLPY_FLASH_OPTIONS} ${ESPTOOLPY_ELF2IMAGE_OPTIONS}
-    DEPENDS ${PROJECT_NAME}.elf
+        -o "${build_dir}/${unsigned_project_binary}" "${elf}"
+    COMMAND ${CMAKE_COMMAND} -E echo "Generated ${build_dir}/${unsigned_project_binary}"
+    COMMAND ${CMAKE_COMMAND} -E md5sum "${build_dir}/${unsigned_project_binary}" > "${build_dir}/.bin_timestamp"
+    DEPENDS ${elf}
     VERBATIM
+    WORKING_DIRECTORY ${build_dir}
+    COMMENT "Generating binary image from built executable"
     )
-add_custom_target(app ALL DEPENDS "${PROJECT_NAME}.bin")
+add_custom_target(gen_project_binary DEPENDS "${build_dir}/.bin_timestamp")
+
+set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES
+    "${build_dir}/${unsigned_project_binary}"
+    )
+
+add_custom_target(app ALL DEPENDS gen_project_binary)
+
+if(NOT BOOTLOADER_BUILD AND CONFIG_SECURE_SIGNED_APPS)
+    if(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)
+        # for locally signed secure boot image, add a signing step to get from unsigned app to signed app
+        add_custom_command(OUTPUT "${build_dir}/.signed_bin_timestamp"
+            COMMAND ${ESPSECUREPY} sign_data --keyfile ${secure_boot_signing_key}
+                -o "${build_dir}/${PROJECT_BIN}" "${build_dir}/${unsigned_project_binary}"
+            COMMAND ${CMAKE_COMMAND} -E echo "Generated signed binary image ${build_dir}/${PROJECT_BIN}"
+                                    "from ${build_dir}/${unsigned_project_binary}"
+            COMMAND ${CMAKE_COMMAND} -E md5sum "${build_dir}/${PROJECT_BIN}" > "${build_dir}/.signed_bin_timestamp"
+            DEPENDS "${build_dir}/.bin_timestamp"
+            VERBATIM
+            COMMENT "Generating signed binary image"
+            )
+        add_custom_target(gen_signed_project_binary DEPENDS "${build_dir}/.signed_bin_timestamp")
+        add_dependencies(gen_project_binary gen_signed_project_binary)
+
+        set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+            APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES
+            "${build_dir}/${PROJECT_BIN}"
+            )
+    else()
+        string(REPLACE ";" " " espsecurepy "${ESPSECUREPY}")
+        add_custom_command(TARGET app POST_BUILD
+            COMMAND ${CMAKE_COMMAND} -E echo
+                "App built but not signed. Sign app before flashing"
+            COMMAND ${CMAKE_COMMAND} -E echo
+                "\t${espsecurepy} sign_data --keyfile KEYFILE ${build_dir}/${PROJECT_BIN}"
+            VERBATIM)
+    endif()
+endif()
 
 #
 # Add 'flash' target - not all build systems can run this directly
 #
 function(esptool_py_custom_target target_name flasher_filename dependencies)
+    idf_build_get_property(idf_path IDF_PATH)
     add_custom_target(${target_name} DEPENDS ${dependencies}
-        COMMAND ${ESPTOOLPY} -p ${CONFIG_ESPTOOLPY_PORT} -b ${CONFIG_ESPTOOLPY_BAUD}
+        COMMAND ${CMAKE_COMMAND}
-        write_flash @flash_${flasher_filename}_args
+        -D IDF_PATH="${idf_path}"
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+        -D ESPTOOLPY="${ESPTOOLPY}"
+        -D ESPTOOL_ARGS="write_flash;@flash_${flasher_filename}_args"
+        -D ESPTOOL_WORKING_DIR="${build_dir}"
+        -P run_esptool.cmake
+        WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+        USES_TERMINAL
         )
 endfunction()
 
 esptool_py_custom_target(flash project "app;partition_table;bootloader")
 esptool_py_custom_target(app-flash app "app")
-esptool_py_custom_target(bootloader-flash bootloader "bootloader")
+
+if(CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT)
+    esptool_py_custom_target(encrypted-flash encrypted_project "app;partition_table;bootloader")
+    esptool_py_custom_target(encrypted-app-flash encrypted_app "app")
+endif()
+
+# esptool_py_flash_project_args
+#
+# Add file to the flasher args list, to be flashed at a particular offset.
+#
+# When a template FLASH_FILE_TEMPLATE is given, the variables OFFSET and IMAGE
+# hold the value of arguments offset and image, respectively.
+function(esptool_py_flash_project_args entry offset image)
+    set(options FLASH_IN_PROJECT)  # flash the image when flashing the project
+    set(single_value FLASH_FILE_TEMPLATE) # template file to use to be able to
+                                        # flash the image individually using esptool
+    cmake_parse_arguments(_ "${options}" "${single_value}" "" "${ARGN}")
+
+    if(${entry} IN_LIST flash_project_entries)
+        message(FATAL_ERROR "entry '${entry}' has already been added to flash project entries")
+    endif()
+
+    idf_component_set_property(esptool_py FLASH_PROJECT_ENTRIES "${entry}" APPEND)
+
+    idf_build_get_property(build_dir BUILD_DIR)
+    file(RELATIVE_PATH image ${build_dir} ${image})
+
+    # Generate the standalone flash file to flash the image individually using esptool
+    set(entry_flash_args ${build_dir}/flash_${entry}_args)
+    if(NOT __FLASH_FILE_TEMPLATE)
+        file(GENERATE OUTPUT ${entry_flash_args} CONTENT "${offset} ${image}")
+    else()
+        set(OFFSET ${offset})
+        set(IMAGE ${image})
+
+        get_filename_component(template_in "${__FLASH_FILE_TEMPLATE}" ABSOLUTE)
+        get_filename_component(template_name "${template_in}" NAME)
+        set(template_partial "${CMAKE_CURRENT_BINARY_DIR}/${template_name}.in2")
+
+        configure_file("${template_in}" "${template_partial}")
+        file(GENERATE OUTPUT ${entry_flash_args} INPUT "${template_partial}")
+        set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+                    APPEND PROPERTY
+                    ADDITIONAL_MAKE_CLEAN_FILES "${template_partial}")
+        unset(OFFSET)
+        unset(IMAGE)
+    endif()
+
+    set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+                APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${entry_flash_args})
+
+    # Generate standalone entries in the flasher args json file
+    idf_component_set_property(esptool_py FLASH_PROJECT_ARGS_ENTRY_JSON
+                "\"${entry}\" : { \"offset\" : \"${offset}\", \"file\" : \"${image}\" }" APPEND)
+
+    # Generate entries in the flasher args json file
+    if(__FLASH_IN_PROJECT)
+        idf_component_set_property(esptool_py FLASH_PROJECT_ARGS
+                                "${offset} ${image}" APPEND)
+
+        idf_component_set_property(esptool_py FLASH_PROJECT_ARGS_JSON
+                                "\"${offset}\" : \"${image}\"" APPEND)
+    endif()
+endfunction()

components/esptool_py/sdkconfig.rename

@@ -0,0 +1,17 @@
+# sdkconfig replacement configurations for deprecated options formatted as
+# CONFIG_DEPRECATED_OPTION CONFIG_NEW_OPTION
+
+CONFIG_FLASHMODE_QIO            CONFIG_ESPTOOLPY_FLASHMODE_QIO
+CONFIG_FLASHMODE_QOUT           CONFIG_ESPTOOLPY_FLASHMODE_QOUT
+CONFIG_FLASHMODE_DIO            CONFIG_ESPTOOLPY_FLASHMODE_DIO
+CONFIG_FLASHMODE_DOUT           CONFIG_ESPTOOLPY_FLASHMODE_DOUT
+
+CONFIG_MONITOR_BAUD             CONFIG_ESPTOOLPY_MONITOR_BAUD
+CONFIG_MONITOR_BAUD_9600B       CONFIG_ESPTOOLPY_MONITOR_BAUD_9600B
+CONFIG_MONITOR_BAUD_57600B      CONFIG_ESPTOOLPY_MONITOR_BAUD_57600B
+CONFIG_MONITOR_BAUD_115200B     CONFIG_ESPTOOLPY_MONITOR_BAUD_115200B
+CONFIG_MONITOR_BAUD_230400B     CONFIG_ESPTOOLPY_MONITOR_BAUD_230400B
+CONFIG_MONITOR_BAUD_921600B     CONFIG_ESPTOOLPY_MONITOR_BAUD_921600B
+CONFIG_MONITOR_BAUD_2MB         CONFIG_ESPTOOLPY_MONITOR_BAUD_2MB
+CONFIG_MONITOR_BAUD_OTHER       CONFIG_ESPTOOLPY_MONITOR_BAUD_OTHER
+CONFIG_MONITOR_BAUD_OTHER_VAL   CONFIG_ESPTOOLPY_MONITOR_BAUD_OTHER_VAL

components/freertos/CMakeLists.txt

@@ -1,10 +1,9 @@
 set(COMPONENT_ADD_INCLUDEDIRS include include/freertos include/freertos/private port/esp8266/include port/esp8266/include/freertos)
 set(COMPONENT_SRCDIRS "freertos" "port/esp8266")
-set(COMPONENT_REQUIRES "esp8266")
 
 register_component()
 
-target_link_libraries(freertos "-Wl,--undefined=uxTopUsedPriority")
+target_link_libraries(${COMPONENT_LIB} "-Wl,--undefined=uxTopUsedPriority")
 
 set_source_files_properties(
     tasks.c

components/lwip/CMakeLists.txt

@@ -29,6 +29,7 @@ if(CONFIG_USING_ESP_VFS)
 set(COMPONENT_SRCDIRS ${COMPONENT_SRCDIRS} "port")
 endif()
 
+set(COMPONENT_REQUIRES vfs)
 set(COMPONENT_PRIV_REQUIRES tcpip_adapter)
 
 register_component()

components/mqtt/CMakeLists.txt

@@ -26,5 +26,5 @@ set(COMPONENT_REQUIRES lwip http_parser ssl tcp_transport freertos lwip ssl)
 register_component()
 
 if(CONFIG_MQTT_USING_IBM)
-target_compile_options(${COMPONENT_NAME} PUBLIC -DMQTT_TASK -DMQTTCLIENT_PLATFORM_HEADER=MQTTFreeRTOS.h)
+target_compile_options(${COMPONENT_LIB} PUBLIC -DMQTT_TASK -DMQTTCLIENT_PLATFORM_HEADER=MQTTFreeRTOS.h)
 endif()

components/newlib/CMakeLists.txt

@@ -10,13 +10,27 @@ endif()
 
 set(LIBM m)
 
-set(COMPONENT_REQUIRES vfs)  # for sys/ioctl.h
+set(COMPONENT_PRIV_REQUIRES "vfs" "lwip")  # for sys/ioctl.h
 
 register_component()
 
-target_compile_definitions(newlib PUBLIC
+target_compile_definitions(${COMPONENT_LIB} PUBLIC
     -D_CLOCKS_PER_SEC_=CONFIG_FREERTOS_HZ -D_POSIX_THREADS=1 -D_UNIX98_THREAD_MUTEX_ATTRIBUTES=1
     )
 
-target_link_libraries(newlib "-L ${CMAKE_CURRENT_SOURCE_DIR}/newlib/lib")
+target_link_libraries(${COMPONENT_LIB} PUBLIC "-L ${CMAKE_CURRENT_SOURCE_DIR}/newlib/lib")
-target_link_libraries(newlib "${LIBC}" "${LIBM}")
+
+set(blobs "${LIBC}" "${LIBM}" "gcc")
+foreach(blob ${blobs})
+    add_library(${blob} STATIC IMPORTED)
+    set_property(TARGET ${blob} PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_SOURCE_DIR}/newlib/lib/lib${blob}.a)
+    target_link_libraries(${COMPONENT_LIB} PUBLIC ${blob})
+
+    foreach(_blob ${blobs})
+        if(NOT _blob STREQUAL ${blob})
+            set_property(TARGET ${blob} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${_blob})
+        endif()
+    endforeach()
+
+    set_property(TARGET ${blob} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${COMPONENT_LIB})
+endforeach()

components/nvs_flash/CMakeLists.txt

@@ -1,12 +1,6 @@
 set(COMPONENT_SRCDIRS "src")
 set(COMPONENT_ADD_INCLUDEDIRS "include")
 
-set(COMPONENT_REQUIRES "spi_flash" "util")
+set(COMPONENT_PRIV_REQUIRES "spi_flash" "util")
 
 register_component()
-
-target_link_libraries(nvs_flash stdc++)
-
-component_compile_definitions(
-    NVS_CRC_HEADER_FILE="crc.h"
-)

components/nvs_flash/src/nvs_page.cpp

@@ -13,11 +13,7 @@
 // limitations under the License.
 #include "nvs_page.hpp"
 #if defined(ESP_PLATFORM)
-#if defined(NVS_CRC_HEADER_FILE)
-#include NVS_CRC_HEADER_FILE
-#else
 #include <rom/crc.h>
-#endif
 #else
 #include "crc.h"
 #endif

components/nvs_flash/src/nvs_types.cpp

@@ -14,11 +14,7 @@
 #include "nvs_types.hpp"
 
 #if defined(ESP_PLATFORM)
-#if defined(NVS_CRC_HEADER_FILE)
-#include NVS_CRC_HEADER_FILE
-#else
 #include <rom/crc.h>
-#endif
 #else
 #include "crc.h"
 #endif

components/partition_table/CMakeLists.txt

@@ -1,6 +1,8 @@
-register_config_only_component()
+idf_component_register()
 
-if(NOT BOOTLOADER_BUILD)
+if(BOOTLOADER_BUILD)
+    return()
+endif()
 
 set(partition_csv "${PARTITION_CSV_PATH}")
 
@@ -14,63 +16,67 @@ else()
     set(final_partition_target "build_partition_table")
 endif()
 
-if(CONFIG_PARTITION_TABLE_MD5)
+set(md5_opt --disable-md5sum)
-    set(md5_opt --disable-md5sum)
-endif()
 
 if(CONFIG_ESPTOOLPY_FLASHSIZE)
     set(flashsize_opt --flash-size ${CONFIG_ESPTOOLPY_FLASHSIZE})
 endif()
 
-set(PARTITION_MD5_OPT "--disable-md5sum")
+if(CONFIG_SECURE_BOOT_ENABLED AND NOT CONFIG_SECURE_BOOT_ALLOW_SHORT_APP_PARTITION)
-set(PARTITION_FLASHSIZE_OPT "--flash-size" "${CONFIG_ESPTOOLPY_FLASHSIZE}")
+    set(partition_secure_opt --secure)
-set(PARTITION_TABLE_OFFSET "${CONFIG_PARTITION_TABLE_OFFSET}")
+else()
-set(PARTITION_TABLE_OFFSET_ARG "--offset" "${PARTITION_TABLE_OFFSET}")
+    set(partition_secure_opt "")
+endif()
 
-add_custom_command(OUTPUT "${unsigned_partition_bin}"
+idf_build_get_property(build_dir BUILD_DIR)
-    COMMAND "${PYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/gen_esp32part.py" -q
+idf_build_get_property(python PYTHON)
-    ${PARTITION_MD5_OPT} ${PARTITION_FLASHSIZE_OPT} ${PARTITION_TABLE_OFFSET_ARG}
+
-    ${partition_csv} ${unsigned_partition_bin}
+add_custom_command(OUTPUT "${build_dir}/partition_table/${unsigned_partition_bin}"
+    COMMAND "${python}" "${CMAKE_CURRENT_SOURCE_DIR}/gen_esp32part.py"
+    -q --offset ${PARTITION_TABLE_OFFSET} ${md5_opt} ${flashsize_opt}
+    ${partition_secure_opt} ${partition_csv} ${build_dir}/partition_table/${unsigned_partition_bin}
     DEPENDS ${partition_csv} "${CMAKE_CURRENT_SOURCE_DIR}/gen_esp32part.py"
     VERBATIM)
 
-# Add signing steps
-if(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)
-    get_filename_component(secure_boot_signing_key
-        "${CONFIG_SECURE_BOOT_SIGNING_KEY}"
-        ABSOLUTE BASE_DIR "${PROJECT_PATH}")
-
-    add_custom_command(OUTPUT "${final_partition_bin}"
-        COMMAND "${PYTHON}" "${ESPSECUREPY}" sign_data --keyfile "${secure_boot_signing_key}"
-        -o "${final_partition_bin}" "${unsigned_partition_bin}"
-        DEPENDS "${unsigned_partition_bin}"
-        VERBATIM)
-
-endif()
-
 if(EXISTS ${partition_csv})
-    add_custom_target(partition_table ALL DEPENDS "${final_partition_bin}")
+    add_custom_target(partition_table ALL DEPENDS "${build_dir}/partition_table/${final_partition_bin}")
 else()
-    # This is a bit of a hack: If the partition input CSV is not found, create a phony partition_table target that
+    # If the partition input CSV is not found, create a phony partition_table target that
-    # fails the build. Have it also touch CMakeCache.txt to cause a cmake run next time
+    # fails the build. fail_at_build_time also touches CMakeCache.txt to cause a cmake run next time
     # (to pick up a new CSV if one exists, etc.)
-    #
+    fail_at_build_time(partition_table
-    # This is because partition CSV is required at CMake runtime (to generate metadata files with flashing data, etc) but we can't
+        "Partition table CSV ${partition_csv} does not exist."
-    # fail the build if it is not found, because the "menuconfig" target may be required to fix the problem. CMAKE_CONFIGURE_DEPENDS
+        "Either change partition table in menuconfig or create this input file.")
-    # only works for files which exist at CMake runtime.
-    add_custom_target(partition_table ALL
-        COMMAND ${CMAKE_COMMAND} -E echo "Partition table CSV ${partition_csv} does not exist. Either change partition table in menuconfig or create this input file."
-        COMMAND ${CMAKE_COMMAND} -E touch "${CMAKE_BINARY_DIR}/CMakeCache.txt"
-        COMMAND ${CMAKE_COMMAND} -P ${IDF_PATH}/tools/cmake/scripts/fail.cmake)
 endif()
 
-add_dependencies(bootloader partition_table)
+# Add signing steps
-add_dependencies(app partition_table)
+if(CONFIG_SECURE_SIGNED_APPS)
+    if(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)
+        add_custom_target(gen_unsigned_partition_bin ALL DEPENDS
+                        "${build_dir}/partition_table/${unsigned_partition_bin}")
+
+        add_custom_command(OUTPUT "${build_dir}/partition_table/${final_partition_bin}"
+            COMMAND ${ESPSECUREPY} sign_data --keyfile "${SECURE_BOOT_SIGNING_KEY}"
+            -o "${build_dir}/partition_table/${final_partition_bin}"
+            "${build_dir}/partition_table/${unsigned_partition_bin}"
+            DEPENDS "${build_dir}/partition_table/${unsigned_partition_bin}"
+            VERBATIM)
+    else()
+        string(REPLACE ";" " " espsecurepy "${ESPSECUREPY}")
+        add_custom_command(TARGET partition_table POST_BUILD
+            COMMAND ${CMAKE_COMMAND} -E echo
+                "Partition table built but not signed. Sign partition data before flashing:"
+            COMMAND ${CMAKE_COMMAND} -E echo 
+                "\t${espsecurepy} sign_data --keyfile KEYFILE ${build_dir}/partition_table/${final_partition_bin}"
+            VERBATIM)
+    endif()
+endif()
 
 # Use global properties ESPTOOL_WRITE_FLASH_ARGS to pass this info to build
 # the list of esptool write arguments for flashing
 set_property(GLOBAL APPEND_STRING PROPERTY
     ESPTOOL_WRITE_FLASH_ARGS
-    "${PARTITION_TABLE_OFFSET} ${final_partition_bin} ")
+    "${PARTITION_TABLE_OFFSET}  ${build_dir}/partition_table/${final_partition_bin} ")
 
-endif()
+esptool_py_flash_project_args(partition_table ${PARTITION_TABLE_OFFSET}
+                              ${build_dir}/partition_table/partition-table.bin FLASH_IN_PROJECT)

components/partition_table/Makefile.projbuild

@@ -56,13 +56,27 @@ $(PARTITION_TABLE_BIN_UNSIGNED): $(PARTITION_TABLE_CSV_PATH) $(SDKCONFIG_MAKEFIL
 all_binaries: $(PARTITION_TABLE_BIN) partition_table_get_info
 
 partition_table_get_info: $(PARTITION_TABLE_BIN)
-	$(eval OTA_DATA_SIZE := $(shell $(GET_PART_INFO) --type data --subtype ota --size $(PARTITION_TABLE_BIN) || echo 0))
+	$(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO)  --partition-table-file $(PARTITION_TABLE_BIN) \
-	$(eval OTA_DATA_OFFSET := $(shell $(GET_PART_INFO) --type data --subtype ota --offset $(PARTITION_TABLE_BIN)))
+							--partition-table-offset $(PARTITION_TABLE_OFFSET) \
-	$(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO) --type data --subtype phy --offset $(PARTITION_TABLE_BIN)))
+							get_partition_info --partition-type data --partition-subtype phy --info offset))
-	$(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --default-boot-partition --offset $(PARTITION_TABLE_BIN)))
+	$(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \
-	$(eval APP_SIZE:=$(shell $(GET_PART_INFO) --default-boot-partition --size $(PARTITION_TABLE_BIN)))
+							--partition-table-offset $(PARTITION_TABLE_OFFSET) \
-	$(eval APP2_OFFSET:=$(shell $(GET_PART_INFO) --type app --subtype ota_1 --offset $(PARTITION_TABLE_BIN)))
+							get_partition_info --partition-boot-default --info offset))
-	$(eval APP2_SIZE:=$(shell $(GET_PART_INFO) --type app --subtype ota_1 --size $(PARTITION_TABLE_BIN)))
+	$(eval APP_SIZE:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \
+							--partition-table-offset $(PARTITION_TABLE_OFFSET) \
+							get_partition_info --partition-boot-default --info size))
+	$(eval OTA_DATA_OFFSET:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \
+							--partition-table-offset $(PARTITION_TABLE_OFFSET) \
+							get_partition_info --partition-type data --partition-subtype ota --info offset))
+	$(eval OTA_DATA_SIZE:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \
+							--partition-table-offset $(PARTITION_TABLE_OFFSET) \
+							get_partition_info --partition-type data --partition-subtype ota --info size))
+	$(eval APP2_OFFSET:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \
+							--partition-table-offset $(PARTITION_TABLE_OFFSET) \
+							get_partition_info --partition-type app --partition-subtype ota_1 --info offset))
+	$(eval APP2_SIZE:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \
+							--partition-table-offset $(PARTITION_TABLE_OFFSET) \
+							get_partition_info --partition-type app --partition-subtype ota_1 --info size))
 	$(eval CFLAGS += -DAPP_OFFSET=$(APP_OFFSET) -DAPP_SIZE=$(APP_SIZE))
 
 export OTA_DATA_SIZE OTA_DATA_OFFSET PHY_DATA_OFFSET APP_OFFSET APP_SIZE APP2_OFFSET APP2_SIZE

components/partition_table/gen_empty_partition.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 #
 # generates an empty binary file
-# 
+#
 # This tool generates an empty binary file of the required size.
 #
 # Copyright 2018 Espressif Systems (Shanghai) PTE LTD
@@ -20,27 +20,12 @@
 from __future__ import print_function, division
 from __future__ import unicode_literals
 import argparse
-import os
-import re
-import struct
 import sys
-import hashlib
-import binascii
 
 __version__ = '1.0'
 
 quiet = False
 
-def status(msg):
-    """ Print status message to stderr """
-    if not quiet:
-        critical(msg)
-
-def critical(msg):
-    """ Print critical message to stderr """
-    if not quiet:
-        sys.stderr.write(msg)
-        sys.stderr.write('\n')
 
 def generate_blanked_file(size, output_path):
     output = b"\xFF" * size
@@ -51,28 +36,25 @@ def generate_blanked_file(size, output_path):
     with stdout_binary if output_path == '-' else open(output_path, 'wb') as f:
         f.write(output)
 
+
 def main():
-    global quiet
     parser = argparse.ArgumentParser(description='Generates an empty binary file of the required size.')
+    parser.add_argument('size', help='Size of generated the file', type=str)
 
-    parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
-
-    parser.add_argument('--size', help='Size of generated the file',  type=str, required=True)
-    
     parser.add_argument('output', help='Path for binary file.', nargs='?', default='-')
     args = parser.parse_args()
 
-    quiet = args.quiet
-
     size = int(args.size, 0)
-    if size > 0 :
+    if size > 0:
         generate_blanked_file(size, args.output)
     return 0
-    
+
+
 class InputError(RuntimeError):
     def __init__(self, e):
         super(InputError, self).__init__(e)
 
+
 if __name__ == '__main__':
     try:
         r = main()

components/partition_table/gen_esp32part.py

@@ -21,6 +21,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from __future__ import print_function, division
+from __future__ import unicode_literals
 import argparse
 import os
 import re
@@ -28,28 +29,61 @@ import struct
 import sys
 import hashlib
 import binascii
+import errno
 
 MAX_PARTITION_LENGTH = 0xC00   # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature
-MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum
+MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14  # The first 2 bytes are like magic numbers for MD5 sum
 PARTITION_TABLE_SIZE  = 0x1000  # Size of partition table
 
-__version__ = '1.1'
+MIN_PARTITION_SUBTYPE_APP_OTA = 0x10
+NUM_PARTITION_SUBTYPE_APP_OTA = 16
+
+__version__ = '1.2'
+
+APP_TYPE = 0x00
+DATA_TYPE = 0x01
+
+TYPES = {
+    "app": APP_TYPE,
+    "data": DATA_TYPE,
+}
+
+# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h
+SUBTYPES = {
+    APP_TYPE: {
+        "factory": 0x00,
+        "test": 0x20,
+    },
+    DATA_TYPE: {
+        "ota": 0x00,
+        "phy": 0x01,
+        "nvs": 0x02,
+        "coredump": 0x03,
+        "nvs_keys": 0x04,
+        "efuse": 0x05,
+        "esphttpd": 0x80,
+        "fat": 0x81,
+        "spiffs": 0x82,
+    },
+}
 
 quiet = False
 md5sum = True
 secure = False
 offset_part_table = 0
 
+
 def status(msg):
     """ Print status message to stderr """
     if not quiet:
         critical(msg)
 
+
 def critical(msg):
     """ Print critical message to stderr """
-    if not quiet:
+    sys.stderr.write(msg)
-        sys.stderr.write(msg)
+    sys.stderr.write('\n')
-        sys.stderr.write('\n')
+
 
 class PartitionTable(list):
     def __init__(self):
@@ -72,21 +106,25 @@ class PartitionTable(list):
             if line.startswith("#") or len(line) == 0:
                 continue
             try:
-                res.append(PartitionDefinition.from_csv(line))
+                res.append(PartitionDefinition.from_csv(line, line_no + 1))
             except InputError as e:
-                raise InputError("Error at line %d: %s" % (line_no+1, e))
+                raise InputError("Error at line %d: %s" % (line_no + 1, e))
             except Exception:
-                critical("Unexpected error parsing line %d: %s" % (line_no+1, line))
+                critical("Unexpected error parsing CSV line %d: %s" % (line_no + 1, line))
                 raise
 
         # fix up missing offsets & negative sizes
-        last_end = offset_part_table + PARTITION_TABLE_SIZE # first offset after partition table
+        last_end = offset_part_table + PARTITION_TABLE_SIZE  # first offset after partition table
         for e in res:
-            if offset_part_table != 0 and e.offset is not None and e.offset < last_end:
+            if e.offset is not None and e.offset < last_end:
-                critical("WARNING: 0x%x address in the partition table is below 0x%x" % (e.offset, last_end))
+                if e == res[0]:
-                e.offset = None
+                    raise InputError("CSV Error: First partition offset 0x%x overlaps end of partition table 0x%x"
+                                     % (e.offset, last_end))
+                else:
+                    raise InputError("CSV Error: Partitions overlap. Partition at line %d sets offset 0x%x. Previous partition ends 0x%x"
+                                     % (e.line_no, e.offset, last_end))
             if e.offset is None:
-                pad_to = 0x10000 if e.type == PartitionDefinition.APP_TYPE else 4
+                pad_to = 0x1000 if e.type == APP_TYPE else 4
                 if last_end % pad_to != 0:
                     last_end += pad_to - (last_end % pad_to)
                 e.offset = last_end
@@ -110,21 +148,19 @@ class PartitionTable(list):
     def find_by_type(self, ptype, subtype):
         """ Return a partition by type & subtype, returns
         None if not found """
-        TYPES = PartitionDefinition.TYPES
-        SUBTYPES = PartitionDefinition.SUBTYPES
         # convert ptype & subtypes names (if supplied this way) to integer values
         try:
             ptype = TYPES[ptype]
         except KeyError:
             try:
-                ptypes = int(ptype, 0)
+                ptype = int(ptype, 0)
             except TypeError:
                 pass
         try:
             subtype = SUBTYPES[int(ptype)][subtype]
         except KeyError:
             try:
-                ptypes = int(ptype, 0)
+                ptype = int(ptype, 0)
             except TypeError:
                 pass
 
@@ -143,13 +179,26 @@ class PartitionTable(list):
         # verify each partition individually
         for p in self:
             p.verify()
+
+        # check on duplicate name
+        names = [p.name for p in self]
+        duplicates = set(n for n in names if names.count(n) > 1)
+
+        # print sorted duplicate partitions by name
+        if len(duplicates) != 0:
+            print("A list of partitions that have the same name:")
+            for p in sorted(self, key=lambda x:x.name):
+                if len(duplicates.intersection([p.name])) != 0:
+                    print("%s" % (p.to_csv()))
+            raise InputError("Partition names must be unique")
+
         # check for overlaps
         last = None
         for p in sorted(self, key=lambda x:x.offset):
             if p.offset < offset_part_table + PARTITION_TABLE_SIZE:
                 raise InputError("Partition offset 0x%x is below 0x%x" % (p.offset, offset_part_table + PARTITION_TABLE_SIZE))
             if last is not None and p.offset < last.offset + last.size:
-                raise InputError("Partition at 0x%x overlaps 0x%x-0x%x" % (p.offset, last.offset, last.offset+last.size-1))
+                raise InputError("Partition at 0x%x overlaps 0x%x-0x%x" % (p.offset, last.offset, last.offset + last.size - 1))
             last = p
 
     def flash_size(self):
@@ -164,17 +213,17 @@ class PartitionTable(list):
 
     @classmethod
     def from_binary(cls, b):
-        md5 = hashlib.md5();
+        md5 = hashlib.md5()
         result = cls()
         for o in range(0,len(b),32):
-            data = b[o:o+32]
+            data = b[o:o + 32]
             if len(data) != 32:
                 raise InputError("Partition table length must be a multiple of 32 bytes")
-            if data == b'\xFF'*32:
+            if data == b'\xFF' * 32:
                 return result  # got end marker
-            if md5sum and data[:2] == MD5_PARTITION_BEGIN[:2]: #check only the magic number part
+            if md5sum and data[:2] == MD5_PARTITION_BEGIN[:2]:  # check only the magic number part
                 if data[16:] == md5.digest():
-                    continue # the next iteration will check for the end marker
+                    continue  # the next iteration will check for the end marker
                 else:
                     raise InputError("MD5 checksums don't match! (computed: 0x%s, parsed: 0x%s)" % (md5.hexdigest(), binascii.hexlify(data[16:])))
             else:
@@ -186,59 +235,35 @@ class PartitionTable(list):
         result = b"".join(e.to_binary() for e in self)
         if md5sum:
             result += MD5_PARTITION_BEGIN + hashlib.md5(result).digest()
-        if len(result )>= MAX_PARTITION_LENGTH:
+        if len(result) >= MAX_PARTITION_LENGTH:
             raise InputError("Binary partition table length (%d) longer than max" % len(result))
         result += b"\xFF" * (MAX_PARTITION_LENGTH - len(result))  # pad the sector, for signing
         return result
 
     def to_csv(self, simple_formatting=False):
-        rows = [ "# Espressif ESP32 Partition Table",
+        rows = ["# Espressif ESP32 Partition Table",
-                 "# Name, Type, SubType, Offset, Size, Flags" ]
+                "# Name, Type, SubType, Offset, Size, Flags"]
-        rows += [ x.to_csv(simple_formatting) for x in self ]
+        rows += [x.to_csv(simple_formatting) for x in self]
         return "\n".join(rows) + "\n"
 
+
 class PartitionDefinition(object):
-    APP_TYPE = 0x00
-    DATA_TYPE = 0x01
-    TYPES = {
-        "app" : APP_TYPE,
-        "data" : DATA_TYPE,
-    }
-
-    # Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h 
-    SUBTYPES = {
-        APP_TYPE : {
-            "factory" : 0x00,
-            "test" : 0x20,
-            },
-        DATA_TYPE : {
-            "ota" : 0x00,
-            "phy" : 0x01,
-            "nvs" : 0x02,
-            "coredump" : 0x03,
-            "nvs_keys" : 0x04,
-            "esphttpd" : 0x80,
-            "fat" : 0x81,
-            "spiffs" : 0x82,
-            },
-    }
-
     MAGIC_BYTES = b"\xAA\x50"
 
     ALIGNMENT = {
-        APP_TYPE : 0x1000,
+        APP_TYPE: 0x1000,
-        DATA_TYPE : 0x04,
+        DATA_TYPE: 0x04,
     }
 
     # dictionary maps flag name (as used in CSV flags list, property name)
     # to bit set in flags words in binary format
     FLAGS = {
-        "encrypted" : 0
+        "encrypted": 0
     }
 
     # add subtypes for the 16 OTA slot values ("ota_XX, etc.")
-    for ota_slot in range(16):
+    for ota_slot in range(NUM_PARTITION_SUBTYPE_APP_OTA):
-        SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = 0x10 + ota_slot
+        SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = MIN_PARTITION_SUBTYPE_APP_OTA + ota_slot
 
     def __init__(self):
         self.name = ""
@@ -249,12 +274,13 @@ class PartitionDefinition(object):
         self.encrypted = False
 
     @classmethod
-    def from_csv(cls, line):
+    def from_csv(cls, line, line_no):
         """ Parse a line from the CSV """
         line_w_defaults = line + ",,,,"  # lazy way to support default fields
-        fields = [ f.strip() for f in line_w_defaults.split(",") ]
+        fields = [f.strip() for f in line_w_defaults.split(",")]
 
         res = PartitionDefinition()
+        res.line_no = line_no
         res.name = fields[0]
         res.type = res.parse_type(fields[1])
         res.subtype = res.parse_subtype(fields[2])
@@ -281,7 +307,7 @@ class PartitionDefinition(object):
         def maybe_hex(x):
             return "0x%x" % x if x is not None else "None"
         return "PartitionDefinition('%s', 0x%x, 0x%x, %s, %s)" % (self.name, self.type, self.subtype or 0,
-                                                              maybe_hex(self.offset), maybe_hex(self.size))
+                                                                  maybe_hex(self.offset), maybe_hex(self.size))
 
     def __str__(self):
         return "Part '%s' %d/%d @ 0x%x size 0x%x" % (self.name, self.type, self.subtype, self.offset or -1, self.size or -1)
@@ -304,12 +330,12 @@ class PartitionDefinition(object):
     def parse_type(self, strval):
         if strval == "":
             raise InputError("Field 'type' can't be left empty.")
-        return parse_int(strval, self.TYPES)
+        return parse_int(strval, TYPES)
 
     def parse_subtype(self, strval):
         if strval == "":
-            return 0 # default
+            return 0  # default
-        return parse_int(strval, self.SUBTYPES.get(self.type, {}))
+        return parse_int(strval, SUBTYPES.get(self.type, {}))
 
     def parse_address(self, strval):
         if strval == "":
@@ -331,7 +357,17 @@ class PartitionDefinition(object):
         if self.size is None:
             raise ValidationError(self, "Size field is not set")
 
-    STRUCT_FORMAT = "<2sBBLL16sL"
+        if self.name in TYPES and TYPES.get(self.name, "") != self.type:
+            critical("WARNING: Partition has name '%s' which is a partition type, but does not match this partition's "
+                     "type (0x%x). Mistake in partition table?" % (self.name, self.type))
+        all_subtype_names = []
+        for names in (t.keys() for t in SUBTYPES.values()):
+            all_subtype_names += names
+        if self.name in all_subtype_names and SUBTYPES.get(self.type, {}).get(self.name, "") != self.subtype:
+            critical("WARNING: Partition has name '%s' which is a partition subtype, but this partition has "
+                     "non-matching type 0x%x and subtype 0x%x. Mistake in partition table?" % (self.name, self.type, self.subtype))
+
+    STRUCT_FORMAT = b"<2sBBLL16sL"
 
     @classmethod
     def from_binary(cls, b):
@@ -340,21 +376,21 @@ class PartitionDefinition(object):
         res = cls()
         (magic, res.type, res.subtype, res.offset,
          res.size, res.name, flags) = struct.unpack(cls.STRUCT_FORMAT, b)
-        if b"\x00" in res.name: # strip null byte padding from name string
+        if b"\x00" in res.name:  # strip null byte padding from name string
             res.name = res.name[:res.name.index(b"\x00")]
         res.name = res.name.decode()
         if magic != cls.MAGIC_BYTES:
             raise InputError("Invalid magic bytes (%r) for partition definition" % magic)
         for flag,bit in cls.FLAGS.items():
-            if flags & (1<<bit):
+            if flags & (1 << bit):
                 setattr(res, flag, True)
-                flags &= ~(1<<bit)
+                flags &= ~(1 << bit)
         if flags != 0:
             critical("WARNING: Partition definition had unknown flag(s) 0x%08x. Newer binary format?" % flags)
         return res
 
     def get_flags_list(self):
-        return [ flag for flag in self.FLAGS.keys() if getattr(self, flag) ]
+        return [flag for flag in self.FLAGS.keys() if getattr(self, flag)]
 
     def to_binary(self):
         flags = sum((1 << self.FLAGS[flag]) for flag in self.get_flags_list())
@@ -368,14 +404,14 @@ class PartitionDefinition(object):
     def to_csv(self, simple_formatting=False):
         def addr_format(a, include_sizes):
             if not simple_formatting and include_sizes:
-                for (val, suffix) in [ (0x100000, "M"), (0x400, "K") ]:
+                for (val, suffix) in [(0x100000, "M"), (0x400, "K")]:
                     if a % val == 0:
                         return "%d%s" % (a // val, suffix)
             return "0x%x" % a
 
         def lookup_keyword(t, keywords):
             for k,v in keywords.items():
-                if simple_formatting == False and t == v:
+                if simple_formatting is False and t == v:
                     return k
             return "%d" % t
 
@@ -383,12 +419,12 @@ class PartitionDefinition(object):
             """ colon-delimited list of flags """
             return ":".join(self.get_flags_list())
 
-        return ",".join([ self.name,
+        return ",".join([self.name,
-                          lookup_keyword(self.type, self.TYPES),
+                         lookup_keyword(self.type, TYPES),
-                          lookup_keyword(self.subtype, self.SUBTYPES.get(self.type, {})),
+                         lookup_keyword(self.subtype, SUBTYPES.get(self.type, {})),
-                          addr_format(self.offset, False),
+                         addr_format(self.offset, False),
-                          addr_format(self.size, True),
+                         addr_format(self.size, True),
-                          generate_text_flags()])
+                         generate_text_flags()])
 
 
 def parse_int(v, keywords={}):
@@ -396,7 +432,7 @@ def parse_int(v, keywords={}):
     k/m/K/M suffixes and 'keyword' value lookup.
     """
     try:
-        for letter, multiplier in [ ("k",1024), ("m",1024*1024) ]:
+        for letter, multiplier in [("k", 1024), ("m", 1024 * 1024)]:
             if v.lower().endswith(letter):
                 return parse_int(v[:-1], keywords) * multiplier
         return int(v, 0)
@@ -408,6 +444,7 @@ def parse_int(v, keywords={}):
         except KeyError:
             raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords)))
 
+
 def main():
     global quiet
     global md5sum
@@ -416,16 +453,17 @@ def main():
     parser = argparse.ArgumentParser(description='ESP32 partition table utility')
 
     parser.add_argument('--flash-size', help='Optional flash size limit, checks partition table fits in flash',
-                        nargs='?', choices=[ '1MB', '2MB', '4MB', '8MB', '16MB' ])
+                        nargs='?', choices=['1MB', '2MB', '4MB', '8MB', '16MB'])
     parser.add_argument('--disable-md5sum', help='Disable md5 checksum for the partition table', default=False, action='store_true')
-    parser.add_argument('--verify', '-v', help='Verify partition table fields', default=True, action='store_false')
+    parser.add_argument('--no-verify', help="Don't verify partition table fields", action='store_true')
-    parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
+    parser.add_argument('--verify', '-v', help="Verify partition table fields (deprecated, this behaviour is "
+                                               "enabled by default and this flag does nothing.", action='store_true')
+    parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true')
     parser.add_argument('--offset', '-o', help='Set offset partition table', default='0x8000')
     parser.add_argument('--secure', help="Require app partitions to be suitable for secure boot", action='store_true')
-    parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', type=argparse.FileType('rb'), default=sys.stdin)
+    parser.add_argument('input', help='Path to CSV or binary file to parse.', type=argparse.FileType('rb'))
-    parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted, unless the --display argument is also passed (in which case only the summary is printed.)',
+    parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted.',
-                        nargs='?',
+                        nargs='?', default='-')
-                        default='-')
 
     args = parser.parse_args()
 
@@ -443,7 +481,7 @@ def main():
         status("Parsing CSV input...")
         table = PartitionTable.from_csv(input)
 
-    if args.verify:
+    if not args.no_verify:
         status("Verifying table...")
         table.verify()
 
@@ -452,16 +490,31 @@ def main():
         size = size_mb * 1024 * 1024  # flash memory uses honest megabytes!
         table_size = table.flash_size()
         if size < table_size:
-            raise InputError("Partitions defined in '%s' occupy %.1fMB of flash (%d bytes) which does not fit in configured flash size %dMB. Change the flash size in menuconfig under the 'Serial Flasher Config' menu." %
+            raise InputError("Partitions defined in '%s' occupy %.1fMB of flash (%d bytes) which does not fit in configured "
+                             "flash size %dMB. Change the flash size in menuconfig under the 'Serial Flasher Config' menu." %
                              (args.input.name, table_size / 1024.0 / 1024.0, table_size, size_mb))
 
+    # Make sure that the output directory is created
+    output_dir = os.path.abspath(os.path.dirname(args.output))
+
+    if not os.path.exists(output_dir):
+        try:
+            os.makedirs(output_dir)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+
     if input_is_binary:
         output = table.to_csv()
         with sys.stdout if args.output == '-' else open(args.output, 'w') as f:
             f.write(output)
     else:
         output = table.to_binary()
-        with sys.stdout.buffer if args.output == '-' else open(args.output, 'wb') as f:
+        try:
+            stdout_binary = sys.stdout.buffer  # Python 3
+        except AttributeError:
+            stdout_binary = sys.stdout
+        with stdout_binary if args.output == '-' else open(args.output, 'wb') as f:
             f.write(output)
 
 

components/partition_table/parttool.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 #
-# parttool returns info about the required partition.
+# parttool is used to perform partition level operations - reading,
-#
+# writing, erasing and getting info about the partition.
-# This utility is used by the make system to get information 
-# about the start addresses: partition table, factory area, phy area.
 #
 # Copyright 2018 Espressif Systems (Shanghai) PTE LTD
 #
@@ -21,113 +19,320 @@
 from __future__ import print_function, division
 import argparse
 import os
-import re
-import struct
 import sys
-import hashlib
+import subprocess
-import binascii
+import tempfile
+import re
 import gen_esp32part as gen
 
-__version__ = '1.0'
+
+__version__ = '2.0'
+
+COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components"))
+ESPTOOL_PY = os.path.join(COMPONENTS_PATH, "esptool_py", "esptool", "esptool.py")
+
+PARTITION_TABLE_OFFSET = 0x8000
+
 
 quiet = False
 
-def status(msg):
-    """ Print status message to stderr """
-    if not quiet:
-        critical(msg)
 
-def critical(msg):
+def status(msg):
-    """ Print critical message to stderr """
     if not quiet:
-        sys.stderr.write(msg)
+        print(msg)
-        sys.stderr.write('\n')
+
-               
+
+class _PartitionId():
+
+    def __init__(self, name=None, type=None, subtype=None):
+        self.name = name
+        self.type = type
+        self.subtype = subtype
+
+
+class PartitionName(_PartitionId):
+
+    def __init__(self, name):
+        _PartitionId.__init__(self, name=name)
+
+
+class PartitionType(_PartitionId):
+
+    def __init__(self, type, subtype):
+        _PartitionId.__init__(self, type=type, subtype=subtype)
+
+
+PARTITION_BOOT_DEFAULT = _PartitionId()
+
+
+class ParttoolTarget():
+
+    def __init__(self, port=None, baud=None, partition_table_offset=PARTITION_TABLE_OFFSET, partition_table_file=None,
+                 esptool_args=[], esptool_write_args=[], esptool_read_args=[], esptool_erase_args=[]):
+        self.port = port
+        self.baud = baud
+
+        gen.offset_part_table = partition_table_offset
+
+        def parse_esptool_args(esptool_args):
+            results = list()
+            for arg in esptool_args:
+                pattern = re.compile(r"(.+)=(.+)")
+                result = pattern.match(arg)
+                try:
+                    key = result.group(1)
+                    value = result.group(2)
+                    results.extend(["--" + key, value])
+                except AttributeError:
+                    results.extend(["--" + arg])
+            return results
+
+        self.esptool_args = parse_esptool_args(esptool_args)
+        self.esptool_write_args = parse_esptool_args(esptool_write_args)
+        self.esptool_read_args = parse_esptool_args(esptool_read_args)
+        self.esptool_erase_args = parse_esptool_args(esptool_erase_args)
+
+        if partition_table_file:
+            try:
+                with open(partition_table_file, "rb") as f:
+                    partition_table = gen.PartitionTable.from_binary(f.read())
+            except (gen.InputError, IOError, TypeError):
+                with open(partition_table_file, "r") as f:
+                    f.seek(0)
+                    partition_table = gen.PartitionTable.from_csv(f.read())
+        else:
+            temp_file = tempfile.NamedTemporaryFile(delete=False)
+            temp_file.close()
+
+            try:
+                self._call_esptool(["read_flash", str(partition_table_offset), str(gen.MAX_PARTITION_LENGTH), temp_file.name])
+                with open(temp_file.name, "rb") as f:
+                    partition_table = gen.PartitionTable.from_binary(f.read())
+            finally:
+                os.unlink(temp_file.name)
+
+        self.partition_table = partition_table
+
+    def _call_esptool(self, args, out=None):
+        esptool_args = [sys.executable, ESPTOOL_PY] + self.esptool_args
+
+        if self.port:
+            esptool_args += ["--port", self.port]
+
+        if self.baud:
+            esptool_args += ["--baud", str(self.baud)]
+
+        esptool_args += args
+
+        with open(os.devnull, "w") as null_file:
+            subprocess.check_call(esptool_args, stdout=null_file, stderr=null_file)
+
+    def get_partition_info(self, partition_id):
+        partition = None
+
+        if partition_id.name:
+            partition = self.partition_table.find_by_name(partition_id.name)
+        elif partition_id.type and partition_id.subtype:
+            partition = self.partition_table.find_by_type(partition_id.type, partition_id.subtype)
+        else:  # default boot partition
+            search = ["factory"] + ["ota_{}".format(d) for d in range(16)]
+            for subtype in search:
+                partition = self.partition_table.find_by_type("app", subtype)
+                if partition:
+                    break
+
+        if not partition:
+            raise Exception("Partition does not exist")
+
+        return partition
+
+    def erase_partition(self, partition_id):
+        partition = self.get_partition_info(partition_id)
+        self._call_esptool(["erase_region", str(partition.offset),  str(partition.size)] + self.esptool_erase_args)
+
+    def read_partition(self, partition_id, output):
+        partition = self.get_partition_info(partition_id)
+        self._call_esptool(["read_flash", str(partition.offset), str(partition.size), output] + self.esptool_read_args)
+
+    def write_partition(self, partition_id, input):
+        self.erase_partition(partition_id)
+
+        partition = self.get_partition_info(partition_id)
+
+        with open(input, "rb") as input_file:
+            content_len = len(input_file.read())
+
+            if content_len > partition.size:
+                raise Exception("Input file size exceeds partition size")
+
+        self._call_esptool(["write_flash", str(partition.offset), input] + self.esptool_write_args)
+
+
+def _write_partition(target, partition_id, input):
+    target.write_partition(partition_id, input)
+    partition = target.get_partition_info(partition_id)
+    status("Written contents of file '{}' at offset 0x{:x}".format(input, partition.offset))
+
+
+def _read_partition(target, partition_id, output):
+    target.read_partition(partition_id, output)
+    partition = target.get_partition_info(partition_id)
+    status("Read partition '{}' contents from device at offset 0x{:x} to file '{}'"
+           .format(partition.name, partition.offset, output))
+
+
+def _erase_partition(target, partition_id):
+    target.erase_partition(partition_id)
+    partition = target.get_partition_info(partition_id)
+    status("Erased partition '{}' at offset 0x{:x}".format(partition.name, partition.offset))
+
+
+def _get_partition_info(target, partition_id, info):
+    try:
+        partition = target.get_partition_info(partition_id)
+    except Exception:
+        return
+
+    info_dict = {
+        "offset": '0x{:x}'.format(partition.offset),
+        "size": '0x{:x}'.format(partition.size)
+    }
+
+    infos = []
+
+    try:
+        for i in info:
+            infos += [info_dict[i]]
+    except KeyError:
+        raise RuntimeError("Request for unknown partition info {}".format(i))
+
+    print(" ".join(infos))
+
+
 def main():
     global quiet
-    parser = argparse.ArgumentParser(description='Returns info about the required partition.')
 
-    parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
+    parser = argparse.ArgumentParser("ESP-IDF Partitions Tool")
 
-    parser.add_argument('--partition-table-offset', help='The offset of the partition table in flash. Only consulted if partition table is in CSV format.', type=str, default='0x8000')
+    parser.add_argument("--quiet", "-q", help="suppress stderr messages", action="store_true")
+    parser.add_argument("--esptool-args", help="additional main arguments for esptool", nargs="+")
+    parser.add_argument("--esptool-write-args", help="additional subcommand arguments when writing to flash", nargs="+")
+    parser.add_argument("--esptool-read-args", help="additional subcommand arguments when reading flash", nargs="+")
+    parser.add_argument("--esptool-erase-args", help="additional subcommand arguments when erasing regions of flash", nargs="+")
 
-    search_type = parser.add_mutually_exclusive_group()
+    # By default the device attached to the specified port is queried for the partition table. If a partition table file
-    search_type.add_argument('--partition-name', '-p', help='The name of the required partition', type=str, default=None)
+    # is specified, that is used instead.
-    search_type.add_argument('--type', '-t', help='The type of the required partition', type=str, default=None)
+    parser.add_argument("--port", "-p", help="port where the target device of the command is connected to; the partition table is sourced from this device \
-    search_type.add_argument('--default-boot-partition', help='Select the default boot partition, '+
+                                            when the partition table file is not defined")
-                             'using the same fallback logic as the IDF bootloader', action="store_true")
+    parser.add_argument("--baud", "-b", help="baudrate to use", type=int)
 
-    parser.add_argument('--subtype', '-s', help='The subtype of the required partition', type=str, default=None)
+    parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from", type=str)
+    parser.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from; \
+                                                            overrides device attached to specified port as the partition table source when defined")
 
-    parser.add_argument('--offset', '-o', help='Return offset of required partition', action="store_true")
+    partition_selection_parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument('--size', help='Return size of required partition', action="store_true")
 
-    parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.',
+    # Specify what partition to perform the operation on. This can either be specified using the
-                        type=argparse.FileType('rb'), default=sys.stdin)
+    # partition name or the first partition that matches the specified type/subtype
+    partition_selection_args = partition_selection_parser.add_mutually_exclusive_group()
+
+    partition_selection_args.add_argument("--partition-name", "-n", help="name of the partition")
+    partition_selection_args.add_argument("--partition-type", "-t", help="type of the partition")
+    partition_selection_args.add_argument('--partition-boot-default', "-d", help='select the default boot partition \
+                                           using the same fallback logic as the IDF bootloader', action="store_true")
+
+    partition_selection_parser.add_argument("--partition-subtype", "-s", help="subtype of the partition")
+
+    subparsers = parser.add_subparsers(dest="operation", help="run parttool -h for additional help")
+
+    # Specify the supported operations
+    read_part_subparser = subparsers.add_parser("read_partition", help="read partition from device and dump contents into a file",
+                                                parents=[partition_selection_parser])
+    read_part_subparser.add_argument("--output", help="file to dump the read partition contents to")
+
+    write_part_subparser = subparsers.add_parser("write_partition", help="write contents of a binary file to partition on device",
+                                                 parents=[partition_selection_parser])
+    write_part_subparser.add_argument("--input", help="file whose contents are to be written to the partition offset")
+
+    subparsers.add_parser("erase_partition", help="erase the contents of a partition on the device", parents=[partition_selection_parser])
+
+    print_partition_info_subparser = subparsers.add_parser("get_partition_info", help="get partition information", parents=[partition_selection_parser])
+    print_partition_info_subparser.add_argument("--info", help="type of partition information to get", nargs="+")
 
     args = parser.parse_args()
-
-    if args.type is not None and args.subtype is None:
-        status("If --type is specified, --subtype is required")
-        return 2
-    if args.type is None and args.subtype is not None:
-        status("--subtype is only used with --type")
-        return 2
-
     quiet = args.quiet
 
-    gen.offset_part_table = int(args.partition_table_offset, 0)
+    # No operation specified, display help and exit
+    if args.operation is None:
+        if not quiet:
+            parser.print_help()
+        sys.exit(1)
 
-    input = args.input.read()
+    # Prepare the partition to perform operation on
-    input_is_binary = input[0:2] == gen.PartitionDefinition.MAGIC_BYTES
+    if args.partition_name:
-    if input_is_binary:
+        partition_id = PartitionName(args.partition_name)
-        status("Parsing binary partition input...")
+    elif args.partition_type:
-        table = gen.PartitionTable.from_binary(input)
+        if not args.partition_subtype:
+            raise RuntimeError("--partition-subtype should be defined when --partition-type is defined")
+        partition_id = PartitionType(args.partition_type, args.partition_subtype)
+    elif args.partition_boot_default:
+        partition_id = PARTITION_BOOT_DEFAULT
     else:
-        input = input.decode()
+        raise RuntimeError("Partition to operate on should be defined using --partition-name OR \
-        status("Parsing CSV input...")
+                            partition-type,--partition-subtype OR partition-boot-default")
-        table = gen.PartitionTable.from_csv(input)
 
-    found_partition = None
+    # Prepare the device to perform operation on
+    target_args = {}
 
-    if args.default_boot_partition:
+    if args.port:
-        search = [ "factory" ] + [ "ota_%d" % d for d in range(16) ]
+        target_args["port"] = args.port
-        for subtype in search:
+
-            found_partition = table.find_by_type("app", subtype)
+    if args.baud:
-            if found_partition is not None:
+        target_args["baud"] = args.baud
-                break
+
-    elif args.partition_name is not None:
+    if args.partition_table_file:
-        found_partition = table.find_by_name(args.partition_name)
+        target_args["partition_table_file"] = args.partition_table_file
-    elif args.type is not None:
+
-        found_partition = table.find_by_type(args.type, args.subtype)
+    if args.partition_table_offset:
+        target_args["partition_table_offset"] = int(args.partition_table_offset, 0)
+
+    if args.esptool_args:
+        target_args["esptool_args"] = args.esptool_args
+
+    if args.esptool_write_args:
+        target_args["esptool_write_args"] = args.esptool_write_args
+
+    if args.esptool_read_args:
+        target_args["esptool_read_args"] = args.esptool_read_args
+
+    if args.esptool_erase_args:
+        target_args["esptool_erase_args"] = args.esptool_erase_args
+
+    target = ParttoolTarget(**target_args)
+
+    # Create the operation table and execute the operation
+    common_args = {'target':target, 'partition_id':partition_id}
+    parttool_ops = {
+        'erase_partition':(_erase_partition, []),
+        'read_partition':(_read_partition, ["output"]),
+        'write_partition':(_write_partition, ["input"]),
+        'get_partition_info':(_get_partition_info, ["info"])
+    }
+
+    (op, op_args) = parttool_ops[args.operation]
+
+    for op_arg in op_args:
+        common_args.update({op_arg:vars(args)[op_arg]})
+
+    if quiet:
+        # If exceptions occur, suppress and exit quietly
+        try:
+            op(**common_args)
+        except Exception:
+            sys.exit(2)
     else:
-        raise RuntimeError("invalid partition selection choice")
+        op(**common_args)
-
-    if found_partition is None:
-        return 1  # nothing found
-
-    if args.offset:
-        print('0x%x ' % (found_partition.offset))
-    if args.size:
-        print('0x%x' % (found_partition.size))
-
-    return 0
-
-class InputError(RuntimeError):
-    def __init__(self, e):
-        super(InputError, self).__init__(e)
-
-
-class ValidationError(InputError):
-    def __init__(self, partition, message):
-        super(ValidationError, self).__init__(
-            "Partition %s invalid: %s" % (partition.name, message))
 
 
 if __name__ == '__main__':
-    try:
+    main()
-        r = main()
-        sys.exit(r)
-    except InputError as e:
-        print(e, file=sys.stderr)
-        sys.exit(2)

components/partition_table/project_include.cmake

@@ -1,65 +1,51 @@
 if(NOT BOOTLOADER_BUILD)
+    set(PARTITION_TABLE_OFFSET ${CONFIG_PARTITION_TABLE_OFFSET})
 
-set(PARTITION_TABLE_OFFSET ${CONFIG_PARTITION_TABLE_OFFSET})
+    # Set PARTITION_CSV_PATH to the configured partition CSV file
+    # absolute path
+    if(CONFIG_PARTITION_TABLE_CUSTOM)
+        idf_build_get_property(project_dir PROJECT_DIR)
+        # Custom filename expands any path relative to the project
+        get_filename_component(PARTITION_CSV_PATH "${CONFIG_PARTITION_TABLE_FILENAME}"
+                                ABSOLUTE BASE_DIR "${project_dir}")
 
-# Set PARTITION_CSV_PATH to the configured partition CSV file
+        if(NOT EXISTS "${PARTITION_CSV_PATH}")
-# absolute path
+            message(WARNING "Partition table CSV file ${PARTITION_CSV_PATH} not found. "
-if(CONFIG_PARTITION_TABLE_CUSTOM)
+                "Change custom partition CSV path in menuconfig.")
-    # Custom filename expands any path relative to the project
+            # Note: partition_table CMakeLists.txt contains some logic to create a dummy
-    get_filename_component(PARTITION_CSV_PATH "${CONFIG_PARTITION_TABLE_FILENAME}" ABSOLUTE BASE_DIR "${PROJECT_PATH}")
+            # partition_table target in this case, see comments in that file.
+        endif()
+    else()
+        # Other .csv files are always in the component directory
+        get_filename_component(PARTITION_CSV_PATH "${COMPONENT_DIR}/${CONFIG_PARTITION_TABLE_FILENAME}" ABSOLUTE)
 
-    if(NOT EXISTS "${PARTITION_CSV_PATH}")
+        if(NOT EXISTS "${PARTITION_CSV_PATH}")
-        message(WARNING "Partition table CSV file ${PARTITION_CSV_PATH} not found. "
+            message(FATAL_ERROR "Internal error, built-in ${PARTITION_CSV_PATH} not found.")
-            "Change custom partition CSV path in menuconfig.")
+        endif()
-        # Note: partition_table CMakeLists.txt contains some logic to create a dummy
-        # partition_table target in this case, see comments in that file.
     endif()
-else()
-    # Other .csv files are always in the component directory
-    get_filename_component(PARTITION_CSV_PATH "${COMPONENT_PATH}/${CONFIG_PARTITION_TABLE_FILENAME}" ABSOLUTE)
 
-    if(NOT EXISTS "${PARTITION_CSV_PATH}")
+    # need to re-run CMake if the partition CSV changes, as the offsets/sizes of partitions may change
-        message(FATAL_ERROR "Internal error, built-in ${PARTITION_CSV_PATH} not found.")
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${PARTITION_CSV_PATH})
-    endif()
 endif()
 
-# need to re-run CMake if the partition CSV changes, as the offsets/sizes of partitions may change
+# partition_table_get_partition_info
-set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${PARTITION_CSV_PATH})
+#
-
+# Get information about a partition from the partition table
-# Parse the partition table to get variable partition offsets & sizes which must be known at CMake runtime
+function(partition_table_get_partition_info result get_part_info_args part_info)
-function(get_partition_info variable get_part_info_args)
+    idf_build_get_property(python PYTHON)
+    idf_build_get_property(idf_path IDF_PATH)
     separate_arguments(get_part_info_args)
-    execute_process(COMMAND ${PYTHON}
+    execute_process(COMMAND ${python}
-        ${COMPONENT_PATH}/parttool.py -q
+        ${idf_path}/components/partition_table/parttool.py -q
         --partition-table-offset ${PARTITION_TABLE_OFFSET}
-        ${get_part_info_args}
+        --partition-table-file ${PARTITION_CSV_PATH}
-        ${PARTITION_CSV_PATH}
+        get_partition_info ${get_part_info_args} --info ${part_info}
-        OUTPUT_VARIABLE result
+        OUTPUT_VARIABLE info
         RESULT_VARIABLE exit_code
         OUTPUT_STRIP_TRAILING_WHITESPACE)
     if(NOT ${exit_code} EQUAL 0 AND NOT ${exit_code} EQUAL 1)
         # can't fail here as it would prevent the user from running 'menuconfig' again
         message(WARNING "parttool.py execution failed (${result}), problem with partition CSV file (see above)")
     endif()
-    set(${variable} ${result} PARENT_SCOPE)
+    set(${result} ${info} PARENT_SCOPE)
 endfunction()
-
-if(CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION)
-    get_partition_info(PHY_PARTITION_OFFSET "--type data --subtype phy --offset")
-    set(PHY_PARTITION_BIN_FILE "${CMAKE_BINARY_DIR}/esp32/phy_init_data.bin")
-endif()
-
-get_partition_info(APP_PARTITION_OFFSET "--default-boot-partition --offset")
-
-get_partition_info(APP_PARTITION_SIZE "--default-boot-partition --size")
-
-get_partition_info(OTADATA_PARTITION_OFFSET "--type data --subtype ota --offset")
-
-get_partition_info(OTADATA_PARTITION_SIZE "--type data --subtype ota --size")
-
-set(APP_OFFSET ${APP_PARTITION_OFFSET})
-set(APP_SIZE ${APP_PARTITION_SIZE})
-
-set(CFLAGS "${CFLAGS}" "-DAPP_OFFSET=${APP_OFFSET}" "-DAPP_SIZE=${APP_SIZE}")
-
-endif()

components/spiffs/CMakeLists.txt

@@ -1,17 +1,21 @@
+set(include_dirs "include")
+
 if(CONFIG_USING_SPIFFS)
-set(COMPONENT_ADD_INCLUDEDIRS "include")
+set(priv_include_dirs "." "spiffs/src")
-set(COMPONENT_PRIV_INCLUDEDIRS "." "spiffs/src")
+set(srcs "esp_spiffs.c"
-set(COMPONENT_SRCS "esp_spiffs.c"
+         "spiffs_api.c"
-                   "spiffs_api.c"
+         "spiffs/src/spiffs_cache.c"
-                   "spiffs/src/spiffs_cache.c"
+         "spiffs/src/spiffs_check.c"
-                   "spiffs/src/spiffs_check.c"
+         "spiffs/src/spiffs_gc.c"
-                   "spiffs/src/spiffs_gc.c"
+         "spiffs/src/spiffs_hydrogen.c"
-                   "spiffs/src/spiffs_hydrogen.c"
+         "spiffs/src/spiffs_nucleus.c")
-                   "spiffs/src/spiffs_nucleus.c")
 endif()
 
-set(COMPONENT_REQUIRES  "freertos" "spi_flash")
+set(requires "spi_flash" "vfs")
-set(COMPONENT_PRIV_REQUIRES "esp8266" "bootloader_support")
+set(priv_requires "bootloader_support")
-
-register_component()
 
+idf_component_register(SRCS "${srcs}"
+                       INCLUDE_DIRS "${include_dirs}"
+                       PRIV_INCLUDE_DIRS "${priv_include_dirs}"
+                       REQUIRES "${requires}"
+                       PRIV_REQUIRES "${priv_requires}")

components/ssl/CMakeLists.txt

@@ -27,12 +27,12 @@ set(COMPONENT_REQUIRES "lwip" "esp8266" "util")
 register_component()
 
 if(CONFIG_SSL_USING_WOLFSSL)
-target_compile_options(${COMPONENT_NAME} PUBLIC -DWOLFSSL_USER_SETTINGS)
+target_compile_options(${COMPONENT_LIB} PUBLIC -DWOLFSSL_USER_SETTINGS)
 
-target_link_libraries(${COMPONENT_NAME} "-L ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/lib")
+target_link_libraries(${COMPONENT_LIB} "-L ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/lib")
-target_link_libraries(${COMPONENT_NAME} wolfssl)
+target_link_libraries(${COMPONENT_LIB} wolfssl)
 else()
 if(CONFIG_SSL_USING_MBEDTLS)
-target_compile_options(${COMPONENT_NAME} PUBLIC -DMBEDTLS_CONFIG_FILE="mbedtls/esp_config.h")
+target_compile_options(${COMPONENT_LIB} PUBLIC -DMBEDTLS_CONFIG_FILE="mbedtls/esp_config.h")
 endif()
 endif()

components/util/CMakeLists.txt

@@ -1,10 +1,8 @@
 set(COMPONENT_SRCDIRS "src")
 set(COMPONENT_ADD_INCLUDEDIRS "include")
 
-set(COMPONENT_REQUIRES)
-
 register_component()
 
-if(NOT IS_BOOTLOADER_BUILD)
+if(NOT BOOTLOADER_BUILD)
-target_compile_definitions(${COMPONENT_NAME} PRIVATE -DUSING_IBUS_FASTER_GET)
+target_compile_definitions(${COMPONENT_LIB} PUBLIC -DUSING_IBUS_FASTER_GET)
 endif()

components/util/include/crc.h

@@ -1,4 +1,4 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+// Copyright 2019-2020 Espressif Systems (Shanghai) PTE LTD
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,127 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef __CRC_H_
+#pragma once
-#define __CRC_H_
 
-#include <stdint.h>
+#include "esp_crc.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/** \defgroup uart_apis, uart configuration and communication related apis
-  * @brief uart apis
-  */
-
-/** @addtogroup uart_apis
-  * @{
-  */
-
-
-/* Standard CRC8/16/32 algorithms. */
-// CRC-8        x8+x2+x1+1              0x07
-// CRC16-CCITT  x16+x12+x5+1   1021   ISO HDLC, ITU X.25, V.34/V.41/V.42, PPP-FCS
-// CRC32:
-//G(x) = x32 +x26 + x23 + x22 + x16 + x12 + x11 + x10 + x8 + x7 + x5 + x4 + x2 + x1 + 1
-//If your buf is not continuous, you can use the first result to be the second parameter.
-
-/**
-  * @brief Crc32 value that is in little endian.
-  *
-  * @param  uint32_t crc : init crc value, use 0 at the first use.
-  *
-  * @param  uint8_t const *buf : buffer to start calculate crc.
-  *
-  * @param  uint32_t len : buffer length in byte.
-  *
-  * @return None
-  */
-uint32_t crc32_le(uint32_t crc, uint8_t const *buf, uint32_t len);
-
-/**
-  * @brief Crc32 value that is in big endian.
-  *
-  * @param  uint32_t crc : init crc value, use 0 at the first use.
-  *
-  * @param  uint8_t const *buf : buffer to start calculate crc.
-  *
-  * @param  uint32_t len : buffer length in byte.
-  *
-  * @return None
-  */
-uint32_t crc32_be(uint32_t crc, uint8_t const *buf, uint32_t len);
-
-/**
-  * @brief Crc16 value that is in little endian.
-  *
-  * @param  uint16_t crc : init crc value, use 0 at the first use.
-  *
-  * @param  uint8_t const *buf : buffer to start calculate crc.
-  *
-  * @param  uint32_t len : buffer length in byte.
-  *
-  * @return None
-  */
-uint16_t crc16_le(uint16_t crc, uint8_t const *buf, uint32_t len);
-
-/**
-  * @brief Crc16 value that is in big endian.
-  *
-  * @param  uint16_t crc : init crc value, use 0 at the first use.
-  *
-  * @param  uint8_t const *buf : buffer to start calculate crc.
-  *
-  * @param  uint32_t len : buffer length in byte.
-  *
-  * @return None
-  */
-uint16_t crc16_be(uint16_t crc, uint8_t const *buf, uint32_t len);
-
-/**
-  * @brief Crc8 value that is in little endian.
-  *
-  * @param  uint8_t crc : init crc value, use 0 at the first use.
-  *
-  * @param  uint8_t const *buf : buffer to start calculate crc.
-  *
-  * @param  uint32_t len : buffer length in byte.
-  *
-  * @return None
-  */
-uint8_t crc8_le(uint8_t crc, uint8_t const *buf, uint32_t len);
-
-/**
-  * @brief Crc8 value that is in big endian.
-  *
-  * @param  uint32_t crc : init crc value, use 0 at the first use.
-  *
-  * @param  uint8_t const *buf : buffer to start calculate crc.
-  *
-  * @param  uint32_t len : buffer length in byte.
-  *
-  * @return None
-  */
-uint8_t crc8_be(uint8_t crc, uint8_t const *buf, uint32_t len);
-
-/**
-  * @brief  A crc8 algorithm used in efuse check.
-  *
-  * @param  uint8_t const *p : Pointer to original data.
-  *
-  * @param  uint32_t len : Data length in byte.
-  *
-  * @return uint8_t: Crc value.
-  */
-uint8_t esp_crc8(uint8_t const *p, uint32_t len);
-
-/**
-  * @}
-  */
-
-#ifdef __cplusplus
-}
-#endif
-
-
-#endif

components/util/include/esp_crc.h

@@ -0,0 +1,138 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __CRC_H_
+#define __CRC_H_
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** \defgroup uart_apis, uart configuration and communication related apis
+  * @brief uart apis
+  */
+
+/** @addtogroup uart_apis
+  * @{
+  */
+
+
+/* Standard CRC8/16/32 algorithms. */
+// CRC-8        x8+x2+x1+1              0x07
+// CRC16-CCITT  x16+x12+x5+1   1021   ISO HDLC, ITU X.25, V.34/V.41/V.42, PPP-FCS
+// CRC32:
+//G(x) = x32 +x26 + x23 + x22 + x16 + x12 + x11 + x10 + x8 + x7 + x5 + x4 + x2 + x1 + 1
+//If your buf is not continuous, you can use the first result to be the second parameter.
+
+/**
+  * @brief Crc32 value that is in little endian.
+  *
+  * @param  uint32_t crc : init crc value, use 0 at the first use.
+  *
+  * @param  uint8_t const *buf : buffer to start calculate crc.
+  *
+  * @param  uint32_t len : buffer length in byte.
+  *
+  * @return None
+  */
+uint32_t crc32_le(uint32_t crc, uint8_t const *buf, uint32_t len);
+
+/**
+  * @brief Crc32 value that is in big endian.
+  *
+  * @param  uint32_t crc : init crc value, use 0 at the first use.
+  *
+  * @param  uint8_t const *buf : buffer to start calculate crc.
+  *
+  * @param  uint32_t len : buffer length in byte.
+  *
+  * @return None
+  */
+uint32_t crc32_be(uint32_t crc, uint8_t const *buf, uint32_t len);
+
+/**
+  * @brief Crc16 value that is in little endian.
+  *
+  * @param  uint16_t crc : init crc value, use 0 at the first use.
+  *
+  * @param  uint8_t const *buf : buffer to start calculate crc.
+  *
+  * @param  uint32_t len : buffer length in byte.
+  *
+  * @return None
+  */
+uint16_t crc16_le(uint16_t crc, uint8_t const *buf, uint32_t len);
+
+/**
+  * @brief Crc16 value that is in big endian.
+  *
+  * @param  uint16_t crc : init crc value, use 0 at the first use.
+  *
+  * @param  uint8_t const *buf : buffer to start calculate crc.
+  *
+  * @param  uint32_t len : buffer length in byte.
+  *
+  * @return None
+  */
+uint16_t crc16_be(uint16_t crc, uint8_t const *buf, uint32_t len);
+
+/**
+  * @brief Crc8 value that is in little endian.
+  *
+  * @param  uint8_t crc : init crc value, use 0 at the first use.
+  *
+  * @param  uint8_t const *buf : buffer to start calculate crc.
+  *
+  * @param  uint32_t len : buffer length in byte.
+  *
+  * @return None
+  */
+uint8_t crc8_le(uint8_t crc, uint8_t const *buf, uint32_t len);
+
+/**
+  * @brief Crc8 value that is in big endian.
+  *
+  * @param  uint32_t crc : init crc value, use 0 at the first use.
+  *
+  * @param  uint8_t const *buf : buffer to start calculate crc.
+  *
+  * @param  uint32_t len : buffer length in byte.
+  *
+  * @return None
+  */
+uint8_t crc8_be(uint8_t crc, uint8_t const *buf, uint32_t len);
+
+/**
+  * @brief  A crc8 algorithm used in efuse check.
+  *
+  * @param  uint8_t const *p : Pointer to original data.
+  *
+  * @param  uint32_t len : Data length in byte.
+  *
+  * @return uint8_t: Crc value.
+  */
+uint8_t esp_crc8(uint8_t const *p, uint32_t len);
+
+/**
+  * @}
+  */
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

components/wpa_supplicant/CMakeLists.txt

@@ -1,13 +1,12 @@
 set(COMPONENT_SRCDIRS "src/crypto" "port")
 set(COMPONENT_ADD_INCLUDEDIRS "include" "port/include")
 
-set(COMPONENT_REQUIRES "")
 set(COMPONENT_PRIV_REQUIRES "freertos" "heap" "newlib" "util")
 
 register_component()
 
-component_compile_options(-Wno-strict-aliasing)
+target_compile_options(${COMPONENT_LIB} PRIVATE -Wno-strict-aliasing)
-component_compile_definitions(
+target_compile_definitions(${COMPONENT_LIB} PRIVATE
     __ets__
     EMBEDDED_SUPP
     ESPRESSIF_USE

examples/protocols/aws_iot/subscribe_publish/CMakeLists.txt

@@ -4,3 +4,9 @@ cmake_minimum_required(VERSION 3.5)
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(subscribe_publish)
+
+if(CONFIG_EXAMPLE_EMBEDDED_CERTS)
+target_add_binary_data(${CMAKE_PROJECT_NAME}.elf "main/certs/aws-root-ca.pem" TEXT)
+target_add_binary_data(${CMAKE_PROJECT_NAME}.elf "main/certs/certificate.pem.crt" TEXT)
+target_add_binary_data(${CMAKE_PROJECT_NAME}.elf "main/certs/private.pem.key" TEXT)
+endif()

examples/protocols/aws_iot/subscribe_publish/main/CMakeLists.txt

@@ -3,9 +3,3 @@ set(COMPONENT_ADD_INCLUDEDIRS ".")
 
 
 register_component()
-
-if(CONFIG_EXAMPLE_EMBEDDED_CERTS)
-target_add_binary_data(${COMPONENT_NAME} "certs/aws-root-ca.pem" TEXT)
-target_add_binary_data(${COMPONENT_NAME} "certs/certificate.pem.crt" TEXT)
-target_add_binary_data(${COMPONENT_NAME} "certs/private.pem.key" TEXT)
-endif()

examples/protocols/aws_iot/thing_shadow/CMakeLists.txt

@@ -4,3 +4,9 @@ cmake_minimum_required(VERSION 3.5)
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(thing_shadow)
+
+if(CONFIG_EXAMPLE_EMBEDDED_CERTS)
+target_add_binary_data(${CMAKE_PROJECT_NAME}.elf "main/certs/aws-root-ca.pem" TEXT)
+target_add_binary_data(${CMAKE_PROJECT_NAME}.elf "main/certs/certificate.pem.crt" TEXT)
+target_add_binary_data(${CMAKE_PROJECT_NAME}.elf "main/certs/private.pem.key" TEXT)
+endif()

examples/protocols/aws_iot/thing_shadow/main/CMakeLists.txt

@@ -3,9 +3,3 @@ set(COMPONENT_ADD_INCLUDEDIRS ".")
 
 
 register_component()
-
-if(CONFIG_EXAMPLE_EMBEDDED_CERTS)
-target_add_binary_data(${COMPONENT_NAME} "certs/aws-root-ca.pem" TEXT)
-target_add_binary_data(${COMPONENT_NAME} "certs/certificate.pem.crt" TEXT)
-target_add_binary_data(${COMPONENT_NAME} "certs/private.pem.key" TEXT)
-endif()

examples/system/ota/simple_ota_example/main/CMakeLists.txt

@@ -1,8 +1,4 @@
-set(COMPONENT_SRCS "simple_ota_example.c")
+idf_build_get_property(project_dir PROJECT_DIR)
-set(COMPONENT_ADD_INCLUDEDIRS ".")
+idf_component_register(SRCS "simple_ota_example.c"
-
+                    INCLUDE_DIRS "."
-
+                    EMBED_TXTFILES ${project_dir}/server_certs/ca_cert.pem)
-# Embed the server root certificate into the final binary
-set(COMPONENT_EMBED_TXTFILES ${PROJECT_PATH}/server_certs/ca_cert.pem)
-
-register_component()

sdkconfig.rename

@@ -0,0 +1,6 @@
+# sdkconfig replacement configurations for deprecated options formatted as
+# CONFIG_DEPRECATED_OPTION CONFIG_NEW_OPTION
+
+# SDK tool configuration
+CONFIG_TARGET_PLATFORM                 CONFIG_IDF_TARGET
+CONFIG_TOOLPREFIX                      CONFIG_SDK_TOOLPREFIX

tools/cmake/build.cmake

@@ -154,7 +154,7 @@ function(__build_init idf_path)
     endforeach()
 
     # Set components required by all other components in the build
-    set(requires_common cxx newlib freertos heap log soc esp_rom esp_common xtensa)
+    set(requires_common newlib freertos heap log)
     idf_build_set_property(__COMPONENT_REQUIRES_COMMON "${requires_common}")
 
     __build_get_idf_git_revision()

tools/cmake/project.cmake

@@ -399,7 +399,7 @@ macro(project project_name)
     target_link_libraries(${project_elf} ${build_components})
 
     set(mapfile "${CMAKE_BINARY_DIR}/${CMAKE_PROJECT_NAME}.map")
-    target_link_libraries(${project_elf} "-Wl,--cref -Wl,--Map=${mapfile}")
+    target_link_libraries(${project_elf} "-Wl,--cref -Wl,--Map=${mapfile} -Wl,--start-group")
 
     set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY
         ADDITIONAL_MAKE_CLEAN_FILES

tools/cmake/project_description.json.in

@@ -7,7 +7,7 @@
     "app_elf":            "${PROJECT_EXECUTABLE}",
     "app_bin":            "${PROJECT_BIN}",
     "git_revision":       "${IDF_VER}",
-    "phy_data_partition": "${CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION}",
+    "phy_data_partition": "${CONFIG_ESP8266_PHY_INIT_DATA_IN_PARTITION}",
     "monitor_baud" : "${CONFIG_ESPTOOLPY_MONITOR_BAUD}",
     "config_environment" : {
         "COMPONENT_KCONFIGS" : "${COMPONENT_KCONFIGS}",

tools/cmake/targets.cmake

@@ -10,7 +10,7 @@ macro(__target_init)
         if(IDF_TARGET)
             set(env_idf_target ${IDF_TARGET})
         else()
-            set(env_idf_target esp32)
+            set(env_idf_target esp8266)
             message(STATUS "IDF_TARGET not set, using default target: ${env_idf_target}")
         endif()
     else()

tools/cmake/toolchain-esp8266.cmake

@@ -0,0 +1,11 @@
+set(CMAKE_SYSTEM_NAME Generic)
+
+set(CMAKE_C_COMPILER xtensa-lx106-elf-gcc)
+set(CMAKE_CXX_COMPILER xtensa-lx106-elf-g++)
+set(CMAKE_ASM_COMPILER xtensa-lx106-elf-gcc)
+
+set(CMAKE_C_FLAGS "-mlongcalls -Wno-frame-address" CACHE STRING "C Compiler Base Flags")
+set(CMAKE_CXX_FLAGS "-mlongcalls -Wno-frame-address" CACHE STRING "C++ Compiler Base Flags")
+
+# Can be removed after gcc 5.2.0 support is removed (ref GCC_NOT_5_2_0)
+set(CMAKE_EXE_LINKER_FLAGS "-nostdlib -Wl,--gc-sections" CACHE STRING "Linker Base Flags")