espressif/FreeRTOS
1
0
Fork 0
mirror of https://github.com/FreeRTOS/FreeRTOS.git synced 2025-09-12 18:02:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Autorelease/manifest (#469)

Browse Source 
new release flow
This commit is contained in:
David Chalco
2020-12-11 21:15:16 -08:00
committed by GitHub
parent a503a3a7a0
commit cc922e03a8
4 changed files with 191 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
.github/scripts/packager.py vendored
View File

@ -37,30 +37,6 @@ LABS_RELATIVE_EXCLUDE_FILES = [
os.path.join('.git')
]
'''
- Take inputs
- version will be specified
- This will be used to name directory
- baseline zip. From last release
- Used to compare contents of last release
- setup stage
- Create 'zipper-output' directory
- This will house the zip
- and the directory used to zip. 'out/unzipped-package'
- Unzip the input zip into 'out/baseline'
- Git clone recursive into latest FreeRTOS master from Git into 'out/head-master
- process stage
- remove all RELATIVE_FILE_EXCLUDES from 'out/unzipped-package'
- perform a filetree diff between 'out/unzipped-package' and 'out/head-master'
- Save to a file, to be confirmed by user
- Present and query user to authorize the diff
- zip contents of 'out/unzipped-package' --> 'out/FreeRTOSVXX.YY.ZZ.zip
- Use 7z compression, with compression set to max
- calculate zip file size diff, present to user
- Done
'''
# -------------------------------------------------------------------------------------------------
# Helpers
# -------------------------------------------------------------------------------------------------
@ -105,36 +81,37 @@ def download_git_tree(git_link, root_dir, dir_name, ref='master', commit_id='HEA
Download HEAD from Git Master. Place into working files dir
'''
args = ['git', '-C', root_dir, 'clone', '-b', ref, git_link, dir_name]
rc = subprocess.run(args).returncode
rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'checkout', '-f', commit_id]).returncode
rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'clean', '-fd']).returncode
subprocess.run(args, check=True)
subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'checkout', '-f', commit_id], check=True)
subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'clean', '-fd'], check=True)
if recurse:
rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'submodule', 'update', '--init', '--recursive']).returncode
return os.path.join(root_dir, dir_name) if rc == 0 else None
subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'submodule', 'update', '--init', '--recursive'], check=True)
return os.path.join(root_dir, dir_name)
def commit_git_tree_changes(repo_dir, commit_message=''):
rc = subprocess.run(['git', '-C', repo_dir, 'add', '-u']).returncode
rc += subprocess.run(['git', '-C', repo_dir, 'commit', '-m', commit_message]).returncode
subprocess.run(['git', '-C', repo_dir, 'add', '-u'], check=True)
subprocess.run(['git', '-C', repo_dir, 'commit', '-m', commit_message], check=True)
return rc
return 0
def push_git_tree_changes(repo_dir, tag=None, force_tag=False):
rc = subprocess.run(['git', '-C', repo_dir, 'push']).returncode
subprocess.run(['git', '-C', repo_dir, 'push'], check=True)
if tag != None:
force_tag_arg = '-f' if force_tag else ''
rc += subprocess.run(['git', '-C', repo_dir, 'tag', force_tag_arg, tag]).returncode
rc += subprocess.run(['git', '-C', repo_dir, 'push', force_tag_arg, '--tags']).returncode
subprocess.run(['git', '-C', repo_dir, 'tag', force_tag_arg, tag], check=True)
subprocess.run(['git', '-C', repo_dir, 'push', force_tag_arg, '--tags'], check=True)
return rc
return 0
def update_submodule_pointer(repo_dir, rel_submodule_path, new_submodule_ref):
rc = subprocess.run(['git', '-C', repo_dir, 'submodule', 'update', '--init']).returncode
rc += subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'fetch']).returncode
rc += subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'checkout', new_submodule_ref]).returncode
rc += subprocess.run(['git', '-C', repo_dir, 'add', rel_submodule_path]).returncode
subprocess.run(['git', '-C', repo_dir, 'submodule', 'update', '--init'], check=True)
subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'fetch'], check=True)
subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'checkout', new_submodule_ref], check=True)
subprocess.run(['git', '-C', repo_dir, 'add', rel_submodule_path], check=True)
return 0
def setup_intermediate_files(scratch_dir, intree_dir, outtree_dir):
cleanup_intermediate_files(scratch_dir)
@ -206,6 +183,7 @@ def create_package(path_ziproot, path_outtree, package_name, exclude_files=[]):
print('Done.')
return path_outzip
# -------------------------------------------------------------------------------------------------
# CLI
# -------------------------------------------------------------------------------------------------

196
.github/scripts/release.py vendored
View File

@ -1,5 +1,10 @@
#!/usr/bin/env python3
import os, shutil
from yaml import load, dump
try:
from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
from yaml import Loader, Dumper
from argparse import ArgumentParser
import re
@ -22,24 +27,6 @@ from packager import RELATIVE_FILE_EXCLUDES as FREERTOS_RELATIVE_FILE_EXCLUDES
# PyGithub Docs - https://pygithub.readthedocs.io/en/latest/github_objects
# REST API used by PyGithub - https://developer.github.com/v3/
'''
FUTURE ENHANCEMENTS
- Add mechanism that restores state of all affected to repos to BEFORE this script was run
- Input sanitizing
- Include regex patterns that MUST be honored for version strings, etc.
- Create a companion dependencies file that can be piped to pip3
- Ease of HTTPS authentication
- This should make it very easy to port to Github action. Currently, Github action mostly operates with
via https endpoints, rather than SSH
- Break down some functions and any repeated work --> more granular (reasonably), less duplicated code
- Unit tests
- Theres already an option and some desired tests laid out via comments. See bottom
- All of the scratch-work directories/files should be placed under a single directory the name of which makes obvious
that it's a scratch-work dir (Ex. tmp-*, scratch-*, etc.)
- Intermediate checks
-
'''
def info(msg, indent_level=0):
print('%s[INFO]: %s' % (' ' * indent_level, str(msg)))
@ -59,7 +46,9 @@ class BaseRelease:
self.commit = commit
self.git_ssh = git_ssh
self.git_org = git_org
self.repo_path = None
self.commit_msg_prefix = '[AUTO][RELEASE]: '
self.description = ''
self.mGit = mGit # Save a handle to the authed git session
@ -97,6 +86,10 @@ class BaseRelease:
for r in releases:
print(r)
def pushAutoCommits(self):
rc = push_git_tree_changes(self.repo_path, tag=self.tag, force_tag=True)
assert rc == 0, 'Failed to upload git tree changes'
class KernelRelease(BaseRelease):
def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'):
super().__init__(mGit, version, commit, git_ssh=git_ssh, git_org=git_org)
@ -105,7 +98,19 @@ class KernelRelease(BaseRelease):
self.repo = mGit.get_repo(self.repo_name)
self.tag = 'V%s' % version
def updateFileHeaderVersions(self, old_version_prefix):
# Download a local git repo for pushing commits
remote_name = self.getRemoteEndpoint(self.repo_name)
self.repo_path = 'tmp-release-freertos-kernel'
# Clean up any old work from previous runs
if os.path.exists(self.repo_path):
shutil.rmtree(self.repo_path)
# Download master:HEAD. Update its file header versions and kernel macros
self.repo_path = download_git_tree(remote_name, '.', self.repo_path, 'master', 'HEAD')
assert self.repo_path != None, 'Failed to download git tree'
def updateFileHeaderVersions(self):
'''
Adds changes for two commits
1.) Updates to file headers
@ -113,28 +118,35 @@ class KernelRelease(BaseRelease):
Then tags commit #2 with the new tag version. Notes this will overwrite a tag it already exists
Finally pushes all these changes
'''
remote_name = self.getRemoteEndpoint(self.repo_name)
rel_repo_path = 'tmp-versioning-freertos-kernel'
# Clean up any old work from previous runs
if os.path.exists(rel_repo_path):
shutil.rmtree(rel_repo_path)
# Download master:HEAD. Update its file header versions and kernel macros
repo_path = download_git_tree(remote_name, '.', rel_repo_path, 'master', 'HEAD')
assert repo_path != None, 'Failed to download git tree'
update_version_number_in_freertos_component(repo_path, '.', old_version_prefix, 'FreeRTOS Kernel V%s' % self.version)
commit_git_tree_changes(rel_repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version)
target_version_prefixes = ['FreeRTOS Kernel V']
update_version_number_in_freertos_component(self.repo_path, '.', target_version_prefixes, 'FreeRTOS Kernel V%s' % self.version)
commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version)
def updateVersionMacros(self):
(major, minor, build) = self.version.split('.')
update_freertos_version_macros(os.path.join(repo_path, 'include', 'task.h'), major, minor, build)
commit_git_tree_changes(rel_repo_path, commit_message=self.commit_msg_prefix + 'Bump task.h version macros to "%s"' % self.version)
update_freertos_version_macros(os.path.join(self.repo_path, 'include', 'task.h'), major, minor, build)
commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix + 'Bump task.h version macros to "%s"' % self.version)
# Commit the versioning, tag it, and upload all to remote
rc = push_git_tree_changes(repo_path, tag=self.tag, force_tag=True)
assert rc == 0, 'Failed to upload git tree changes'
def createGitRelease(self):
'''
Creates/Overwrites release identified by target tag
'''
# If this release already exists, delete it
try:
release_queried = self.repo.get_release(self.tag)
info('Deleting existing release "%s"...' % self.tag)
release_queried.delete_release()
except UnknownObjectException:
info('Creating release/tag "%s"...' % self.tag)
# Create the new release endpoint at upload assets
release = self.repo.create_git_release(tag = self.tag,
name = 'V%s' % (self.version),
message = self.description,
draft = False,
prerelease = False)
class FreertosRelease(BaseRelease):
def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'):
@ -146,29 +158,46 @@ class FreertosRelease(BaseRelease):
self.description = 'Contains source code and example projects for the FreeRTOS Kernel and FreeRTOS+ libraries.'
self.zip = None
def updateFileHeaderVersions(self, old_version_prefix, new_kernel_ref):
remote_name = self.getRemoteEndpoint(self.repo_name)
rel_repo_path = 'tmp-versioning-freertos'
self.repo_path = 'tmp-release-freertos'
# Clean up any old work from previous runs
if os.path.exists(rel_repo_path):
shutil.rmtree(rel_repo_path)
if os.path.exists(self.repo_path):
shutil.rmtree(self.repo_path)
# Download master:HEAD. Update its file header versions and kernel submodule pointer
repo_path = download_git_tree(remote_name, '.', rel_repo_path, 'master', 'HEAD')
assert repo_path != None, 'Failed to download git tree'
self.repo_path = download_git_tree(remote_name, '.', self.repo_path, 'master', 'HEAD')
assert self.repo_path != None, 'Failed to download git tree'
update_version_number_in_freertos_component(repo_path, '.', old_version_prefix, 'FreeRTOS V%s' % self.version)
commit_git_tree_changes(repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version)
def updateFileHeaderVersions(self):
target_version_substrings = ['FreeRTOS Kernel V', 'FreeRTOS V']
update_version_number_in_freertos_component(self.repo_path, '.', target_version_substrings, 'FreeRTOS V%s' % self.version)
commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version)
update_submodule_pointer(repo_path, os.path.join('FreeRTOS', 'Source'), new_kernel_ref)
commit_git_tree_changes(repo_path, commit_message=self.commit_msg_prefix + 'Bump kernel pointer "%s"' % new_kernel_ref)
def updateSubmodulePointers(self):
'''
Reads the 'manifest.yml' file from the local FreeRTOS clone that is being used to stage the commits
'''
path_manifest = os.path.join(self.repo_path, 'manifest.yml')
assert os.path.exists(path_manifest), 'Missing manifest.yml'
# Commit the versioning, tag it, and upload all to remote
rc = push_git_tree_changes(repo_path, tag=self.tag, force_tag=True)
assert rc == 0, 'Failed to upload git tree changes'
with open(path_manifest, 'r') as fp:
manifest_data = fp.read()
yml = load(manifest_data, Loader=Loader)
assert 'dependencies' in yml, 'Manifest YML parsing error'
for dep in yml['dependencies']:
assert 'version' in dep, 'Failed to parse submodule tag from manifest'
assert 'repository' in dep and 'path' in dep['repository'], 'Failed to parse submodule path from manifest'
submodule_path = dep['repository']['path']
submodule_tag = dep['version']
def CreateReleaseZip(self):
# Update the submodule to point to version noted in manifest file
update_submodule_pointer(self.repo_path, submodule_path, submodule_tag)
commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix
+ 'Bump submodules per manifest.yml for V%s' % self.version)
def createReleaseZip(self):
'''
At the moment, the only asset we upload is the
'''
@ -196,7 +225,7 @@ class FreertosRelease(BaseRelease):
'FreeRTOSv%s' % self.version,
exclude_files=FREERTOS_RELATIVE_FILE_EXCLUDES)
def Upload(self):
def createGitRelease(self):
'''
Creates/Overwrites release identified by target tag
'''
@ -222,25 +251,14 @@ class FreertosRelease(BaseRelease):
def configure_argparser():
parser = ArgumentParser(description='FreeRTOS Release tool')
parser.add_argument('--old-core-version',
default=None,
required=True,
help='FreeRTOS Version to match and replace. (Ex. FreeRTOS V202011.00)')
parser.add_argument('--new-core-version',
default=None,
required=True,
help='FreeRTOS Version to replace old version. (Ex. FreeRTOS V202011.00)')
parser.add_argument('--old-kernel-version',
default=None,
required=True,
help='FreeRTOS-Kernel Version to match and replace. (Ex. "FreeRTOS Kernel V10.4.1")')
required=False,
help='FreeRTOS-Kernel Version to replace old version. (Ex. "FreeRTOS Kernel V10.4.1")')
parser.add_argument('--new-kernel-version',
default=None,
required=True,
required=False,
help='FreeRTOS-Kernel Version to replace old version. (Ex. "FreeRTOS Kernel V10.4.1")')
parser.add_argument('--git-org',
@ -253,7 +271,6 @@ def configure_argparser():
action='store_true',
help='Use SSH endpoints to interface git remotes, instead of HTTPS')
parser.add_argument('--unit-test',
action='store_true',
default=False,
@ -261,50 +278,35 @@ def configure_argparser():
return parser
def sanitize_cmd_args(args):
info('TODO: Add cmdline input sanitizing')
def main():
# CLI
cmd = configure_argparser()
# Setup
args = cmd.parse_args()
sanitize_cmd_args(args)
# Auth
assert 'GITHUB_TOKEN' in os.environ, 'You must set env variable GITHUB_TOKEN to an authorized git PAT'
assert 'GITHUB_TOKEN' in os.environ, 'Set env{GITHUB_TOKEN} to an authorized git PAT'
mGit = Github(os.environ.get('GITHUB_TOKEN'))
# Create release or test
if args.unit_test:
pass
else:
# Update versions
return
if args.new_kernel_version:
rel_kernel = KernelRelease(mGit, args.new_kernel_version, None, git_ssh=args.use_git_ssh, git_org=args.git_org)
rel_kernel.updateFileHeaderVersions(args.old_kernel_version)
rel_kernel.updateFileHeaderVersions()
rel_kernel.updateVersionMacros()
rel_kernel.pushAutoCommits()
rel_kernel.createGitRelease()
if args.new_core_version:
rel_freertos = FreertosRelease(mGit, args.new_core_version, None, git_ssh=args.use_git_ssh, git_org=args.git_org)
rel_freertos.updateFileHeaderVersions(args.old_core_version, 'V%s' % args.new_kernel_version)
# Package contents of FreeRTOS/FreeRTOS and upload release assets to Git
rel_freertos.CreateReleaseZip()
rel_freertos.Upload()
rel_freertos.updateFileHeaderVersions()
rel_freertos.updateSubmodulePointers()
rel_freertos.pushAutoCommits()
rel_freertos.createReleaseZip()
rel_freertos.createGitRelease()
info('Review script output for any unexpected behaviour.')
info('Release done.')
if __name__ == '__main__':
main()
#--------------------------------------------------------------------
# TESTING
#--------------------------------------------------------------------
# Create new tag, verify creation
# Create release endpoint, delete it, verify deletion
# Overwrite an existing tag
# Perform full operation, restore to state before operation, verify restored state
# Run zipping operation, check versions, pathing, etc

8
.github/scripts/versioning.py vendored
View File

@ -224,7 +224,8 @@ def update_freertos_version_macros(path_macrofile, major, minor, build):
print('Done. Replaced "%s" --> "V%s.%s.%s".' % (old_version_number, major, minor, build))
def update_version_number_in_freertos_component(component, root_dir, old_version_prefix, new_version, verbose=False):
def update_version_number_in_freertos_component(component, root_dir, old_version_prefix_list, new_version, verbose=False):
assert isinstance(old_version_prefix_list, list), 'Expected a list for arg(old_version_prefix_list)'
print('Updating "%s"...' % component)
component_files = list_files_in_a_component(component, root_dir, ext_filter=None)
version_numbers = defaultdict(list)
@ -242,8 +243,9 @@ def update_version_number_in_freertos_component(component, root_dir, old_version
old_version_string = vkey[0]
new_version_string = new_version
if old_version_prefix in old_version_string:
if old_version_string != new_version_string:
# Check if any of the associated versioning strings are present. Update if so
for old_prefix in old_version_prefix_list:
if old_prefix in old_version_string and old_version_string != new_version_string:
files_using_old_version = version_numbers[vkey]
if verbose:

68
manifest.yml Normal file
View File

@ -0,0 +1,68 @@
name : "FreeRTOS"
version: "202012.00"
description: "This is the standard distribution of FreeRTOS."
dependencies:
- name: "FreeRTOS-Kernel"
version: "V10.4.3"
repository:
type: "git"
url: "https://github.com/FreeRTOS/FreeRTOS-Kernel.git"
path: "FreeRTOS/Source"
- name: "FreeRTOS-Plus-TCP"
version: "V2.3.2"
repository:
type: "git"
url: "https://github.com/FreeRTOS/FreeRTOS-Plus-TCP.git"
path: "FreeRTOS-Plus/Source/FreeRTOS-Plus-TCP"
- name: "coreJSON"
version: "v3.0.0"
repository:
type: "git"
url: "https://github.com/FreeRTOS/coreJSON.git"
path: "FreeRTOS-Plus/Source/coreJSON"
- name: "coreHTTP"
version: "v1.0.1"
repository:
type: "git"
url: "https://github.com/FreeRTOS/coreHTTP.git"
path: "FreeRTOS-Plus/Source/Application-Protocols/coreHTTP"
- name: "coreMQTT"
version: "v1.0.2"
repository:
type: "git"
url: "https://github.com/FreeRTOS/coreMQTT.git"
path: "FreeRTOS-Plus/Source/Application-Protocols/coreMQTT"
- name: "corePKCS11"
version: "v3.0.0"
repository:
type: "git"
url: "https://github.com/FreeRTOS/corePKCS11.git"
path: "FreeRTOS-Plus/Source/corePKCS11"
- name: "device-defender"
version: "v1.0.1"
repository:
type: "git"
url: "https://github.com/aws/Device-Defender-for-AWS-IoT-embedded-sdk.git"
path: "FreeRTOS-Plus/Source/AWS/device-defender"
- name: "device-shadow"
version: "v1.0.2"
repository:
type: "git"
url: "https://github.com/aws/Device-Shadow-for-AWS-IoT-embedded-sdk.git"
path: "FreeRTOS-Plus/Source/AWS/device-shadow"
- name: "jobs"
version: "v1.0.1"
repository:
type: "git"
url: "https://github.com/aws/Jobs-for-AWS-IoT-embedded-sdk.git"
path: "FreeRTOS-Plus/Source/AWS/jobs"