Generate workflow files with a Jinja template (#2687)

* Generate workflow files with a Jinja template

Fixes #2686

* Remove sudo from allowlist_externals

* Update workflows

* Add condition to skip generate-workflows

* Update workflows

* Update .github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2

Co-authored-by: Emídio Neto <9735060+emdneto@users.noreply.github.com>

* Update .github/workflows/generate_workflows_lib/src/generate_workflows_lib/misc.yml.j2

Co-authored-by: Emídio Neto <9735060+emdneto@users.noreply.github.com>

* Update .github/workflows/generate_workflows_lib/src/generate_workflows_lib/lint.yml.j2

Co-authored-by: Emídio Neto <9735060+emdneto@users.noreply.github.com>

* Update .github/workflows/generate_workflows_lib/src/generate_workflows_lib/contrib.yml.j2

Co-authored-by: Emídio Neto <9735060+emdneto@users.noreply.github.com>

* Update workflows

* Update workflows

---------

Co-authored-by: Emídio Neto <9735060+emdneto@users.noreply.github.com>

.github/workflows/generate_workflows_lib/hatch_build.py

@@ -0,0 +1,15 @@
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+from pathlib import Path
+
+
+class CustomBuildHook(BuildHookInterface):
+
+    def initialize(self, version, build_data):
+
+        with open(
+            Path(__file__).parent.parent.parent.parent.joinpath("tox.ini")
+        ) as tox_ini_file_0:
+            with open(
+                Path(__file__).parent.joinpath("src/generate_workflows_lib/tox.ini"), "w"
+            ) as tox_ini_file_1:
+                tox_ini_file_1.write(tox_ini_file_0.read())

.github/workflows/generate_workflows_lib/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "generate-workflows-lib"
+dynamic = ["version"]
+description = "A library to generate workflows"
+license = "Apache-2.0"
+requires-python = ">=3.8"
+authors = [
+  { name = "OpenTelemetry Authors", email = "cncf-opentelemetry-contributors@lists.cncf.io" },
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: Apache Software License",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Typing :: Typed",
+]
+dependencies = ["Jinja2", "tox"]
+
+[tool.hatch.version]
+path = "src/generate_workflows_lib/version.py"
+
+[tool.hatch.build.targets.wheel.hooks.custom]

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/.gitignore

@@ -0,0 +1 @@
+tox.ini

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py

@@ -0,0 +1,267 @@
+from re import compile as re_compile
+from jinja2 import Environment, FileSystemLoader
+from pathlib import Path
+from tox.config.cli.parse import get_options
+from tox.session.state import State
+from tox.config.sets import CoreConfigSet
+from tox.config.source.tox_ini import ToxIni
+from collections import defaultdict
+
+
+_tox_test_env_regex = re_compile(
+    r"(?P<python_version>py\w+)-test-"
+    r"(?P<name>[-\w]+\w)-?(?P<test_requirements>\d+)?"
+)
+_tox_lint_env_regex = re_compile(r"lint-(?P<name>[-\w]+)")
+_tox_contrib_env_regex = re_compile(
+    r"py38-test-(?P<name>[-\w]+\w)-?(?P<contrib_requirements>\d+)?"
+)
+
+
+def get_tox_envs(tox_ini_path: Path) -> list:
+
+    tox_ini = ToxIni(tox_ini_path)
+
+    conf = State(get_options(), []).conf
+
+    tox_section = next(tox_ini.sections())
+
+    core_config_set = (
+        CoreConfigSet(conf, tox_section, tox_ini_path.parent, tox_ini_path)
+    )
+
+    (
+        core_config_set.
+        loaders.
+        extend(
+            tox_ini.
+            get_loaders(
+                tox_section,
+                base=[],
+                override_map=defaultdict(list, {}),
+                conf=core_config_set
+            )
+        )
+    )
+
+    return core_config_set.load("env_list")
+
+
+def get_test_job_datas(tox_envs: list, operating_systems: list) -> list:
+
+    os_alias = {
+        "ubuntu-latest": "Ubuntu",
+        "windows-latest": "Windows"
+    }
+
+    python_version_alias = {
+        "pypy3": "pypy-3.8",
+        "py38": "3.8",
+        "py39": "3.9",
+        "py310": "3.10",
+        "py311": "3.11",
+        "py312": "3.12",
+    }
+
+    test_job_datas = []
+
+    for operating_system in operating_systems:
+        for tox_env in tox_envs:
+
+            tox_test_env_match = _tox_test_env_regex.match(tox_env)
+
+            if tox_test_env_match is None:
+                continue
+
+            groups = tox_test_env_match.groupdict()
+
+            aliased_python_version = (
+                python_version_alias[groups["python_version"]]
+            )
+            tox_env = tox_test_env_match.string
+
+            test_requirements = groups["test_requirements"]
+
+            if test_requirements is None:
+                test_requirements = " "
+
+            else:
+                test_requirements = f"-{test_requirements} "
+
+            test_job_datas.append(
+                {
+                    "name": f"{tox_env}_{operating_system}",
+                    "ui_name": (
+                        f"{groups['name']}"
+                        f"{test_requirements}"
+                        f"{aliased_python_version} "
+                        f"{os_alias[operating_system]}"
+                    ),
+                    "python_version": aliased_python_version,
+                    "tox_env": tox_env,
+                    "os": operating_system
+                }
+
+            )
+
+    return test_job_datas
+
+
+def get_lint_job_datas(tox_envs: list) -> list:
+
+    lint_job_datas = []
+
+    for tox_env in tox_envs:
+
+        tox_lint_env_match = _tox_lint_env_regex.match(tox_env)
+
+        if tox_lint_env_match is None:
+            continue
+
+        tox_env = tox_lint_env_match.string
+
+        lint_job_datas.append(
+            {
+                "name": f"{tox_env}",
+                "ui_name": f"{tox_lint_env_match.groupdict()['name']}",
+                "tox_env": tox_env,
+            }
+
+        )
+
+    return lint_job_datas
+
+
+def get_contrib_job_datas(tox_envs: list) -> list:
+
+    contrib_job_datas = []
+
+    for tox_env in tox_envs:
+
+        tox_contrib_env_match = _tox_contrib_env_regex.match(tox_env)
+
+        if tox_contrib_env_match is None:
+            continue
+
+        groups = tox_contrib_env_match.groupdict()
+
+        tox_env = tox_contrib_env_match.string
+
+        contrib_requirements = groups["contrib_requirements"]
+
+        if contrib_requirements is None:
+            contrib_requirements = " "
+
+        else:
+            contrib_requirements = f"-{contrib_requirements} "
+
+        contrib_job_datas.append(
+            {
+                "ui_name": (
+                    f"{groups['name']}"
+                    f"{contrib_requirements}"
+                ),
+                "tox_env": tox_env,
+            }
+
+        )
+
+    return contrib_job_datas
+
+
+def get_misc_job_datas(tox_envs: list) -> list:
+
+    misc_job_datas = []
+
+    _tox_benchmark_env_regex = re_compile(r"benchmark.+")
+
+    for tox_env in tox_envs:
+        if (
+            _tox_test_env_regex.match(tox_env) is not None or
+            _tox_lint_env_regex.match(tox_env) is not None or
+            _tox_contrib_env_regex.match(tox_env) is not None or
+            _tox_benchmark_env_regex.match(tox_env) is not None
+        ):
+            continue
+
+        misc_job_datas.append(tox_env)
+
+    return misc_job_datas
+
+
+def _generate_workflow(
+    job_datas: list, name: str, workflow_directory_path: Path
+):
+
+    # Github seems to limit the amount of jobs in a workflow file, that is why
+    # they are split in groups of 250 per workflow file.
+    for file_number, job_datas in enumerate(
+        [
+            job_datas[index:index + 250]
+            for index in range(0, len(job_datas), 250)
+        ]
+    ):
+
+        with open(
+            workflow_directory_path.joinpath(f"{name}_{file_number}.yml"),
+            "w"
+        ) as test_yml_file:
+
+            test_yml_file.write(
+                Environment(
+                    loader=FileSystemLoader(Path(__file__).parent)
+                ).get_template(f"{name}.yml.j2").render(
+                    job_datas=job_datas, file_number=file_number
+                )
+            )
+            test_yml_file.write("\n")
+
+
+def generate_test_workflow(
+    tox_ini_path: Path,
+    workflow_directory_path: Path,
+    *operating_systems
+) -> None:
+
+    _generate_workflow(
+        get_test_job_datas(get_tox_envs(tox_ini_path), operating_systems),
+        "test",
+        workflow_directory_path
+    )
+
+
+def generate_lint_workflow(
+    tox_ini_path: Path,
+    workflow_directory_path: Path,
+) -> None:
+
+    _generate_workflow(
+        get_lint_job_datas(get_tox_envs(tox_ini_path)),
+        "lint",
+        workflow_directory_path
+    )
+
+
+def generate_contrib_workflow(
+    workflow_directory_path: Path,
+) -> None:
+
+    _generate_workflow(
+        get_contrib_job_datas(
+            get_tox_envs(Path(__file__).parent.joinpath("tox.ini"))
+        ),
+        "contrib",
+        workflow_directory_path
+    )
+
+
+def generate_misc_workflow(
+    tox_ini_path: Path,
+    workflow_directory_path: Path,
+) -> None:
+
+    _generate_workflow(
+        get_misc_job_datas(get_tox_envs(tox_ini_path)),
+        "misc",
+        workflow_directory_path
+    )

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/contrib.yml.j2

@@ -0,0 +1,47 @@
+# Do not edit this file.
+# This file is generated automatically by executing tox -e generate-workflows
+
+name: Contrib {{ file_number }}
+
+on:
+  push:
+    branches-ignore:
+    - 'release/*'
+  pull_request:
+
+env:
+  CORE_REPO_SHA: ${% raw %}{{ github.sha }}{% endraw %}
+  CONTRIB_REPO_SHA: main
+  PIP_EXISTS_ACTION: w
+
+jobs:
+  {%- for job_data in job_datas %}
+
+  {{ job_data.tox_env }}:
+    name: {{ job_data.ui_name }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout contrib repo @ SHA - ${% raw %}{{ env.CONTRIB_REPO_SHA }}{% endraw %}
+        uses: actions/checkout@v4
+        with:
+          repository: open-telemetry/opentelemetry-python-contrib
+          ref: ${% raw %}{{ env.CONTRIB_REPO_SHA }}{% endraw %}
+
+      - name: Checkout core repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
+        uses: actions/checkout@v4
+        with:
+          repository: open-telemetry/opentelemetry-python
+          path: opentelemetry-python-core
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
+          architecture: "x64"
+
+      - name: Install tox
+        run: pip install tox
+
+      - name: Run tests
+        run: tox -e {{ job_data.tox_env }} -- -ra
+  {%- endfor %}

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/lint.yml.j2

@@ -0,0 +1,37 @@
+# Do not edit this file.
+# This file is generated automatically by executing tox -e generate-workflows
+
+name: Lint {{ file_number }}
+
+on:
+  push:
+    branches-ignore:
+    - 'release/*'
+  pull_request:
+
+env:
+  CORE_REPO_SHA: main
+  CONTRIB_REPO_SHA: main
+  PIP_EXISTS_ACTION: w
+
+jobs:
+  {%- for job_data in job_datas %}
+
+  {{ job_data.name }}:
+    name: {{ job_data.ui_name }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install tox
+        run: pip install tox
+
+      - name: Run tests
+        run: tox -e {{ job_data.tox_env }}
+  {%- endfor %}

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/misc.yml.j2

@@ -0,0 +1,67 @@
+# Do not edit this file.
+# This file is generated automatically by executing tox -e generate-workflows
+
+name: Misc {{ file_number }}
+
+on:
+  push:
+    branches-ignore:
+    - 'release/*'
+  pull_request:
+
+env:
+  CORE_REPO_SHA: main
+  CONTRIB_REPO_SHA: main
+  PIP_EXISTS_ACTION: w
+
+jobs:
+  {%- for job_data in job_datas %}
+
+  {{ job_data }}:
+    name: {{ job_data }}
+    runs-on: ubuntu-latest
+    {%- if job_data == "generate-workflows" %}
+    if: |
+      !contains(github.event.pull_request.labels.*.name, 'Skip generate-workflows')
+      && github.actor != 'opentelemetrybot'
+    {%- endif %}
+    steps:
+      - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
+        uses: actions/checkout@v4
+      {%- if job_data == "public-symbols-check" %}
+        with:
+          fetch-depth: 0
+
+      - name: Checkout main
+        run: git checkout main
+
+      - name: Pull origin
+        run: git pull --rebase=false origin main
+
+      - name: Checkout pull request
+        run: git checkout ${% raw %}{{ github.event.pull_request.head.sha }}{% endraw %}
+      {%- endif %}
+      {%- if job_data != "shellcheck" %}
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      {%- endif %}
+
+      - name: Install tox
+        run: pip install tox
+
+      - name: Run tests
+        run: tox -e {{ job_data }}
+      {%- if job_data == "generate-workflows" %}
+
+      - name: Check workflows are up to date
+        run: git diff --exit-code || (echo 'Generated workflows are out of date, run "tox -e generate-workflows" and commit the changes in this PR.' && exit 1)
+      {%- endif %}
+      {%- if job_data == "generate" %}
+
+      - name: Check workflows are up to date
+        run: git diff --exit-code || (echo 'Generated code is out of date, run "tox -e generate" and commit the changes in this PR.' && exit 1)
+      {%- endif %}
+  {%- endfor %}

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2

@@ -0,0 +1,42 @@
+# Do not edit this file.
+# This file is generated automatically by executing tox -e generate-workflows
+
+name: Test {{ file_number }}
+
+on:
+  push:
+    branches-ignore:
+    - 'release/*'
+  pull_request:
+
+env:
+  CORE_REPO_SHA: main
+  CONTRIB_REPO_SHA: main
+  PIP_EXISTS_ACTION: w
+
+jobs:
+  {%- for job_data in job_datas %}
+
+  {{ job_data.name }}:
+    name: {{ job_data.ui_name }}
+    runs-on: {{ job_data.os }}
+    steps:
+      - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
+        uses: actions/checkout@v4
+
+      - name: Set up Python {{ job_data.python_version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: "{{ job_data.python_version }}"
+
+      - name: Install tox
+        run: pip install tox
+      {%- if job_data.os == "windows-latest" %}
+
+      - name: Configure git to support long filenames
+        run: git config --system core.longpaths true
+      {%- endif %}
+
+      - name: Run tests
+        run: tox -e {{ job_data.tox_env }} -- -ra
+  {%- endfor %}

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"