Separate jobs per instrumentation (#2121)

* Separate jobs per instrumentation

Fixes #2036

* Separate in 2 workflows

* Added explanation for separation

.github/workflows/instrumentations_0.yml

@@ -0,0 +1,95 @@
+name: Contrib Repo Tests
+
+on:
+  push:
+    branches-ignore:
+    - 'release/*'
+  pull_request:
+env:
+  CORE_REPO_SHA: 84c0e4f38d4fcdb8c13fd3988469fbb8cda28150
+
+jobs:
+  instrumentations-0:
+    env:
+      # We use these variables to convert between tox and GHA version literals
+      py37: 3.7
+      py38: 3.8
+      py39: 3.9
+      py310: "3.10"
+      py311: "3.11"
+      pypy3: pypy-3.7
+      RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false  # ensures the entire test matrix is run, even if one permutation fails
+      matrix:
+        python-version: [py37, py38, py39, py310, py311, pypy3]
+        package:
+          # Do not add more instrumentations here, add them in instrumentations_1.yml.
+          # The reason for this separation of instrumentations into more than one YAML file is
+          # the limit of jobs that can be run from a Github actions matrix:
+          # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
+          # "A matrix will generate a maximum of 256 jobs per workflow run. This limit applies
+          # to both GitHub-hosted and self-hosted runners."
+          - "aiohttp-client"
+          - "aiohttp-server"
+          - "aiopg"
+          - "aio-pika"
+          - "asgi"
+          - "asyncpg"
+          - "aws-lambda"
+          - "boto"
+          - "boto3sqs"
+          - "botocore"
+          - "cassandra"
+          - "celery"
+          - "confluent-kafka"
+          - "dbapi"
+          - "django"
+          - "elasticsearch"
+          - "falcon"
+          - "fastapi"
+          - "flask"
+          - "grpc"
+          - "httpx"
+          - "jinja2"
+          - "kafka-python"
+          - "logging"
+          - "mysql"
+          - "mysqlclient"
+          - "pika"
+          - "psycopg2"
+          - "pymemcache"
+          - "pymongo"
+          - "pymysql"
+          - "pyramid"
+          - "redis"
+          - "remoulade"
+          - "requests"
+          - "sklearn"
+          - "sqlalchemy"
+          - "sqlite3"
+          - "starlette"
+          - "system-metrics"
+          - "tornado"
+          - "tortoiseorm"
+        os: [ubuntu-20.04]
+    steps:
+      - name: Checkout Contrib Repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@v2
+      - name: Set up Python ${{ env[matrix.python-version] }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env[matrix.python-version] }}
+      - name: Install tox
+        run: pip install tox==3.27.1 tox-factor
+      - name: Cache tox environment
+        # Preserves .tox directory between runs for faster installs
+        uses: actions/cache@v1
+        with:
+          path: |
+            .tox
+            ~/.cache/pip
+          key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }}
+      - name: run tox
+        run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra --benchmark-json=${{ env.RUN_MATRIX_COMBINATION }}-benchmark.json

.github/workflows/instrumentations_1.yml

@@ -0,0 +1,57 @@
+name: Contrib Repo Tests
+
+on:
+  push:
+    branches-ignore:
+    - 'release/*'
+  pull_request:
+env:
+  CORE_REPO_SHA: 84c0e4f38d4fcdb8c13fd3988469fbb8cda28150
+
+jobs:
+  instrumentations-1:
+    env:
+      # We use these variables to convert between tox and GHA version literals
+      py37: 3.7
+      py38: 3.8
+      py39: 3.9
+      py310: "3.10"
+      py311: "3.11"
+      pypy3: pypy-3.7
+      RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false  # ensures the entire test matrix is run, even if one permutation fails
+      matrix:
+        python-version: [py37, py38, py39, py310, py311, pypy3]
+        package:
+          - "urllib"
+          - "urllib3"
+          - "wsgi"
+          - "distro"
+          - "richconsole"
+          - "prometheus-remote-write"
+          - "sdkextension-aws"
+          - "propagator-aws-xray"
+          - "propagator-ot-trace"
+          - "resource-detector-container"
+        os: [ubuntu-20.04]
+    steps:
+      - name: Checkout Contrib Repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@v2
+      - name: Set up Python ${{ env[matrix.python-version] }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env[matrix.python-version] }}
+      - name: Install tox
+        run: pip install tox==3.27.1 tox-factor
+      - name: Cache tox environment
+        # Preserves .tox directory between runs for faster installs
+        uses: actions/cache@v1
+        with:
+          path: |
+            .tox
+            ~/.cache/pip
+          key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }}
+      - name: run tox
+        run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra --benchmark-json=${{ env.RUN_MATRIX_COMBINATION }}-benchmark.json

.github/workflows/test.yml

@@ -9,89 +9,6 @@ env:
   CORE_REPO_SHA: 84c0e4f38d4fcdb8c13fd3988469fbb8cda28150
 
 jobs:
-  build:
-    env:
-      # We use these variables to convert between tox and GHA version literals
-      py37: 3.7
-      py38: 3.8
-      py39: 3.9
-      py310: "3.10"
-      py311: "3.11"
-      pypy3: "pypy3.7"
-      RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false  # ensures the entire test matrix is run, even if one permutation fails
-      matrix:
-        python-version: [ py37, py38, py39, py310, py311, pypy3 ]
-        package: ["instrumentation", "distro", "exporter", "sdkextension", "propagator", "resource"]
-        os: [ ubuntu-20.04 ]
-    steps:
-      - name: Checkout Contrib Repo @ SHA - ${{ github.sha }}
-        uses: actions/checkout@v2
-      - name: Set up Python ${{ env[matrix.python-version] }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env[matrix.python-version] }}
-      - name: Install tox
-        run: pip install tox==3.27.1 tox-factor
-      - name: Cache tox environment
-        # Preserves .tox directory between runs for faster installs
-        uses: actions/cache@v1
-        with:
-          path: |
-            .tox
-            ~/.cache/pip
-          key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }}
-      - name: run tox
-        run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra --benchmark-json=${{ env.RUN_MATRIX_COMBINATION }}-benchmark.json
-  #     - name: Find and merge ${{ matrix.package }} benchmarks
-  #       # TODO: Add at least one benchmark to every package type to remove this (#249)
-  #       if: matrix.package == 'sdkextension' || matrix.package == 'propagator'
-  #       run: >-
-  #         mkdir -p benchmarks;
-  #         jq -s '.[0].benchmarks = ([.[].benchmarks] | add)
-  #         | if .[0].benchmarks == null then null else .[0] end'
-  #         **/**/tests/*${{ matrix.package }}*-benchmark.json > benchmarks/output_${{ matrix.package }}.json
-  #     - name: Upload all benchmarks under same key as an artifact
-  #       if: ${{ success() }}
-  #       uses: actions/upload-artifact@v2
-  #       with:
-  #         name: benchmarks
-  #         path: benchmarks/output_${{ matrix.package }}.json
-  # combine-benchmarks:
-  #   runs-on: ubuntu-latest
-  #   needs: build
-  #   if: ${{ always() }}
-  #   name: Combine benchmarks from previous build job
-  #   steps:
-  #     - name: Checkout Contrib Repo @ SHA - ${{ github.sha }}
-  #       uses: actions/checkout@v2
-  #     - name: Download all benchmarks as artifact using key
-  #       uses: actions/download-artifact@v2
-  #       with:
-  #         name: benchmarks
-  #         path: benchmarks
-  #     - name: Find and merge all benchmarks
-  #       run: >-
-  #         jq -s '.[0].benchmarks = ([.[].benchmarks] | add)
-  #         | if .[0].benchmarks == null then null else .[0] end'
-  #         benchmarks/output_*.json > output.json;
-  #     - name: Report on benchmark results
-  #       uses: benchmark-action/github-action-benchmark@v1
-  #       with:
-  #         name: OpenTelemetry Python Benchmarks - Python ${{ env[matrix.python-version ]}} - ${{ matrix.package }}
-  #         tool: pytest
-  #         output-file-path: output.json
-  #         github-token: ${{ secrets.GITHUB_TOKEN }}
-  #         max-items-in-chart: 100
-  #         # Alert with a commit comment on possible performance regression
-  #         alert-threshold: 200%
-  #         fail-on-alert: true
-  #         # Make a commit on `gh-pages` with benchmarks from previous step
-  #         auto-push: ${{ github.ref == 'refs/heads/main' }}
-  #         gh-pages-branch: gh-pages
-  #         benchmark-data-dir-path: benchmarks
   misc:
     strategy:
       fail-fast: false