Move benchmarks outside of tests directory (#2670)

Fixes #2669

.github/workflows/instrumentations_0.yml

@@ -126,5 +126,5 @@ jobs:
             .tox
             ~/.cache/pip
           key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }}
-      - name: run pytest without --benchmark-skip
+      - name: run tox
         run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra

.github/workflows/instrumentations_1.yml

@@ -25,7 +25,6 @@ jobs:
       matrix:
         python-version: [py38, py39, py310, py311, py312, pypy3]
         package:
-          # Only add here packages that do not have benchmark tests
           - "urllib"
           - "urllib3"
           - "wsgi"
@@ -33,6 +32,8 @@ jobs:
           - "richconsole"
           - "psycopg"
           - "prometheus-remote-write"
+          - "sdk-extension-aws"
+          - "propagator-aws-xray"
           - "propagator-ot-trace"
           - "resource-detector-azure"
           - "resource-detector-container"
@@ -58,5 +59,5 @@ jobs:
             .tox
             ~/.cache/pip
           key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }}
-      - name: run pytest without --benchmark-skip
+      - name: run tox
         run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra

.github/workflows/instrumentations_2.yml

@@ -1,50 +0,0 @@
-name: Contrib Repo Tests
-
-on:
-  push:
-    branches-ignore:
-    - 'release/*'
-  pull_request:
-env:
-  CORE_REPO_SHA: 141a6a2e473ef7f0ec4915dfb71e3c0fa595283e
-
-jobs:
-  instrumentations-2:
-    env:
-      # We use these variables to convert between tox and GHA version literals
-      py38: 3.8
-      py39: 3.9
-      py310: "3.10"
-      py311: "3.11"
-      py312: "3.12"
-      pypy3: pypy-3.8
-      RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false  # ensures the entire test matrix is run, even if one permutation fails
-      matrix:
-        python-version: [py38, py39, py310, py311, py312, pypy3]
-        package:
-          # Only add here packages that have benchmark tests
-          - "sdk-extension-aws"
-          - "propagator-aws-xray"
-        os: [ubuntu-20.04]
-    steps:
-      - name: Checkout Contrib Repo @ SHA - ${{ github.sha }}
-        uses: actions/checkout@v4
-      - name: Set up Python ${{ env[matrix.python-version] }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env[matrix.python-version] }}
-      - name: Install tox
-        run: pip install tox
-      - name: Cache tox environment
-        # Preserves .tox directory between runs for faster installs
-        uses: actions/cache@v4
-        with:
-          path: |
-            .tox
-            ~/.cache/pip
-          key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }}
-      - name: run pytest with --benchmark-skip
-        run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra --benchmark-skip

.gitignore

@@ -58,3 +58,6 @@ _build/
 # mypy
 .mypy_cache/
 target
+
+# Benchmark result files
+*-benchmark.json

CONTRIBUTING.md

@@ -96,9 +96,7 @@ for more detail on available tox commands.
 
 ### Benchmarks
 
-Performance progression of benchmarks for packages distributed by OpenTelemetry Python can be viewed as a [graph of throughput vs commit history](https://opentelemetry-python-contrib.readthedocs.io/en/latest/performance/benchmarks.html). From the linked page, you can download a JSON file with the performance results.
+Some packages have benchmark tests. To run them, run `tox -f benchmark`. Benchmark tests use `pytest-benchmark` and they output a table with results to the console.
-
-Running the `tox` tests also runs the performance tests if any are available. Benchmarking tests are done with `pytest-benchmark` and they output a table with results to the console.
 
 To write benchmarks, simply use the [pytest benchmark fixture](https://pytest-benchmark.readthedocs.io/en/latest/usage.html#usage) like the following:
 
@@ -114,10 +112,10 @@ def test_simple_start_span(benchmark):
     benchmark(benchmark_start_as_current_span, "benchmarkedSpan", 42)
 ```
 
-Make sure the test file is under the `tests/performance/benchmarks/` folder of
+Make sure the test file is under the `benchmarks/` folder of
 the package it is benchmarking and further has a path that corresponds to the
 file in the package it is testing. Make sure that the file name begins with
-`test_benchmark_`. (e.g. `propagator/opentelemetry-propagator-aws-xray/tests/performance/benchmarks/trace/propagation/test_benchmark_aws_xray_propagator.py`)
+`test_benchmark_`. (e.g. `propagator/opentelemetry-propagator-aws-xray/benchmarks/trace/propagation/test_benchmark_aws_xray_propagator.py`)
 
 ## Pull Requests
 

propagator/opentelemetry-propagator-aws-xray/benchmark-requirements.txt

@@ -0,0 +1 @@
+pytest-benchmark==4.0.0

sdk-extension/opentelemetry-sdk-extension-aws/benchmark-requirements.txt

@@ -0,0 +1 @@
+pytest-benchmark==4.0.0

tox.ini

@@ -20,6 +20,7 @@ envlist =
     py3{8,9,10,11,12}-test-sdk-extension-aws
     pypy3-test-sdk-extension-aws
     lint-sdk-extension-aws
+    benchmark-sdk-extension-aws
 
     ; opentelemetry-distro
     py3{8,9,10,11,12}-test-distro
@@ -314,6 +315,7 @@ envlist =
     py3{8,9,10,11,12}-test-propagator-aws-xray
     pypy3-test-propagator-aws-xray
     lint-propagator-aws-xray
+    benchmark-propagator-aws-xray
 
     ; opentelemetry-propagator-ot-trace
     py3{8,9,10,11,12}-test-propagator-ot-trace
@@ -741,6 +743,7 @@ commands_pre =
   sdk-extension-aws: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk
   sdk-extension-aws: pip install opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils
   sdk-extension-aws: pip install -r {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws/test-requirements.txt
+  benchmark-sdk-extension-aws: pip install -r {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws/benchmark-requirements.txt
 
   resource-detector-container: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api
   resource-detector-container: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions
@@ -765,6 +768,7 @@ commands_pre =
   propagator-aws-xray: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk
   propagator-aws-xray: pip install opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils
   propagator-aws-xray: pip install -r {toxinidir}/propagator/opentelemetry-propagator-aws-xray/test-requirements.txt
+  benchmark-propagator-aws-xray: pip install -r {toxinidir}/propagator/opentelemetry-propagator-aws-xray/benchmark-requirements.txt
 
   processor-baggage: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api
   processor-baggage: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions
@@ -1094,6 +1098,7 @@ commands =
   lint-sdk-extension-aws: isort --diff --check-only --settings-path {toxinidir}/.isort.cfg {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws
   lint-sdk-extension-aws: flake8 --config {toxinidir}/.flake8 {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws
   lint-sdk-extension-aws: sh -c "cd sdk-extension && pylint --rcfile ../.pylintrc opentelemetry-sdk-extension-aws"
+  benchmark-sdk-extension-aws: pytest {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws/benchmarks {posargs} --benchmark-json=sdk-extension-aws-benchmark.json
 
   test-resource-detector-container: pytest {toxinidir}/resource/opentelemetry-resource-detector-container/tests {posargs}
   lint-resource-detector-container: black --diff --check --config {toxinidir}/pyproject.toml {toxinidir}/resource/opentelemetry-resource-detector-container
@@ -1118,6 +1123,7 @@ commands =
   lint-propagator-aws-xray: isort --diff --check-only --settings-path {toxinidir}/.isort.cfg {toxinidir}/propagator/opentelemetry-propagator-aws-xray
   lint-propagator-aws-xray: flake8 --config {toxinidir}/.flake8 {toxinidir}/propagator/opentelemetry-propagator-aws-xray
   lint-propagator-aws-xray: sh -c "cd propagator && pylint --rcfile ../.pylintrc opentelemetry-propagator-aws-xray"
+  benchmark-propagator-aws-xray: pytest {toxinidir}/propagator/opentelemetry-propagator-aws-xray/benchmarks {posargs} --benchmark-json=propagator-aws-xray-benchmark.json
 
   test-propagator-ot-trace: pytest {toxinidir}/propagator/opentelemetry-propagator-ot-trace/tests {posargs}
   lint-propagator-ot-trace: black --diff --check --config {toxinidir}/pyproject.toml {toxinidir}/propagator/opentelemetry-propagator-ot-trace