Files
Diego Hurtado 3e36dc282d Mark test case as flaky (#197)
Fixes #196

This marks the test case as flaky, making it run at most 3 times. It is
enough for one of this runs to pass to consider this test case passed
and ran no more. If 3 consecutive runs of this test case fail, the test
case will be considered failed. It has been reported that running this
test case again makes it pass, usually. This approach is preferred over
marking it as xfail(strict=False) because most of the times the test
ends up passing after another run, so that in most of the cases we
can still benefit from running this test case (since if it is actually
failing because of a bug it will be reported as such after failing 3
times, making the team aware of an actual issue happening here).
2020-11-30 14:30:58 -08:00

340 lines
14 KiB
INI

[tox]
skipsdist = True
skip_missing_interpreters = True
envlist =
; Environments are organized by individual package, allowing
; for specifying supported Python versions per package.
; opentelemetry-sdk-extension-aws
py3{5,6,7,8}-test-sdkextension-aws
pypy3-test-sdkextension-aws
; opentelemetry-instrumentation-aiohttp-client
py3{5,6,7,8}-test-instrumentation-aiohttp-client
pypy3-test-instrumentation-aiohttp-client
; opentelemetry-instrumentation-aiopg
py3{5,6,7,8}-test-instrumentation-aiopg
; instrumentation-aiopg intentionally excluded from pypy3
; opentelemetry-instrumentation-botocore
py3{6,7,8}-test-instrumentation-botocore
pypy3-test-instrumentation-botocore
; opentelemetry-instrumentation-django
py3{5,6,7,8}-test-instrumentation-django
pypy3-test-instrumentation-django
; opentelemetry-instrumentation-dbapi
py3{5,6,7,8}-test-instrumentation-dbapi
pypy3-test-instrumentation-dbapi
; opentelemetry-instrumentation-boto
py3{5,6,7,8}-test-instrumentation-boto
pypy3-test-instrumentation-boto
; opentelemetry-instrumentation-elasticsearch
py3{5,6,7,8}-test-instrumentation-elasticsearch{2,5,6,7}
pypy3-test-instrumentation-elasticsearch{2,5,6,7}
; opentelemetry-instrumentation-falcon
py3{4,5,6,7,8}-test-instrumentation-falcon
pypy3-test-instrumentation-falcon
; opentelemetry-instrumentation-fastapi
; fastapi only supports 3.6 and above.
py3{6,7,8}-test-instrumentation-fastapi
pypy3-test-instrumentation-fastapi
; opentelemetry-instrumentation-flask
py3{5,6,7,8}-test-instrumentation-flask
pypy3-test-instrumentation-flask
; opentelemetry-instrumentation-requests
py3{5,6,7,8}-test-instrumentation-requests
pypy3-test-instrumentation-requests
; opentelemetry-instrumentation-starlette.
; starlette only supports 3.6 and above.
py3{6,7,8}-test-instrumentation-starlette
pypy3-test-instrumentation-starlette
; opentelemetry-instrumentation-jinja2
py3{5,6,7,8}-test-instrumentation-jinja2
pypy3-test-instrumentation-jinja2
; opentelemetry-exporter-datadog
py3{5,6,7,8}-test-exporter-datadog
; opentelemetry-instrumentation-mysql
py3{5,6,7,8}-test-instrumentation-mysql
pypy3-test-instrumentation-mysql
; opentelemetry-instrumentation-psycopg2
py3{5,6,7,8}-test-instrumentation-psycopg2
; ext-psycopg2 intentionally excluded from pypy3
; opentelemetry-instrumentation-pymemcache
py3{5,6,7,8}-test-instrumentation-pymemcache
pypy3-test-instrumentation-pymemcache
; opentelemetry-instrumentation-pymongo
py3{5,6,7,8}-test-instrumentation-pymongo
pypy3-test-instrumentation-pymongo
; opentelemetry-instrumentation-pymysql
py3{5,6,7,8}-test-instrumentation-pymysql
pypy3-test-instrumentation-pymysql
; opentelemetry-instrumentation-pyramid
py3{5,6,7,8}-test-instrumentation-pyramid
pypy3-test-instrumentation-pyramid
; opentelemetry-instrumentation-asgi
py3{5,6,7,8}-test-instrumentation-asgi
pypy3-test-instrumentation-asgi
; opentelemetry-instrumentation-asyncpg
py3{5,6,7,8}-test-instrumentation-asyncpg
; ext-asyncpg intentionally excluded from pypy3
; opentelemetry-instrumentation-sqlite3
py3{5,6,7,8}-test-instrumentation-sqlite3
pypy3-test-instrumentation-sqlite3
; opentelemetry-instrumentation-wsgi
py3{5,6,7,8}-test-instrumentation-wsgi
pypy3-test-instrumentation-wsgi
; opentelemetry-instrumentation-grpc
py3{5,6,7,8}-test-instrumentation-grpc
; opentelemetry-instrumentation-sqlalchemy
py3{5,6,7,8}-test-instrumentation-sqlalchemy
pypy3-test-instrumentation-sqlalchemy
; opentelemetry-instrumentation-redis
py3{5,6,7,8}-test-instrumentation-redis
pypy3-test-instrumentation-redis
; opentelemetry-instrumentation-celery
py3{5,6,7,8}-test-instrumentation-celery
pypy3-test-instrumentation-celery
; opentelemetry-instrumentation-sklearn
py3{5,6,7,8}-test-instrumentation-sklearn
; opentelemetry-instrumentation-system-metrics
py3{5,6,7,8}-test-instrumentation-system-metrics
; instrumentation-system-metrics intentionally excluded from pypy3
; known limitation: gc.get_count won't work under pypy
; opentelemetry-instrumentation-tornado
; instrumentation supports >=6 on Py 3.5 and above.
py3{5,6,7,8}-test-instrumentation-tornado
pypy3-test-instrumentation-tornado
lint
docker-tests
[testenv]
deps =
-c dev-requirements.txt
test: pytest
coverage: pytest
coverage: pytest-cov
elasticsearch2: elasticsearch-dsl>=2.0,<3.0
elasticsearch2: elasticsearch>=2.0,<3.0
elasticsearch5: elasticsearch-dsl>=5.0,<6.0
elasticsearch5: elasticsearch>=5.0,<6.0
elasticsearch6: elasticsearch-dsl>=6.0,<7.0
elasticsearch6: elasticsearch>=6.0,<7.0
elasticsearch7: elasticsearch-dsl>=7.0,<8.0
elasticsearch7: elasticsearch>=7.0,<8.0
; FIXME: add coverage testing
; FIXME: add mypy testing
changedir =
test-instrumentation-aiohttp-client: instrumentation/opentelemetry-instrumentation-aiohttp-client/tests
test-instrumentation-aiopg: instrumentation/opentelemetry-instrumentation-aiopg/tests
test-instrumentation-asgi: instrumentation/opentelemetry-instrumentation-asgi/tests
test-instrumentation-asyncpg: instrumentation/opentelemetry-instrumentation-asyncpg/tests
test-instrumentation-boto: instrumentation/opentelemetry-instrumentation-boto/tests
test-instrumentation-botocore: instrumentation/opentelemetry-instrumentation-botocore/tests
test-instrumentation-celery: instrumentation/opentelemetry-instrumentation-celery/tests
test-instrumentation-dbapi: instrumentation/opentelemetry-instrumentation-dbapi/tests
test-instrumentation-django: instrumentation/opentelemetry-instrumentation-django/tests
test-instrumentation-elasticsearch{2,5,6,7}: instrumentation/opentelemetry-instrumentation-elasticsearch/tests
test-instrumentation-falcon: instrumentation/opentelemetry-instrumentation-falcon/tests
test-instrumentation-fastapi: instrumentation/opentelemetry-instrumentation-fastapi/tests
test-instrumentation-flask: instrumentation/opentelemetry-instrumentation-flask/tests
test-instrumentation-grpc: instrumentation/opentelemetry-instrumentation-grpc/tests
test-instrumentation-jinja2: instrumentation/opentelemetry-instrumentation-jinja2/tests
test-instrumentation-mysql: instrumentation/opentelemetry-instrumentation-mysql/tests
test-instrumentation-psycopg2: instrumentation/opentelemetry-instrumentation-psycopg2/tests
test-instrumentation-pymemcache: instrumentation/opentelemetry-instrumentation-pymemcache/tests
test-instrumentation-pymongo: instrumentation/opentelemetry-instrumentation-pymongo/tests
test-instrumentation-pymysql: instrumentation/opentelemetry-instrumentation-pymysql/tests
test-instrumentation-pyramid: instrumentation/opentelemetry-instrumentation-pyramid/tests
test-instrumentation-redis: instrumentation/opentelemetry-instrumentation-redis/tests
test-instrumentation-requests: instrumentation/opentelemetry-instrumentation-requests/tests
test-instrumentation-sklearn: instrumentation/opentelemetry-instrumentation-sklearn/tests
test-instrumentation-sqlalchemy: instrumentation/opentelemetry-instrumentation-sqlalchemy/tests
test-instrumentation-sqlite3: instrumentation/opentelemetry-instrumentation-sqlite3/tests
test-instrumentation-starlette: instrumentation/opentelemetry-instrumentation-starlette/tests
test-instrumentation-system-metrics: instrumentation/opentelemetry-instrumentation-system-metrics/tests
test-instrumentation-tornado: instrumentation/opentelemetry-instrumentation-tornado/tests
test-instrumentation-wsgi: instrumentation/opentelemetry-instrumentation-wsgi/tests
test-sdkextension-aws: sdk-extension/opentelemetry-sdk-extension-aws/tests
test-exporter-datadog: exporter/opentelemetry-exporter-datadog/tests
commands_pre =
; Install without -e to test the actual installation
py3{5,6,7,8}: python -m pip install -U pip setuptools wheel
; Install common packages for all the tests. These are not needed in all the
; cases but it saves a lot of boilerplate in this file.
test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-api {toxinidir}/opentelemetry-python-core/opentelemetry-sdk {toxinidir}/opentelemetry-python-core/tests/util
test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation
celery: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-celery[test]
grpc: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-grpc[test]
wsgi,falcon,flask,django,pyramid: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-wsgi
asgi,starlette,fastapi: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-asgi
asyncpg: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-asyncpg
boto: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-botocore[test]
boto: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-boto[test]
falcon: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-falcon[test]
flask: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-flask[test]
botocore: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-botocore[test]
dbapi: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi[test]
django: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-django[test]
fastapi: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-fastapi[test]
mysql: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi {toxinidir}/instrumentation/opentelemetry-instrumentation-mysql[test]
pymemcache: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-pymemcache[test]
pymongo: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-pymongo[test]
psycopg2: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi {toxinidir}/instrumentation/opentelemetry-instrumentation-psycopg2[test]
pymysql: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi {toxinidir}/instrumentation/opentelemetry-instrumentation-pymysql[test]
pyramid: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-pyramid[test]
sqlite3: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi {toxinidir}/instrumentation/opentelemetry-instrumentation-sqlite3[test]
redis: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-redis[test]
requests: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-requests[test]
starlette: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-starlette[test]
tornado: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-tornado
jinja2: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-jinja2[test]
aiohttp-client: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-aiohttp-client
aiopg: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-aiopg[test]
datadog: pip install flaky {toxinidir}/exporter/opentelemetry-exporter-datadog
sklearn: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-sklearn[test]
sqlalchemy: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-sqlalchemy
system-metrics: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-system-metrics[test]
elasticsearch{2,5,6,7}: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation {toxinidir}/instrumentation/opentelemetry-instrumentation-elasticsearch[test]
aws: pip install requests {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws
; In order to get a healthy coverage report,
; we have to install packages in editable mode.
coverage: python {toxinidir}/scripts/eachdist.py install --editable
commands =
test: pytest {posargs}
coverage: {toxinidir}/scripts/coverage.sh
[testenv:lint]
basepython: python3.8
recreate = True
deps =
-c dev-requirements.txt
flaky
pylint
flake8
isort
black
psutil
readme_renderer
httpretty
commands_pre =
python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-api
python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-sdk
python -m pip install {toxinidir}/opentelemetry-python-core/tests/util
python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation
python scripts/eachdist.py install --editable --with-test-deps
commands =
python scripts/eachdist.py lint --check-only
[testenv:docker-tests]
deps =
pytest
asyncpg==0.20.1
docker-compose >= 1.25.2
mysql-connector-python ~= 8.0
pymongo ~= 3.1
pymysql ~= 0.9.3
psycopg2-binary ~= 2.8.4
aiopg >= 0.13.0
sqlalchemy ~= 1.3.16
redis ~= 3.3.11
celery ~= 4.0, != 4.4.4
changedir =
tests/opentelemetry-docker-tests/tests
commands_pre =
pip install -e {toxinidir}/opentelemetry-python-core/opentelemetry-api \
-e {toxinidir}/opentelemetry-python-core/opentelemetry-sdk \
-e {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation \
-e {toxinidir}/opentelemetry-python-core/tests/util \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-asyncpg \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-celery \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-mysql \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-psycopg2 \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-pymongo \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-pymysql \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-sqlalchemy \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-aiopg \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-redis \
-e {toxinidir}/instrumentation/opentelemetry-instrumentation-system-metrics \
-e {toxinidir}/opentelemetry-python-core/exporter/opentelemetry-exporter-opencensus
docker-compose up -d
python check_availability.py
commands =
pytest {posargs}
commands_post =
docker-compose down -v