From 1bc956f705d421e049e0f8658e1be7e39e979397 Mon Sep 17 00:00:00 2001 From: Aliaksei Urbanski Date: Thu, 7 Nov 2019 03:11:45 +0300 Subject: [PATCH 01/24] Add test coverage collecting (#128) --- scripts/coverage.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 scripts/coverage.sh diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100755 index 000000000..bddf39a90 --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +function cov { + pytest \ + --ignore-glob=*/setup.py \ + --cov ${1} \ + --cov-append \ + --cov-branch \ + --cov-report='' \ + ${1} +} + + +coverage erase + +cov opentelemetry-api +cov opentelemetry-sdk +cov ext/opentelemetry-ext-http-requests +cov ext/opentelemetry-ext-jaeger +cov ext/opentelemetry-ext-opentracing-shim +cov ext/opentelemetry-ext-wsgi +cov examples/opentelemetry-example-app + +coverage report +coverage xml From d8434bf2da03b75385eb4d1f2d931f63166de67f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Neum=C3=BCller?= Date: Tue, 10 Dec 2019 02:54:56 +0100 Subject: [PATCH 02/24] Add eachdist.py to simplify build (#291) --- scripts/eachdist.py | 508 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 508 insertions(+) create mode 100755 scripts/eachdist.py diff --git a/scripts/eachdist.py b/scripts/eachdist.py new file mode 100755 index 000000000..8d41315fc --- /dev/null +++ b/scripts/eachdist.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 + +import argparse +import shlex +import shutil +import subprocess +import sys +from collections import namedtuple +from configparser import ConfigParser +from inspect import cleandoc +from itertools import chain +from pathlib import Path, PurePath + +DEFAULT_ALLSEP = " " +DEFAULT_ALLFMT = "{rel}" + + +def unique(elems): + seen = set() + for elem in elems: + if elem not in seen: + yield elem + seen.add(elem) + + +try: + subprocess_run = subprocess.run +except AttributeError: # Py < 3.5 compat + CompletedProcess = namedtuple("CompletedProcess", "returncode") + + def subprocess_run(*args, **kwargs): + check = kwargs.pop("check", False) + if check: + subprocess.check_call(*args, **kwargs) + return CompletedProcess(returncode=0) + return CompletedProcess(returncode=subprocess.call(*args, **kwargs)) + + +def extraargs_help(calledcmd): + return cleandoc( + """ + Additional arguments to pass on to {}. + + This is collected from any trailing arguments passed to `%(prog)s`. + Use an initial `--` to separate them from regular arguments. + """.format( + calledcmd + ) + ) + + +def parse_args(args=None): + parser = argparse.ArgumentParser(description="Development helper script.") + parser.set_defaults(parser=parser) + parser.add_argument( + "--dry-run", + action="store_true", + help="Only display what would be done, don't actually do anything.", + ) + subparsers = parser.add_subparsers(metavar="COMMAND") + subparsers.required = True + + excparser = subparsers.add_parser( + "exec", + help="Run a command for each or all targets.", + formatter_class=argparse.RawTextHelpFormatter, + description=cleandoc( + """Run a command according to the `format` argument for each or all targets. + + This is an advanced command that is used internally by other commands. + + For example, to install all distributions in this repository + editable, you could use: + + scripts/eachdist.py exec "python -m pip install -e {}" + + This will run pip for all distributions which is quite slow. It gets + a bit faster if we only invoke pip once but with all the paths + gathered together, which can be achieved by using `--all`: + + scripts/eachdist.py exec "python -m pip install {}" --all "-e {}" + + The sortfirst option in the DEFAULT section of eachdist.ini makes + sure that dependencies are installed before their dependents. + + Search for usages of `parse_subargs` in the source code of this script + to see more examples. + + This command first collects target paths and then executes + commands according to `format` and `--all`. + + Target paths are initially all Python distribution root paths + (as determined by the existence of setup.py, etc. files). + They are then augmented according to the section of the + `PROJECT_ROOT/eachdist.ini` config file specified by the `--mode` option. + + The following config options are available (and processed in that order): + + - `extraroots`: List of project root-relative glob expressions. + The resulting paths will be added. + - `sortfirst`: List of glob expressions. + Any matching paths will be put to the front of the path list, + in the same order they appear in this option. If more than one + glob matches, ordering is according to the first. + - `subglob`: List of glob expressions. Each path added so far is removed + and replaced with the result of all glob expressions relative to it (in + order of the glob expressions). + + After all this, any duplicate paths are removed (the first occurrence remains). + """ + ), + ) + excparser.set_defaults(func=execute_args) + excparser.add_argument( + "format", + help=cleandoc( + """Format string for the command to execute. + + The available replacements depend on whether `--all` is specified. + If `--all` was specified, there is only a single replacement, + `{}`, that is replaced with the string that is generated from + joining all targets formatted with `--all` to a single string + with the value of `--allsep` as separator. + + If `--all` was not specified, the following replacements are available: + + - `{}`: the absolute path to the current target in POSIX format + (with forward slashes) + - `{rel}`: like `{}` but relative to the project root. + - `{raw}`: the absolute path to the current target in native format + (thus exactly the same as `{}` on Unix but with backslashes on Windows). + - `{rawrel}`: like `{raw}` but relative to the project root. + + The resulting string is then split according to POSIX shell rules + (so you can use quotation marks or backslashes to handle arguments + containing spaces). + + The first token is the name of the executable to run, the remaining + tokens are the arguments. + + Note that a shell is *not* involved by default. + You can add bash/sh/cmd/powershell yourself to the format if you want. + + If `--all` was specified, the resulting command is simply executed once. + Otherwise, the command is executed for each found target. In both cases, + the project root is the working directory. + """ + ), + ) + excparser.add_argument( + "--all", + nargs="?", + const=DEFAULT_ALLFMT, + metavar="ALLFORMAT", + help=cleandoc( + """Instead of running the command for each target, join all target + paths together to run a single command. + + This option optionally takes a format string to apply to each path. The + available replacements are the ones that would be available for `format` + if `--all` was not specified. + + Default ALLFORMAT if this flag is specified: `%(const)s`. + """ + ), + ) + excparser.add_argument( + "--allsep", + help=cleandoc( + """Separator string for the strings resulting from `--all`. + Only valid if `--all` is specified. + """ + ), + ) + excparser.add_argument( + "--allowexitcode", + type=int, + action="append", + default=[0], + help=cleandoc( + """The given command exit code is treated as success and does not abort execution. + Can be specified multiple times. + """ + ), + ) + excparser.add_argument( + "--mode", + "-m", + default="DEFAULT", + help=cleandoc( + """Section of config file to use for target selection configuration. + See description of exec for available options.""" + ), + ) + + instparser = subparsers.add_parser( + "install", help="Install all distributions." + ) + + def setup_instparser(instparser): + instparser.set_defaults(func=install_args) + instparser.add_argument( + "pipargs", nargs=argparse.REMAINDER, help=extraargs_help("pip") + ) + + setup_instparser(instparser) + instparser.add_argument("--editable", "-e", action="store_true") + instparser.add_argument("--with-dev-deps", action="store_true") + instparser.add_argument("--eager-upgrades", action="store_true") + + devparser = subparsers.add_parser( + "develop", + help="Install all distributions editable + dev dependencies.", + ) + setup_instparser(devparser) + devparser.set_defaults( + editable=True, with_dev_deps=True, eager_upgrades=True + ) + + lintparser = subparsers.add_parser( + "lint", help="Lint everything, autofixing if possible." + ) + lintparser.add_argument("--check-only", action="store_true") + lintparser.set_defaults(func=lint_args) + + testparser = subparsers.add_parser( + "test", + help="Test everything (run pytest yourself for more complex operations).", + ) + testparser.set_defaults(func=test_args) + testparser.add_argument( + "pytestargs", nargs=argparse.REMAINDER, help=extraargs_help("pytest") + ) + + return parser.parse_args(args) + + +def find_projectroot(search_start=Path(".")): + root = search_start.resolve() + for root in chain((root,), root.parents): + if any((root / marker).exists() for marker in (".git", "tox.ini")): + return root + return None + + +def find_targets_unordered(rootpath): + for subdir in rootpath.iterdir(): + if not subdir.is_dir(): + continue + if subdir.name.startswith(".") or subdir.name.startswith("venv"): + continue + if any( + (subdir / marker).exists() + for marker in ("setup.py", "pyproject.toml") + ): + yield subdir + else: + yield from find_targets_unordered(subdir) + + +def getlistcfg(strval): + return [ + val.strip() + for line in strval.split("\n") + for val in line.split(",") + if val.strip() + ] + + +def find_targets(mode, rootpath): + if not rootpath: + sys.exit("Could not find a root directory.") + + cfg = ConfigParser() + cfg.read(str(rootpath / "eachdist.ini")) + mcfg = cfg[mode] + + targets = list(find_targets_unordered(rootpath)) + if "extraroots" in mcfg: + targets += [ + path + for extraglob in getlistcfg(mcfg["extraroots"]) + for path in rootpath.glob(extraglob) + ] + if "sortfirst" in mcfg: + sortfirst = getlistcfg(mcfg["sortfirst"]) + + def keyfunc(path): + path = path.relative_to(rootpath) + for idx, pattern in enumerate(sortfirst): + if path.match(pattern): + return idx + return float("inf") + + targets.sort(key=keyfunc) + + subglobs = getlistcfg(mcfg.get("subglob", "")) + if subglobs: + targets = [ + newentry + for newentry in ( + target / subdir + for target in targets + for subglob in subglobs + # We need to special-case the dot, because glob fails to parse that with an IndexError. + for subdir in ( + (target,) if subglob == "." else target.glob(subglob) + ) + ) + if ".egg-info" not in str(newentry) and newentry.exists() + ] + + return list(unique(targets)) + + +def runsubprocess(dry_run, params, *args, **kwargs): + cmdstr = join_args(params) + if dry_run: + print(cmdstr) + return None + + # Py < 3.6 compat. + cwd = kwargs.get("cwd") + if cwd and isinstance(cwd, PurePath): + kwargs["cwd"] = str(cwd) + + check = kwargs.pop("check") # Enforce specifying check + + print(">>>", cmdstr, file=sys.stderr) + + # This is a workaround for subprocess.run(['python']) leaving the virtualenv on Win32. + # The cause for this is that when running the python.exe in a virtualenv, + # the wrapper executable launches the global python as a subprocess and the search sequence + # for CreateProcessW which subprocess.run and Popen use is a follows + # (https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw): + # > 1. The directory from which the application loaded. + # This will be the directory of the global python.exe, not the venv directory, due to the suprocess mechanism. + # > 6. The directories that are listed in the PATH environment variable. + # Only this would find the "correct" python.exe. + + params = list(params) + executable = shutil.which(params[0]) # On Win32, pytho + if executable: + params[0] = executable + try: + return subprocess_run(params, *args, check=check, **kwargs) + except OSError as exc: + raise ValueError( + "Failed executing " + repr(params) + ": " + str(exc) + ) from exc + + +def execute_args(args): + if args.allsep and not args.all: + args.parser.error("--allsep specified but not --all.") + + if args.all and not args.allsep: + args.allsep = DEFAULT_ALLSEP + + rootpath = find_projectroot() + targets = find_targets(args.mode, rootpath) + if not targets: + sys.exit("Error: No targets selected (root: {})".format(rootpath)) + + def fmt_for_path(fmt, path): + return fmt.format( + path.as_posix(), + rel=path.relative_to(rootpath).as_posix(), + raw=path, + rawrel=path.relative_to(rootpath), + ) + + def _runcmd(cmd): + result = runsubprocess( + args.dry_run, shlex.split(cmd), cwd=rootpath, check=False + ) + if result is not None and result.returncode not in args.allowexitcode: + print( + "'{}' failed with code {}".format(cmd, result.returncode), + file=sys.stderr, + ) + sys.exit(result.returncode) + + if args.all: + allstr = args.allsep.join( + fmt_for_path(args.all, path) for path in targets + ) + cmd = args.format.format(allstr) + _runcmd(cmd) + else: + for target in targets: + cmd = fmt_for_path(args.format, target) + _runcmd(cmd) + + +def clean_remainder_args(remainder_args): + if remainder_args and remainder_args[0] == "--": + del remainder_args[0] + + +def join_args(arglist): + return " ".join(map(shlex.quote, arglist)) + + +def install_args(args): + clean_remainder_args(args.pipargs) + if args.eager_upgrades: + args.pipargs += ["--upgrade-strategy=eager"] + + if args.with_dev_deps: + runsubprocess( + args.dry_run, + [ + "python", + "-m", + "pip", + "install", + "--upgrade", + "pip", + "setuptools", + "wheel", + ] + + args.pipargs, + check=True, + ) + + allfmt = "-e 'file://{}'" if args.editable else "'file://{}'" + execute_args( + parse_subargs( + args, + ( + "exec", + "python -m pip install {} " + join_args(args.pipargs), + "--all", + allfmt, + ), + ) + ) + if args.with_dev_deps: + rootpath = find_projectroot() + runsubprocess( + args.dry_run, + [ + "python", + "-m", + "pip", + "install", + "--upgrade", + "-r", + str(rootpath / "dev-requirements.txt"), + ] + + args.pipargs, + check=True, + ) + + +def parse_subargs(parentargs, args): + subargs = parse_args(args) + subargs.dry_run = parentargs.dry_run or subargs.dry_run + return subargs + + +def lint_args(args): + rootdir = str(find_projectroot()) + + runsubprocess( + args.dry_run, + ("black", ".") + (("--diff", "--check") if args.check_only else ()), + cwd=rootdir, + check=True, + ) + runsubprocess( + args.dry_run, + ("isort", "--recursive", ".") + + (("--diff", "--check-only") if args.check_only else ()), + cwd=rootdir, + check=True, + ) + runsubprocess(args.dry_run, ("flake8", rootdir), check=True) + execute_args( + parse_subargs( + args, ("exec", "pylint {}", "--all", "--mode", "lintroots",), + ) + ) + + +def test_args(args): + clean_remainder_args(args.pytestargs) + execute_args( + parse_subargs( + args, + ( + "exec", + "pytest {} " + join_args(args.pytestargs), + "--mode", + "testroots", + ), + ) + ) + + +def main(): + args = parse_args() + args.func(args) + + +if __name__ == "__main__": + main() From 331aeb37516725fac465de8f95a1289dfe4e2a1f Mon Sep 17 00:00:00 2001 From: alrex Date: Sun, 29 Dec 2019 22:02:55 -0800 Subject: [PATCH 03/24] Adding Zipkin exporter (#320) Signed-off-by: Alex Boten --- scripts/coverage.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index bddf39a90..9b981b081 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -17,10 +17,12 @@ coverage erase cov opentelemetry-api cov opentelemetry-sdk +cov ext/opentelemetry-ext-flask cov ext/opentelemetry-ext-http-requests cov ext/opentelemetry-ext-jaeger cov ext/opentelemetry-ext-opentracing-shim cov ext/opentelemetry-ext-wsgi +cov ext/opentelemetry-ext-zipkin cov examples/opentelemetry-example-app coverage report From 5b282c3951f1cfe138e6da29859564faddd8def6 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Tue, 11 Feb 2020 11:07:11 -0800 Subject: [PATCH 04/24] Metrics export pipeline + metrics stdout exporter (#341) Initial implementation of the end-to-end metrics pipeline. --- scripts/eachdist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/eachdist.py b/scripts/eachdist.py index 8d41315fc..406afb6eb 100755 --- a/scripts/eachdist.py +++ b/scripts/eachdist.py @@ -479,7 +479,7 @@ def lint_args(args): runsubprocess(args.dry_run, ("flake8", rootdir), check=True) execute_args( parse_subargs( - args, ("exec", "pylint {}", "--all", "--mode", "lintroots",), + args, ("exec", "pylint {}", "--all", "--mode", "lintroots") ) ) From 6a56ce413245e989ec423f2fa2846cf218d5f8de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mauricio=20V=C3=A1squez?= Date: Tue, 10 Mar 2020 11:50:16 -0500 Subject: [PATCH 05/24] Improve docs structure (#467) - Creates a tree structure for documentation, it allows to organize it better from a developer point of view and also the rendered documentation should be easier to navigate. - Moves partially the main readme to be included in the online docs, the main readme will be updated to have a link to avoid duplicated content) - Moves the examples folder to the docs, so they can be accessed through the online documentation. Creates a new pair of "macros" to create links to specific versions, scm_web & scm_raw_web. Co-authored-by: Chris Kleinknecht --- scripts/coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 9b981b081..b9f9b6493 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -23,7 +23,7 @@ cov ext/opentelemetry-ext-jaeger cov ext/opentelemetry-ext-opentracing-shim cov ext/opentelemetry-ext-wsgi cov ext/opentelemetry-ext-zipkin -cov examples/opentelemetry-example-app +cov docs/examples/opentelemetry-example-app coverage report coverage xml From 9278d5ebf9c98b0afb5716ddcd03c5e1c5484137 Mon Sep 17 00:00:00 2001 From: Daniel <61800298+ffe4@users.noreply.github.com> Date: Sat, 28 Mar 2020 05:45:15 +0100 Subject: [PATCH 06/24] lint: Add test for package readme syntax errors (#492) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a test to ensure readmes render properly Also adds README.rst for testutil package to pass new test. Co-authored-by: Christian Neumüller --- scripts/check_for_valid_readme.py | 51 +++++++++++++++++++++++++++++++ scripts/eachdist.py | 6 ++++ 2 files changed, 57 insertions(+) create mode 100644 scripts/check_for_valid_readme.py diff --git a/scripts/check_for_valid_readme.py b/scripts/check_for_valid_readme.py new file mode 100644 index 000000000..edf94d9c3 --- /dev/null +++ b/scripts/check_for_valid_readme.py @@ -0,0 +1,51 @@ +"""Test script to check given paths for valid README.rst files.""" +import argparse +import sys +from pathlib import Path + +import readme_renderer.rst + + +def is_valid_rst(path): + """Checks if RST can be rendered on PyPI.""" + with open(path) as readme_file: + markup = readme_file.read() + return readme_renderer.rst.render(markup) is not None + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Checks README.rst file in path for syntax errors." + ) + parser.add_argument( + "paths", nargs="+", help="paths containing a README.rst to test" + ) + parser.add_argument("-v", "--verbose", action="store_true") + return parser.parse_args() + + +def main(): + args = parse_args() + error = False + + for path in map(Path, args.paths): + readme = path / "README.rst" + try: + if not is_valid_rst(readme): + error = True + print("FAILED: RST syntax errors in", readme) + continue + except FileNotFoundError: + error = True + print("FAILED: README.rst not found in", path) + continue + if args.verbose: + print("PASSED:", readme) + + if error: + sys.exit(1) + print("All clear.") + + +if __name__ == "__main__": + main() diff --git a/scripts/eachdist.py b/scripts/eachdist.py index 406afb6eb..f1c5e18b6 100755 --- a/scripts/eachdist.py +++ b/scripts/eachdist.py @@ -482,6 +482,12 @@ def lint_args(args): args, ("exec", "pylint {}", "--all", "--mode", "lintroots") ) ) + execute_args( + parse_subargs( + args, + ("exec", "python scripts/check_for_valid_readme.py {}", "--all",), + ) + ) def test_args(args): From da77a731d8d6961ff330db605ee0e8613aceb85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mauricio=20V=C3=A1squez?= Date: Sat, 2 May 2020 23:31:15 -0500 Subject: [PATCH 07/24] requests: Rename http_requests to requests (#619) The requests integration is named http-requests because at the time it was created there were some problems with pylint. other integrations are using opentelemetry.ext.integration without problems, tests are passing without issue, even renamed. --- scripts/coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index b9f9b6493..839380c27 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -18,7 +18,7 @@ coverage erase cov opentelemetry-api cov opentelemetry-sdk cov ext/opentelemetry-ext-flask -cov ext/opentelemetry-ext-http-requests +cov ext/opentelemetry-ext-requests cov ext/opentelemetry-ext-jaeger cov ext/opentelemetry-ext-opentracing-shim cov ext/opentelemetry-ext-wsgi From 35f1e5bbdfb1b1ae2da74f9cc9d70b6ea41895d3 Mon Sep 17 00:00:00 2001 From: joshuahlang Date: Wed, 6 May 2020 20:40:47 -0700 Subject: [PATCH 08/24] aiohttp: aiohttp client (#421) Adding initial aiohttp client. This module is only supported on Python3.5, which is the oldest supported by aiohttp. Co-authored-by: Yusuke Tsutsumi --- scripts/coverage.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 839380c27..1ff42d9e5 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -12,6 +12,8 @@ function cov { ${1} } +PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:3])))') +PYTHON_VERSION_INFO=(${PYTHON_VERSION//./ }) coverage erase @@ -25,5 +27,10 @@ cov ext/opentelemetry-ext-wsgi cov ext/opentelemetry-ext-zipkin cov docs/examples/opentelemetry-example-app -coverage report +# aiohttp is only supported on Python 3.5+. +if [ ${PYTHON_VERSION_INFO[1]} -gt 4 ]; then + cov ext/opentelemetry-ext-aiohttp-client +fi + +coverage report --show-missing coverage xml From 157f6ba035328ed4b75b7530fe5d7709f8b28142 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Wed, 13 May 2020 17:51:28 -0400 Subject: [PATCH 09/24] Add exporter to Datadog (#572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an exporter to Datadog. This implementation makes use of ddtrace to handle the creation of Datadog traces and writing them to the Datadog agent. Co-Authored-By: Mauricio Vásquez --- scripts/coverage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 1ff42d9e5..248e5faea 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -19,6 +19,7 @@ coverage erase cov opentelemetry-api cov opentelemetry-sdk +cov ext/opentelemetry-ext-datadog cov ext/opentelemetry-ext-flask cov ext/opentelemetry-ext-requests cov ext/opentelemetry-ext-jaeger From 45a003f3076a808ae8dd9149257a4a2dbd9bd712 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Wed, 27 May 2020 12:17:35 -0400 Subject: [PATCH 10/24] asgi: Add ASGI middleware (#716) Adding an ASGI extension. Co-authored-by: Emil Madsen Co-authored-by: alrex Co-authored-by: Florimond Manca --- scripts/coverage.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 248e5faea..8e09ae23a 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -31,6 +31,9 @@ cov docs/examples/opentelemetry-example-app # aiohttp is only supported on Python 3.5+. if [ ${PYTHON_VERSION_INFO[1]} -gt 4 ]; then cov ext/opentelemetry-ext-aiohttp-client +# ext-asgi is only supported on Python 3.5+. +if [ ${PYTHON_VERSION_INFO[1]} -gt 4 ]; then + cov ext/opentelemetry-ext-asgi fi coverage report --show-missing From 4047cf4b15766151b2e299aea38673d6a9d1abc1 Mon Sep 17 00:00:00 2001 From: Yusuke Tsutsumi Date: Tue, 2 Jun 2020 20:37:36 -0700 Subject: [PATCH 11/24] opentracing-shim: add testbed for otshim (#727) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit ports the OpenTracing testbed[1] to check that the ot-shim is working as expected using different frameworks. Gevent doesn't support context vars yet[2], so those tests are not compatible with opentelemetry and were not ported. [1] https://github.com/opentracing/opentracing-python/tree/master/testbed [2] https://github.com/gevent/gevent/issues/1407 Co-authored-by: Mauricio Vásquez Co-authored-by: alrex --- scripts/coverage.sh | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 8e09ae23a..0b45fbf64 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -3,13 +3,25 @@ set -e function cov { - pytest \ - --ignore-glob=*/setup.py \ - --cov ${1} \ - --cov-append \ - --cov-branch \ - --cov-report='' \ - ${1} + if [ ${TOX_ENV_NAME:0:4} == "py34" ] + then + pytest \ + --ignore-glob=*/setup.py \ + --ignore-glob=ext/opentelemetry-ext-opentracing-shim/tests/testbed/* \ + --cov ${1} \ + --cov-append \ + --cov-branch \ + --cov-report='' \ + ${1} + else + pytest \ + --ignore-glob=*/setup.py \ + --cov ${1} \ + --cov-append \ + --cov-branch \ + --cov-report='' \ + ${1} + fi } PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:3])))') From 1536a863da95c5c9b04999bab140a19656dee9b1 Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Thu, 4 Jun 2020 00:33:36 -0400 Subject: [PATCH 12/24] cloud-trace: Cloud Trace exporter (#698) Co-authored-by: Cheng-Lung Sung --- scripts/coverage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 0b45fbf64..1794cdf01 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -36,6 +36,7 @@ cov ext/opentelemetry-ext-flask cov ext/opentelemetry-ext-requests cov ext/opentelemetry-ext-jaeger cov ext/opentelemetry-ext-opentracing-shim +cov ext/opentelemetry-exporter-cloud-trace cov ext/opentelemetry-ext-wsgi cov ext/opentelemetry-ext-zipkin cov docs/examples/opentelemetry-example-app From d672ca2cb07656ff3e6e418188a33cc9cc4d5408 Mon Sep 17 00:00:00 2001 From: Yusuke Tsutsumi Date: Wed, 10 Jun 2020 16:14:33 -0700 Subject: [PATCH 13/24] chore: installation of test packages in eachdist (#794) eachdist.py did not support the installation of test packages, (as defined by the extra_requires:test package group). As a result, test packages were being added to dev-requirements.txt By having eachdist.py develop install test packages, and moving develop/test package definitions to the individual instrumentations, it is easier to determine which packages require which dependencies for testing purposes, and enables support for existing dependencies that follow the extra_requires:test pattern. --- scripts/eachdist.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/eachdist.py b/scripts/eachdist.py index f1c5e18b6..15b9b8edc 100755 --- a/scripts/eachdist.py +++ b/scripts/eachdist.py @@ -205,6 +205,7 @@ def parse_args(args=None): setup_instparser(instparser) instparser.add_argument("--editable", "-e", action="store_true") + instparser.add_argument("--with-test-deps", action="store_true") instparser.add_argument("--with-dev-deps", action="store_true") instparser.add_argument("--eager-upgrades", action="store_true") @@ -214,7 +215,10 @@ def parse_args(args=None): ) setup_instparser(devparser) devparser.set_defaults( - editable=True, with_dev_deps=True, eager_upgrades=True + editable=True, + with_dev_deps=True, + eager_upgrades=True, + with_test_deps=True, ) lintparser = subparsers.add_parser( @@ -424,7 +428,16 @@ def install_args(args): check=True, ) - allfmt = "-e 'file://{}'" if args.editable else "'file://{}'" + allfmt = "-e 'file://{}" if args.editable else "'file://{}" + # packages should provide an extra_requires that is named + # 'test', to denote test dependencies. + extras = [] + if args.with_test_deps: + extras.append("test") + if extras: + allfmt += "[{}]".format(",".join(extras)) + # note the trailing single quote, to close the quote opened above. + allfmt += "'" execute_args( parse_subargs( args, From 34f0e88967a9ba40edeff1371333a8725632e283 Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Thu, 11 Jun 2020 17:41:22 -0400 Subject: [PATCH 14/24] chore: add test coverage for Cloud Monitoring exporter (#804) Previously cloud monitoring was missing coverage. --- scripts/coverage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 1794cdf01..a88aca747 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -37,6 +37,7 @@ cov ext/opentelemetry-ext-requests cov ext/opentelemetry-ext-jaeger cov ext/opentelemetry-ext-opentracing-shim cov ext/opentelemetry-exporter-cloud-trace +cov ext/opentelemetry-exporter-cloud-monitoring cov ext/opentelemetry-ext-wsgi cov ext/opentelemetry-ext-zipkin cov docs/examples/opentelemetry-example-app From b2aa82cd953eb129ef85994a38b2e965732d9d09 Mon Sep 17 00:00:00 2001 From: Yusuke Tsutsumi Date: Mon, 15 Jun 2020 13:59:57 -0700 Subject: [PATCH 15/24] starlette instrumentation (#777) adding an initial starlette instrumentation. tox does exact match on fields delimited by a dash. Thus, any instrumentation that includes "instrumentation" in the name would collide with testing of the "opentelemetry-instrumentation" package. Renaming opentelemetry-instrumentation to opentelemetry-instrumentation-base to fix that. Co-authored-by: Leighton Chen Co-authored-by: alrex --- scripts/check_for_valid_readme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_for_valid_readme.py b/scripts/check_for_valid_readme.py index edf94d9c3..d555b4fa2 100644 --- a/scripts/check_for_valid_readme.py +++ b/scripts/check_for_valid_readme.py @@ -10,7 +10,7 @@ def is_valid_rst(path): """Checks if RST can be rendered on PyPI.""" with open(path) as readme_file: markup = readme_file.read() - return readme_renderer.rst.render(markup) is not None + return readme_renderer.rst.render(markup, stream=sys.stderr) is not None def parse_args(): From d219699d0bd904ff49ef72c0b2fa729f202a3030 Mon Sep 17 00:00:00 2001 From: alrex Date: Tue, 16 Jun 2020 15:51:56 -0700 Subject: [PATCH 16/24] chore: moving code to prepare the release to eachdist (#808) --- scripts/eachdist.py | 124 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/scripts/eachdist.py b/scripts/eachdist.py index 15b9b8edc..c8b7f7c3f 100755 --- a/scripts/eachdist.py +++ b/scripts/eachdist.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 import argparse +import os +import re import shlex import shutil import subprocess import sys from collections import namedtuple from configparser import ConfigParser +from datetime import datetime from inspect import cleandoc from itertools import chain from pathlib import Path, PurePath @@ -236,6 +239,15 @@ def parse_args(args=None): "pytestargs", nargs=argparse.REMAINDER, help=extraargs_help("pytest") ) + releaseparser = subparsers.add_parser( + "release", help="Prepares release, used by maintainers and CI", + ) + releaseparser.set_defaults(func=release_args) + releaseparser.add_argument("--version", required=True) + releaseparser.add_argument( + "releaseargs", nargs=argparse.REMAINDER, help=extraargs_help("pytest") + ) + return parser.parse_args(args) @@ -503,6 +515,118 @@ def lint_args(args): ) +def update_changelog(path, version, new_entry): + unreleased_changes = False + try: + with open(path) as changelog: + text = changelog.read() + if "## Version {}".format(version) in text: + raise AttributeError( + "{} already contans version {}".format(path, version) + ) + with open(path) as changelog: + for line in changelog: + if line.startswith("## Unreleased"): + unreleased_changes = False + elif line.startswith("## "): + break + elif len(line.strip()) > 0: + unreleased_changes = True + + except FileNotFoundError: + print("file missing: {}".format(path)) + return + + if unreleased_changes: + print("updating: {}".format(path)) + text = re.sub("## Unreleased", new_entry, text) + with open(path, "w") as changelog: + changelog.write(text) + + +def update_changelogs(targets, version): + print("updating CHANGELOG") + today = datetime.now().strftime("%Y-%m-%d") + new_entry = "## Unreleased\n\n## Version {}\n\nReleased {}".format( + version, today + ) + errors = False + for target in targets: + try: + update_changelog( + "{}/CHANGELOG.md".format(target), version, new_entry + ) + except Exception as err: # pylint: disable=broad-except + print(str(err)) + errors = True + + if errors: + sys.exit(1) + + +def find(name, path): + for root, _, files in os.walk(path): + if name in files: + return os.path.join(root, name) + return None + + +def update_version_files(targets, version): + print("updating version.py files") + update_files( + targets, + version, + "version.py", + "__version__ .*", + '__version__ = "{}"'.format(version), + ) + + +def update_dependencies(targets, version): + print("updating dependencies") + update_files( + targets, + version, + "setup.cfg", + r"(opentelemetry-.*)= (.*)", + r"\1= " + version, + ) + + +def update_files(targets, version, filename, search, replace): + errors = False + for target in targets: + curr_file = find(filename, target) + if curr_file is None: + print("file missing: {}/{}".format(target, filename)) + continue + + with open(curr_file) as _file: + text = _file.read() + + if version in text: + print("{} already contans version {}".format(curr_file, version)) + errors = True + continue + + with open(curr_file, "w") as _file: + _file.write(re.sub(search, replace, text)) + + if errors: + sys.exit(1) + + +def release_args(args): + print("preparing release") + + rootpath = find_projectroot() + targets = list(find_targets_unordered(rootpath)) + version = args.version + update_dependencies(targets, version) + update_version_files(targets, version) + update_changelogs(targets, version) + + def test_args(args): clean_remainder_args(args.pytestargs) execute_args( From 277bcee29ab49929b8660e9c2555326c3520ba40 Mon Sep 17 00:00:00 2001 From: Owais Lone Date: Wed, 17 Jun 2020 10:39:43 +0530 Subject: [PATCH 17/24] Initial elasticsearch instrumentation (#747) This commit adds auto-instrumentation for elasticsearch. The instrumentation has been mostly ported over from OpenTracing elasticsearch instrumentation. Co-authored-by: Yusuke Tsutsumi Co-authored-by: alrex --- scripts/eachdist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/eachdist.py b/scripts/eachdist.py index c8b7f7c3f..57560b9b8 100755 --- a/scripts/eachdist.py +++ b/scripts/eachdist.py @@ -450,6 +450,7 @@ def install_args(args): allfmt += "[{}]".format(",".join(extras)) # note the trailing single quote, to close the quote opened above. allfmt += "'" + execute_args( parse_subargs( args, From aab6c9c00bebbe7be4c1c509d83458171bb8245c Mon Sep 17 00:00:00 2001 From: Yusuke Tsutsumi Date: Thu, 2 Jul 2020 09:45:56 -0700 Subject: [PATCH 18/24] chore: Making eachdist release catch more deps (#867) In the last release, eachdist missed updating dependencies on ext-datadog, which has a slightly different, but valid, way of specifying the version (omitting some spaces). Making the eachdist regex more lenient to catch other valid version specifications. Also modifying the ext-datadog dependency specification to match the format of all the others. Co-authored-by: Leighton Chen --- scripts/eachdist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/eachdist.py b/scripts/eachdist.py index 57560b9b8..724061690 100755 --- a/scripts/eachdist.py +++ b/scripts/eachdist.py @@ -589,8 +589,8 @@ def update_dependencies(targets, version): targets, version, "setup.cfg", - r"(opentelemetry-.*)= (.*)", - r"\1= " + version, + r"(opentelemetry-.*)==(.*)", + r"\1== " + version, ) From 52c2719b7aa8c9fd0cf2581f24457fa7acde7664 Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Thu, 16 Jul 2020 16:12:44 -0400 Subject: [PATCH 19/24] remove google exporter files (#918) --- scripts/coverage.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index a88aca747..0b45fbf64 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -36,8 +36,6 @@ cov ext/opentelemetry-ext-flask cov ext/opentelemetry-ext-requests cov ext/opentelemetry-ext-jaeger cov ext/opentelemetry-ext-opentracing-shim -cov ext/opentelemetry-exporter-cloud-trace -cov ext/opentelemetry-exporter-cloud-monitoring cov ext/opentelemetry-ext-wsgi cov ext/opentelemetry-ext-zipkin cov docs/examples/opentelemetry-example-app From 9367a2588d7ea2819f0553d1e27bab248090f3af Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Wed, 29 Jul 2020 10:03:46 -0700 Subject: [PATCH 20/24] Rename exporter packages from "ext" to "exporter" (#953) --- scripts/coverage.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 0b45fbf64..5cf393e7e 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -31,13 +31,13 @@ coverage erase cov opentelemetry-api cov opentelemetry-sdk -cov ext/opentelemetry-ext-datadog +cov exporter/opentelemetry-exporter-datadog cov ext/opentelemetry-ext-flask cov ext/opentelemetry-ext-requests -cov ext/opentelemetry-ext-jaeger +cov exporter/opentelemetry-exporter-jaeger cov ext/opentelemetry-ext-opentracing-shim cov ext/opentelemetry-ext-wsgi -cov ext/opentelemetry-ext-zipkin +cov exporter/opentelemetry-exporter-zipkin cov docs/examples/opentelemetry-example-app # aiohttp is only supported on Python 3.5+. From 76d1cee3fc468527d60bd4f4a40cc1a324325393 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 3 Aug 2020 10:10:45 -0700 Subject: [PATCH 21/24] Rename web framework packages from "ext" to "instrumentation" (#961) --- scripts/coverage.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 5cf393e7e..830381225 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -32,20 +32,20 @@ coverage erase cov opentelemetry-api cov opentelemetry-sdk cov exporter/opentelemetry-exporter-datadog -cov ext/opentelemetry-ext-flask -cov ext/opentelemetry-ext-requests +cov instrumentation/opentelemetry-instrumentation-flask +cov instrumentation/opentelemetry-instrumentation-requests cov exporter/opentelemetry-exporter-jaeger cov ext/opentelemetry-ext-opentracing-shim -cov ext/opentelemetry-ext-wsgi +cov instrumentation/opentelemetry-instrumentation-wsgi cov exporter/opentelemetry-exporter-zipkin cov docs/examples/opentelemetry-example-app # aiohttp is only supported on Python 3.5+. if [ ${PYTHON_VERSION_INFO[1]} -gt 4 ]; then - cov ext/opentelemetry-ext-aiohttp-client + cov instrumentation/opentelemetry-instrumentation-aiohttp-client # ext-asgi is only supported on Python 3.5+. if [ ${PYTHON_VERSION_INFO[1]} -gt 4 ]; then - cov ext/opentelemetry-ext-asgi + cov instrumentation/opentelemetry-instrumentation-asgi fi coverage report --show-missing From 55064ba489ca64510dff1360df823e9b2c2676c0 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Tue, 4 Aug 2020 19:10:51 -0700 Subject: [PATCH 22/24] Rename remaining framework packages from "ext" to "instrumentation" (#969) --- scripts/coverage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 830381225..7d3daa2af 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -7,7 +7,7 @@ function cov { then pytest \ --ignore-glob=*/setup.py \ - --ignore-glob=ext/opentelemetry-ext-opentracing-shim/tests/testbed/* \ + --ignore-glob=instrumentation/opentelemetry-instrumentation-opentracing-shim/tests/testbed/* \ --cov ${1} \ --cov-append \ --cov-branch \ @@ -35,7 +35,7 @@ cov exporter/opentelemetry-exporter-datadog cov instrumentation/opentelemetry-instrumentation-flask cov instrumentation/opentelemetry-instrumentation-requests cov exporter/opentelemetry-exporter-jaeger -cov ext/opentelemetry-ext-opentracing-shim +cov instrumentation/opentelemetry-instrumentation-opentracing-shim cov instrumentation/opentelemetry-instrumentation-wsgi cov exporter/opentelemetry-exporter-zipkin cov docs/examples/opentelemetry-example-app From 7de931f047cbb175fdeaf81ce56936e7e760d35c Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Wed, 30 Sep 2020 08:22:53 -0600 Subject: [PATCH 23/24] Add Global Error Handler (#1080) --- scripts/check_for_valid_readme.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/check_for_valid_readme.py b/scripts/check_for_valid_readme.py index d555b4fa2..7eff0ae58 100644 --- a/scripts/check_for_valid_readme.py +++ b/scripts/check_for_valid_readme.py @@ -29,6 +29,7 @@ def main(): error = False for path in map(Path, args.paths): + readme = path / "README.rst" try: if not is_valid_rst(readme): From 1dd16c33255140ea45777cc114139c6c076b3e37 Mon Sep 17 00:00:00 2001 From: Nathaniel Ruiz Nowell Date: Mon, 2 Nov 2020 14:56:14 -0800 Subject: [PATCH 24/24] Update scripts for contrib repo --- scripts/coverage.sh | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 7d3daa2af..8055adb0d 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -3,25 +3,13 @@ set -e function cov { - if [ ${TOX_ENV_NAME:0:4} == "py34" ] - then - pytest \ - --ignore-glob=*/setup.py \ - --ignore-glob=instrumentation/opentelemetry-instrumentation-opentracing-shim/tests/testbed/* \ - --cov ${1} \ - --cov-append \ - --cov-branch \ - --cov-report='' \ - ${1} - else - pytest \ - --ignore-glob=*/setup.py \ - --cov ${1} \ - --cov-append \ - --cov-branch \ - --cov-report='' \ - ${1} - fi + pytest \ + --ignore-glob=*/setup.py \ + --cov ${1} \ + --cov-append \ + --cov-branch \ + --cov-report='' \ + ${1} } PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:3])))') @@ -29,16 +17,10 @@ PYTHON_VERSION_INFO=(${PYTHON_VERSION//./ }) coverage erase -cov opentelemetry-api -cov opentelemetry-sdk cov exporter/opentelemetry-exporter-datadog cov instrumentation/opentelemetry-instrumentation-flask cov instrumentation/opentelemetry-instrumentation-requests -cov exporter/opentelemetry-exporter-jaeger -cov instrumentation/opentelemetry-instrumentation-opentracing-shim cov instrumentation/opentelemetry-instrumentation-wsgi -cov exporter/opentelemetry-exporter-zipkin -cov docs/examples/opentelemetry-example-app # aiohttp is only supported on Python 3.5+. if [ ${PYTHON_VERSION_INFO[1]} -gt 4 ]; then