opensource/opentelemetry-python-contrib
1
0
Fork 0
mirror of https://github.com/open-telemetry/opentelemetry-python-contrib.git synced 2025-07-31 14:11:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into flask-custom-span

Browse Source
This commit is contained in:
Leighton Chen
2020-11-11 09:32:50 -05:00
committed by GitHub
parent 4567715032 79b519f5cc
commit ee7596447a
38 changed files with 2521 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/test.yml vendored
View File

@ -22,7 +22,7 @@ jobs:
fail-fast: false # ensures the entire test matrix is run, even if one permutation fails
matrix:
python-version: [ py35, py36, py37, py38, pypy3 ]
package: ["instrumentation", "exporter"]
package: ["instrumentation", "exporter", "sdkextension"]
os: [ ubuntu-latest ]
include:
# py35-instrumentation segfaults on 18.04 so we instead run on 20.04

4
dev-requirements.txt
View File

@ -8,3 +8,7 @@ sphinx-rtd-theme~=0.4
sphinx-autodoc-typehints~=1.10.2
pytest!=5.2.3
pytest-cov>=2.8
readme-renderer~=24.0
grpcio-tools==1.29.0
mypy-protobuf>=1.23
protobuf>=3.13.0

6
instrumentation/opentelemetry-instrumentation-aiopg/tests/test_aiopg_integration.py
View File

@ -199,7 +199,11 @@ class TestAiopgIntegration(TestBase):
"user": "user",
}
db_integration = AiopgIntegration(
self.tracer, "testcomponent", "testtype", connection_attributes
self.tracer,
"testcomponent",
"testtype",
connection_attributes,
capture_parameters=True,
)
mock_connection = async_call(
db_integration.wrapped_connection(

3
instrumentation/opentelemetry-instrumentation-dbapi/CHANGELOG.md
View File

@ -2,6 +2,9 @@
## Unreleased
Stop capturing query parameters by default
([#156](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/156))
## Version 0.13b0
Released 2020-09-17

15
instrumentation/opentelemetry-instrumentation-dbapi/src/opentelemetry/instrumentation/dbapi/__init__.py
View File

@ -62,6 +62,7 @@ def trace_integration(
database_type: str = "",
connection_attributes: typing.Dict = None,
tracer_provider: typing.Optional[TracerProvider] = None,
capture_parameters: bool = False,
):
"""Integrate with DB API library.
https://www.python.org/dev/peps/pep-0249/
@ -76,6 +77,7 @@ def trace_integration(
user in Connection object.
tracer_provider: The :class:`opentelemetry.trace.TracerProvider` to
use. If ommited the current configured one is used.
capture_parameters: Configure if db.statement.parameters should be captured.
"""
wrap_connect(
__name__,
@ -86,6 +88,7 @@ def trace_integration(
connection_attributes,
version=__version__,
tracer_provider=tracer_provider,
capture_parameters=capture_parameters,
)
@ -98,6 +101,7 @@ def wrap_connect(
connection_attributes: typing.Dict = None,
version: str = "",
tracer_provider: typing.Optional[TracerProvider] = None,
capture_parameters: bool = False,
):
"""Integrate with DB API library.
https://www.python.org/dev/peps/pep-0249/
@ -111,6 +115,8 @@ def wrap_connect(
database_type: The Database type. For any SQL database, "sql".
connection_attributes: Attribute names for database, port, host and
user in Connection object.
capture_parameters: Configure if db.statement.parameters should be captured.
"""
# pylint: disable=unused-argument
@ -127,6 +133,7 @@ def wrap_connect(
connection_attributes=connection_attributes,
version=version,
tracer_provider=tracer_provider,
capture_parameters=capture_parameters,
)
return db_integration.wrapped_connection(wrapped, args, kwargs)
@ -159,6 +166,7 @@ def instrument_connection(
connection_attributes: typing.Dict = None,
version: str = "",
tracer_provider: typing.Optional[TracerProvider] = None,
capture_parameters=False,
):
"""Enable instrumentation in a database connection.
@ -170,7 +178,7 @@ def instrument_connection(
database_type: The Database type. For any SQL database, "sql".
connection_attributes: Attribute names for database, port, host and
user in a connection object.
capture_parameters: Configure if db.statement.parameters should be captured.
Returns:
An instrumented connection.
"""
@ -181,6 +189,7 @@ def instrument_connection(
connection_attributes=connection_attributes,
version=version,
tracer_provider=tracer_provider,
capture_parameters=capture_parameters,
)
db_integration.get_connection_attributes(connection)
return get_traced_connection_proxy(connection, db_integration)
@ -211,6 +220,7 @@ class DatabaseApiIntegration:
connection_attributes=None,
version: str = "",
tracer_provider: typing.Optional[TracerProvider] = None,
capture_parameters: bool = False,
):
self.connection_attributes = connection_attributes
if self.connection_attributes is None:
@ -223,6 +233,7 @@ class DatabaseApiIntegration:
self._name = name
self._version = version
self._tracer_provider = tracer_provider
self.capture_parameters = capture_parameters
self.database_component = database_component
self.database_type = database_type
self.connection_props = {}
@ -327,7 +338,7 @@ class TracedCursor:
) in self._db_api_integration.span_attributes.items():
span.set_attribute(attribute_key, attribute_value)
if len(args) > 1:
if self._db_api_integration.capture_parameters and len(args) > 1:
span.set_attribute("db.statement.parameters", str(args[1]))
def traced_execution(

43
instrumentation/opentelemetry-instrumentation-dbapi/tests/test_dbapi_integration.py
View File

@ -53,6 +53,49 @@ class TestDBApiIntegration(TestBase):
self.assertEqual(span.name, "testcomponent.testdatabase")
self.assertIs(span.kind, trace_api.SpanKind.CLIENT)
self.assertEqual(span.attributes["component"], "testcomponent")
self.assertEqual(span.attributes["db.type"], "testtype")
self.assertEqual(span.attributes["db.instance"], "testdatabase")
self.assertEqual(span.attributes["db.statement"], "Test query")
self.assertFalse("db.statement.parameters" in span.attributes)
self.assertEqual(span.attributes["db.user"], "testuser")
self.assertEqual(span.attributes["net.peer.name"], "testhost")
self.assertEqual(span.attributes["net.peer.port"], 123)
self.assertIs(
span.status.status_code, trace_api.status.StatusCode.UNSET,
)
def test_span_succeeded_with_capture_of_statement_parameters(self):
connection_props = {
"database": "testdatabase",
"server_host": "testhost",
"server_port": 123,
"user": "testuser",
}
connection_attributes = {
"database": "database",
"port": "server_port",
"host": "server_host",
"user": "user",
}
db_integration = dbapi.DatabaseApiIntegration(
self.tracer,
"testcomponent",
"testtype",
connection_attributes,
capture_parameters=True,
)
mock_connection = db_integration.wrapped_connection(
mock_connect, {}, connection_props
)
cursor = mock_connection.cursor()
cursor.execute("Test query", ("param1Value", False))
spans_list = self.memory_exporter.get_finished_spans()
self.assertEqual(len(spans_list), 1)
span = spans_list[0]
self.assertEqual(span.name, "testcomponent.testdatabase")
self.assertIs(span.kind, trace_api.SpanKind.CLIENT)
self.assertEqual(span.attributes["component"], "testcomponent")
self.assertEqual(span.attributes["db.type"], "testtype")
self.assertEqual(span.attributes["db.instance"], "testdatabase")

4
instrumentation/opentelemetry-instrumentation-elasticsearch/src/opentelemetry/instrumentation/elasticsearch/__init__.py
View File

@ -22,14 +22,14 @@ Usage
.. code-block:: python
from opentelemetry import trace
from opentelemetry.instrumentation.elasticsearch import ElasticSearchInstrumentor
from opentelemetry.instrumentation.elasticsearch import ElasticsearchInstrumentor
from opentelemetry.sdk.trace import TracerProvider
import elasticsearch
trace.set_tracer_provider(TracerProvider())
# instrument elasticsearch
ElasticSearchInstrumentor().instrument(tracer_provider=trace.get_tracer_provider())
ElasticsearchInstrumentor().instrument(tracer_provider=trace.get_tracer_provider())
# Using elasticsearch as normal now will automatically generate spans
es = elasticsearch.Elasticsearch()

3
instrumentation/opentelemetry-instrumentation-grpc/CHANGELOG.md
View File

@ -2,6 +2,9 @@
## Unreleased
- Update protobuf versions
([#1356](https://github.com/open-telemetry/opentelemetry-python/pull/1356))
## Version 0.15b0
Released 2020-11-02

2
instrumentation/opentelemetry-instrumentation-grpc/setup.cfg
View File

@ -47,7 +47,7 @@ install_requires =
test =
opentelemetry-test == 0.16.dev0
opentelemetry-sdk == 0.16.dev0
protobuf == 3.12.2
protobuf >= 3.13.0
[options.packages.find]
where = src

1
instrumentation/opentelemetry-instrumentation-jinja2/src/opentelemetry/instrumentation/jinja2/__init__.py
View File

@ -52,7 +52,6 @@ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
from opentelemetry.instrumentation.jinja2.version import __version__
from opentelemetry.instrumentation.utils import unwrap
from opentelemetry.trace import SpanKind, get_tracer
from opentelemetry.trace.status import Status, StatusCode
logger = logging.getLogger(__name__)

5
instrumentation/opentelemetry-instrumentation-sklearn/CHANGELOG.md Normal file
View File

@ -0,0 +1,5 @@
# Changelog
## Unreleased
- Initial release ([#151](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/151))

201
instrumentation/opentelemetry-instrumentation-sklearn/LICENSE Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

9
instrumentation/opentelemetry-instrumentation-sklearn/MANIFEST.in Normal file
View File

@ -0,0 +1,9 @@
graft src
graft tests
global-exclude *.pyc
global-exclude *.pyo
global-exclude __pycache__/*
include CHANGELOG.md
include MANIFEST.in
include README.rst
include LICENSE

23
instrumentation/opentelemetry-instrumentation-sklearn/README.rst Normal file
View File

@ -0,0 +1,23 @@
OpenTelemetry Scikit-Learn Instrumentation
==========================================
|pypi|
.. |pypi| image:: https://badge.fury.io/py/opentelemetry-instrumentation-sklearn.svg
:target: https://pypi.org/project/opentelemetry-instrumentation-sklearn/
This library allows tracing HTTP requests made by the
`scikit-learn <https://scikit-learn.org/stable/>`_ library.
Installation
------------
::
pip install opentelemetry-instrumentation-sklearn
References
----------
* `OpenTelemetry sklearn Instrumentation <https://opentelemetry-python.readthedocs.io/en/latest/instrumentation/sklearn/sklearn.html>`_
* `OpenTelemetry Project <https://opentelemetry.io/>`_

55
instrumentation/opentelemetry-instrumentation-sklearn/setup.cfg Normal file
View File

@ -0,0 +1,55 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[metadata]
name = opentelemetry-instrumentation-sklearn
description = OpenTelemetry sklearn instrumentation
long_description = file: README.rst
long_description_content_type = text/x-rst
author = OpenTelemetry Authors
author_email = cncf-opentelemetry-contributors@lists.cncf.io
url = https://github.com/open-telemetry/opentelemetry-python-contrib/tree/master/instrumentation/opentelemetry-instrumentation-sklearn
platforms = any
license = Apache-2.0
classifiers =
Development Status :: 4 - Beta
Intended Audience :: Developers
License :: OSI Approved :: Apache Software License
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
[options]
python_requires = >=3.5
package_dir=
=src
packages=find_namespace:
install_requires =
opentelemetry-api == 0.16.dev0
opentelemetry-instrumentation == 0.16.dev0
scikit-learn ~= 0.22.0
[options.extras_require]
test =
opentelemetry-test == 0.16.dev0
[options.packages.find]
where = src
[options.entry_points]
opentelemetry_instrumentor =
sklearn = opentelemetry.instrumentation.sklearn:SklearnInstrumentor

31
instrumentation/opentelemetry-instrumentation-sklearn/setup.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import setuptools
BASE_DIR = os.path.dirname(__file__)
VERSION_FILENAME = os.path.join(
BASE_DIR,
"src",
"opentelemetry",
"instrumentation",
"sklearn",
"version.py",
)
PACKAGE_INFO = {}
with open(VERSION_FILENAME) as f:
exec(f.read(), PACKAGE_INFO)
setuptools.setup(version=PACKAGE_INFO["__version__"])

759
instrumentation/opentelemetry-instrumentation-sklearn/src/opentelemetry/instrumentation/sklearn/__init__.py Normal file
View File

@ -0,0 +1,759 @@
# Copyright 2020, OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
The integration with sklearn supports the scikit-learn compatible libraries,
it can be enabled by using ``SklearnInstrumentor``.
.. sklearn: https://github.com/scikit-learn/scikit-learn
Usage
-----
Package instrumentation example:
.. code-block:: python
from opentelemetry.instrumentation.sklearn import SklearnInstrumentor
# instrument the sklearn library
SklearnInstrumentor().instrument()
# instrument sklearn and other libraries
SklearnInstrumentor(
packages=["sklearn", "lightgbm", "xgboost"]
).instrument()
Model intrumentation example:
.. code-block:: python
from opentelemetry.instrumentation.sklearn import SklearnInstrumentor
from sklearn.datasets import load_iris
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
X, y = load_iris(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y)
model = Pipeline(
[
("class", RandomForestClassifier(n_estimators=10)),
]
)
model.fit(X_train, y_train)
SklearnInstrumentor().instrument_estimator(model)
"""
import logging
import os
from functools import wraps
from importlib import import_module
from inspect import isclass
from pkgutil import iter_modules
from typing import Callable, Dict, List, MutableMapping, Sequence, Type, Union
from sklearn.base import BaseEstimator
from sklearn.pipeline import FeatureUnion, Pipeline
from sklearn.tree import BaseDecisionTree
from sklearn.utils.metaestimators import _IffHasAttrDescriptor
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
from opentelemetry.instrumentation.sklearn.version import __version__
from opentelemetry.trace import get_tracer
from opentelemetry.util.types import Attributes
logger = logging.getLogger(__name__)
def implement_span_estimator(
func: Callable,
estimator: Union[BaseEstimator, Type[BaseEstimator]],
attributes: Attributes = None,
):
"""Wrap the method call with a span.
Args:
func: A callable to be wrapped in a span
estimator: An instance or class of an estimator
attributes: Attributes to apply to the span
Returns:
The passed function wrapped in a span.
"""
if isclass(estimator):
name = estimator.__name__
else:
name = estimator.__class__.__name__
logger.debug("Instrumenting: %s.%s", name, func.__name__)
attributes = attributes or {}
name = "{cls}.{func}".format(cls=name, func=func.__name__)
return implement_span_function(func, name, attributes)
def implement_span_function(func: Callable, name: str, attributes: Attributes):
"""Wrap the function with a span.
Args:
func: A callable to be wrapped in a span
name: The name of the span
attributes: Attributes to apply to the span
Returns:
The passed function wrapped in a span.
"""
@wraps(func)
def wrapper(*args, **kwargs):
with get_tracer(__name__, __version__).start_as_current_span(
name=name
) as span:
if span.is_recording():
for key, val in attributes.items():
span.set_attribute(key, val)
return func(*args, **kwargs)
return wrapper
def implement_span_delegator(
obj: _IffHasAttrDescriptor, attributes: Attributes = None
):
"""Wrap the descriptor's fn with a span.
Args:
obj: An instance of _IffHasAttrDescriptor
attributes: Attributes to apply to the span
"""
# Don't instrument inherited delegators
if hasattr(obj, "_otel_original_fn"):
logger.debug("Already instrumented: %s", obj.fn.__qualname__)
return
logger.debug("Instrumenting: %s", obj.fn.__qualname__)
attributes = attributes or {}
setattr(obj, "_otel_original_fn", getattr(obj, "fn"))
setattr(
obj,
"fn",
implement_span_function(obj.fn, obj.fn.__qualname__, attributes),
)
def get_delegator(
estimator: Type[BaseEstimator], method_name: str
) -> Union[_IffHasAttrDescriptor, None]:
"""Get the delegator from a class method or None.
Args:
estimator: A class derived from ``sklearn``'s ``BaseEstimator``.
method_name (str): The method name of the estimator on which to
check for delegation.
Returns:
The delegator, if one exists, otherwise None.
"""
class_attr = getattr(estimator, method_name)
if getattr(class_attr, "__closure__", None) is not None:
for cell in class_attr.__closure__:
if isinstance(cell.cell_contents, _IffHasAttrDescriptor):
return cell.cell_contents
return None
def get_base_estimators(packages: List[str]) -> Dict[str, Type[BaseEstimator]]:
"""Walk package hierarchies to get BaseEstimator-derived classes.
Args:
packages (list(str)): A list of package names to instrument.
Returns:
A dictionary of qualnames and classes inheriting from
``BaseEstimator``.
"""
klasses = dict()
for package_name in packages:
lib = import_module(package_name)
package_dir = os.path.dirname(lib.__file__)
for (_, module_name, _) in iter_modules([package_dir]):
# import the module and iterate through its attributes
try:
module = import_module(package_name + "." + module_name)
except ImportError:
logger.warning(
"Unable to import %s.%s", package_name, module_name
)
continue
for attribute_name in dir(module):
attrib = getattr(module, attribute_name)
if isclass(attrib) and issubclass(attrib, BaseEstimator):
klasses[
".".join([package_name, module_name, attribute_name])
] = attrib
return klasses
# Methods on which spans should be applied.
DEFAULT_METHODS = [
"fit",
"transform",
"predict",
"predict_proba",
"_fit",
"_transform",
"_predict",
"_predict_proba",
]
# Classes and their attributes which contain a list of tupled estimators
# through which we should walk recursively for estimators.
DEFAULT_NAMEDTUPLE_ATTRIBS = {
Pipeline: ["steps"],
FeatureUnion: ["transformer_list"],
}
# Classes and their attributes which contain an estimator or sequence of
# estimators through which we should walk recursively for estimators.
DEFAULT_ATTRIBS = {}
# Classes (including children) explicitly excluded from autoinstrumentation
DEFAULT_EXCLUDE_CLASSES = [BaseDecisionTree]
# Default packages for autoinstrumentation
DEFAULT_PACKAGES = ["sklearn"]
class SklearnInstrumentor(BaseInstrumentor):
"""Instrument a fitted sklearn model with opentelemetry spans.
Instrument methods of ``BaseEstimator``-derived components in a sklearn
model. The assumption is that a machine learning model ``Pipeline`` (or
class descendent) is being instrumented with opentelemetry. Within a
``Pipeline`` is some hierarchy of estimators and transformers.
The ``instrument_estimator`` method walks this hierarchy of estimators,
implementing each of the defined methods with its own span.
Certain estimators in the sklearn ecosystem contain other estimators as
instance attributes. Support for walking this embedded sub-hierarchy is
supported with ``recurse_attribs``. This argument is a dictionary
with classes as keys, and a list of attributes representing embedded
estimators as values. By default, ``recurse_attribs`` is empty.
Similar to Pipelines, there are also estimators which have class attributes
as a list of 2-tuples; for instance, the ``FeatureUnion`` and its attribute
``transformer_list``. Instrumenting estimators like this is also
supported through the ``recurse_namedtuple_attribs`` argument. This
argument is a dictionary with classes as keys, and a list of attribute
names representing the namedtuple list(s). By default, the
``recurse_namedtuple_attribs`` dictionary supports
``Pipeline`` with ``steps``, and ``FeatureUnion`` with
``transformer_list``.
Note that spans will not be generated for any child transformer whose
parent transformer has ``n_jobs`` parameter set to anything besides
``None`` or ``1``.
Package instrumentation example:
.. code-block:: python
from opentelemetry.instrumentation.sklearn import SklearnInstrumentor
# instrument the sklearn library
SklearnInstrumentor().instrument()
# instrument several sklearn-compatible libraries
packages = ["sklearn", "lightgbm", "xgboost"]
SklearnInstrumentor(packages=packages).instrument()
Model intrumentation example:
.. code-block:: python
from opentelemetry.instrumentation.sklearn import SklearnInstrumentor
from sklearn.datasets import load_iris
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
X, y = load_iris(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y)
model = Pipeline(
[
("class", RandomForestClassifier(n_estimators=10)),
]
)
model.fit(X_train, y_train)
SklearnInstrumentor().instrument_estimator(model)
Args:
methods (list): A list of method names on which to instrument a span.
This list of methods will be checked on all estimators in the model
hierarchy. Used in package and model instrumentation
recurse_attribs (dict): A dictionary of ``BaseEstimator``-derived
sklearn classes as keys, with values being a list of attributes. Each
attribute represents either an estimator or list of estimators on
which to also implement spans. An example is
``RandomForestClassifier`` and its attribute ``estimators_``. Used
in model instrumentation only.
recurse_namedtuple_attribs (dict): A dictionary of ``BaseEstimator``-
derived sklearn types as keys, with values being a list of
attribute names. Each attribute represents a list of 2-tuples in
which the first element is the estimator name, and the second
element is the estimator. Defaults include sklearn's ``Pipeline``
and its attribute ``steps``, and the ``FeatureUnion`` and its
attribute ``transformer_list``. Used in model instrumentation only.
packages: A list of sklearn-compatible packages to
instrument. Used with package instrumentation only.
exclude_classes: A list of classes to exclude from instrumentation.
Child classes are also excluded. Default is sklearn's
``[BaseDecisionTree]``.
"""
def __new__(cls, *args, **kwargs):
"""Override new.
The base class' new method passes args and kwargs. We override because
we init the class with configuration and Python raises TypeError when
additional arguments are passed to the object.__new__() method.
"""
if cls._instance is None:
cls._instance = object.__new__(cls)
return cls._instance
def __init__(
self,
methods: List[str] = None,
recurse_attribs: Dict[Type[BaseEstimator], List[str]] = None,
recurse_namedtuple_attribs: Dict[
Type[BaseEstimator], List[str]
] = None,
packages: List[str] = None,
exclude_classes: List[Type] = None,
):
self.methods = methods or DEFAULT_METHODS
self.recurse_attribs = recurse_attribs or DEFAULT_ATTRIBS
self.recurse_namedtuple_attribs = (
recurse_namedtuple_attribs or DEFAULT_NAMEDTUPLE_ATTRIBS
)
self.packages = packages or DEFAULT_PACKAGES
if exclude_classes is None:
self.exclude_classes = tuple(DEFAULT_EXCLUDE_CLASSES)
else:
self.exclude_classes = tuple(exclude_classes)
def _instrument(self, **kwargs):
"""Instrument the library, and any additional specified on init."""
klasses = get_base_estimators(packages=self.packages)
attributes = kwargs.get("attributes")
for _, klass in klasses.items():
if issubclass(klass, self.exclude_classes):
logger.debug("Not instrumenting (excluded): %s", str(klass))
else:
logger.debug("Instrumenting: %s", str(klass))
for method_name in self.methods:
if hasattr(klass, method_name):
self._instrument_class_method(
estimator=klass,
method_name=method_name,
attributes=attributes,
)
def _uninstrument(self, **kwargs):
"""Uninstrument the library"""
klasses = get_base_estimators(packages=self.packages)
for _, klass in klasses.items():
logger.debug("Uninstrumenting: %s", str(klass))
for method_name in self.methods:
if hasattr(klass, method_name):
self._uninstrument_class_method(
estimator=klass, method_name=method_name
)
def instrument_estimator(
self, estimator: BaseEstimator, attributes: Attributes = None
):
"""Instrument a fitted estimator and its hierarchy where configured.
Args:
estimator (sklearn.base.BaseEstimator): A fitted ``sklearn``
estimator, typically a ``Pipeline`` instance.
attributes (dict): Attributes to attach to the spans.
"""
if isinstance(estimator, self.exclude_classes):
logger.debug(
"Not instrumenting (excluded): %s",
estimator.__class__.__name__,
)
return
if isinstance(
estimator, tuple(self.recurse_namedtuple_attribs.keys())
):
self._instrument_estimator_namedtuple(
estimator=estimator, attributes=attributes
)
if isinstance(estimator, tuple(self.recurse_attribs.keys())):
self._instrument_estimator_attribute(
estimator=estimator, attributes=attributes
)
for method_name in self.methods:
if hasattr(estimator, method_name):
self._instrument_instance_method(
estimator=estimator,
method_name=method_name,
attributes=attributes,
)
def uninstrument_estimator(self, estimator: BaseEstimator):
"""Uninstrument a fitted estimator and its hierarchy where configured.
Args:
estimator (sklearn.base.BaseEstimator): A fitted ``sklearn``
estimator, typically a ``Pipeline`` instance.
"""
if isinstance(estimator, self.exclude_classes):
logger.debug(
"Not uninstrumenting (excluded): %s",
estimator.__class__.__name__,
)
return
if isinstance(
estimator, tuple(self.recurse_namedtuple_attribs.keys())
):
self._uninstrument_estimator_namedtuple(estimator=estimator)
if isinstance(estimator, tuple(self.recurse_attribs.keys())):
self._uninstrument_estimator_attribute(estimator=estimator)
for method_name in self.methods:
if hasattr(estimator, method_name):
self._uninstrument_instance_method(
estimator=estimator, method_name=method_name
)
def _check_instrumented(
self,
estimator: Union[BaseEstimator, Type[BaseEstimator]],
method_name: str,
) -> bool:
"""Check an estimator-method is instrumented.
Args:
estimator (BaseEstimator): A class or instance of an ``sklearn``
estimator.
method_name (str): The method name of the estimator on which to
check for instrumentation.
"""
orig_method_name = "_otel_original_" + method_name
has_original = hasattr(estimator, orig_method_name)
orig_class, orig_method = getattr(
estimator, orig_method_name, (None, None)
)
same_class = orig_class == estimator
if has_original and same_class:
class_method = self._unwrap_function(
getattr(estimator, method_name)
)
# if they match then the subclass doesn't override
# if they don't then the overridden method needs instrumentation
if class_method.__name__ == orig_method.__name__:
return True
return False
def _uninstrument_class_method(
self, estimator: Type[BaseEstimator], method_name: str
):
"""Uninstrument a class method.
Replaces the patched method with the original, and deletes the
attribute which stored the original method.
Args:
estimator (BaseEstimator): A class or instance of an ``sklearn``
estimator.
method_name (str): The method name of the estimator on which to
apply a span.
"""
orig_method_name = "_otel_original_" + method_name
if isclass(estimator):
qualname = estimator.__qualname__
else:
qualname = estimator.__class__.__qualname__
delegator = get_delegator(estimator, method_name)
if self._check_instrumented(estimator, method_name):
logger.debug(
"Uninstrumenting: %s.%s", qualname, method_name,
)
_, orig_method = getattr(estimator, orig_method_name)
setattr(
estimator, method_name, orig_method,
)
delattr(estimator, orig_method_name)
elif delegator is not None:
if not hasattr(delegator, "_otel_original_fn"):
logger.debug(
"Already uninstrumented: %s.%s", qualname, method_name,
)
return
setattr(
delegator, "fn", getattr(delegator, "_otel_original_fn"),
)
delattr(delegator, "_otel_original_fn")
else:
logger.debug(
"Already uninstrumented: %s.%s", qualname, method_name,
)
def _uninstrument_instance_method(
self, estimator: BaseEstimator, method_name: str
):
"""Uninstrument an instance method.
Replaces the patched method with the original, and deletes the
attribute which stored the original method.
Args:
estimator (BaseEstimator): A class or instance of an ``sklearn``
estimator.
method_name (str): The method name of the estimator on which to
apply a span.
"""
orig_method_name = "_otel_original_" + method_name
if isclass(estimator):
qualname = estimator.__qualname__
else:
qualname = estimator.__class__.__qualname__
if self._check_instrumented(estimator, method_name):
logger.debug(
"Uninstrumenting: %s.%s", qualname, method_name,
)
_, orig_method = getattr(estimator, orig_method_name)
setattr(
estimator, method_name, orig_method,
)
delattr(estimator, orig_method_name)
else:
logger.debug(
"Already uninstrumented: %s.%s", qualname, method_name,
)
def _instrument_class_method(
self,
estimator: Type[BaseEstimator],
method_name: str,
attributes: Attributes = None,
):
"""Instrument an estimator method with a span.
When instrumenting we attach a tuple of (Class, method) to the
attribute ``_otel_original_<method_name>`` for each method. This allows
us to replace the patched with the original in uninstrumentation, but
also allows proper instrumentation of child classes without
instrumenting inherited methods twice.
Args:
estimator (BaseEstimator): A ``BaseEstimator``-derived
class
method_name (str): The method name of the estimator on which to
apply a span.
attributes (dict): Attributes to attach to the spans.
"""
if self._check_instrumented(estimator, method_name):
logger.debug(
"Already instrumented: %s.%s",
estimator.__qualname__,
method_name,
)
return
class_attr = getattr(estimator, method_name)
delegator = get_delegator(estimator, method_name)
if isinstance(class_attr, property):
logger.debug(
"Not instrumenting found property: %s.%s",
estimator.__qualname__,
method_name,
)
elif delegator is not None:
implement_span_delegator(delegator)
else:
setattr(
estimator,
"_otel_original_" + method_name,
(estimator, class_attr),
)
setattr(
estimator,
method_name,
implement_span_estimator(class_attr, estimator, attributes),
)
def _unwrap_function(self, function):
"""Fetch the function underlying any decorators"""
if hasattr(function, "__wrapped__"):
return self._unwrap_function(function.__wrapped__)
return function
def _instrument_instance_method(
self,
estimator: BaseEstimator,
method_name: str,
attributes: Attributes = None,
):
"""Instrument an estimator instance method with a span.
When instrumenting we attach a tuple of (Class, method) to the
attribute ``_otel_original_<method_name>`` for each method. This allows
us to replace the patched with the original in unstrumentation.
Args:
estimator (BaseEstimator): A fitted ``sklearn`` estimator.
method_name (str): The method name of the estimator on which to
apply a span.
attributes (dict): Attributes to attach to the spans.
"""
if self._check_instrumented(estimator, method_name):
logger.debug(
"Already instrumented: %s.%s",
estimator.__class__.__qualname__,
method_name,
)
return
class_attr = getattr(type(estimator), method_name, None)
if isinstance(class_attr, property):
logger.debug(
"Not instrumenting found property: %s.%s",
estimator.__class__.__qualname__,
method_name,
)
else:
method = getattr(estimator, method_name)
setattr(
estimator, "_otel_original_" + method_name, (estimator, method)
)
setattr(
estimator,
method_name,
implement_span_estimator(method, estimator, attributes),
)
def _instrument_estimator_attribute(
self, estimator: BaseEstimator, attributes: Attributes = None
):
"""Instrument instance attributes which also contain estimators.
Handle instance attributes which are also estimators, are a list
(Sequence) of estimators, or are mappings (dictionary) in which
the values are estimators.
Examples include ``RandomForestClassifier`` and
``MultiOutputRegressor`` instances which have attributes
``estimators_`` attributes.
Args:
estimator (BaseEstimator): A fitted ``sklearn`` estimator, with an
attribute which also contains an estimator or collection of
estimators.
attributes (dict): Attributes to attach to the spans.
"""
attribs = self.recurse_attribs.get(estimator.__class__, [])
for attrib in attribs:
attrib_value = getattr(estimator, attrib)
if isinstance(attrib_value, Sequence):
for value in attrib_value:
self.instrument_estimator(
estimator=value, attributes=attributes
)
elif isinstance(attrib_value, MutableMapping):
for value in attrib_value.values():
self.instrument_estimator(
estimator=value, attributes=attributes
)
else:
self.instrument_estimator(
estimator=attrib_value, attributes=attributes
)
def _instrument_estimator_namedtuple(
self, estimator: BaseEstimator, attributes: Attributes = None
):
"""Instrument attributes with (name, estimator) tupled components.
Examples include Pipeline and FeatureUnion instances which
have attributes steps and transformer_list, respectively.
Args:
estimator: A fitted sklearn estimator, with an attribute which also
contains an estimator or collection of estimators.
attributes (dict): Attributes to attach to the spans.
"""
attribs = self.recurse_namedtuple_attribs.get(estimator.__class__, [])
for attrib in attribs:
for _, est in getattr(estimator, attrib):
self.instrument_estimator(estimator=est, attributes=attributes)
def _uninstrument_estimator_attribute(self, estimator: BaseEstimator):
"""Uninstrument instance attributes which also contain estimators.
Handle instance attributes which are also estimators, are a list
(Sequence) of estimators, or are mappings (dictionary) in which
the values are estimators.
Examples include ``RandomForestClassifier`` and
``MultiOutputRegressor`` instances which have attributes
``estimators_`` attributes.
Args:
estimator (BaseEstimator): A fitted ``sklearn`` estimator, with an
attribute which also contains an estimator or collection of
estimators.
"""
attribs = self.recurse_attribs.get(estimator.__class__, [])
for attrib in attribs:
attrib_value = getattr(estimator, attrib)
if isinstance(attrib_value, Sequence):
for value in attrib_value:
self.uninstrument_estimator(estimator=value)
elif isinstance(attrib_value, MutableMapping):
for value in attrib_value.values():
self.uninstrument_estimator(estimator=value)
else:
self.uninstrument_estimator(estimator=attrib_value)
def _uninstrument_estimator_namedtuple(self, estimator: BaseEstimator):
"""Uninstrument attributes with (name, estimator) tupled components.
Examples include Pipeline and FeatureUnion instances which
have attributes steps and transformer_list, respectively.
Args:
estimator: A fitted sklearn estimator, with an attribute which also
contains an estimator or collection of estimators.
"""
attribs = self.recurse_namedtuple_attribs.get(estimator.__class__, [])
for attrib in attribs:
for _, est in getattr(estimator, attrib):
self.uninstrument_estimator(estimator=est)

15
instrumentation/opentelemetry-instrumentation-sklearn/src/opentelemetry/instrumentation/sklearn/version.py Normal file
View File

@ -0,0 +1,15 @@
# Copyright 2020, OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = "0.16.dev0"

0
instrumentation/opentelemetry-instrumentation-sklearn/tests/__init__.py Normal file
View File

54
instrumentation/opentelemetry-instrumentation-sklearn/tests/fixtures.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright 2020, OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
from sklearn.datasets import load_iris
from sklearn.decomposition import PCA, TruncatedSVD
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.pipeline import FeatureUnion, Pipeline
from sklearn.preprocessing import Normalizer, StandardScaler
X, y = load_iris(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y)
def pipeline():
"""A dummy model that has a bunch of components that we can test."""
model = Pipeline(
[
("scaler", StandardScaler()),
("normal", Normalizer()),
(
"union",
FeatureUnion(
[
("pca", PCA(n_components=1)),
("svd", TruncatedSVD(n_components=2)),
],
n_jobs=1, # parallelized components won't generate spans
),
),
("class", RandomForestClassifier(n_estimators=10)),
]
)
model.fit(X_train, y_train)
return model
def random_input():
"""A random record from the feature set."""
rows = X.shape[0]
random_row = np.random.choice(rows, size=1)
return X[random_row, :]

189
instrumentation/opentelemetry-instrumentation-sklearn/tests/test_sklearn.py Normal file
View File

@ -0,0 +1,189 @@
# Copyright 2020, OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sklearn.ensemble import RandomForestClassifier
from opentelemetry.instrumentation.sklearn import (
DEFAULT_EXCLUDE_CLASSES,
DEFAULT_METHODS,
SklearnInstrumentor,
get_base_estimators,
get_delegator,
)
from opentelemetry.test.test_base import TestBase
from opentelemetry.trace import SpanKind
from .fixtures import pipeline, random_input
def assert_instrumented(base_estimators):
for _, estimator in base_estimators.items():
for method_name in DEFAULT_METHODS:
original_method_name = "_otel_original_" + method_name
if issubclass(estimator, tuple(DEFAULT_EXCLUDE_CLASSES)):
assert not hasattr(estimator, original_method_name)
continue
class_attr = getattr(estimator, method_name, None)
if isinstance(class_attr, property):
assert not hasattr(estimator, original_method_name)
continue
delegator = None
if hasattr(estimator, method_name):
delegator = get_delegator(estimator, method_name)
if delegator is not None:
assert hasattr(delegator, "_otel_original_fn")
elif hasattr(estimator, method_name):
assert hasattr(estimator, original_method_name)
def assert_uninstrumented(base_estimators):
for _, estimator in base_estimators.items():
for method_name in DEFAULT_METHODS:
original_method_name = "_otel_original_" + method_name
if issubclass(estimator, tuple(DEFAULT_EXCLUDE_CLASSES)):
assert not hasattr(estimator, original_method_name)
continue
class_attr = getattr(estimator, method_name, None)
if isinstance(class_attr, property):
assert not hasattr(estimator, original_method_name)
continue
delegator = None
if hasattr(estimator, method_name):
delegator = get_delegator(estimator, method_name)
if delegator is not None:
assert not hasattr(delegator, "_otel_original_fn")
elif hasattr(estimator, method_name):
assert not hasattr(estimator, original_method_name)
class TestSklearn(TestBase):
def test_package_instrumentation(self):
ski = SklearnInstrumentor()
base_estimators = get_base_estimators(packages=["sklearn"])
model = pipeline()
ski.instrument()
assert_instrumented(base_estimators)
x_test = random_input()
model.predict(x_test)
spans = self.memory_exporter.get_finished_spans()
self.assertEqual(len(spans), 8)
self.memory_exporter.clear()
ski.uninstrument()
assert_uninstrumented(base_estimators)
model = pipeline()
x_test = random_input()
model.predict(x_test)
spans = self.memory_exporter.get_finished_spans()
self.assertEqual(len(spans), 0)
def test_span_properties(self):
"""Test that we get all of the spans we expect."""
model = pipeline()
ski = SklearnInstrumentor()
ski.instrument_estimator(estimator=model)
x_test = random_input()
model.predict(x_test)
spans = self.memory_exporter.get_finished_spans()
self.assertEqual(len(spans), 8)
span = spans[0]
self.assertEqual(span.name, "StandardScaler.transform")
self.assertEqual(span.kind, SpanKind.INTERNAL)
self.assertEqual(span.parent.span_id, spans[-1].context.span_id)
span = spans[1]
self.assertEqual(span.name, "Normalizer.transform")
self.assertEqual(span.kind, SpanKind.INTERNAL)
self.assertEqual(span.parent.span_id, spans[-1].context.span_id)
span = spans[2]
self.assertEqual(span.name, "PCA.transform")
self.assertEqual(span.kind, SpanKind.INTERNAL)
self.assertEqual(span.parent.span_id, spans[4].context.span_id)
span = spans[3]
self.assertEqual(span.name, "TruncatedSVD.transform")
self.assertEqual(span.kind, SpanKind.INTERNAL)
self.assertEqual(span.parent.span_id, spans[4].context.span_id)
span = spans[4]
self.assertEqual(span.name, "FeatureUnion.transform")
self.assertEqual(span.kind, SpanKind.INTERNAL)
self.assertEqual(span.parent.span_id, spans[-1].context.span_id)
span = spans[5]
self.assertEqual(span.name, "RandomForestClassifier.predict_proba")
self.assertEqual(span.kind, SpanKind.INTERNAL)
self.assertEqual(span.parent.span_id, spans[6].context.span_id)
span = spans[6]
self.assertEqual(span.name, "RandomForestClassifier.predict")
self.assertEqual(span.kind, SpanKind.INTERNAL)
self.assertEqual(span.parent.span_id, spans[-1].context.span_id)
span = spans[7]
self.assertEqual(span.name, "Pipeline.predict")
self.assertEqual(span.kind, SpanKind.INTERNAL)
self.memory_exporter.clear()
# uninstrument
ski.uninstrument_estimator(estimator=model)
x_test = random_input()
model.predict(x_test)
spans = self.memory_exporter.get_finished_spans()
self.assertEqual(len(spans), 0)
def test_attrib_config(self):
"""Test that the attribute config makes spans on the decision trees."""
model = pipeline()
attrib_config = {RandomForestClassifier: ["estimators_"]}
ski = SklearnInstrumentor(
recurse_attribs=attrib_config,
exclude_classes=[], # decision trees excluded by default
)
ski.instrument_estimator(estimator=model)
x_test = random_input()
model.predict(x_test)
spans = self.memory_exporter.get_finished_spans()
self.assertEqual(len(spans), 8 + model.steps[-1][-1].n_estimators)
self.memory_exporter.clear()
ski.uninstrument_estimator(estimator=model)
x_test = random_input()
model.predict(x_test)
spans = self.memory_exporter.get_finished_spans()
self.assertEqual(len(spans), 0)
def test_span_attributes(self):
model = pipeline()
attributes = {"model_name": "random_forest_model"}
ski = SklearnInstrumentor()
ski.instrument_estimator(estimator=model, attributes=attributes)
x_test = random_input()
model.predict(x_test)
spans = self.memory_exporter.get_finished_spans()
for span in spans:
assert span.attributes["model_name"] == "random_forest_model"

6
sdk-extension/opentelemetry-sdk-extension-aws/CHANGELOG.md Normal file
View File

@ -0,0 +1,6 @@
# Changelog
## Unreleased
- Provide components needed to Configure OTel SDK for Tracing with AWS X-Ray
([#130](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/130))

201
sdk-extension/opentelemetry-sdk-extension-aws/LICENSE Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

9
sdk-extension/opentelemetry-sdk-extension-aws/MANIFEST.rst Normal file
View File

@ -0,0 +1,9 @@
graft src
graft tests
global-exclude *.pyc
global-exclude *.pyo
global-exclude __pycache__/*
include CHANGELOG.md
include MANIFEST.in
include README.rst
include LICENSE

51
sdk-extension/opentelemetry-sdk-extension-aws/README.rst Normal file
View File

@ -0,0 +1,51 @@
OpenTelemetry SDK Extension for AWS X-Ray Compatibility
=======================================================
|pypi|
.. |pypi| image:: https://badge.fury.io/py/opentelemetry-sdk-extension-aws.svg
:target: https://pypi.org/project/opentelemetry-sdk-extension-aws/
This library provides components necessary to configure the OpenTelemetry SDK
for tracing with AWS X-Ray.
Installation
------------
::
pip install opentelemetry-sdk-extension-aws
Usage (AWS X-Ray IDs Generator)
-------------------------------
Configure the OTel SDK TracerProvider with the provided custom IDs Generator to
make spans compatible with the AWS X-Ray backend tracing service.
.. code-block:: python
from opentelemetry.sdk.extension.aws.trace import AwsXRayIdsGenerator
trace.set_tracer_provider(
TracerProvider(ids_generator=AwsXRayIdsGenerator())
)
Usage (AWS X-Ray Propagator)
----------------------------
Set this environment variable to have the OTel SDK use the provided AWS X-Ray
Propagator:
::
export OTEL_PROPAGATORS = aws_xray
References
----------
* `OpenTelemetry Project <https://opentelemetry.io/>`_
* `AWS X-Ray Trace IDs Format <https://docs.aws.amazon.com/xray/latest/devguide/xray-api-sendingdata.html#xray-api-traceids>`_

53
sdk-extension/opentelemetry-sdk-extension-aws/setup.cfg Normal file
View File

@ -0,0 +1,53 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[metadata]
name = opentelemetry-sdk-extension-aws
description = AWS SDK extension for OpenTelemetry
long_description = file: README.rst
long_description_content_type = text/x-rst
author = OpenTelemetry Authors
author_email = cncf-opentelemetry-contributors@lists.cncf.io
url = https://github.com/open-telemetry/opentelemetry-python-contrib/tree/master/sdk-extension/opentelemetry-sdk-extension-aws
platforms = any
license = Apache-2.0
classifiers =
Development Status :: 4 - Beta
Intended Audience :: Developers
License :: OSI Approved :: Apache Software License
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
[options]
python_requires = >=3.5
package_dir=
=src
packages=find_namespace:
install_requires =
opentelemetry-api == 0.16.dev0
[options.entry_points]
opentelemetry_propagator =
aws_xray = opentelemetry.sdk.extension.aws.trace.propagation.aws_xray_format:AwsXRayFormat
[options.extras_require]
test =
opentelemetry-test == 0.16.dev0
[options.packages.find]
where = src

26
sdk-extension/opentelemetry-sdk-extension-aws/setup.py Normal file
View File

@ -0,0 +1,26 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import setuptools
BASE_DIR = os.path.dirname(__file__)
VERSION_FILENAME = os.path.join(
BASE_DIR, "src", "opentelemetry", "sdk", "extension", "aws", "version.py"
)
PACKAGE_INFO = {}
with open(VERSION_FILENAME) as f:
exec(f.read(), PACKAGE_INFO)
setuptools.setup(version=PACKAGE_INFO["__version__"])

19
sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/trace/__init__.py Normal file
View File

@ -0,0 +1,19 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from opentelemetry.sdk.extension.aws.trace.aws_xray_ids_generator import (
AwsXRayIdsGenerator,
)
__all__ = ["AwsXRayIdsGenerator"]

40
sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/trace/aws_xray_ids_generator.py Normal file
View File

@ -0,0 +1,40 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import random
import time
from opentelemetry import trace
class AwsXRayIdsGenerator(trace.IdsGenerator):
"""Generates tracing IDs compatible with the AWS X-Ray tracing service. In
the X-Ray system, the first 32 bits of the `TraceId` are the Unix epoch time
in seconds. Since spans (AWS calls them segments) with an embedded timestamp
more than 30 days ago are rejected, a purely random `TraceId` risks being
rejected by the service.
See: https://docs.aws.amazon.com/xray/latest/devguide/xray-api-sendingdata.html#xray-api-traceids
"""
random_ids_generator = trace.RandomIdsGenerator()
def generate_span_id(self) -> int:
return self.random_ids_generator.generate_span_id()
@staticmethod
def generate_trace_id() -> int:
trace_time = int(time.time())
trace_identifier = random.getrandbits(96)
return (trace_time << 96) + trace_identifier

0
sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/trace/propagation/__init__.py Normal file
View File

276
sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/trace/propagation/aws_xray_format.py Normal file
View File

@ -0,0 +1,276 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import typing
import opentelemetry.trace as trace
from opentelemetry.context import Context
from opentelemetry.trace.propagation.textmap import (
Getter,
Setter,
TextMapPropagator,
TextMapPropagatorT,
)
TRACE_HEADER_KEY = "X-Amzn-Trace-Id"
KV_PAIR_DELIMITER = ";"
KEY_AND_VALUE_DELIMITER = "="
TRACE_ID_KEY = "Root"
TRACE_ID_LENGTH = 35
TRACE_ID_VERSION = "1"
TRACE_ID_DELIMITER = "-"
TRACE_ID_DELIMITER_INDEX_1 = 1
TRACE_ID_DELIMITER_INDEX_2 = 10
TRACE_ID_FIRST_PART_LENGTH = 8
PARENT_ID_KEY = "Parent"
PARENT_ID_LENGTH = 16
SAMPLED_FLAG_KEY = "Sampled"
SAMPLED_FLAG_LENGTH = 1
IS_SAMPLED = "1"
NOT_SAMPLED = "0"
_logger = logging.getLogger(__name__)
class AwsParseTraceHeaderError(Exception):
def __init__(self, message):
super().__init__()
self.message = message
class AwsXRayFormat(TextMapPropagator):
"""Propagator for the AWS X-Ray Trace Header propagation protocol.
See: https://docs.aws.amazon.com/xray/latest/devguide/xray-concepts.html#xray-concepts-tracingheader
"""
# AWS
def extract(
self,
getter: Getter[TextMapPropagatorT],
carrier: TextMapPropagatorT,
context: typing.Optional[Context] = None,
) -> Context:
trace_header_list = getter.get(carrier, TRACE_HEADER_KEY)
if not trace_header_list or len(trace_header_list) != 1:
return trace.set_span_in_context(
trace.INVALID_SPAN, context=context
)
trace_header = trace_header_list[0]
if not trace_header:
return trace.set_span_in_context(
trace.INVALID_SPAN, context=context
)
try:
(
trace_id,
span_id,
sampled,
) = AwsXRayFormat._extract_span_properties(trace_header)
except AwsParseTraceHeaderError as err:
_logger.debug(err.message)
return trace.set_span_in_context(
trace.INVALID_SPAN, context=context
)
options = 0
if sampled:
options |= trace.TraceFlags.SAMPLED
span_context = trace.SpanContext(
trace_id=trace_id,
span_id=span_id,
is_remote=True,
trace_flags=trace.TraceFlags(options),
trace_state=trace.TraceState(),
)
if not span_context.is_valid:
_logger.debug(
"Invalid Span Extracted. Insertting INVALID span into provided context."
)
return trace.set_span_in_context(
trace.INVALID_SPAN, context=context
)
return trace.set_span_in_context(
trace.DefaultSpan(span_context), context=context
)
@staticmethod
def _extract_span_properties(trace_header):
trace_id = trace.INVALID_TRACE_ID
span_id = trace.INVALID_SPAN_ID
sampled = False
for kv_pair_str in trace_header.split(KV_PAIR_DELIMITER):
try:
key_str, value_str = kv_pair_str.split(KEY_AND_VALUE_DELIMITER)
key, value = key_str.strip(), value_str.strip()
except ValueError as ex:
raise AwsParseTraceHeaderError(
(
"Error parsing X-Ray trace header. Invalid key value pair: %s. Returning INVALID span context.",
kv_pair_str,
)
) from ex
if key == TRACE_ID_KEY:
if not AwsXRayFormat._validate_trace_id(value):
raise AwsParseTraceHeaderError(
(
"Invalid TraceId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
TRACE_HEADER_KEY,
trace_header,
)
)
try:
trace_id = AwsXRayFormat._parse_trace_id(value)
except ValueError as ex:
raise AwsParseTraceHeaderError(
(
"Invalid TraceId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
TRACE_HEADER_KEY,
trace_header,
)
) from ex
elif key == PARENT_ID_KEY:
if not AwsXRayFormat._validate_span_id(value):
raise AwsParseTraceHeaderError(
(
"Invalid ParentId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
TRACE_HEADER_KEY,
trace_header,
)
)
try:
span_id = AwsXRayFormat._parse_span_id(value)
except ValueError as ex:
raise AwsParseTraceHeaderError(
(
"Invalid TraceId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
TRACE_HEADER_KEY,
trace_header,
)
) from ex
elif key == SAMPLED_FLAG_KEY:
if not AwsXRayFormat._validate_sampled_flag(value):
raise AwsParseTraceHeaderError(
(
"Invalid Sampling flag in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
TRACE_HEADER_KEY,
trace_header,
)
)
sampled = AwsXRayFormat._parse_sampled_flag(value)
return trace_id, span_id, sampled
@staticmethod
def _validate_trace_id(trace_id_str):
return (
len(trace_id_str) == TRACE_ID_LENGTH
and trace_id_str.startswith(TRACE_ID_VERSION)
and trace_id_str[TRACE_ID_DELIMITER_INDEX_1] == TRACE_ID_DELIMITER
and trace_id_str[TRACE_ID_DELIMITER_INDEX_2] == TRACE_ID_DELIMITER
)
@staticmethod
def _parse_trace_id(trace_id_str):
timestamp_subset = trace_id_str[
TRACE_ID_DELIMITER_INDEX_1 + 1 : TRACE_ID_DELIMITER_INDEX_2
]
unique_id_subset = trace_id_str[
TRACE_ID_DELIMITER_INDEX_2 + 1 : TRACE_ID_LENGTH
]
return int(timestamp_subset + unique_id_subset, 16)
@staticmethod
def _validate_span_id(span_id_str):
return len(span_id_str) == PARENT_ID_LENGTH
@staticmethod
def _parse_span_id(span_id_str):
return int(span_id_str, 16)
@staticmethod
def _validate_sampled_flag(sampled_flag_str):
return len(
sampled_flag_str
) == SAMPLED_FLAG_LENGTH and sampled_flag_str in (
IS_SAMPLED,
NOT_SAMPLED,
)
@staticmethod
def _parse_sampled_flag(sampled_flag_str):
return sampled_flag_str[0] == IS_SAMPLED
def inject(
self,
set_in_carrier: Setter[TextMapPropagatorT],
carrier: TextMapPropagatorT,
context: typing.Optional[Context] = None,
) -> None:
span = trace.get_current_span(context=context)
span_context = span.get_span_context()
if not span_context.is_valid:
return
otel_trace_id = "{:032x}".format(span_context.trace_id)
xray_trace_id = TRACE_ID_DELIMITER.join(
[
TRACE_ID_VERSION,
otel_trace_id[:TRACE_ID_FIRST_PART_LENGTH],
otel_trace_id[TRACE_ID_FIRST_PART_LENGTH:],
]
)
parent_id = "{:016x}".format(span_context.span_id)
sampling_flag = (
IS_SAMPLED
if span_context.trace_flags & trace.TraceFlags.SAMPLED
else NOT_SAMPLED
)
# TODO: Add OT trace state to the X-Ray trace header
trace_header = KV_PAIR_DELIMITER.join(
[
KEY_AND_VALUE_DELIMITER.join([key, value])
for key, value in [
(TRACE_ID_KEY, xray_trace_id),
(PARENT_ID_KEY, parent_id),
(SAMPLED_FLAG_KEY, sampling_flag),
]
]
)
set_in_carrier(
carrier, TRACE_HEADER_KEY, trace_header,
)

15
sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/version.py Normal file
View File

@ -0,0 +1,15 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = "0.16.dev0"

0
sdk-extension/opentelemetry-sdk-extension-aws/tests/__init__.py Normal file
View File

0
sdk-extension/opentelemetry-sdk-extension-aws/tests/trace/__init__.py Normal file
View File

0
sdk-extension/opentelemetry-sdk-extension-aws/tests/trace/propagation/__init__.py Normal file
View File

359
sdk-extension/opentelemetry-sdk-extension-aws/tests/trace/propagation/test_aws_xray_format.py Normal file
View File

@ -0,0 +1,359 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from requests.structures import CaseInsensitiveDict
import opentelemetry.trace as trace_api
from opentelemetry.sdk.extension.aws.trace.propagation.aws_xray_format import (
TRACE_HEADER_KEY,
AwsXRayFormat,
)
from opentelemetry.trace import (
DEFAULT_TRACE_OPTIONS,
DEFAULT_TRACE_STATE,
INVALID_SPAN_CONTEXT,
SpanContext,
TraceFlags,
TraceState,
set_span_in_context,
)
from opentelemetry.trace.propagation.textmap import DictGetter
TRACE_ID_BASE16 = "8a3c60f7d188f8fa79d48a391a778fa6"
SPAN_ID_BASE16 = "53995c3f42cd8ad8"
# Propagators Usage Methods
def get_as_list(dict_object, key):
value = dict_object.get(key)
return [value] if value is not None else []
# Inject Methods
def build_test_current_context(
trace_id=int(TRACE_ID_BASE16, 16),
span_id=int(SPAN_ID_BASE16, 16),
is_remote=True,
trace_flags=DEFAULT_TRACE_OPTIONS,
trace_state=DEFAULT_TRACE_STATE,
):
return set_span_in_context(
trace_api.DefaultSpan(
build_test_span_context(
trace_id, span_id, is_remote, trace_flags, trace_state
)
)
)
# Extract Methods
def get_nested_span_context(parent_context):
return trace_api.get_current_span(parent_context).get_span_context()
# Helper Methods
def build_test_span_context(
trace_id=int(TRACE_ID_BASE16, 16),
span_id=int(SPAN_ID_BASE16, 16),
is_remote=True,
trace_flags=DEFAULT_TRACE_OPTIONS,
trace_state=DEFAULT_TRACE_STATE,
):
return SpanContext(trace_id, span_id, is_remote, trace_flags, trace_state,)
class AwsXRayPropagatorTest(unittest.TestCase):
carrier_setter = CaseInsensitiveDict.__setitem__
carrier_getter = DictGetter()
XRAY_PROPAGATOR = AwsXRayFormat()
# Inject Tests
def test_inject_into_non_sampled_context(self):
carrier = CaseInsensitiveDict()
AwsXRayPropagatorTest.XRAY_PROPAGATOR.inject(
AwsXRayPropagatorTest.carrier_setter,
carrier,
build_test_current_context(),
)
injected_items = set(carrier.items())
expected_items = set(
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0"
}
).items()
)
self.assertEqual(injected_items, expected_items)
def test_inject_into_sampled_context(self):
carrier = CaseInsensitiveDict()
AwsXRayPropagatorTest.XRAY_PROPAGATOR.inject(
AwsXRayPropagatorTest.carrier_setter,
carrier,
build_test_current_context(
trace_flags=TraceFlags(TraceFlags.SAMPLED)
),
)
injected_items = set(carrier.items())
expected_items = set(
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=1"
}
).items()
)
self.assertEqual(injected_items, expected_items)
def test_inject_into_context_with_non_default_state(self):
carrier = CaseInsensitiveDict()
AwsXRayPropagatorTest.XRAY_PROPAGATOR.inject(
AwsXRayPropagatorTest.carrier_setter,
carrier,
build_test_current_context(trace_state=TraceState({"foo": "bar"})),
)
# TODO: (NathanielRN) Assert trace state when the propagator supports it
injected_items = set(carrier.items())
expected_items = set(
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0"
}
).items()
)
self.assertEqual(injected_items, expected_items)
# Extract Tests
def test_extract_empty_carrier_from_invalid_context(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter, CaseInsensitiveDict()
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
INVALID_SPAN_CONTEXT,
)
def test_extract_not_sampled_context(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0"
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
build_test_span_context(),
)
def test_extract_sampled_context(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=1"
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
build_test_span_context(
trace_flags=TraceFlags(TraceFlags.SAMPLED)
),
)
def test_extract_different_order(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Sampled=0;Parent=53995c3f42cd8ad8;Root=1-8a3c60f7-d188f8fa79d48a391a778fa6"
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
build_test_span_context(),
)
def test_extract_with_additional_fields(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0;Foo=Bar"
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
build_test_span_context(),
)
def test_extract_with_extra_whitespace(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: " Root = 1-8a3c60f7-d188f8fa79d48a391a778fa6 ; Parent = 53995c3f42cd8ad8 ; Sampled = 0 "
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
build_test_span_context(),
)
def test_extract_invalid_xray_trace_header(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict({TRACE_HEADER_KEY: ""}),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
INVALID_SPAN_CONTEXT,
)
def test_extract_invalid_trace_id(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-12345678-abcdefghijklmnopqrstuvwx;Parent=53995c3f42cd8ad8;Sampled=0"
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
INVALID_SPAN_CONTEXT,
)
def test_extract_invalid_trace_id_size(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa600;Parent=53995c3f42cd8ad8;Sampled=0="
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
INVALID_SPAN_CONTEXT,
)
def test_extract_invalid_span_id(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=abcdefghijklmnop;Sampled=0"
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
INVALID_SPAN_CONTEXT,
)
def test_extract_invalid_span_id_size(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad800;Sampled=0"
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
INVALID_SPAN_CONTEXT,
)
def test_extract_invalid_empty_sampled_flag(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled="
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
INVALID_SPAN_CONTEXT,
)
def test_extract_invalid_sampled_flag_size(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=011"
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
INVALID_SPAN_CONTEXT,
)
def test_extract_invalid_non_numeric_sampled_flag(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict(
{
TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=a"
}
),
)
self.assertEqual(
get_nested_span_context(context_with_extracted),
INVALID_SPAN_CONTEXT,
)

42
sdk-extension/opentelemetry-sdk-extension-aws/tests/trace/test_aws_xray_ids_generator.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import time
import unittest
from opentelemetry.sdk.extension.aws.trace import AwsXRayIdsGenerator
from opentelemetry.trace.span import INVALID_TRACE_ID
class AwsXRayIdsGeneratorTest(unittest.TestCase):
def test_ids_are_valid(self):
ids_generator = AwsXRayIdsGenerator()
for _ in range(1000):
trace_id = ids_generator.generate_trace_id()
self.assertTrue(trace_id != INVALID_TRACE_ID)
span_id = ids_generator.generate_span_id()
self.assertTrue(span_id != INVALID_TRACE_ID)
def test_id_timestamps_are_acceptable_for_xray(self):
ids_generator = AwsXRayIdsGenerator()
for _ in range(1000):
trace_id = ids_generator.generate_trace_id()
trace_id_time = trace_id >> 96
current_time = int(time.time())
self.assertLessEqual(trace_id_time, current_time)
one_month_ago_time = int(
(datetime.datetime.now() - datetime.timedelta(30)).timestamp()
)
self.assertGreater(trace_id_time, one_month_ago_time)

8
tox.ini
View File

@ -5,6 +5,10 @@ envlist =
; Environments are organized by individual package, allowing
; for specifying supported Python versions per package.
; opentelemetry-sdk-extension-aws
py3{5,6,7,8}-test-sdkextension-aws
pypy3-test-sdkextension-aws
; opentelemetry-instrumentation-aiohttp-client
py3{5,6,7,8}-test-instrumentation-aiohttp-client
pypy3-test-instrumentation-aiohttp-client
@ -178,6 +182,7 @@ changedir =
test-instrumentation-system-metrics: instrumentation/opentelemetry-instrumentation-system-metrics/tests
test-instrumentation-tornado: instrumentation/opentelemetry-instrumentation-tornado/tests
test-instrumentation-wsgi: instrumentation/opentelemetry-instrumentation-wsgi/tests
test-sdkextension-aws: sdk-extension/opentelemetry-sdk-extension-aws/tests
test-exporter-datadog: exporter/opentelemetry-exporter-datadog/tests
@ -250,6 +255,9 @@ commands_pre =
elasticsearch{2,5,6,7}: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation {toxinidir}/instrumentation/opentelemetry-instrumentation-elasticsearch[test]
aws: pip install requests {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws
; In order to get a healthy coverage report,
; we have to install packages in editable mode.
coverage: python {toxinidir}/scripts/eachdist.py install --editable