Rename web framework packages from "ext" to "instrumentation" (#961)

instrumentation/opentelemetry-instrumentation-asgi/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+## Unreleased
+
+- Change package name to opentelemetry-instrumentation-asgi
+  ([#961](https://github.com/open-telemetry/opentelemetry-python/pull/961))
+
+## 0.8b0
+
+Released 2020-05-27
+
+- Add ASGI middleware ([#716](https://github.com/open-telemetry/opentelemetry-python/pull/716))

instrumentation/opentelemetry-instrumentation-asgi/README.rst

@@ -0,0 +1,60 @@
+OpenTelemetry ASGI Middleware
+=============================
+
+|pypi|
+
+.. |pypi| image:: https://badge.fury.io/py/opentelemetry-instrumentation-asgi.svg
+   :target: https://pypi.org/project/opentelemetry-instrumentation-asgi/
+
+
+This library provides a ASGI middleware that can be used on any ASGI framework
+(such as Django, Starlette, FastAPI or Quart) to track requests timing through OpenTelemetry.
+
+Installation
+------------
+
+::
+
+    pip install opentelemetry-instrumentation-asgi
+
+
+Usage (Quart)
+-------------
+
+.. code-block:: python
+
+    from quart import Quart
+    from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware
+
+    app = Quart(__name__)
+    app.asgi_app = OpenTelemetryMiddleware(app.asgi_app)
+
+    @app.route("/")
+    async def hello():
+        return "Hello!"
+
+    if __name__ == "__main__":
+        app.run(debug=True)
+
+
+Usage (Django 3.0)
+------------------
+
+Modify the application's ``asgi.py`` file as shown below.
+
+.. code-block:: python
+
+    import os
+    from django.core.asgi import get_asgi_application
+    from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware
+
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'asgi_example.settings')
+
+    application = get_asgi_application()
+    application = OpenTelemetryMiddleware(application)
+
+
+References
+----------
+
+* `OpenTelemetry Project <https://opentelemetry.io/>`_

instrumentation/opentelemetry-instrumentation-asgi/setup.cfg

@@ -0,0 +1,51 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[metadata]
+name = opentelemetry-instrumentation-asgi
+description = ASGI instrumentation for OpenTelemetry
+long_description = file: README.rst
+long_description_content_type = text/x-rst
+author = OpenTelemetry Authors
+author_email = cncf-opentelemetry-contributors@lists.cncf.io
+url = https://github.com/open-telemetry/opentelemetry-python/instrumentation/opentelemetry-instrumentation-asgi
+platforms = any
+license = Apache-2.0
+classifiers =
+    Development Status :: 4 - Beta
+    Intended Audience :: Developers
+    License :: OSI Approved :: Apache Software License
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.5
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+
+[options]
+python_requires = >=3.5
+package_dir=
+    =src
+packages=find_namespace:
+install_requires =
+    opentelemetry-api == 0.12.dev0
+    opentelemetry-instrumentation == 0.12.dev0
+    asgiref ~= 3.0
+
+[options.extras_require]
+test =
+    opentelemetry-test
+
+[options.packages.find]
+where = src

instrumentation/opentelemetry-instrumentation-asgi/setup.py

@@ -0,0 +1,26 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+import setuptools
+
+BASE_DIR = os.path.dirname(__file__)
+VERSION_FILENAME = os.path.join(
+    BASE_DIR, "src", "opentelemetry", "instrumentation", "asgi", "version.py"
+)
+PACKAGE_INFO = {}
+with open(VERSION_FILENAME) as f:
+    exec(f.read(), PACKAGE_INFO)
+
+setuptools.setup(version=PACKAGE_INFO["__version__"])

instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py

@@ -0,0 +1,195 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+The opentelemetry-instrumentation-asgi package provides an ASGI middleware that can be used
+on any ASGI framework (such as Django-channels / Quart) to track requests
+timing through OpenTelemetry.
+"""
+
+import operator
+import typing
+import urllib
+from functools import wraps
+from typing import Tuple
+
+from asgiref.compatibility import guarantee_single_callable
+
+from opentelemetry import context, propagators, trace
+from opentelemetry.instrumentation.asgi.version import __version__  # noqa
+from opentelemetry.instrumentation.utils import http_status_to_canonical_code
+from opentelemetry.trace.status import Status, StatusCanonicalCode
+
+
+def get_header_from_scope(scope: dict, header_name: str) -> typing.List[str]:
+    """Retrieve a HTTP header value from the ASGI scope.
+
+    Returns:
+        A list with a single string with the header value if it exists, else an empty list.
+    """
+    headers = scope.get("headers")
+    return [
+        value.decode("utf8")
+        for (key, value) in headers
+        if key.decode("utf8") == header_name
+    ]
+
+
+def collect_request_attributes(scope):
+    """Collects HTTP request attributes from the ASGI scope and returns a
+    dictionary to be used as span creation attributes."""
+    server = scope.get("server") or ["0.0.0.0", 80]
+    port = server[1]
+    server_host = server[0] + (":" + str(port) if port != 80 else "")
+    full_path = scope.get("root_path", "") + scope.get("path", "")
+    http_url = scope.get("scheme", "http") + "://" + server_host + full_path
+    query_string = scope.get("query_string")
+    if query_string and http_url:
+        if isinstance(query_string, bytes):
+            query_string = query_string.decode("utf8")
+        http_url = http_url + ("?" + urllib.parse.unquote(query_string))
+
+    result = {
+        "component": scope["type"],
+        "http.scheme": scope.get("scheme"),
+        "http.host": server_host,
+        "host.port": port,
+        "http.flavor": scope.get("http_version"),
+        "http.target": scope.get("path"),
+        "http.url": http_url,
+    }
+    http_method = scope.get("method")
+    if http_method:
+        result["http.method"] = http_method
+    http_host_value = ",".join(get_header_from_scope(scope, "host"))
+    if http_host_value:
+        result["http.server_name"] = http_host_value
+    http_user_agent = get_header_from_scope(scope, "user-agent")
+    if len(http_user_agent) > 0:
+        result["http.user_agent"] = http_user_agent[0]
+
+    if "client" in scope and scope["client"] is not None:
+        result["net.peer.ip"] = scope.get("client")[0]
+        result["net.peer.port"] = scope.get("client")[1]
+
+    # remove None values
+    result = {k: v for k, v in result.items() if v is not None}
+
+    return result
+
+
+def set_status_code(span, status_code):
+    """Adds HTTP response attributes to span using the status_code argument."""
+    try:
+        status_code = int(status_code)
+    except ValueError:
+        span.set_status(
+            Status(
+                StatusCanonicalCode.UNKNOWN,
+                "Non-integer HTTP status: " + repr(status_code),
+            )
+        )
+    else:
+        span.set_attribute("http.status_code", status_code)
+        span.set_status(Status(http_status_to_canonical_code(status_code)))
+
+
+def get_default_span_details(scope: dict) -> Tuple[str, dict]:
+    """Default implementation for span_details_callback
+
+    Args:
+        scope: the asgi scope dictionary
+
+    Returns:
+        a tuple of the span, and any attributes to attach to the
+        span.
+    """
+    method_or_path = scope.get("method") or scope.get("path")
+
+    return method_or_path, {}
+
+
+class OpenTelemetryMiddleware:
+    """The ASGI application middleware.
+
+    This class is an ASGI middleware that starts and annotates spans for any
+    requests it is invoked with.
+
+    Args:
+        app: The ASGI application callable to forward requests to.
+        span_details_callback: Callback which should return a string
+            and a tuple, representing the desired span name and a
+            dictionary with any additional span attributes to set.
+            Optional: Defaults to get_default_span_details.
+    """
+
+    def __init__(self, app, span_details_callback=None):
+        self.app = guarantee_single_callable(app)
+        self.tracer = trace.get_tracer(__name__, __version__)
+        self.span_details_callback = (
+            span_details_callback or get_default_span_details
+        )
+
+    async def __call__(self, scope, receive, send):
+        """The ASGI application
+
+        Args:
+            scope: A ASGI environment.
+            receive: An awaitable callable yielding dictionaries
+            send: An awaitable callable taking a single dictionary as argument.
+        """
+        if scope["type"] not in ("http", "websocket"):
+            return await self.app(scope, receive, send)
+
+        token = context.attach(
+            propagators.extract(get_header_from_scope, scope)
+        )
+        span_name, additional_attributes = self.span_details_callback(scope)
+        attributes = collect_request_attributes(scope)
+        attributes.update(additional_attributes)
+
+        try:
+            with self.tracer.start_as_current_span(
+                span_name + " asgi",
+                kind=trace.SpanKind.SERVER,
+                attributes=attributes,
+            ):
+
+                @wraps(receive)
+                async def wrapped_receive():
+                    with self.tracer.start_as_current_span(
+                        span_name + " asgi." + scope["type"] + ".receive"
+                    ) as receive_span:
+                        message = await receive()
+                        if message["type"] == "websocket.receive":
+                            set_status_code(receive_span, 200)
+                        receive_span.set_attribute("type", message["type"])
+                    return message
+
+                @wraps(send)
+                async def wrapped_send(message):
+                    with self.tracer.start_as_current_span(
+                        span_name + " asgi." + scope["type"] + ".send"
+                    ) as send_span:
+                        if message["type"] == "http.response.start":
+                            status_code = message["status"]
+                            set_status_code(send_span, status_code)
+                        elif message["type"] == "websocket.send":
+                            set_status_code(send_span, 200)
+                        send_span.set_attribute("type", message["type"])
+                        await send(message)
+
+                await self.app(scope, wrapped_receive, wrapped_send)
+        finally:
+            context.detach(token)

instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/version.py

@@ -0,0 +1,15 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.12.dev0"

instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py

@@ -0,0 +1,344 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import unittest
+import unittest.mock as mock
+
+import opentelemetry.instrumentation.asgi as otel_asgi
+from opentelemetry import trace as trace_api
+from opentelemetry.test.asgitestutil import (
+    AsgiTestBase,
+    setup_testing_defaults,
+)
+
+
+async def http_app(scope, receive, send):
+    message = await receive()
+    assert scope["type"] == "http"
+    if message.get("type") == "http.request":
+        await send(
+            {
+                "type": "http.response.start",
+                "status": 200,
+                "headers": [[b"Content-Type", b"text/plain"]],
+            }
+        )
+        await send({"type": "http.response.body", "body": b"*"})
+
+
+async def websocket_app(scope, receive, send):
+    assert scope["type"] == "websocket"
+    while True:
+        message = await receive()
+        if message.get("type") == "websocket.connect":
+            await send({"type": "websocket.accept"})
+
+        if message.get("type") == "websocket.receive":
+            if message.get("text") == "ping":
+                await send({"type": "websocket.send", "text": "pong"})
+
+        if message.get("type") == "websocket.disconnect":
+            break
+
+
+async def simple_asgi(scope, receive, send):
+    assert isinstance(scope, dict)
+    if scope["type"] == "http":
+        await http_app(scope, receive, send)
+    elif scope["type"] == "websocket":
+        await websocket_app(scope, receive, send)
+
+
+async def error_asgi(scope, receive, send):
+    assert isinstance(scope, dict)
+    assert scope["type"] == "http"
+    message = await receive()
+    if message.get("type") == "http.request":
+        try:
+            raise ValueError
+        except ValueError:
+            scope["hack_exc_info"] = sys.exc_info()
+        await send(
+            {
+                "type": "http.response.start",
+                "status": 200,
+                "headers": [[b"Content-Type", b"text/plain"]],
+            }
+        )
+        await send({"type": "http.response.body", "body": b"*"})
+
+
+class TestAsgiApplication(AsgiTestBase):
+    def validate_outputs(self, outputs, error=None, modifiers=None):
+        # Ensure modifiers is a list
+        modifiers = modifiers or []
+        # Check for expected outputs
+        self.assertEqual(len(outputs), 2)
+        response_start = outputs[0]
+        response_body = outputs[1]
+        self.assertEqual(response_start["type"], "http.response.start")
+        self.assertEqual(response_body["type"], "http.response.body")
+
+        # Check http response body
+        self.assertEqual(response_body["body"], b"*")
+
+        # Check http response start
+        self.assertEqual(response_start["status"], 200)
+        self.assertEqual(
+            response_start["headers"], [[b"Content-Type", b"text/plain"]]
+        )
+
+        exc_info = self.scope.get("hack_exc_info")
+        if error:
+            self.assertIs(exc_info[0], error)
+            self.assertIsInstance(exc_info[1], error)
+            self.assertIsNotNone(exc_info[2])
+        else:
+            self.assertIsNone(exc_info)
+
+        # Check spans
+        span_list = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(span_list), 4)
+        expected = [
+            {
+                "name": "GET asgi.http.receive",
+                "kind": trace_api.SpanKind.INTERNAL,
+                "attributes": {"type": "http.request"},
+            },
+            {
+                "name": "GET asgi.http.send",
+                "kind": trace_api.SpanKind.INTERNAL,
+                "attributes": {
+                    "http.status_code": 200,
+                    "type": "http.response.start",
+                },
+            },
+            {
+                "name": "GET asgi.http.send",
+                "kind": trace_api.SpanKind.INTERNAL,
+                "attributes": {"type": "http.response.body"},
+            },
+            {
+                "name": "GET asgi",
+                "kind": trace_api.SpanKind.SERVER,
+                "attributes": {
+                    "component": "http",
+                    "http.method": "GET",
+                    "http.scheme": "http",
+                    "host.port": 80,
+                    "http.host": "127.0.0.1",
+                    "http.flavor": "1.0",
+                    "http.target": "/",
+                    "http.url": "http://127.0.0.1/",
+                    "net.peer.ip": "127.0.0.1",
+                    "net.peer.port": 32767,
+                },
+            },
+        ]
+        # Run our expected modifiers
+        for modifier in modifiers:
+            expected = modifier(expected)
+        # Check that output matches
+        for span, expected in zip(span_list, expected):
+            self.assertEqual(span.name, expected["name"])
+            self.assertEqual(span.kind, expected["kind"])
+            self.assertDictEqual(dict(span.attributes), expected["attributes"])
+
+    def test_basic_asgi_call(self):
+        """Test that spans are emitted as expected."""
+        app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+        self.seed_app(app)
+        self.send_default_request()
+        outputs = self.get_all_output()
+        self.validate_outputs(outputs)
+
+    def test_asgi_exc_info(self):
+        """Test that exception information is emitted as expected."""
+        app = otel_asgi.OpenTelemetryMiddleware(error_asgi)
+        self.seed_app(app)
+        self.send_default_request()
+        outputs = self.get_all_output()
+        self.validate_outputs(outputs, error=ValueError)
+
+    def test_override_span_name(self):
+        """Test that span_names can be overwritten by our callback function."""
+        span_name = "Dymaxion"
+
+        def get_predefined_span_details(_):
+            return span_name, {}
+
+        def update_expected_span_name(expected):
+            for entry in expected:
+                entry["name"] = " ".join(
+                    [span_name] + entry["name"].split(" ")[-1:]
+                )
+            return expected
+
+        app = otel_asgi.OpenTelemetryMiddleware(
+            simple_asgi, span_details_callback=get_predefined_span_details
+        )
+        self.seed_app(app)
+        self.send_default_request()
+        outputs = self.get_all_output()
+        self.validate_outputs(outputs, modifiers=[update_expected_span_name])
+
+    def test_behavior_with_scope_server_as_none(self):
+        """Test that middleware is ok when server is none in scope."""
+
+        def update_expected_server(expected):
+            expected[3]["attributes"].update(
+                {
+                    "http.host": "0.0.0.0",
+                    "host.port": 80,
+                    "http.url": "http://0.0.0.0/",
+                }
+            )
+            return expected
+
+        self.scope["server"] = None
+        app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+        self.seed_app(app)
+        self.send_default_request()
+        outputs = self.get_all_output()
+        self.validate_outputs(outputs, modifiers=[update_expected_server])
+
+    def test_host_header(self):
+        """Test that host header is converted to http.server_name."""
+        hostname = b"server_name_1"
+
+        def update_expected_server(expected):
+            expected[3]["attributes"].update(
+                {"http.server_name": hostname.decode("utf8")}
+            )
+            return expected
+
+        self.scope["headers"].append([b"host", hostname])
+        app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+        self.seed_app(app)
+        self.send_default_request()
+        outputs = self.get_all_output()
+        self.validate_outputs(outputs, modifiers=[update_expected_server])
+
+    def test_user_agent(self):
+        """Test that host header is converted to http.server_name."""
+        user_agent = b"test-agent"
+
+        def update_expected_user_agent(expected):
+            expected[3]["attributes"].update(
+                {"http.user_agent": user_agent.decode("utf8")}
+            )
+            return expected
+
+        self.scope["headers"].append([b"user-agent", user_agent])
+        app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+        self.seed_app(app)
+        self.send_default_request()
+        outputs = self.get_all_output()
+        self.validate_outputs(outputs, modifiers=[update_expected_user_agent])
+
+    def test_websocket(self):
+        self.scope = {
+            "type": "websocket",
+            "http_version": "1.1",
+            "scheme": "ws",
+            "path": "/",
+            "query_string": b"",
+            "headers": [],
+            "client": ("127.0.0.1", 32767),
+            "server": ("127.0.0.1", 80),
+        }
+        app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+        self.seed_app(app)
+        self.send_input({"type": "websocket.connect"})
+        self.send_input({"type": "websocket.receive", "text": "ping"})
+        self.send_input({"type": "websocket.disconnect"})
+        self.get_all_output()
+        span_list = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(span_list), 6)
+        expected = [
+            "/ asgi.websocket.receive",
+            "/ asgi.websocket.send",
+            "/ asgi.websocket.receive",
+            "/ asgi.websocket.send",
+            "/ asgi.websocket.receive",
+            "/ asgi",
+        ]
+        actual = [span.name for span in span_list]
+        self.assertListEqual(actual, expected)
+
+    def test_lifespan(self):
+        self.scope["type"] = "lifespan"
+        app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+        self.seed_app(app)
+        self.send_default_request()
+        span_list = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(span_list), 0)
+
+
+class TestAsgiAttributes(unittest.TestCase):
+    def setUp(self):
+        self.scope = {}
+        setup_testing_defaults(self.scope)
+        self.span = mock.create_autospec(trace_api.Span, spec_set=True)
+
+    def test_request_attributes(self):
+        self.scope["query_string"] = b"foo=bar"
+
+        attrs = otel_asgi.collect_request_attributes(self.scope)
+        self.assertDictEqual(
+            attrs,
+            {
+                "component": "http",
+                "http.method": "GET",
+                "http.host": "127.0.0.1",
+                "http.target": "/",
+                "http.url": "http://127.0.0.1/?foo=bar",
+                "host.port": 80,
+                "http.scheme": "http",
+                "http.flavor": "1.0",
+                "net.peer.ip": "127.0.0.1",
+                "net.peer.port": 32767,
+            },
+        )
+
+    def test_query_string(self):
+        self.scope["query_string"] = b"foo=bar"
+        attrs = otel_asgi.collect_request_attributes(self.scope)
+        self.assertEqual(attrs["http.url"], "http://127.0.0.1/?foo=bar")
+
+    def test_query_string_percent_bytes(self):
+        self.scope["query_string"] = b"foo%3Dbar"
+        attrs = otel_asgi.collect_request_attributes(self.scope)
+        self.assertEqual(attrs["http.url"], "http://127.0.0.1/?foo=bar")
+
+    def test_query_string_percent_str(self):
+        self.scope["query_string"] = "foo%3Dbar"
+        attrs = otel_asgi.collect_request_attributes(self.scope)
+        self.assertEqual(attrs["http.url"], "http://127.0.0.1/?foo=bar")
+
+    def test_response_attributes(self):
+        otel_asgi.set_status_code(self.span, 404)
+        expected = (mock.call("http.status_code", 404),)
+        self.assertEqual(self.span.set_attribute.call_count, 1)
+        self.assertEqual(self.span.set_attribute.call_count, 1)
+        self.span.set_attribute.assert_has_calls(expected, any_order=True)
+
+    def test_response_attributes_invalid_status_code(self):
+        otel_asgi.set_status_code(self.span, "Invalid Status Code")
+        self.assertEqual(self.span.set_status.call_count, 1)
+
+
+if __name__ == "__main__":
+    unittest.main()