Integrating sql commenter into otel_django_instrumentation (#896)

* Integrating sql commenter into otel_django_instrumentation

* Added test cases for django

* - Linting changes
- Added Changelog

* - Linting changes

* - Linting changes

* - Linting changes

* - Linting changes

* - Linting changes

* - Linting changes

* - Linting changes

* PR changes

* PR changes

* Linting changes

* Linting changes

* Linting changes

* Linting changes

* PR changes

* PR changes

* PR changes

* linting changes

* PR changes

* linting changes

* PR changes

* PR changes

* PR changes

* PR changes

* PR changes

Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>
Co-authored-by: Diego Hurtado <ocelotl@users.noreply.github.com>

CHANGELOG.md

@@ -39,7 +39,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add metric instrumentation for WSGI
   ([#1128](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1128))
 - `opentelemetry-instrumentation-requests` Restoring metrics in requests
-  ([#1110](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1110)
+  ([#1110](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1110))
+- Integrated sqlcommenter plugin into opentelemetry-instrumentation-django
+  ([#896](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/896))
 
 
 ## [1.12.0rc1-0.31b0](https://github.com/open-telemetry/opentelemetry-python/releases/tag/v1.12.0rc1-0.31b0) - 2022-05-17

instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py

@@ -17,6 +17,68 @@ Instrument `django`_ to trace Django applications.
 
 .. _django: https://pypi.org/project/django/
 
+SQLCOMMENTER
+*****************************************
+You can optionally configure Django instrumentation to enable sqlcommenter which enriches
+the query with contextual information.
+
+Usage
+-----
+
+.. code:: python
+
+    from opentelemetry.instrumentation.django import DjangoInstrumentor
+
+    DjangoInstrumentor().instrument(is_sql_commentor_enabled=True)
+
+
+For example,
+::
+
+   Invoking Users().objects.all() will lead to sql query "select * from auth_users" but when SQLCommenter is enabled
+   the query will get appended with some configurable tags like "select * from auth_users /*metrics=value*/;"
+
+
+SQLCommenter Configurations
+***************************
+We can configure the tags to be appended to the sqlquery log by adding below variables to the settings.py
+
+SQLCOMMENTER_WITH_FRAMEWORK = True(Default) or False
+
+For example,
+::
+Enabling this flag will add django framework and it's version which is /*framework='django%3A2.2.3*/
+
+SQLCOMMENTER_WITH_CONTROLLER = True(Default) or False
+
+For example,
+::
+Enabling this flag will add controller name that handles the request /*controller='index'*/
+
+SQLCOMMENTER_WITH_ROUTE = True(Default) or False
+
+For example,
+::
+Enabling this flag will add url path that handles the request /*route='polls/'*/
+
+SQLCOMMENTER_WITH_APP_NAME = True(Default) or False
+
+For example,
+::
+Enabling this flag will add app name that handles the request /*app_name='polls'*/
+
+SQLCOMMENTER_WITH_OPENTELEMETRY = True(Default) or False
+
+For example,
+::
+Enabling this flag will add opentelemetry traceparent /*traceparent='00-fd720cffceba94bbf75940ff3caaf3cc-4fd1a2bdacf56388-01'*/
+
+SQLCOMMENTER_WITH_DB_DRIVER = True(Default) or False
+
+For example,
+::
+Enabling this flag will add name of the db driver /*db_driver='django.db.backends.postgresql'*/
+
 Usage
 -----
 
@@ -124,6 +186,7 @@ Example of the added span attribute,
 
 API
 ---
+
 """
 
 from logging import getLogger
@@ -136,7 +199,9 @@ from django.conf import settings
 from opentelemetry.instrumentation.django.environment_variables import (
     OTEL_PYTHON_DJANGO_INSTRUMENT,
 )
-from opentelemetry.instrumentation.django.middleware import _DjangoMiddleware
+from opentelemetry.instrumentation.django.middleware.otel_middleware import (
+    _DjangoMiddleware,
+)
 from opentelemetry.instrumentation.django.package import _instruments
 from opentelemetry.instrumentation.django.version import __version__
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
@@ -166,6 +231,8 @@ class DjangoInstrumentor(BaseInstrumentor):
         [_DjangoMiddleware.__module__, _DjangoMiddleware.__qualname__]
     )
 
+    _sql_commenter_middleware = "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter"
+
     def instrumentation_dependencies(self) -> Collection[str]:
         return _instruments
 
@@ -204,7 +271,13 @@ class DjangoInstrumentor(BaseInstrumentor):
         if isinstance(settings_middleware, tuple):
             settings_middleware = list(settings_middleware)
 
+        is_sql_commentor_enabled = kwargs.pop("is_sql_commentor_enabled", None)
+
+        if is_sql_commentor_enabled:
+            settings_middleware.insert(0, self._sql_commenter_middleware)
+
         settings_middleware.insert(0, self._opentelemetry_middleware)
+
         setattr(settings, _middleware_setting, settings_middleware)
 
     def _uninstrument(self, **kwargs):

instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/sqlcommenter_middleware.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+#
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from logging import getLogger
+from typing import Any, Type, TypeVar
+from urllib.parse import quote as urllib_quote
+
+# pylint: disable=no-name-in-module
+from django import conf, get_version
+from django.db import connection
+from django.db.backends.utils import CursorDebugWrapper
+
+from opentelemetry.trace.propagation.tracecontext import (
+    TraceContextTextMapPropagator,
+)
+
+_propagator = TraceContextTextMapPropagator()
+
+_django_version = get_version()
+_logger = getLogger(__name__)
+
+T = TypeVar("T")  # pylint: disable-msg=invalid-name
+
+
+class SqlCommenter:
+    """
+    Middleware to append a comment to each database query with details about
+    the framework and the execution context.
+    """
+
+    def __init__(self, get_response) -> None:
+        self.get_response = get_response
+
+    def __call__(self, request) -> Any:
+        with connection.execute_wrapper(_QueryWrapper(request)):
+            return self.get_response(request)
+
+
+class _QueryWrapper:
+    def __init__(self, request) -> None:
+        self.request = request
+
+    def __call__(self, execute: Type[T], sql, params, many, context) -> T:
+        # pylint: disable-msg=too-many-locals
+        with_framework = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_FRAMEWORK", True
+        )
+        with_controller = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_CONTROLLER", True
+        )
+        with_route = getattr(conf.settings, "SQLCOMMENTER_WITH_ROUTE", True)
+        with_app_name = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_APP_NAME", True
+        )
+        with_opentelemetry = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_OPENTELEMETRY", True
+        )
+        with_db_driver = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_DB_DRIVER", True
+        )
+
+        db_driver = context["connection"].settings_dict.get("ENGINE", "")
+        resolver_match = self.request.resolver_match
+
+        sql_comment = _generate_sql_comment(
+            # Information about the controller.
+            controller=resolver_match.view_name
+            if resolver_match and with_controller
+            else None,
+            # route is the pattern that matched a request with a controller i.e. the regex
+            # See https://docs.djangoproject.com/en/stable/ref/urlresolvers/#django.urls.ResolverMatch.route
+            # getattr() because the attribute doesn't exist in Django < 2.2.
+            route=getattr(resolver_match, "route", None)
+            if resolver_match and with_route
+            else None,
+            # app_name is the application namespace for the URL pattern that matches the URL.
+            # See https://docs.djangoproject.com/en/stable/ref/urlresolvers/#django.urls.ResolverMatch.app_name
+            app_name=(resolver_match.app_name or None)
+            if resolver_match and with_app_name
+            else None,
+            # Framework centric information.
+            framework=f"django:{_django_version}" if with_framework else None,
+            # Information about the database and driver.
+            db_driver=db_driver if with_db_driver else None,
+            **_get_opentelemetry_values() if with_opentelemetry else {},
+        )
+
+        # TODO: MySQL truncates logs > 1024B so prepend comments
+        # instead of statements, if the engine is MySQL.
+        # See:
+        #  * https://github.com/basecamp/marginalia/issues/61
+        #  * https://github.com/basecamp/marginalia/pull/80
+        sql += sql_comment
+
+        # Add the query to the query log if debugging.
+        if context["cursor"].__class__ is CursorDebugWrapper:
+            context["connection"].queries_log.append(sql)
+
+        return execute(sql, params, many, context)
+
+
+def _generate_sql_comment(**meta) -> str:
+    """
+    Return a SQL comment with comma delimited key=value pairs created from
+    **meta kwargs.
+    """
+    key_value_delimiter = ","
+
+    if not meta:  # No entries added.
+        return ""
+
+    # Sort the keywords to ensure that caching works and that testing is
+    # deterministic. It eases visual inspection as well.
+    return (
+        " /*"
+        + key_value_delimiter.join(
+            f"{_url_quote(key)}={_url_quote(value)!r}"
+            for key, value in sorted(meta.items())
+            if value is not None
+        )
+        + "*/"
+    )
+
+
+def _url_quote(value) -> str:
+    if not isinstance(value, (str, bytes)):
+        return value
+    _quoted = urllib_quote(value)
+    # Since SQL uses '%' as a keyword, '%' is a by-product of url quoting
+    # e.g. foo,bar --> foo%2Cbar
+    # thus in our quoting, we need to escape it too to finally give
+    #      foo,bar --> foo%%2Cbar
+    return _quoted.replace("%", "%%")
+
+
+def _get_opentelemetry_values() -> dict or None:
+    """
+    Return the OpenTelemetry Trace and Span IDs if Span ID is set in the
+    OpenTelemetry execution context.
+    """
+    return _propagator.inject({})

instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py

@@ -102,11 +102,11 @@ class TestMiddleware(WsgiTestBase):
         )
         self.env_patch.start()
         self.exclude_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._excluded_urls",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._excluded_urls",
             get_excluded_urls("DJANGO"),
         )
         self.traced_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._traced_request_attrs",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._traced_request_attrs",
             get_traced_request_attrs("DJANGO"),
         )
         self.exclude_patch.start()

instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py

@@ -104,11 +104,11 @@ class TestMiddlewareAsgi(SimpleTestCase, TestBase):
         )
         self.env_patch.start()
         self.exclude_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._excluded_urls",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._excluded_urls",
             get_excluded_urls("DJANGO"),
         )
         self.traced_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._traced_request_attrs",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._traced_request_attrs",
             get_traced_request_attrs("DJANGO"),
         )
         self.exclude_patch.start()

instrumentation/opentelemetry-instrumentation-django/tests/test_sqlcommenter.py

@@ -0,0 +1,100 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=no-name-in-module
+
+from unittest.mock import MagicMock, patch
+
+from django import VERSION, conf
+from django.http import HttpResponse
+from django.test.utils import setup_test_environment, teardown_test_environment
+
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware import (
+    _QueryWrapper,
+)
+from opentelemetry.test.wsgitestutil import WsgiTestBase
+
+DJANGO_2_0 = VERSION >= (2, 0)
+
+_django_instrumentor = DjangoInstrumentor()
+
+
+class TestMiddleware(WsgiTestBase):
+    @classmethod
+    def setUpClass(cls):
+        conf.settings.configure(
+            SQLCOMMENTER_WITH_FRAMEWORK=False,
+            SQLCOMMENTER_WITH_DB_DRIVER=False,
+        )
+        super().setUpClass()
+
+    def setUp(self):
+        super().setUp()
+        setup_test_environment()
+        _django_instrumentor.instrument(is_sql_commentor_enabled=True)
+
+    def tearDown(self):
+        super().tearDown()
+        teardown_test_environment()
+        _django_instrumentor.uninstrument()
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+        conf.settings = conf.LazySettings()
+
+    @patch(
+        "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter"
+    )
+    def test_middleware_added(self, sqlcommenter_middleware):
+        instance = sqlcommenter_middleware.return_value
+        instance.get_response = HttpResponse()
+        if DJANGO_2_0:
+            middleware = conf.settings.MIDDLEWARE
+        else:
+            middleware = conf.settings.MIDDLEWARE_CLASSES
+
+        self.assertTrue(
+            "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter"
+            in middleware
+        )
+
+    @patch(
+        "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware._get_opentelemetry_values"
+    )
+    def test_query_wrapper(self, trace_capture):
+        requests_mock = MagicMock()
+        requests_mock.resolver_match.view_name = "view"
+        requests_mock.resolver_match.route = "route"
+        requests_mock.resolver_match.app_name = "app"
+
+        trace_capture.return_value = {
+            "traceparent": "*traceparent='00-000000000000000000000000deadbeef-000000000000beef-00"
+        }
+        qw_instance = _QueryWrapper(requests_mock)
+        execute_mock_obj = MagicMock()
+        qw_instance(
+            execute_mock_obj,
+            "Select 1",
+            MagicMock("test"),
+            MagicMock("test1"),
+            MagicMock(),
+        )
+        output_sql = execute_mock_obj.call_args[0][0]
+        self.assertEqual(
+            output_sql,
+            "Select 1 /*app_name='app',controller='view',route='route',traceparent='%%2Atraceparent%%3D%%2700-0000000"
+            "00000000000000000deadbeef-000000000000beef-00'*/",
+        )