Adding tracked_url_callback to RequestsInstrumentor (#714)

* Adding `tracked_url_callback` to `RequestsInstrumentor`

* linting fixes and CHANGELOG update

* Switching to `get_excluded_urls`

* Update CHANGELOG.md

Co-authored-by: Leighton Chen <lechen@microsoft.com>

* Fixing linting

* Stop patch in tearDown

* Fixing lint

* Fixing lint

* Fixing lint

Co-authored-by: Leighton Chen <lechen@microsoft.com>
Co-authored-by: Owais Lone <owais@users.noreply.github.com>

CHANGELOG.md

@@ -23,6 +23,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   ([#686](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/686))
 - `opentelemetry-instrumentation-tornado` now sets `http.client_ip` and `tornado.handler` attributes
   ([#706](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/706))
+- `opentelemetry-instrumentation-requests` added exclude urls functionality
+  ([#714](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/714))
 
 ### Changed
 - `opentelemetry-instrumentation-botocore` Make common span attributes compliant with semantic conventions

instrumentation/opentelemetry-instrumentation-requests/src/opentelemetry/instrumentation/requests/__init__.py

@@ -52,7 +52,11 @@ from opentelemetry.propagate import inject
 from opentelemetry.semconv.trace import SpanAttributes
 from opentelemetry.trace import SpanKind, get_tracer
 from opentelemetry.trace.status import Status
-from opentelemetry.util.http import remove_url_credentials
+from opentelemetry.util.http import (
+    get_excluded_urls,
+    parse_excluded_urls,
+    remove_url_credentials,
+)
 from opentelemetry.util.http.httplib import set_ip_on_next_http_connection
 
 # A key to a context variable to avoid creating duplicate spans when instrumenting
@@ -61,10 +65,14 @@ _SUPPRESS_HTTP_INSTRUMENTATION_KEY = context.create_key(
     "suppress_http_instrumentation"
 )
 
+_excluded_urls_from_env = get_excluded_urls("REQUESTS")
+
 
 # pylint: disable=unused-argument
 # pylint: disable=R0915
-def _instrument(tracer, span_callback=None, name_callback=None):
+def _instrument(
+    tracer, span_callback=None, name_callback=None, excluded_urls=None
+):
     """Enables tracing of all requests calls that go through
     :code:`requests.session.Session.request` (this includes
     :code:`requests.get`, etc.)."""
@@ -81,6 +89,9 @@ def _instrument(tracer, span_callback=None, name_callback=None):
 
     @functools.wraps(wrapped_request)
     def instrumented_request(self, method, url, *args, **kwargs):
+        if excluded_urls and excluded_urls.url_disabled(url):
+            return wrapped_request(self, method, url, *args, **kwargs)
+
         def get_or_create_headers():
             headers = kwargs.get("headers")
             if headers is None:
@@ -98,6 +109,9 @@ def _instrument(tracer, span_callback=None, name_callback=None):
 
     @functools.wraps(wrapped_send)
     def instrumented_send(self, request, **kwargs):
+        if excluded_urls and excluded_urls.url_disabled(request.url):
+            return wrapped_send(self, request, **kwargs)
+
         def get_or_create_headers():
             request.headers = (
                 request.headers
@@ -223,13 +237,19 @@ class RequestsInstrumentor(BaseInstrumentor):
                 ``name_callback``: Callback which calculates a generic span name for an
                     outgoing HTTP request based on the method and url.
                     Optional: Defaults to get_default_span_name.
+                ``excluded_urls``: A string containing a comma-delimitted
+                    list of regexes used to exclude URLs from tracking
         """
         tracer_provider = kwargs.get("tracer_provider")
         tracer = get_tracer(__name__, __version__, tracer_provider)
+        excluded_urls = kwargs.get("excluded_urls")
         _instrument(
             tracer,
             span_callback=kwargs.get("span_callback"),
             name_callback=kwargs.get("name_callback"),
+            excluded_urls=_excluded_urls_from_env
+            if excluded_urls is None
+            else parse_excluded_urls(excluded_urls),
         )
 
     def _uninstrument(self, **kwargs):

instrumentation/opentelemetry-instrumentation-requests/tests/test_requests_integration.py

@@ -30,6 +30,7 @@ from opentelemetry.semconv.trace import SpanAttributes
 from opentelemetry.test.mock_textmap import MockTextMapPropagator
 from opentelemetry.test.test_base import TestBase
 from opentelemetry.trace import StatusCode
+from opentelemetry.util.http import get_excluded_urls
 
 
 class TransportMock:
@@ -64,6 +65,21 @@ class RequestsIntegrationTestBase(abc.ABC):
     # pylint: disable=invalid-name
     def setUp(self):
         super().setUp()
+
+        self.env_patch = mock.patch.dict(
+            "os.environ",
+            {
+                "OTEL_PYTHON_REQUESTS_EXCLUDED_URLS": "http://localhost/env_excluded_arg/123,env_excluded_noarg"
+            },
+        )
+        self.env_patch.start()
+
+        self.exclude_patch = mock.patch(
+            "opentelemetry.instrumentation.requests._excluded_urls_from_env",
+            get_excluded_urls("REQUESTS"),
+        )
+        self.exclude_patch.start()
+
         RequestsInstrumentor().instrument()
         httpretty.enable()
         httpretty.register_uri(httpretty.GET, self.URL, body="Hello!")
@@ -71,6 +87,7 @@ class RequestsIntegrationTestBase(abc.ABC):
     # pylint: disable=invalid-name
     def tearDown(self):
         super().tearDown()
+        self.env_patch.stop()
         RequestsInstrumentor().uninstrument()
         httpretty.disable()
 
@@ -125,6 +142,32 @@ class RequestsIntegrationTestBase(abc.ABC):
 
         self.assertEqual(span.name, "GET" + self.URL)
 
+    def test_excluded_urls_explicit(self):
+        url_404 = "http://httpbin.org/status/404"
+        httpretty.register_uri(
+            httpretty.GET, url_404, status=404,
+        )
+
+        RequestsInstrumentor().uninstrument()
+        RequestsInstrumentor().instrument(excluded_urls=".*/404")
+        self.perform_request(self.URL)
+        self.perform_request(url_404)
+
+        self.assert_span(num_spans=1)
+
+    def test_excluded_urls_from_env(self):
+        url = "http://localhost/env_excluded_arg/123"
+        httpretty.register_uri(
+            httpretty.GET, url, status=200,
+        )
+
+        RequestsInstrumentor().uninstrument()
+        RequestsInstrumentor().instrument()
+        self.perform_request(self.URL)
+        self.perform_request(url)
+
+        self.assert_span(num_spans=1)
+
     def test_name_callback_default(self):
         def name_callback(method, url):
             return 123