feat(stdlib): Support data_collection filtering for URL query params

sentry_sdk/integrations/stdlib.py

@@ -7,6 +7,9 @@
 
 import sentry_sdk
 from sentry_sdk.consts import OP, SPANDATA
+from sentry_sdk.data_collection import (
+    _apply_data_collection_filtering_to_query_string,
+)
 from sentry_sdk.integrations import Integration
 from sentry_sdk.scope import add_global_event_processor, should_send_default_pii
 from sentry_sdk.traces import StreamedSpan
@@ -24,6 +27,7 @@
     _get_aws_sigv4_signed_headers_from_url_query_string,
     capture_internal_exceptions,
     ensure_integration_enabled,
+    has_data_collection_enabled,
     is_sentry_url,
     logger,
     parse_url,
@@ -33,7 +37,9 @@
 if TYPE_CHECKING:
     from typing import Any, Callable, Dict, List, Optional, Set, Union
 
-    from sentry_sdk._types import Event, Hint
+    from sentry_sdk._types import Attributes, Event, Hint
+    from sentry_sdk.client import BaseClient
+    from sentry_sdk.utils import ParsedUrl
 
 
 _RUNTIME_CONTEXT: "dict[str, object]" = {
@@ -83,6 +89,40 @@ def _complete_span(span: "Union[Span, StreamedSpan]") -> None:
             add_http_request_source(span)
 
 
+def _get_url_attributes(
+    client: "BaseClient", parsed_url: "Optional[ParsedUrl]"
+) -> "Attributes":
+    attributes: "Attributes" = {}
+    if parsed_url is None:
+        return attributes
+
+    query: "Optional[str]"
+    if has_data_collection_enabled(client.options):
+        query = None
+        if parsed_url.query:
+            query = _apply_data_collection_filtering_to_query_string(
+                query_string=parsed_url.query,
+                behaviour=client.options["data_collection"]["url_query_params"],
+            )
+    elif should_send_default_pii():
+        query = parsed_url.query
+    else:
+        return attributes
+
+    url_full = parsed_url.url
+    if query:
+        attributes[SPANDATA.URL_QUERY] = query
+        url_full += "?" + query
+
+    if parsed_url.fragment:
+        attributes[SPANDATA.URL_FRAGMENT] = parsed_url.fragment
+        url_full += "#" + parsed_url.fragment
+
+    attributes[SPANDATA.URL_FULL] = url_full
+
+    return attributes
+
+
 def _get_wrapped_putheader(
     original_putheader: "Callable[..., Any]",
 ) -> "Callable[..., Any]":
@@ -285,15 +325,10 @@ def putrequest(
         breadcrumb: "dict[str, Any]" = {}
 
         if span_streaming:
+            url_attributes = _get_url_attributes(client, parsed_url)
+
             breadcrumb[SPANDATA.HTTP_REQUEST_METHOD] = method
-            if parsed_url is not None and should_send_default_pii():
-                breadcrumb.update(
-                    {
-                        SPANDATA.URL_FRAGMENT: parsed_url.fragment,
-                        SPANDATA.URL_FULL: parsed_url.url,
-                        SPANDATA.URL_QUERY: parsed_url.query,
-                    }
-                )
+            breadcrumb.update(url_attributes)
 
             if sentry_sdk.traces.get_current_span() is not None:
                 span = sentry_sdk.traces.start_span(
@@ -309,10 +344,8 @@ def putrequest(
                     },
                 )
 
-                if parsed_url is not None and should_send_default_pii():
-                    span.set_attribute(SPANDATA.URL_FRAGMENT, parsed_url.fragment)
-                    span.set_attribute(SPANDATA.URL_FULL, parsed_url.url)
-                    span.set_attribute(SPANDATA.URL_QUERY, parsed_url.query)
+                for key, value in url_attributes.items():
+                    span.set_attribute(key, value)
 
                 set_on_span = span.set_attribute
 

tests/integrations/requests/test_requests.py

@@ -59,8 +59,6 @@ def test_crumb_capture_span_streaming(sentry_init, capture_events, send_default_
             {
                 SPANDATA.URL_FULL: url,
                 SPANDATA.HTTP_REQUEST_METHOD: "GET",
-                SPANDATA.URL_FRAGMENT: "",
-                SPANDATA.URL_QUERY: "",
                 SPANDATA.HTTP_STATUS_CODE: response.status_code,
             }
         )
@@ -169,8 +167,6 @@ def test_crumb_capture_client_error_span_streaming(
             {
                 SPANDATA.URL_FULL: url,
                 SPANDATA.HTTP_REQUEST_METHOD: "GET",
-                SPANDATA.URL_FRAGMENT: "",
-                SPANDATA.URL_QUERY: "",
                 SPANDATA.HTTP_STATUS_CODE: response.status_code,
             }
         )

tests/integrations/stdlib/test_httplib.py

@@ -128,8 +128,6 @@ def test_crumb_capture_span_streaming(sentry_init, capture_events, send_default_
                 SPANDATA.URL_FULL: url,
                 SPANDATA.HTTP_REQUEST_METHOD: "GET",
                 SPANDATA.HTTP_STATUS_CODE: 200,
-                SPANDATA.URL_FRAGMENT: "",
-                SPANDATA.URL_QUERY: "",
             }
         )
     else:
@@ -231,8 +229,6 @@ def test_crumb_capture_client_error_span_streaming(
                 SPANDATA.URL_FULL: url,
                 SPANDATA.HTTP_REQUEST_METHOD: "GET",
                 SPANDATA.HTTP_STATUS_CODE: status_code,
-                SPANDATA.URL_FRAGMENT: "",
-                SPANDATA.URL_QUERY: "",
             }
         )
     else:
@@ -311,8 +307,6 @@ def before_breadcrumb(crumb, hint):
                 SPANDATA.HTTP_REQUEST_METHOD: "GET",
                 SPANDATA.HTTP_STATUS_CODE: 200,
                 "extra": "foo",
-                SPANDATA.URL_FRAGMENT: "",
-                SPANDATA.URL_QUERY: "",
             }
         )
     else:
@@ -1408,7 +1402,7 @@ def test_proxy_http_tunnel(
         if send_default_pii:
             assert (
                 span["attributes"][SPANDATA.URL_FULL]
-                == f"http://api.example.com{port_modifier}/foo"
+                == f"http://api.example.com{port_modifier}/foo?bar=1"
             )
             assert span["attributes"][SPANDATA.URL_QUERY] == "bar=1"
         else:
@@ -1480,3 +1474,241 @@ def test_chunked_response_span_covers_body_read(
         end = datetime.datetime.strptime(span["timestamp"], fmt)
         duration = (end - start).total_seconds()
         assert duration >= min_expected_duration
+
+
+@pytest.mark.parametrize(
+    "init_kwargs, expected_query",
+    [
+        pytest.param(
+            {"send_default_pii": True},
+            "toy=tennisball&color=red&auth=secret",
+            id="send_default_pii_true",
+        ),
+        pytest.param(
+            {"send_default_pii": False},
+            None,
+            id="send_default_pii_false",
+        ),
+        pytest.param(
+            {},
+            None,
+            id="defaults",
+        ),
+        pytest.param(
+            {"_experiments": {"data_collection": {}}},
+            "toy=tennisball&color=red&auth=%5BFiltered%5D",
+            id="data_collection_denylist_default",
+        ),
+        pytest.param(
+            {
+                "_experiments": {
+                    "data_collection": {
+                        "url_query_params": {"mode": "denylist", "terms": ["toy"]}
+                    }
+                }
+            },
+            "toy=%5BFiltered%5D&color=red&auth=%5BFiltered%5D",
+            id="data_collection_denylist_custom_terms",
+        ),
+        pytest.param(
+            {
+                "_experiments": {
+                    "data_collection": {
+                        "url_query_params": {"mode": "allowlist", "terms": ["toy"]}
+                    }
+                }
+            },
+            "toy=tennisball&color=%5BFiltered%5D&auth=%5BFiltered%5D",
+            id="data_collection_allowlist",
+        ),
+        pytest.param(
+            {
+                "_experiments": {
+                    "data_collection": {
+                        "url_query_params": {"mode": "allowlist", "terms": ["auth"]}
+                    }
+                }
+            },
+            "toy=%5BFiltered%5D&color=%5BFiltered%5D&auth=%5BFiltered%5D",
+            id="data_collection_allowlist_sensitive_term",
+        ),
+        pytest.param(
+            {
+                "_experiments": {
+                    "data_collection": {"url_query_params": {"mode": "off"}}
+                }
+            },
+            None,
+            id="data_collection_off",
+        ),
+        pytest.param(
+            {
+                "send_default_pii": True,
+                "_experiments": {
+                    "data_collection": {"url_query_params": {"mode": "off"}}
+                },
+            },
+            None,
+            id="data_collection_wins_over_send_default_pii",
+        ),
+    ],
+)
+def test_url_query_data_collection_span_streaming(
+    sentry_init, capture_items, init_kwargs, expected_query
+):
+    sentry_init(
+        integrations=[StdlibIntegration()],
+        traces_sample_rate=1.0,
+        trace_lifecycle="stream",
+        **init_kwargs,
+    )
+
+    items = capture_items("span")
+
+    with sentry_sdk.traces.start_span(name="custom parent"):
+        conn = HTTPConnection("localhost", PORT)
+        conn.request(
+            "GET", "/some/random/url?toy=tennisball&color=red&auth=secret#frag"
+        )
+        conn.getresponse()
+
+    sentry_sdk.flush()
+
+    (span,) = (
+        item.payload
+        for item in items
+        if item.payload["attributes"].get("sentry.origin") == "auto.http.stdlib.httplib"
+    )
+
+    if expected_query is None:
+        assert SPANDATA.URL_QUERY not in span["attributes"]
+    else:
+        assert span["attributes"][SPANDATA.URL_QUERY] == expected_query
+
+
+@pytest.mark.parametrize(
+    "init_kwargs, expected_suffix",
+    [
+        pytest.param(
+            {"_experiments": {"data_collection": {}}},
+            "?toy=tennisball&color=red&auth=%5BFiltered%5D#frag",
+            id="data_collection_denylist_default",
+        ),
+        pytest.param(
+            {
+                "_experiments": {
+                    "data_collection": {
+                        "url_query_params": {"mode": "allowlist", "terms": ["toy"]}
+                    }
+                }
+            },
+            "?toy=tennisball&color=%5BFiltered%5D&auth=%5BFiltered%5D#frag",
+            id="data_collection_allowlist",
+        ),
+        pytest.param(
+            {"send_default_pii": True},
+            "?toy=tennisball&color=red&auth=secret#frag",
+            id="send_default_pii_true",
+        ),
+    ],
+)
+def test_url_full_reassembly_span_streaming(
+    sentry_init, capture_items, init_kwargs, expected_suffix
+):
+    sentry_init(
+        integrations=[StdlibIntegration()],
+        traces_sample_rate=1.0,
+        trace_lifecycle="stream",
+        **init_kwargs,
+    )
+
+    items = capture_items("span")
+
+    with sentry_sdk.traces.start_span(name="custom parent"):
+        conn = HTTPConnection("localhost", PORT)
+        conn.request(
+            "GET", "/some/random/url?toy=tennisball&color=red&auth=secret#frag"
+        )
+        conn.getresponse()
+
+    sentry_sdk.flush()
+
+    (span,) = (
+        item.payload
+        for item in items
+        if item.payload["attributes"].get("sentry.origin") == "auto.http.stdlib.httplib"
+    )
+
+    base_url = "http://localhost:{}/some/random/url".format(PORT)
+    assert span["attributes"][SPANDATA.URL_FULL] == base_url + expected_suffix
+
+
+@pytest.mark.parametrize(
+    "init_kwargs, expected_query, expects_url",
+    [
+        pytest.param(
+            {"send_default_pii": True},
+            "toy=tennisball&color=red&auth=secret",
+            True,
+            id="send_default_pii_true",
+        ),
+        pytest.param(
+            {},
+            None,
+            False,
+            id="defaults",
+        ),
+        pytest.param(
+            {"_experiments": {"data_collection": {}}},
+            "toy=tennisball&color=red&auth=%5BFiltered%5D",
+            True,
+            id="data_collection_denylist_default",
+        ),
+        pytest.param(
+            {
+                "_experiments": {
+                    "data_collection": {"url_query_params": {"mode": "off"}}
+                }
+            },
+            None,
+            True,
+            id="data_collection_off",
+        ),
+    ],
+)
+def test_crumb_url_query_data_collection_span_streaming(
+    sentry_init, capture_events, init_kwargs, expected_query, expects_url
+):
+    sentry_init(
+        integrations=[StdlibIntegration()],
+        trace_lifecycle="stream",
+        **init_kwargs,
+    )
+    events = capture_events()
+
+    conn = HTTPConnection("localhost", PORT)
+    conn.request("GET", "/some/random/url?toy=tennisball&color=red&auth=secret#frag")
+    conn.getresponse()
+
+    capture_message("Testing!")
+
+    (event,) = events
+    (crumb,) = event["breadcrumbs"]["values"]
+
+    base_url = "http://localhost:{}/some/random/url".format(PORT)
+
+    if not expects_url:
+        assert SPANDATA.URL_QUERY not in crumb["data"]
+        assert SPANDATA.URL_FULL not in crumb["data"]
+        return
+
+    assert crumb["data"][SPANDATA.URL_FRAGMENT] == "frag"
+
+    if expected_query is None:
+        assert SPANDATA.URL_QUERY not in crumb["data"]
+        assert crumb["data"][SPANDATA.URL_FULL] == f"{base_url}#frag"
+    else:
+        assert crumb["data"][SPANDATA.URL_QUERY] == expected_query
+        assert (
+            crumb["data"][SPANDATA.URL_FULL] == f"{base_url}?{expected_query}#frag"  # noqa: E231
+        )