diff --git a/.sampo/changesets/django-posthog-js-cookie.md b/.sampo/changesets/django-posthog-js-cookie.md new file mode 100644 index 000000000..260e60d75 --- /dev/null +++ b/.sampo/changesets/django-posthog-js-cookie.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +The Django middleware can read the session ID, and the distinct ID of an identified user, from the posthog-js cookie when a request has no tracing headers, so backend events link to browser sessions on same-site requests without frontend configuration. Turn it on with `POSTHOG_MW_READ_POSTHOG_COOKIE = True`. diff --git a/posthog/integrations/django.py b/posthog/integrations/django.py index 0347c6b86..bbdc8b56b 100644 --- a/posthog/integrations/django.py +++ b/posthog/integrations/django.py @@ -1,5 +1,9 @@ +import json +import math import re +import time from typing import TYPE_CHECKING, Any, Optional, cast +from urllib.parse import unquote from .. import contexts from ..client import Client @@ -56,12 +60,119 @@ def _get_sanitized_tracing_header(request, header_name) -> Optional[str]: return None +_POSTHOG_COOKIE_NAME_RE = re.compile(r"^ph_.+_posthog$") +_POSTHOG_CONSENT_COOKIE_PREFIX = "__ph_opt_in_out_" +_POSTHOG_CONSENT_NO_VALUES = ("false", "0", "no") +# posthog-js defaults. The browser starts a new session after this much inactivity or session length. +_COOKIE_SESSION_IDLE_TIMEOUT_MS = 30 * 60 * 1000 +_COOKIE_SESSION_MAX_LENGTH_MS = 24 * 60 * 60 * 1000 + + +def _default_api_key() -> Optional[str]: + # Read at call time, because the app sets the module-level keys after it imports this module. + from .. import api_key, project_api_key + + return (project_api_key or "").strip() or (api_key or "").strip() or None + + +def _posthog_cookie_name(api_key: str) -> str: + # posthog-js replaces these characters in the token when it names the cookie. + sanitized = api_key.replace("+", "PL").replace("/", "SL").replace("=", "EQ") + return f"ph_{sanitized}_posthog" + + +def _is_recent(timestamp_ms, now_ms: float, max_age_ms: int) -> bool: + # abs, like posthog-js, so a browser clock that runs ahead cannot keep a session alive. + return ( + isinstance(timestamp_ms, (int, float)) + and not isinstance(timestamp_ms, bool) + and math.isfinite(timestamp_ms) + and abs(now_ms - timestamp_ms) <= max_age_ms + ) + + +def _read_posthog_cookie( + request, + api_key, + session_idle_timeout_ms: int = _COOKIE_SESSION_IDLE_TIMEOUT_MS, + opt_out_by_default: bool = False, +) -> "tuple[Optional[str], Optional[str]]": + """Return the identified distinct ID and the live session ID from the posthog-js cookie, if present. + + With its default persistence, posthog-js writes `ph__posthog` as a first-party + cookie, and the browser sends it on every same-site request. This links backend events to the + browser session without `tracing_headers`. A session that is past the posthog-js idle timeout or + length cap is not returned, because the browser starts a new session on its next activity. + An anonymous distinct ID is not returned, so backend events for anonymous visitors stay + personless. Nothing is returned when the visitor's posthog-js consent cookie opts out. + """ + try: + cookies = getattr(request, "COOKIES", None) or {} + api_key = (api_key or "").strip() + if api_key: + raw = cookies.get(_posthog_cookie_name(api_key)) + consent_values = [ + value + for value in [cookies.get(_POSTHOG_CONSENT_COOKIE_PREFIX + api_key)] + if value is not None + ] + else: + # The project is unknown: use the cookie only when it is the only PostHog cookie. + matches = [ + value + for name, value in cookies.items() + if _POSTHOG_COOKIE_NAME_RE.match(name) + ] + raw = matches[0] if len(matches) == 1 else None + consent_values = [ + value + for name, value in cookies.items() + if name.startswith(_POSTHOG_CONSENT_COOKIE_PREFIX) + ] + opted_out = ( + any( + isinstance(value, str) + and value.strip().lower() in _POSTHOG_CONSENT_NO_VALUES + for value in consent_values + ) + # Like posthog-js, a visitor with no consent cookie counts as opted out under this default. + or (opt_out_by_default and not consent_values) + ) + if not raw or opted_out: + return None, None + + data = json.loads(unquote(raw)) + if not isinstance(data, dict): + return None, None + + distinct_id = ( + _sanitize_tracing_header_value(data.get("distinct_id")) + if data.get("$user_state") == "identified" + else None + ) + session_id = None + session = data.get("$sesid") + if isinstance(session, list) and len(session) in (2, 3): + # Older posthog-js versions stored [last activity, session id] and start the session then. + last_activity_ms, candidate = session[0], session[1] + session_start_ms = session[2] if len(session) == 3 else last_activity_ms + now_ms = time.time() * 1000 + if _is_recent( + last_activity_ms, now_ms, session_idle_timeout_ms + ) and _is_recent(session_start_ms, now_ms, _COOKIE_SESSION_MAX_LENGTH_MS): + session_id = _sanitize_tracing_header_value(candidate) + return distinct_id, session_id + except Exception: + return None, None + + class PosthogContextMiddleware: """Middleware to automatically track Django requests. This middleware wraps all calls with a posthog context. It attempts to extract the following from the request: - - Session ID, (extracted from `X-POSTHOG-SESSION-ID`) - - Distinct ID, (extracted from `X-POSTHOG-DISTINCT-ID`, falling back to the authenticated request user ID) + - Session ID, (extracted from `X-POSTHOG-SESSION-ID`, optionally falling back to the posthog-js cookie) + - Distinct ID, (extracted from `X-POSTHOG-DISTINCT-ID`, falling back to the authenticated request user ID, + then optionally to the posthog-js cookie) - Authenticated user email as `email` - Request URL as `$current_url` - Request method as `$request_method` @@ -73,6 +184,15 @@ class PosthogContextMiddleware: `POSTHOG_MW_CAPTURE_EXCEPTIONS` to `False` in your Django settings. The exceptions are captured using the global client, unless the setting `POSTHOG_MW_CLIENT` is set to a custom client instance + Set `POSTHOG_MW_READ_POSTHOG_COOKIE` to `True` to read the session ID, and the distinct ID of an identified + user, from the posthog-js cookie (`ph__posthog`) when a request has neither tracing header. + The browser sends that cookie on same-site requests without any frontend configuration. Turn it on only when + posthog-js stores opt-out consent in a cookie, or when you do not use opt-out: the server cannot read an + opt-out that posthog-js keeps in localStorage. It needs cookie-backed posthog-js persistence, the default + cookie name (no `persistence_name`), and the default `__ph_opt_in_out_` consent cookie name. Set + `POSTHOG_MW_COOKIE_SESSION_IDLE_TIMEOUT_SECONDS` when posthog-js uses a custom `session_idle_timeout_seconds`, + and set `POSTHOG_MW_COOKIE_OPT_OUT_BY_DEFAULT` to `True` when posthog-js uses `opt_out_capturing_by_default`. + The middleware behaviour is customisable through 3 additional functions: - `POSTHOG_MW_EXTRA_TAGS`, which is a Callable[[HttpRequest], Dict[str, Any]] expected to return a dictionary of additional tags to be added to the context. - `POSTHOG_MW_REQUEST_FILTER`, which is a Callable[[HttpRequest], bool] expected to return `False` if the request should not be tracked. @@ -157,6 +277,31 @@ def __init__(self, get_response): else: self.client = None + if hasattr(settings, "POSTHOG_MW_READ_POSTHOG_COOKIE") and isinstance( + settings.POSTHOG_MW_READ_POSTHOG_COOKIE, bool + ): + self.read_posthog_cookie = settings.POSTHOG_MW_READ_POSTHOG_COOKIE + else: + self.read_posthog_cookie = False + + self.cookie_opt_out_by_default = ( + getattr(settings, "POSTHOG_MW_COOKIE_OPT_OUT_BY_DEFAULT", False) is True + ) + + idle_timeout_seconds = getattr( + settings, "POSTHOG_MW_COOKIE_SESSION_IDLE_TIMEOUT_SECONDS", None + ) + # The same bounds posthog-js applies to session_idle_timeout_seconds. + self.cookie_session_idle_timeout_ms = ( + int(min(max(idle_timeout_seconds, 60), 10 * 60 * 60) * 1000) + if isinstance(idle_timeout_seconds, (int, float)) + and not isinstance(idle_timeout_seconds, bool) + and math.isfinite(idle_timeout_seconds) + # Zero falls back to the default, like posthog-js. + and idle_timeout_seconds != 0 + else _COOKIE_SESSION_IDLE_TIMEOUT_MS + ) + def extract_tags(self, request): # type: (HttpRequest) -> Dict[str, Any] """Extract tags from request in sync context.""" @@ -172,15 +317,33 @@ def _build_tags(self, request, user_id, user_email): """ tags = {} - # Extract session ID from X-POSTHOG-SESSION-ID header - session_id = _get_sanitized_tracing_header(request, "X-POSTHOG-SESSION-ID") + header_session_id = _get_sanitized_tracing_header( + request, "X-POSTHOG-SESSION-ID" + ) + header_distinct_id = _get_sanitized_tracing_header( + request, "X-POSTHOG-DISTINCT-ID" + ) + # The cookie is read only without tracing headers, so one request never mixes two identities. + cookie_distinct_id, cookie_session_id = ( + _read_posthog_cookie( + request, + self.client.api_key if self.client else _default_api_key(), + self.cookie_session_idle_timeout_ms, + self.cookie_opt_out_by_default, + ) + if self.read_posthog_cookie + and header_session_id is None + and header_distinct_id is None + else (None, None) + ) + + # Extract session ID from X-POSTHOG-SESSION-ID header or the posthog-js cookie + session_id = header_session_id or cookie_session_id if session_id: contexts.set_context_session(session_id) - # Extract distinct ID from X-POSTHOG-DISTINCT-ID header or request user id - distinct_id = ( - _get_sanitized_tracing_header(request, "X-POSTHOG-DISTINCT-ID") or user_id - ) + # Extract distinct ID from X-POSTHOG-DISTINCT-ID header, request user id, or the posthog-js cookie + distinct_id = header_distinct_id or user_id or cookie_distinct_id if distinct_id: contexts.identify_context(distinct_id) diff --git a/posthog/test/integrations/test_middleware.py b/posthog/test/integrations/test_middleware.py index c76a8779a..297936e03 100644 --- a/posthog/test/integrations/test_middleware.py +++ b/posthog/test/integrations/test_middleware.py @@ -7,6 +7,8 @@ import unittest from unittest.mock import Mock, patch import asyncio +import json +from urllib.parse import quote from parameterized import parameterized # Configure Django settings before importing middleware @@ -24,6 +26,8 @@ from posthog.integrations.django import PosthogContextMiddleware +MINUTE_MS = 60 * 1000 + class MockRequest: """Mock Django HttpRequest object""" @@ -35,8 +39,10 @@ def __init__( path="/test", host="example.com", is_secure=False, + cookies=None, ): self.headers = headers or {} + self.COOKIES = cookies or {} self.method = method self.path = path self._host = host @@ -55,6 +61,9 @@ def create_middleware( tag_map=None, capture_exceptions=True, get_response=None, + read_posthog_cookie=None, + cookie_session_idle_timeout_seconds=None, + cookie_opt_out_by_default=None, ): """Helper to create middleware instance with mock Django settings""" if get_response is None: @@ -67,6 +76,13 @@ def create_middleware( mock_settings.POSTHOG_MW_TAG_MAP = tag_map mock_settings.POSTHOG_MW_CAPTURE_EXCEPTIONS = capture_exceptions mock_settings.POSTHOG_MW_CLIENT = None + mock_settings.POSTHOG_MW_READ_POSTHOG_COOKIE = read_posthog_cookie + mock_settings.POSTHOG_MW_COOKIE_SESSION_IDLE_TIMEOUT_SECONDS = ( + cookie_session_idle_timeout_seconds + ) + mock_settings.POSTHOG_MW_COOKIE_OPT_OUT_BY_DEFAULT = ( + cookie_opt_out_by_default + ) # Make hasattr work correctly def mock_hasattr(obj, name): @@ -76,7 +92,10 @@ def mock_hasattr(obj, name): "POSTHOG_MW_TAG_MAP", "POSTHOG_MW_CAPTURE_EXCEPTIONS", "POSTHOG_MW_CLIENT", - ] + ] or ( + name == "POSTHOG_MW_READ_POSTHOG_COOKIE" + and read_posthog_cookie is not None + ) with patch("builtins.hasattr", side_effect=mock_hasattr): middleware = PosthogContextMiddleware(get_response) @@ -137,6 +156,199 @@ def test_extract_tags_partial_headers(self): self.assertIsNone(get_context_distinct_id()) self.assertEqual(tags["$request_method"], "PUT") + @parameterized.expand( + [ + ( + "live_session", + {}, + None, + {}, + True, + "session-from-cookie", + "anon-from-cookie", + ), + ( + "headers_win", + { + "X-POSTHOG-SESSION-ID": "session-from-header", + "X-POSTHOG-DISTINCT-ID": "user-from-header", + }, + None, + {}, + True, + "session-from-header", + "user-from-header", + ), + ("authenticated_user_wins", {}, 42, {}, True, "session-from-cookie", "42"), + ( + "idle_session_dropped", + {}, + None, + {"idle_ms": 31 * MINUTE_MS}, + True, + None, + "anon-from-cookie", + ), + ( + "long_session_dropped", + {}, + None, + {"length_ms": 25 * 60 * MINUTE_MS}, + True, + None, + "anon-from-cookie", + ), + ( + "future_activity_dropped", + {}, + None, + {"idle_ms": -40 * MINUTE_MS}, + True, + None, + "anon-from-cookie", + ), + ( + "other_project_cookie_ignored", + {}, + None, + {"cookie_key": "other-token"}, + True, + None, + None, + ), + ("opted_out", {}, None, {"consent": "0"}, True, None, None), + ( + "opted_out_by_default", + {}, + None, + {"opt_out_by_default": True}, + True, + None, + None, + ), + ( + "opted_in_under_opt_out_default", + {}, + None, + {"opt_out_by_default": True, "consent": "1"}, + True, + "session-from-cookie", + "anon-from-cookie", + ), + ("reading_disabled", {}, None, {}, False, None, None), + ("off_by_default", {}, None, {}, None, None, None), + ( + "idle_timeout_over_ten_hours_is_clamped", + {}, + None, + { + "idle_ms": 11 * 60 * MINUTE_MS, + "length_ms": 11 * 60 * MINUTE_MS, + "idle_timeout_seconds": 24 * 60 * 60, + }, + True, + None, + "anon-from-cookie", + ), + ( + "zero_idle_timeout_uses_default", + {}, + None, + {"idle_ms": 20 * MINUTE_MS, "idle_timeout_seconds": 0}, + True, + "session-from-cookie", + "anon-from-cookie", + ), + ( + "longer_idle_timeout_keeps_session", + {}, + None, + {"idle_ms": 45 * MINUTE_MS, "idle_timeout_seconds": 60 * 60}, + True, + "session-from-cookie", + "anon-from-cookie", + ), + ( + "anonymous_visitor_gets_session_only", + {}, + None, + {"user_state": "anonymous"}, + True, + "session-from-cookie", + None, + ), + ( + "older_two_item_session", + {}, + None, + {"two_item_session": True}, + True, + "session-from-cookie", + "anon-from-cookie", + ), + ] + ) + def test_extract_tags_reads_posthog_js_cookie( + self, + _name, + headers, + user_pk, + cookie_options, + read_cookie, + expected_session, + expected_distinct, + ): + now_ms = 1_700_000_000_000 + idle_ms = cookie_options.get("idle_ms", MINUTE_MS) + length_ms = cookie_options.get("length_ms", MINUTE_MS) + cookie_key = cookie_options.get("cookie_key", "test-token") + cookies = { + f"ph_{cookie_key}_posthog": quote( + json.dumps( + { + "distinct_id": "anon-from-cookie", + "$user_state": cookie_options.get("user_state", "identified"), + "$sesid": [now_ms - idle_ms, "session-from-cookie"] + if cookie_options.get("two_item_session") + else [ + now_ms - idle_ms, + "session-from-cookie", + now_ms - length_ms, + ], + } + ) + ) + } + if "consent" in cookie_options: + cookies["__ph_opt_in_out_test-token"] = cookie_options["consent"] + + with ( + new_context(), + patch("time.time", return_value=now_ms / 1000), + patch( + "posthog.integrations.django._default_api_key", + return_value="test-token", + ), + ): + middleware = self.create_middleware( + read_posthog_cookie=read_cookie, + cookie_session_idle_timeout_seconds=cookie_options.get( + "idle_timeout_seconds" + ), + cookie_opt_out_by_default=cookie_options.get("opt_out_by_default"), + ) + request = MockRequest(headers=headers, cookies=cookies) + if user_pk is not None: + user = Mock() + user.is_authenticated = True + user.pk = user_pk + request.user = user + + middleware.extract_tags(request) + + self.assertEqual(get_context_session_id(), expected_session) + self.assertEqual(get_context_distinct_id(), expected_distinct) + @parameterized.expand( [ ( diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index 956efda25..35e9224d3 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -878,8 +878,11 @@ attribute posthog.integrations.celery.PosthogCeleryIntegration.task_filter = tas attribute posthog.integrations.django.PosthogContextMiddleware.async_capable = True attribute posthog.integrations.django.PosthogContextMiddleware.capture_exceptions = settings.POSTHOG_MW_CAPTURE_EXCEPTIONS attribute posthog.integrations.django.PosthogContextMiddleware.client = cast('Optional[Client]', settings.POSTHOG_MW_CLIENT) +attribute posthog.integrations.django.PosthogContextMiddleware.cookie_opt_out_by_default = getattr(settings, 'POSTHOG_MW_COOKIE_OPT_OUT_BY_DEFAULT', False) is True +attribute posthog.integrations.django.PosthogContextMiddleware.cookie_session_idle_timeout_ms = int(min(max(idle_timeout_seconds, 60), 10 * 60 * 60) * 1000) if isinstance(idle_timeout_seconds, (int, float)) and not isinstance(idle_timeout_seconds, bool) and math.isfinite(idle_timeout_seconds) and idle_timeout_seconds != 0 else _COOKIE_SESSION_IDLE_TIMEOUT_MS attribute posthog.integrations.django.PosthogContextMiddleware.extra_tags = cast('Optional[Callable[[HttpRequest], Dict[str, Any]]]', settings.POSTHOG_MW_EXTRA_TAGS) attribute posthog.integrations.django.PosthogContextMiddleware.get_response = get_response +attribute posthog.integrations.django.PosthogContextMiddleware.read_posthog_cookie = settings.POSTHOG_MW_READ_POSTHOG_COOKIE attribute posthog.integrations.django.PosthogContextMiddleware.request_filter = cast('Optional[Callable[[HttpRequest], bool]]', settings.POSTHOG_MW_REQUEST_FILTER) attribute posthog.integrations.django.PosthogContextMiddleware.sync_capable = True attribute posthog.integrations.django.PosthogContextMiddleware.tag_map = cast('Optional[Callable[[Dict[str, Any]], Dict[str, Any]]]', settings.POSTHOG_MW_TAG_MAP)