diff --git a/README.md b/README.md index afb265c..237baf0 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,21 @@ Enterprise GitHub App credentials accept either `client_id` or `app_id` as the JWT issuer. When both are configured, `client_id` is preferred. At least one identifier must be supplied together with `key_path` and `enterprise_name`. +### GitHub Enterprise Server endpoints + +GitHub.com is the default deployment and does not require endpoint configuration. +To collect from GitHub Enterprise Server, set both API endpoints in `config.toml`: + +```toml +[sources.github] +rest_api_url = "https://ghe.example/api/v3" +graphql_url = "https://ghe.example/api/graphql" +``` + +Both values must be provided together when overriding the GitHub.com defaults. +Endpoint URLs must use HTTPS and share the same origin. GitHub App +`credentials.api_uri` may override only the path on that same origin. + ### Enterprise SCIM and hybrid correlations A token with enterprise SCIM access is used to collect both `/scim/v2/enterprises/{enterprise}/Users` and `/scim/v2/enterprises/{enterprise}/Groups`. The collector emits normalized `SCIM_Organization`, `SCIM_User`, and `SCIM_Group` nodes plus `SCIM_Contains`, `SCIM_MemberOf`, and `SCIM_Provisioned` relationships. Install the BloodHound SCIM extension alongside this extension to register the shared SCIM kinds. diff --git a/src/openhound_github/auth.py b/src/openhound_github/auth.py index df51819..2c94a43 100644 --- a/src/openhound_github/auth.py +++ b/src/openhound_github/auth.py @@ -293,6 +293,16 @@ def refresh_request(self, request: requests.PreparedRequest) -> bool: return True def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: + try: + request_origin = _normalized_http_origin( + request.url or "", allow_query=True + ) + except ValueError: + return request + + if request_origin != self._api_origin: + return request + request.headers["Authorization"] = f"Bearer {self.token()}" return request diff --git a/src/openhound_github/helpers.py b/src/openhound_github/helpers.py index e6a4210..73f1f44 100644 --- a/src/openhound_github/helpers.py +++ b/src/openhound_github/helpers.py @@ -1,7 +1,7 @@ import logging import time from collections.abc import Iterator -from typing import Any, Optional +from typing import Any, Optional, overload from urllib.parse import urlparse from dlt.common import jsonpath @@ -17,11 +17,38 @@ logger = logging.getLogger(__name__) +DEFAULT_GITHUB_REST_API_URL = "https://api.github.com" +DEFAULT_GITHUB_GRAPHQL_URL = "https://api.github.com/graphql" + class GraphQLPaginationError(RuntimeError): pass +@overload +def graphql_client_and_path( + rest_client: RESTClient, + graphql_client: RESTClient | None, +) -> tuple[RESTClient, str]: ... + + +@overload +def graphql_client_and_path( + rest_client: RESTClient | None, + graphql_client: RESTClient | None, +) -> tuple[RESTClient | None, str]: ... + + +def graphql_client_and_path( + rest_client: RESTClient | None, + graphql_client: RESTClient | None, +) -> tuple[RESTClient | None, str]: + """Return the configured GraphQL client and its request path.""" + if graphql_client: + return graphql_client, "" + return rest_client, "/graphql" + + class AdaptiveGraphQLPageError(RuntimeError): """A terminal GraphQL page failure with adaptive pagination context.""" @@ -184,6 +211,7 @@ def _graphql_gateway_status(exception: BaseException) -> int | None: def adaptive_graphql_paginate( client: RESTClient, *, + graphql_path: str = "/graphql", query: str, variables: dict[str, Any], page_info_path: str, @@ -247,7 +275,7 @@ def adaptive_graphql_paginate( } try: response = client.post( - "/graphql", + graphql_path, json={"query": query, "variables": request_variables}, ) response.raise_for_status() diff --git a/src/openhound_github/resources/enterprise.py b/src/openhound_github/resources/enterprise.py index 389ca47..3eadfb6 100644 --- a/src/openhound_github/resources/enterprise.py +++ b/src/openhound_github/resources/enterprise.py @@ -11,7 +11,11 @@ ENTERPRISE_QUERY, ENTERPRISE_SAML_QUERY, ) -from openhound_github.helpers import GraphQLCursorPaginator, scim_skip_reason +from openhound_github.helpers import ( + GraphQLCursorPaginator, + graphql_client_and_path, + scim_skip_reason, +) from openhound_github.main import app from openhound_github.models import ( BaseUser, @@ -53,7 +57,9 @@ class SourceContext: """Shared context for GitHub API access.""" client: RESTClient + graphql_client: RESTClient | None = None sso_client: RESTClient | None = None + sso_graphql_client: RESTClient | None = None org_name: str | None = None enterprise_name: str | None = None emit_legacy_scim_correlations: bool = False @@ -61,6 +67,14 @@ class SourceContext: github_web_origin: str = DEFAULT_GITHUB_WEB_ORIGIN +def _graphql_client(ctx: SourceContext) -> tuple[RESTClient, str]: + return graphql_client_and_path(ctx.client, ctx.graphql_client) + + +def _sso_graphql_client(ctx: SourceContext) -> tuple[RESTClient | None, str]: + return graphql_client_and_path(ctx.sso_client, ctx.sso_graphql_client) + + def iter_enterprise_scim_resources( client: RESTClient, enterprise_slug: str, @@ -111,8 +125,9 @@ def enterprise(ctx: SourceContext): "variables": {"slug": ctx.enterprise_name, "after": None}, } + client, graphql_path = _graphql_client(ctx) try: - response = ctx.client.post("/graphql", json=data).json() + response = client.post(graphql_path, json=data).json() page_enterprise = (response.get("data") or {}).get("enterprise") if page_enterprise: yield page_enterprise @@ -139,9 +154,10 @@ def enterprise_organizations(enterprise_data: Enterprise, ctx: SourceContext): "variables": {"slug": ctx.enterprise_name, "after": None}, } + client, graphql_path = _graphql_client(ctx) try: - for page_data in ctx.client.paginate( - "/graphql", + for page_data in client.paginate( + graphql_path, method="POST", json=data, paginator=paginator, @@ -263,8 +279,9 @@ def enterprise_members(enterprise_data: Enterprise, ctx: SourceContext): "query": ENTERPRISE_MEMBERS_QUERY, "variables": {"slug": ctx.enterprise_name, "count": 100, "after": None}, } - for page_data in ctx.client.paginate( - "/graphql", + client, graphql_path = _graphql_client(ctx) + for page_data in client.paginate( + graphql_path, method="POST", json=data, paginator=paginator, @@ -635,8 +652,9 @@ def enterprise_admins(enterprise_data: Enterprise, ctx: SourceContext): "query": ENTERPRISE_ADMINS_QUERY, "variables": {"slug": ctx.enterprise_name, "count": 100, "after": None}, } - for page_data in ctx.client.paginate( - "/graphql", + client, graphql_path = _graphql_client(ctx) + for page_data in client.paginate( + graphql_path, method="POST", json=data, paginator=paginator, @@ -665,7 +683,7 @@ def enterprise_admins(enterprise_data: Enterprise, ctx: SourceContext): parallelized=True ) def enterprise_saml_provider(enterprise_data: Enterprise, ctx: SourceContext): - client = ctx.sso_client + client, graphql_path = _sso_graphql_client(ctx) if not client: logger.info( "Skipping enterprise_saml_provider for enterprise '%s': no SSO client configured", @@ -679,7 +697,7 @@ def enterprise_saml_provider(enterprise_data: Enterprise, ctx: SourceContext): } try: - response = client.post("/graphql", json=data).json() + response = client.post(graphql_path, json=data).json() except Exception as e: logger.error( f"Error in resource 'enterprise_saml_provider' processing enterprise '{ctx.enterprise_name}': {e}", @@ -779,7 +797,7 @@ def enterprise_saml_issuer(saml_provider: SamlProvider, ctx: SourceContext): def enterprise_external_identity( saml_provider: SamlProvider, ctx: SourceContext ): - client = ctx.sso_client + client, graphql_path = _sso_graphql_client(ctx) if not client: logger.info( "Skipping enterprise_external_identity for enterprise '%s': no SSO client configured", @@ -801,7 +819,7 @@ def enterprise_external_identity( try: for page_data in client.paginate( - "/graphql", + graphql_path, method="POST", json=data, paginator=paginator, diff --git a/src/openhound_github/resources/organization.py b/src/openhound_github/resources/organization.py index e95e736..d54f17c 100644 --- a/src/openhound_github/resources/organization.py +++ b/src/openhound_github/resources/organization.py @@ -24,8 +24,10 @@ ) from openhound_github.helpers import ( AdaptiveGraphQLPageError, + DEFAULT_GITHUB_REST_API_URL, GraphQLCursorPaginator, adaptive_graphql_paginate, + graphql_client_and_path, scim_skip_reason, ) from openhound_github.main import app @@ -88,11 +90,11 @@ logger = logging.getLogger(__name__) - @dataclass class OrgContext: client: RESTClient org_name: str + graphql_client: RESTClient | None = None enterprise_name: str | None = None github_deployment_id: str = DEFAULT_GITHUB_DEPLOYMENT_ID github_web_origin: str = DEFAULT_GITHUB_WEB_ORIGIN @@ -102,6 +104,7 @@ class OrgContext: class SourceContext: client: RESTClient organizations: list[OrgContext] = field(default_factory=list) + graphql_client: RESTClient | None = None enterprise_name: str | None = None github_deployment_id: str = DEFAULT_GITHUB_DEPLOYMENT_ID github_web_origin: str = DEFAULT_GITHUB_WEB_ORIGIN @@ -138,6 +141,20 @@ def _client_for_org(ctx: SourceContext, org_login: str) -> RESTClient: return ctx.client +def _graphql_client_for_org( + ctx: SourceContext, org_login: str +) -> tuple[RESTClient, str]: + for org in ctx.organizations: + if org.org_name == org_login: + return graphql_client_and_path(org.client, org.graphql_client) + return graphql_client_and_path(ctx.client, ctx.graphql_client) + + +def _rest_api_url(client: RESTClient) -> str: + base_url = getattr(client, "base_url", DEFAULT_GITHUB_REST_API_URL) + return str(base_url).strip().rstrip("/") or DEFAULT_GITHUB_REST_API_URL + + def _encode_path_segment(value: str) -> str: return quote(value, safe="") @@ -464,7 +481,7 @@ def users(ctx: SourceContext) -> Iterator[dict[str, Any]]: for org in ctx.organizations: org_name = org.org_name - client = org.client + client, graphql_path = _graphql_client_for_org(ctx, org_name) try: paginator = GraphQLCursorPaginator( page_info_path="data.organization.membersWithRole.pageInfo", @@ -478,7 +495,7 @@ def users(ctx: SourceContext) -> Iterator[dict[str, Any]]: } for page_data in client.paginate( - "/graphql", + graphql_path, method="POST", json=data, paginator=paginator, @@ -515,7 +532,7 @@ def teams(ctx: SourceContext): for org in ctx.organizations: org_name = org.org_name - client = org.client + client, graphql_path = _graphql_client_for_org(ctx, org_name) try: paginator = GraphQLCursorPaginator( page_info_path="data.organization.teams.pageInfo", @@ -528,7 +545,7 @@ def teams(ctx: SourceContext): "variables": {"login": org_name, "count": 100, "after": None}, } for page_data in client.paginate( - "/graphql", + graphql_path, method="POST", json=data, paginator=paginator, @@ -626,7 +643,7 @@ def team_members(team: Team, ctx: SourceContext): raise RuntimeError( f"GitHub team {team.org_login}/{team.slug} has more members but no endCursor" ) - client = _client_for_org(ctx, team.org_login) + client, graphql_path = _graphql_client_for_org(ctx, team.org_login) data = { "query": TEAM_MEMBERS_OVERFLOW_QUERY, "variables": { @@ -637,7 +654,7 @@ def team_members(team: Team, ctx: SourceContext): }, } for page_data in client.paginate( - "/graphql", + graphql_path, method="POST", json=data, paginator=paginator, @@ -927,12 +944,13 @@ def repositories_graphql(ctx: SourceContext): """ for org in ctx.organizations: org_name = org.org_name - client = org.client + client, graphql_path = _graphql_client_for_org(ctx, org_name) repository_cursor: str | None = None emitted_repositories = 0 try: for page_data in adaptive_graphql_paginate( client, + graphql_path=graphql_path, query=REPO_REFS_QUERY, variables={"login": org_name, "after": None}, page_info_path="data.organization.repositories.pageInfo", @@ -1014,7 +1032,7 @@ def branches(repository: RepositoryQL, ctx: SourceContext): } if repository.refs.page_info.has_next_page: - client = _client_for_org(ctx, repository.org_login) + client, graphql_path = _graphql_client_for_org(ctx, repository.org_login) data = { "query": REF_OVERFLOW_QUERY, "variables": { @@ -1026,7 +1044,7 @@ def branches(repository: RepositoryQL, ctx: SourceContext): } for page_data in client.paginate( - "/graphql", + graphql_path, method="POST", json=data, paginator=paginator, @@ -1068,12 +1086,12 @@ def branch_protection_rules(repository: RepositoryQL, ctx: SourceContext): rule_ids_seen.add(rule_id) rule_ids_list = list(rule_ids_seen) - client = _client_for_org(ctx, repository.org_login) + client, graphql_path = _graphql_client_for_org(ctx, repository.org_login) for i in range(0, len(rule_ids_list), 100): rules_chunk = rule_ids_list[i : i + 100] if rules_chunk: data = {"query": PROTECTION_RULES_QUERY, "variables": {"ids": rules_chunk}} - response = client.post("/graphql", json=data).json() + response = client.post(graphql_path, json=data).json() for rule in response["data"].get("nodes", []): # GitHub can return null actors for deleted or inaccessible allowance actors. for allowance_key in ("bypassPullRequestAllowances", "pushAllowances"): @@ -1657,7 +1675,7 @@ def secret_scanning_alerts(ctx: SourceContext): ): try: resp = requests.get( - "https://api.github.com/user", + f"{_rest_api_url(client)}/user", headers={"Authorization": f"Bearer {secret}"}, timeout=10, ) @@ -1793,14 +1811,14 @@ def saml_provider(ctx: SourceContext): """ for org in ctx.organizations: org_name = org.org_name - client = org.client + client, graphql_path = _graphql_client_for_org(ctx, org_name) try: data = { "query": SAML_QUERY, "variables": {"login": org_name, "count": 100, "after": None}, } - response = client.post("/graphql", json=data).json() + response = client.post(graphql_path, json=data).json() response_data = response.get("data", {}) org_data = response_data.get("organization", {}) if response_data and org_data: @@ -1837,7 +1855,7 @@ def external_identities(ctx: SourceContext): """ for org in ctx.organizations: org_name = org.org_name - client = org.client + client, graphql_path = _graphql_client_for_org(ctx, org_name) github_deployment_id = org.github_deployment_id try: paginator = GraphQLCursorPaginator( @@ -1853,7 +1871,7 @@ def external_identities(ctx: SourceContext): } for page_data in client.paginate( - "/graphql", + graphql_path, method="POST", json=data, paginator=paginator, diff --git a/src/openhound_github/source.py b/src/openhound_github/source.py index c64d693..989e6d0 100644 --- a/src/openhound_github/source.py +++ b/src/openhound_github/source.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from threading import Lock from typing import Any, Union +from urllib.parse import urlparse import dlt from dlt.common.configuration import configspec @@ -20,7 +21,11 @@ GithubInstallation, resolve_github_app_jwt_issuer, ) -from openhound_github.helpers import github_retry_policy +from openhound_github.helpers import ( + DEFAULT_GITHUB_GRAPHQL_URL, + DEFAULT_GITHUB_REST_API_URL, + github_retry_policy, +) from openhound_github.main import app from openhound_github.models.saml_helpers import ( DEFAULT_GITHUB_DEPLOYMENT_ID, @@ -33,11 +38,105 @@ logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class GithubEndpoints: + rest_api_url: str + graphql_url: str + + +def _normalize_endpoint_url(url: str, setting_name: str) -> str: + normalized_url = url.strip().rstrip("/") + parsed = urlparse(normalized_url) + if ( + not normalized_url + or parsed.scheme != "https" + or not parsed.hostname + ): + raise ValueError(f"{setting_name} must be an absolute HTTPS URL") + if ( + parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise ValueError( + f"{setting_name} must not contain user-info, query strings, or fragments" + ) + return normalized_url + + +def _endpoint_origin(url: str) -> tuple[str, str, int | None]: + parsed = urlparse(url) + port = parsed.port + if (parsed.scheme.lower(), port) in {("http", 80), ("https", 443)}: + port = None + return parsed.scheme.lower(), parsed.hostname.lower(), port + + +def _resolve_app_auth_api_uri( + credentials_api_uri: str | None, + rest_api_url: str, +) -> str: + if credentials_api_uri is None: + return rest_api_url + + auth_api_uri = _normalize_endpoint_url(credentials_api_uri, "credentials.api_uri") + if _endpoint_origin(auth_api_uri) != _endpoint_origin(rest_api_url): + raise ValueError( + "credentials.api_uri origin must match rest_api_url origin for GitHub App authentication" + ) + return auth_api_uri + + +def _legacy_graphql_url(rest_api_url: str) -> str: + if rest_api_url.endswith("/api/v3"): + return f"{rest_api_url.removesuffix('/v3')}/graphql" + return f"{rest_api_url}/graphql" + + +def resolve_github_endpoints( + *, + host: str = DEFAULT_GITHUB_REST_API_URL, + rest_api_url: str | None = None, + graphql_url: str | None = None, +) -> GithubEndpoints: + """Resolve GitHub REST and GraphQL endpoints from new and legacy settings.""" + if (rest_api_url is None) != (graphql_url is None): + raise ValueError( + "Both rest_api_url and graphql_url must be set when overriding GitHub endpoints" + ) + + if rest_api_url is not None and graphql_url is not None: + resolved_endpoints = GithubEndpoints( + rest_api_url=_normalize_endpoint_url(rest_api_url, "rest_api_url"), + graphql_url=_normalize_endpoint_url(graphql_url, "graphql_url"), + ) + if _endpoint_origin(resolved_endpoints.rest_api_url) != _endpoint_origin( + resolved_endpoints.graphql_url + ): + raise ValueError( + "rest_api_url origin must match graphql_url origin" + ) + return resolved_endpoints + + legacy_rest_api_url = _normalize_endpoint_url(host, "host") + if legacy_rest_api_url == DEFAULT_GITHUB_REST_API_URL: + return GithubEndpoints( + rest_api_url=DEFAULT_GITHUB_REST_API_URL, + graphql_url=DEFAULT_GITHUB_GRAPHQL_URL, + ) + + return GithubEndpoints( + rest_api_url=legacy_rest_api_url, + graphql_url=_legacy_graphql_url(legacy_rest_api_url), + ) + @dataclass class OrgContext: client: RESTClient org_name: str + graphql_client: RESTClient | None = None enterprise_name: str | None = None github_deployment_id: str = DEFAULT_GITHUB_DEPLOYMENT_ID github_web_origin: str = DEFAULT_GITHUB_WEB_ORIGIN @@ -47,7 +146,9 @@ class OrgContext: class SourceContext: organizations: list[OrgContext] | None = field(default_factory=list) client: RESTClient | None = None + graphql_client: RESTClient | None = None sso_client: RESTClient | None = None + sso_graphql_client: RESTClient | None = None enterprise_name: str | None = None emit_legacy_scim_correlations: bool = False github_deployment_id: str = DEFAULT_GITHUB_DEPLOYMENT_ID @@ -105,7 +206,7 @@ class GithubEnterpriseAppCredentials(CredentialsConfiguration): key_path: str = None enterprise_name: str = None pat_token: str | None = None - api_uri: str = "https://api.github.com" + api_uri: str | None = None @property def auth(self) -> str: @@ -118,7 +219,7 @@ class GithubOrgAppCredentials(CredentialsConfiguration): install_id: str = None key_path: str = None org_name: str = None - api_uri: str = "https://api.github.com" + api_uri: str | None = None @property def auth(self) -> str: @@ -143,20 +244,32 @@ def source( credentials: Union[ GithubEnterpriseAppCredentials, GithubOrgAppCredentials, GithubTokenCredentials ] = dlt.secrets.value, - host: str = "https://api.github.com", + host: str = DEFAULT_GITHUB_REST_API_URL, emit_legacy_scim_correlations: bool | None = dlt.config.value, + rest_api_url: str | None = None, + graphql_url: str | None = None, ): """DLT source, defines GitHub collection resources and transformers. Args: credentials (Union[GithubEnterpriseAppCredentials, GithubOrgAppCredentials, GithubTokenCredentials]): The GitHub credentials. - host (str): The base GitHub API URL used for API calls. + host (str): Legacy base GitHub REST API URL used for API calls. + emit_legacy_scim_correlations (bool | None): Whether to emit legacy SCIM correlation relationships. + rest_api_url (str): The GitHub REST API base URL. + graphql_url (str): The GitHub GraphQL endpoint URL. """ - github_deployment_id, github_web_origin = github_deployment_context(host) - - def client(auth: AuthConfigBase) -> RESTClient: + endpoints = resolve_github_endpoints( + host=host, + rest_api_url=rest_api_url, + graphql_url=graphql_url, + ) + github_deployment_id, github_web_origin = github_deployment_context( + endpoints.rest_api_url + ) + + def api_client(base_url: str, auth: AuthConfigBase) -> RESTClient: return RESTClient( - base_url=host, + base_url=base_url, headers={ "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", @@ -169,10 +282,20 @@ def client(auth: AuthConfigBase) -> RESTClient: ).session, ) - def token_client(token: str) -> RESTClient: - return client(BearerTokenAuth(token=token)) + def clients(auth: AuthConfigBase) -> tuple[RESTClient, RESTClient]: + return ( + api_client(endpoints.rest_api_url, auth), + api_client(endpoints.graphql_url, auth), + ) + + def token_clients(token: str) -> tuple[RESTClient, RESTClient]: + return clients(BearerTokenAuth(token=token)) if credentials.auth == "enterprise_app": + auth_api_uri = _resolve_app_auth_api_uri( + credentials.api_uri, + endpoints.rest_api_url, + ) jwt_issuer = resolve_github_app_jwt_issuer( client_id=credentials.client_id, app_id=credentials.app_id, @@ -184,11 +307,13 @@ def token_client(token: str) -> RESTClient: github_web_origin=github_web_origin, ) if credentials.pat_token: - ctx.sso_client = token_client(credentials.pat_token) + ctx.sso_client, ctx.sso_graphql_client = token_clients( + credentials.pat_token + ) github_app_session = GithubApp( jwt_issuer=jwt_issuer, private_key_path=credentials.key_path, - api_uri=host, + api_uri=auth_api_uri, ) for installation in github_app_session.installations: if installation.target_type == "Organization": @@ -196,17 +321,19 @@ def token_client(token: str) -> RESTClient: installation_id=installation.id, jwt_issuer=jwt_issuer, private_key_path=credentials.key_path, - api_uri=host, + api_uri=auth_api_uri, + ) + org_client, org_graphql_client = clients( + GitHubAppInstallationAuth( + installation=org_installation, + api_uri=auth_api_uri, + ) ) ctx.organizations.append( OrgContext( org_name=installation.account.login, - client=client( - GitHubAppInstallationAuth( - installation=org_installation, - api_uri=host, - ) - ), + client=org_client, + graphql_client=org_graphql_client, enterprise_name=credentials.enterprise_name, github_deployment_id=github_deployment_id, github_web_origin=github_web_origin, @@ -217,18 +344,22 @@ def token_client(token: str) -> RESTClient: installation_id=installation.id, jwt_issuer=jwt_issuer, private_key_path=credentials.key_path, - api_uri=host, + api_uri=auth_api_uri, ) - ctx.client = client( + ctx.client, ctx.graphql_client = clients( GitHubAppInstallationAuth( installation=es_installation, - api_uri=host, + api_uri=auth_api_uri, ) ) return (*enterprise_resources(ctx), *organization_resources(ctx)) elif credentials.auth == "org_app": + auth_api_uri = _resolve_app_auth_api_uri( + credentials.api_uri, + endpoints.rest_api_url, + ) ctx = SourceContext( enterprise_name=None, github_deployment_id=github_deployment_id, @@ -238,17 +369,19 @@ def token_client(token: str) -> RESTClient: installation_id=credentials.install_id, jwt_issuer=credentials.client_id, private_key_path=credentials.key_path, - api_uri=host, + api_uri=auth_api_uri, + ) + org_client, org_graphql_client = clients( + GitHubAppInstallationAuth( + installation=org_installation, + api_uri=auth_api_uri, + ) ) ctx.organizations.append( OrgContext( org_name=credentials.org_name, - client=client( - GitHubAppInstallationAuth( - installation=org_installation, - api_uri=host, - ) - ), + client=org_client, + graphql_client=org_graphql_client, github_deployment_id=github_deployment_id, github_web_origin=github_web_origin, ) @@ -259,10 +392,12 @@ def token_client(token: str) -> RESTClient: else: if credentials.enterprise_name: - token_api_client = token_client(credentials.token) + token_api_client, token_graphql_client = token_clients(credentials.token) ctx = SourceContext( client=token_api_client, + graphql_client=token_graphql_client, sso_client=token_api_client, + sso_graphql_client=token_graphql_client, enterprise_name=credentials.enterprise_name, emit_legacy_scim_correlations=bool(emit_legacy_scim_correlations), github_deployment_id=github_deployment_id, @@ -270,6 +405,7 @@ def token_client(token: str) -> RESTClient: ) return enterprise_resources(ctx) + token_api_client, token_graphql_client = token_clients(credentials.token) ctx = SourceContext( github_deployment_id=github_deployment_id, github_web_origin=github_web_origin, @@ -277,7 +413,8 @@ def token_client(token: str) -> RESTClient: ctx.organizations.append( OrgContext( org_name=credentials.org_name, - client=token_client(credentials.token), + client=token_api_client, + graphql_client=token_graphql_client, github_deployment_id=github_deployment_id, github_web_origin=github_web_origin, ) diff --git a/tests/test_github_app_retry.py b/tests/test_github_app_retry.py index a9e6dca..5b9b1fd 100644 --- a/tests/test_github_app_retry.py +++ b/tests/test_github_app_retry.py @@ -156,6 +156,18 @@ def test_refresh_request_does_not_restore_authorization_on_cross_origin_request( assert installation.token_calls == 0 +def test_auth_does_not_attach_token_to_cross_origin_request() -> None: + installation = FakeInstallation("new-token") + auth = GitHubAppInstallationAuth(installation=installation) + request = prepared_request(None, url="https://attacker.example/redirected") + + authenticated_request = auth(request) + + assert authenticated_request is request + assert "Authorization" not in request.headers + assert installation.token_calls == 0 + + def test_retry_policy_repairs_bad_credentials_for_github_app_auth() -> None: installation = FakeInstallation("new-token") auth = GitHubAppInstallationAuth(installation=installation) diff --git a/tests/test_repository_rulesets.py b/tests/test_repository_rulesets.py index d7be6d7..05bb4c1 100644 --- a/tests/test_repository_rulesets.py +++ b/tests/test_repository_rulesets.py @@ -19,9 +19,11 @@ def __init__(self, *responses: requests.Response | BaseException) -> None: self.responses = list(responses) or [ _graphql_response(_repository_page_data("R_1", "repo", branch_ruleset_count=2)) ] + self.request_paths: list[str] = [] self.request_variables: list[dict[str, object]] = [] - def post(self, _path: str, *, json: dict[str, object]): + def post(self, path: str, *, json: dict[str, object]): + self.request_paths.append(path) variables = json["variables"] assert isinstance(variables, dict) self.request_variables.append({**variables}) @@ -143,6 +145,27 @@ def test_repositories_graphql_flattens_branch_ruleset_count() -> None: ] +def test_repositories_graphql_uses_dedicated_graphql_client_path() -> None: + rest_client = _FakeClient() + graphql_client = _FakeClient() + ctx = SourceContext( + client=rest_client, + organizations=[ + OrgContext( + client=rest_client, + graphql_client=graphql_client, + org_name="org", + ) + ], + ) + + rows = list(repositories_graphql.__wrapped__(ctx)) + + assert [row["id"] for row in rows] == ["R_1"] + assert graphql_client.request_paths == [""] + assert rest_client.request_paths == [] + + def test_repositories_graphql_logs_cursor_and_emitted_count_on_page_failure( caplog, ) -> None: diff --git a/tests/test_source_endpoints.py b/tests/test_source_endpoints.py new file mode 100644 index 0000000..58dea2e --- /dev/null +++ b/tests/test_source_endpoints.py @@ -0,0 +1,474 @@ +import importlib +import inspect +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from openhound_github.resources.enterprise import ( + SourceContext as EnterpriseResourceContext, + enterprise, +) +from openhound_github.resources.organization import ( + OrgContext as OrganizationOrgContext, + SourceContext as OrganizationResourceContext, + secret_scanning_alerts, + users, +) +from openhound_github.source import ( + DEFAULT_GITHUB_GRAPHQL_URL, + DEFAULT_GITHUB_REST_API_URL, + GithubEndpoints, + GithubEnterpriseAppCredentials, + GithubOrgAppCredentials, + GithubTokenCredentials, + resolve_github_endpoints, +) + + +def test_resolve_github_endpoints_uses_dotcom_defaults() -> None: + assert resolve_github_endpoints() == GithubEndpoints( + rest_api_url=DEFAULT_GITHUB_REST_API_URL, + graphql_url=DEFAULT_GITHUB_GRAPHQL_URL, + ) + + +def test_resolve_github_endpoints_uses_explicit_endpoint_pair() -> None: + assert resolve_github_endpoints( + rest_api_url="https://ghe.example/api/v3/", + graphql_url="https://ghe.example/api/graphql/", + ) == GithubEndpoints( + rest_api_url="https://ghe.example/api/v3", + graphql_url="https://ghe.example/api/graphql", + ) + + +@pytest.mark.parametrize( + ("rest_api_url", "graphql_url"), + ( + ("https://ghe.example/api/v3", None), + (None, "https://ghe.example/api/graphql"), + ), +) +def test_resolve_github_endpoints_requires_explicit_pair( + rest_api_url: str | None, + graphql_url: str | None, +) -> None: + with pytest.raises( + ValueError, + match="Both rest_api_url and graphql_url must be set", + ): + resolve_github_endpoints( + rest_api_url=rest_api_url, + graphql_url=graphql_url, + ) + + +@pytest.mark.parametrize( + "graphql_url", + ( + "https://graphql.ghe.example/api/graphql", + "https://ghe.example:8443/api/graphql", + ), +) +def test_resolve_github_endpoints_requires_matching_origins(graphql_url: str) -> None: + with pytest.raises( + ValueError, + match=r"rest_api_url origin must match graphql_url origin", + ): + resolve_github_endpoints( + rest_api_url="https://ghe.example/api/v3", + graphql_url=graphql_url, + ) + + +def test_resolve_github_endpoints_preserves_legacy_host_behavior() -> None: + assert resolve_github_endpoints(host="https://ghe.example/api/v3/") == GithubEndpoints( + rest_api_url="https://ghe.example/api/v3", + graphql_url="https://ghe.example/api/graphql", + ) + + +@pytest.mark.parametrize( + ("setting_name", "kwargs"), + ( + ("host", {"host": "ghe.example/api/v3"}), + ("host", {"host": "http://ghe.example/api/v3"}), + ( + "rest_api_url", + { + "rest_api_url": "https://ghe.example/api/v3?token=secret", + "graphql_url": "https://ghe.example/api/graphql", + }, + ), + ( + "rest_api_url", + { + "rest_api_url": "http://ghe.example/api/v3", + "graphql_url": "https://ghe.example/api/graphql", + }, + ), + ( + "graphql_url", + { + "rest_api_url": "https://ghe.example/api/v3", + "graphql_url": "https://ghe.example/api/graphql#fragment", + }, + ), + ( + "graphql_url", + { + "rest_api_url": "https://ghe.example/api/v3", + "graphql_url": "http://ghe.example/api/graphql", + }, + ), + ), +) +def test_resolve_github_endpoints_rejects_invalid_urls( + setting_name: str, + kwargs: dict[str, str], +) -> None: + with pytest.raises(ValueError, match=setting_name): + resolve_github_endpoints(**kwargs) + + +@pytest.mark.parametrize( + "credentials", + ( + GithubOrgAppCredentials( + client_id="Iv1.example", + install_id="123", + key_path="unused-github-app.pem", + org_name="acme", + api_uri="https://api.github.com", + ), + GithubEnterpriseAppCredentials( + app_id="123456", + key_path="unused-github-app.pem", + enterprise_name="acme-enterprise", + api_uri="https://api.github.com", + ), + ), +) +def test_app_source_rejects_auth_api_uri_on_different_rest_origin(credentials) -> None: + with pytest.raises( + ValueError, + match=r"credentials\.api_uri origin must match rest_api_url origin", + ): + source_module = importlib.import_module("openhound_github.source") + source_module.source.__wrapped__( + credentials=credentials, + emit_legacy_scim_correlations=False, + rest_api_url="https://ghe.example/api/v3", + graphql_url="https://ghe.example/api/graphql", + ) + + +def test_org_source_context_carries_rest_and_graphql_clients( + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_module = importlib.import_module("openhound_github.source") + captured_ctx = None + + class FakeRESTClient: + def __init__(self, **kwargs) -> None: + self.base_url = kwargs["base_url"] + + def capture_context(ctx): + nonlocal captured_ctx + captured_ctx = ctx + return () + + monkeypatch.setattr(source_module, "RESTClient", FakeRESTClient) + monkeypatch.setattr(source_module, "organization_resources", capture_context) + + resources = source_module.source.__wrapped__( + credentials=GithubTokenCredentials(token="token", org_name="acme"), + emit_legacy_scim_correlations=False, + rest_api_url="https://ghe.example/api/v3", + graphql_url="https://ghe.example/api/graphql", + ) + + assert resources == () + assert captured_ctx is not None + assert captured_ctx.organizations[0].client.base_url == "https://ghe.example/api/v3" + assert ( + captured_ctx.organizations[0].graphql_client.base_url + == "https://ghe.example/api/graphql" + ) + + +def test_enterprise_source_context_carries_rest_and_graphql_clients( + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_module = importlib.import_module("openhound_github.source") + captured_ctx = None + + class FakeRESTClient: + def __init__(self, **kwargs) -> None: + self.base_url = kwargs["base_url"] + + def capture_context(ctx): + nonlocal captured_ctx + captured_ctx = ctx + return () + + monkeypatch.setattr(source_module, "RESTClient", FakeRESTClient) + monkeypatch.setattr(source_module, "enterprise_resources", capture_context) + + resources = source_module.source.__wrapped__( + credentials=GithubTokenCredentials( + token="token", + enterprise_name="acme-enterprise", + ), + emit_legacy_scim_correlations=False, + rest_api_url="https://ghe.example/api/v3", + graphql_url="https://ghe.example/api/graphql", + ) + + assert resources == () + assert captured_ctx is not None + assert captured_ctx.client.base_url == "https://ghe.example/api/v3" + assert captured_ctx.graphql_client.base_url == "https://ghe.example/api/graphql" + assert captured_ctx.sso_client.base_url == "https://ghe.example/api/v3" + assert captured_ctx.sso_graphql_client.base_url == "https://ghe.example/api/graphql" + + +def test_org_graphql_resource_uses_dedicated_graphql_client() -> None: + rest_client = MagicMock() + graphql_client = MagicMock() + graphql_client.paginate.return_value = [ + [{"organization": {"membersWithRole": {"edges": []}}}] + ] + ctx = OrganizationResourceContext( + client=rest_client, + organizations=[ + OrganizationOrgContext( + client=rest_client, + graphql_client=graphql_client, + org_name="acme", + ) + ], + ) + + rows = list(inspect.unwrap(users._pipe.gen)(ctx)) + + assert rows == [] + graphql_client.paginate.assert_called_once() + assert graphql_client.paginate.call_args.args[0] == "" + rest_client.paginate.assert_not_called() + + +def test_enterprise_graphql_resource_uses_dedicated_graphql_client() -> None: + rest_client = MagicMock() + graphql_client = MagicMock() + graphql_client.post.return_value.json.return_value = { + "data": {"enterprise": {"id": "E_1", "slug": "acme"}} + } + ctx = EnterpriseResourceContext( + client=rest_client, + graphql_client=graphql_client, + enterprise_name="acme", + ) + + rows = list(inspect.unwrap(enterprise._pipe.gen)(ctx)) + + assert len(rows) == 1 + graphql_client.post.assert_called_once() + assert graphql_client.post.call_args.args[0] == "" + rest_client.post.assert_not_called() + + +def test_secret_scanning_pat_validation_uses_configured_rest_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + organization_module = importlib.import_module("openhound_github.resources.organization") + rest_client = MagicMock() + rest_client.base_url = "https://ghe.example/api/v3/" + rest_client.paginate.return_value = [ + [ + { + "state": "open", + "secret_type": "github_personal_access_token", + "secret": "ghp_example", + } + ] + ] + response = MagicMock() + response.status_code = 200 + response.json.return_value = {"node_id": "U_1"} + get = MagicMock(return_value=response) + monkeypatch.setattr(organization_module.requests, "get", get) + ctx = OrganizationResourceContext( + client=rest_client, + organizations=[OrganizationOrgContext(client=rest_client, org_name="acme")], + ) + + rows = list(secret_scanning_alerts.__wrapped__(ctx)) + + assert rows[0]["valid_token_user_node_id"] == "U_1" + get.assert_called_once_with( + "https://ghe.example/api/v3/user", + headers={"Authorization": "Bearer ghp_example"}, + timeout=10, + ) + + +@pytest.mark.parametrize( + ("credential_api_uri", "expected_auth_api_uri"), + ( + (None, "https://ghe.example/api/v3"), + ("https://ghe.example/custom/api/v3", "https://ghe.example/custom/api/v3"), + ), +) +def test_org_app_source_uses_selected_auth_endpoint_and_both_api_clients( + monkeypatch: pytest.MonkeyPatch, + credential_api_uri: str | None, + expected_auth_api_uri: str, +) -> None: + source_module = importlib.import_module("openhound_github.source") + captured_api_uris: list[str] = [] + captured_ctx = None + + class FakeGithubInstallation: + def __init__( + self, + installation_id: str, + jwt_issuer: str, + private_key_path: str, + api_uri: str, + ) -> None: + captured_api_uris.append(api_uri) + + class FakeRESTClient: + def __init__(self, **kwargs) -> None: + self.base_url = kwargs["base_url"] + + def fake_auth(*, installation, api_uri: str): + captured_api_uris.append(api_uri) + return object() + + def capture_context(ctx): + nonlocal captured_ctx + captured_ctx = ctx + return () + + monkeypatch.setattr(source_module, "GithubInstallation", FakeGithubInstallation) + monkeypatch.setattr(source_module, "GitHubAppInstallationAuth", fake_auth) + monkeypatch.setattr(source_module, "RESTClient", FakeRESTClient) + monkeypatch.setattr(source_module, "organization_resources", capture_context) + + resources = source_module.source.__wrapped__( + credentials=GithubOrgAppCredentials( + client_id="Iv1.example", + install_id="123", + key_path="/tmp/github-app.pem", + org_name="acme", + api_uri=credential_api_uri, + ), + emit_legacy_scim_correlations=False, + rest_api_url="https://ghe.example/api/v3", + graphql_url="https://ghe.example/api/graphql", + ) + + assert resources == () + assert captured_api_uris == [ + expected_auth_api_uri, + expected_auth_api_uri, + ] + assert captured_ctx.organizations[0].client.base_url == "https://ghe.example/api/v3" + assert ( + captured_ctx.organizations[0].graphql_client.base_url + == "https://ghe.example/api/graphql" + ) + + +@pytest.mark.parametrize( + ("credential_api_uri", "expected_auth_api_uri"), + ( + (None, "https://ghe.example/api/v3"), + ("https://ghe.example/custom/api/v3", "https://ghe.example/custom/api/v3"), + ), +) +def test_enterprise_app_source_uses_selected_auth_endpoint_and_both_api_clients( + monkeypatch: pytest.MonkeyPatch, + credential_api_uri: str | None, + expected_auth_api_uri: str, +) -> None: + source_module = importlib.import_module("openhound_github.source") + captured_api_uris: list[str] = [] + captured_ctxs: list[object] = [] + + class FakeGithubApp: + def __init__( + self, jwt_issuer: str, private_key_path: str, api_uri: str + ) -> None: + captured_api_uris.append(api_uri) + self.installations = ( + SimpleNamespace( + id=11, + target_type="Organization", + account=SimpleNamespace(login="acme"), + ), + SimpleNamespace( + id=12, + target_type="Enterprise", + account=SimpleNamespace(slug="acme-enterprise"), + ), + ) + + class FakeGithubInstallation: + def __init__( + self, + installation_id: int, + jwt_issuer: str, + private_key_path: str, + api_uri: str, + ) -> None: + captured_api_uris.append(api_uri) + + class FakeRESTClient: + def __init__(self, **kwargs) -> None: + self.base_url = kwargs["base_url"] + + def fake_auth(*, installation, api_uri: str): + captured_api_uris.append(api_uri) + return object() + + def capture_context(ctx): + captured_ctxs.append(ctx) + return () + + monkeypatch.setattr(source_module, "GithubApp", FakeGithubApp) + monkeypatch.setattr(source_module, "GithubInstallation", FakeGithubInstallation) + monkeypatch.setattr(source_module, "GitHubAppInstallationAuth", fake_auth) + monkeypatch.setattr(source_module, "RESTClient", FakeRESTClient) + monkeypatch.setattr(source_module, "enterprise_resources", capture_context) + monkeypatch.setattr(source_module, "organization_resources", capture_context) + + resources = source_module.source.__wrapped__( + credentials=GithubEnterpriseAppCredentials( + app_id="123456", + key_path="/tmp/github-app.pem", + enterprise_name="acme-enterprise", + api_uri=credential_api_uri, + ), + emit_legacy_scim_correlations=False, + rest_api_url="https://ghe.example/api/v3", + graphql_url="https://ghe.example/api/graphql", + ) + + assert resources == () + assert captured_api_uris == [ + expected_auth_api_uri, + expected_auth_api_uri, + expected_auth_api_uri, + expected_auth_api_uri, + expected_auth_api_uri, + ] + assert len(captured_ctxs) == 2 + ctx = captured_ctxs[0] + assert ctx.client.base_url == "https://ghe.example/api/v3" + assert ctx.graphql_client.base_url == "https://ghe.example/api/graphql" + assert ctx.organizations[0].client.base_url == "https://ghe.example/api/v3" + assert ctx.organizations[0].graphql_client.base_url == "https://ghe.example/api/graphql"