From 47fccd11bb1d7be0fef6fdeba8a1b366e0a2b2b3 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Wed, 14 Oct 2026 11:49:02 +0100 Subject: [PATCH 1/3] :sparkles: Add helpers for managing a project licence file --- .../check_licence_compliance.py | 6 + .../describe_licence_obligations.py | 34 ++ .../spdx_report/project_licensing.py | 465 ++++++++++++++++++ .../spdx_report/spdx_helpers.py | 5 +- .../licence_obligations_report.txt.jinja2 | 24 + .../sync_project_licence.py | 38 ++ .../utils/configuration.py | 3 + .../utils/third_party_licences.py | 7 +- news/202610141745000000001.feature | 1 + news/202610141745000000002.feature | 1 + news/202610141745000000003.bugfix | 1 + setup.py | 2 + .../test_describe_licence_obligations.py | 48 ++ .../test_describe_licence_obligations_cli.py | 21 + tests/licensing/test_opensource_licences.py | 4 + tests/licensing/test_sync_project_licence.py | 174 +++++++ tests/spdx/test_check_licence_compliance.py | 45 +- 17 files changed, 867 insertions(+), 12 deletions(-) create mode 100644 continuous_delivery_scripts/describe_licence_obligations.py create mode 100644 continuous_delivery_scripts/spdx_report/project_licensing.py create mode 100644 continuous_delivery_scripts/spdx_report/templates/licence_obligations_report.txt.jinja2 create mode 100644 continuous_delivery_scripts/sync_project_licence.py create mode 100644 news/202610141745000000001.feature create mode 100644 news/202610141745000000002.feature create mode 100644 news/202610141745000000003.bugfix create mode 100644 tests/licensing/test_describe_licence_obligations.py create mode 100644 tests/licensing/test_describe_licence_obligations_cli.py create mode 100644 tests/licensing/test_sync_project_licence.py diff --git a/continuous_delivery_scripts/check_licence_compliance.py b/continuous_delivery_scripts/check_licence_compliance.py index 41ee6c9..c2887bb 100644 --- a/continuous_delivery_scripts/check_licence_compliance.py +++ b/continuous_delivery_scripts/check_licence_compliance.py @@ -12,6 +12,10 @@ from continuous_delivery_scripts.language_specifics import get_language_specifics from continuous_delivery_scripts.spdx_report.licence_assessment import LicenceAssessment +from continuous_delivery_scripts.spdx_report.project_licensing import ( + ensure_project_licence_file_conforms, + has_single_project_licence_identifier, +) from continuous_delivery_scripts.spdx_report.spdx_summary import _configured_licence_policy from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable from continuous_delivery_scripts.utils.logging import log_exception, set_log_level @@ -100,6 +104,8 @@ def main() -> int: set_log_level(args.verbose) try: + if has_single_project_licence_identifier(): + ensure_project_licence_file_conforms(create_if_missing=False) plugin = get_language_specifics() if not plugin.can_get_project_metadata(): raise ValueError("The selected language plugin cannot provide metadata for licence compliance checks.") diff --git a/continuous_delivery_scripts/describe_licence_obligations.py b/continuous_delivery_scripts/describe_licence_obligations.py new file mode 100644 index 0000000..4be6bc5 --- /dev/null +++ b/continuous_delivery_scripts/describe_licence_obligations.py @@ -0,0 +1,34 @@ +# +# Copyright (C) 2020-2026 Arm Limited or its affiliates and Contributors. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Describe project licence obligations from public SPDX and OSADL sources.""" + +import argparse +import logging +import sys + +from continuous_delivery_scripts.spdx_report.project_licensing import ( + describe_licence_obligations, +) +from continuous_delivery_scripts.utils.logging import log_exception, set_log_level + +logger = logging.getLogger(__name__) + + +def main() -> int: + """Create a CLI for describing project licence obligations.""" + parser = argparse.ArgumentParser(description="Describe licence obligations from SPDX and OSADL data.") + parser.add_argument("-v", "--verbose", action="count", default=0, help="Increase logging verbosity.") + args = parser.parse_args() + set_log_level(args.verbose) + try: + print(describe_licence_obligations(), end="") + return 0 + except Exception as error: + log_exception(logger, error) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/continuous_delivery_scripts/spdx_report/project_licensing.py b/continuous_delivery_scripts/spdx_report/project_licensing.py new file mode 100644 index 0000000..8288444 --- /dev/null +++ b/continuous_delivery_scripts/spdx_report/project_licensing.py @@ -0,0 +1,465 @@ +# +# Copyright (C) 2020-2026 Arm Limited or its affiliates and Contributors. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Project-level licence fetching, validation, and obligations reporting. + +References: +- SPDX licence list and JSON details: https://spdx.org/licenses/ +- SPDX licence expression guidance: https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/ +- SPDX matching guidance: https://spdx.github.io/spdx-spec/v2.3/license-matching-guidelines-and-templates/ +- ScanCode LicenseDB: https://scancode-licensedb.aboutcode.org/ +- GitHub licences API: https://docs.github.com/en/rest/licenses/licenses +- choosealicense.com: https://choosealicense.com/ +- OSADL licence obligations checklists: https://www.osadl.org/License-Checklists.oss-compliance-lists.0.html +""" + +import html +import json +import jinja2 +import logging +import re +from difflib import unified_diff +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, Optional +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from continuous_delivery_scripts.spdx_report.scancode_licence_db import ScanCodeLicenceDB +from continuous_delivery_scripts.utils.configuration import ConfigurationVariable, configuration +from continuous_delivery_scripts.utils.third_party_licences import ( + LICENSE_REF_PROPRIETARY, + OPENSOURCE_LICENCES, + is_proprietary_licence, + normalise_proprietary_licence, +) + +logger = logging.getLogger(__name__) + +REQUEST_TIMEOUT = 10 +MAX_RESPONSE_SIZE = 4 * 1024 * 1024 +USER_AGENT = "continuous-delivery-scripts" +SCANCODE_DB = ScanCodeLicenceDB() +JINJA_TEMPLATE_LICENCE_OBLIGATIONS_TEXT = "licence_obligations_report.txt.jinja2" +SCANCODE_TEXT_KEY = "text" +SPDX_NAME_KEY = "name" +SPDX_IS_OSI_APPROVED_KEY = "isOsiApproved" +SPDX_LICENSE_TEXT_KEY = "licenseText" +GITHUB_BODY_KEY = "body" + + +@dataclass(frozen=True) +class RetrievedProjectLicenceText: + """Text fetched for a project licence file from a public source. + + The text may come from ScanCode LicenseDB, SPDX.org, the GitHub licences + API, or choosealicense.com. + """ + + identifier: str + text: str + source: str + url: str + + +@dataclass(frozen=True) +class ChooseALicenseMetadata: + """Summary information exposed by choosealicense.com. + + Reference: https://choosealicense.com/ + """ + + summary: Optional[str] + permissions: tuple[str, ...] + conditions: tuple[str, ...] + limitations: tuple[str, ...] + text: Optional[str] + url: Optional[str] + + +def get_project_licence_identifier(identifier: Optional[str] = None) -> str: + """Return the project's single SPDX licence identifier. + + This helper accepts one exact SPDX identifier only. Compound expressions are + intentionally rejected for commands whose output assumes one canonical text + or one obligations profile. + + References: + - https://spdx.org/licenses/ + - https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/ + """ + raw_identifier = identifier or str(configuration.get_value(ConfigurationVariable.FILE_LICENCE_IDENTIFIER)) + normalised_identifier = normalise_proprietary_licence(raw_identifier.strip()) + if is_proprietary_licence(raw_identifier): + return normalised_identifier + if not has_single_project_licence_identifier(raw_identifier): + raise ValueError( + "This command only covers one exact SPDX licence identifier and does not cover compound licence expressions such as AND, OR, or WITH." + ) + licence = OPENSOURCE_LICENCES.get_licence(normalised_identifier, allow_fuzzy=False) + if not licence or licence.identifier != normalised_identifier: + raise ValueError(f"'{normalised_identifier}' is not a recognised SPDX licence identifier.") + return normalised_identifier + + +def has_single_project_licence_identifier(identifier: Optional[str] = None) -> bool: + """Return whether the configured project licence is one exact identifier. + + This excludes compound SPDX expressions such as ``AND``, ``OR``, and + ``WITH`` because the project-level licence-file and obligations workflows + in this module only cover one canonical licence at a time. + """ + raw_identifier = identifier or str(configuration.get_value(ConfigurationVariable.FILE_LICENCE_IDENTIFIER)) + normalised_identifier = normalise_proprietary_licence(raw_identifier.strip()) + if is_proprietary_licence(raw_identifier): + return True + return not any(token in normalised_identifier for token in (" AND ", " OR ", " WITH ", "(", ")")) + + +def get_project_licence_file_path() -> Path: + """Resolve the configured path of the project licence file. + + The default configuration is ``License.txt`` at the project root. + """ + configured_path = Path(str(configuration.get_value(ConfigurationVariable.LICENCE_FILE_PATH)).strip()) + if configured_path.is_absolute(): + return configured_path + return Path(str(configuration.get_value(ConfigurationVariable.PROJECT_ROOT))).joinpath(configured_path) + + +def normalise_licence_text(text: str) -> str: + """Normalise licence text while preserving meaningful formatting. + + This is a lightweight normalization step used before comparison against + canonical public licence texts. It is intentionally simpler than the full + SPDX matching guidelines. + + Reference: + - https://spdx.github.io/spdx-spec/v2.3/license-matching-guidelines-and-templates/ + """ + lines = [line.rstrip() for line in html.unescape(text).replace("\r\n", "\n").replace("\r", "\n").split("\n")] + while lines and not lines[0].strip(): + lines.pop(0) + while lines and not lines[-1].strip(): + lines.pop() + return "\n".join(lines) + + +def fetch_project_licence_text(identifier: str) -> Optional[RetrievedProjectLicenceText]: + """Fetch canonical licence text for one SPDX identifier from public sources. + + Source order: + 1. ScanCode LicenseDB + 2. SPDX.org + 3. GitHub licences API + 4. choosealicense.com + """ + for fetcher in ( + _fetch_scancode_licence_text, + _fetch_spdx_licence_text, + _fetch_github_licence_text, + _fetch_choosealicense_licence_text, + ): + try: + result = fetcher(identifier) + except Exception as error: + logger.debug("Could not fetch %s from %s: %s", identifier, fetcher.__name__, error) + continue + if result and result.text.strip(): + return result + return None + + +def fetch_spdx_licence_details(identifier: str) -> Optional[Dict[str, object]]: + """Fetch SPDX licence metadata for one identifier. + + Reference: https://spdx.org/licenses/ + """ + try: + return _read_json(f"https://spdx.org/licenses/{identifier}.json") + except Exception as error: + logger.debug("SPDX licence details unavailable for %s: %s", identifier, error) + return None + + +def fetch_osadl_obligations(identifier: str) -> Optional[str]: + """Fetch the OSADL checklist text for one identifier when available. + + Reference: https://www.osadl.org/License-Checklists.oss-compliance-lists.0.html + """ + try: + return normalise_licence_text( + _read_text(f"https://www.osadl.org/fileadmin/checklists/unreflicenses/{identifier}.txt") + ) + except Exception as error: + logger.debug("OSADL obligations unavailable for %s: %s", identifier, error) + return None + + +def fetch_choosealicense_metadata(identifier: str) -> Optional[ChooseALicenseMetadata]: + """Fetch descriptive choosealicense.com metadata when supported. + + This is used for the user-facing permissions/conditions/limitations summary. + + Reference: https://choosealicense.com/ + """ + for slug in _choosealicense_slugs(identifier): + url = f"https://choosealicense.com/licenses/{slug}/" + try: + page = _read_text(url) + return ChooseALicenseMetadata( + summary=_extract_choosealicense_summary(page), + permissions=tuple(_extract_choosealicense_list(page, "license-permissions")), + conditions=tuple(_extract_choosealicense_list(page, "license-conditions")), + limitations=tuple(_extract_choosealicense_list(page, "license-limitations")), + text=_extract_choosealicense_text(page), + url=url, + ) + except Exception as error: + logger.debug("choosealicense.com metadata unavailable for %s at %s: %s", identifier, url, error) + return None + + +def ensure_project_licence_file_conforms(create_if_missing: bool) -> Path: + """Ensure the configured project licence file exists and matches the configured SPDX identifier. + + For non-proprietary single-identifier projects, the file content is checked + against canonical public licence text fetched from the supported services. + For proprietary projects, this helper only verifies presence because CDS has + no public canonical reference text to compare against. + """ + identifier = get_project_licence_identifier() + licence_file = get_project_licence_file_path() + if is_proprietary_licence(identifier): + if licence_file.exists() and licence_file.is_file(): + return licence_file + raise ValueError(f"The project uses a proprietary licence. The author must provide {licence_file} manually.") + fetched = fetch_project_licence_text(identifier) + if not fetched: + raise ValueError( + f"Could not fetch licence text for {identifier} from ScanCode LicenseDB, SPDX.org, GitHub API, or choosealicense.com." + ) + if licence_file.exists(): + if not licence_file.is_file(): + raise ValueError(f"Configured licence path exists but is not a file: {licence_file}") + actual_text = licence_file.read_text(encoding="utf8") + if _normalise_licence_text_for_comparison(actual_text) != _normalise_licence_text_for_comparison(fetched.text): + _log_licence_text_diff(actual_text, fetched.text, licence_file, fetched.source) + raise ValueError(f"Configured SPDX identifier {identifier} does not match the contents of {licence_file}.") + return licence_file + if not create_if_missing: + raise ValueError(f"Configured SPDX identifier {identifier} requires a project licence file at {licence_file}.") + licence_file.parent.mkdir(parents=True, exist_ok=True) + licence_file.write_text(fetched.text.rstrip() + "\n", encoding="utf8") + print(f"Created {licence_file} from {fetched.source}: {fetched.url}") + return licence_file + + +def sync_project_licence_file() -> Path: + """Create the configured project licence file when it is missing. + + This workflow only supports projects configured with one exact SPDX licence + identifier. + """ + return ensure_project_licence_file_conforms(create_if_missing=True) + + +def describe_licence_obligations(identifier: Optional[str] = None) -> str: + """Return a human-readable obligations summary for one project licence. + + The rendered report combines: + - SPDX.org licence metadata + - choosealicense.com summary data + - OSADL obligations text + + This workflow only supports projects configured with one exact SPDX licence + identifier. + """ + resolved_identifier = get_project_licence_identifier(identifier) + if is_proprietary_licence(resolved_identifier): + return _render_licence_obligations_report( + { + "identifier": LICENSE_REF_PROPRIETARY, + "name": None, + "is_osi_approved": None, + "summary": None, + "permissions": (), + "conditions": (), + "limitations": (), + "osadl_obligations": None, + "is_proprietary": True, + } + ) + + spdx_details = fetch_spdx_licence_details(resolved_identifier) or {} + choosealicense = fetch_choosealicense_metadata(resolved_identifier) + osadl = fetch_osadl_obligations(resolved_identifier) + return _render_licence_obligations_report( + { + "identifier": resolved_identifier, + "name": spdx_details.get(SPDX_NAME_KEY) if isinstance(spdx_details.get(SPDX_NAME_KEY), str) else None, + "is_osi_approved": ( + "yes" + if spdx_details.get(SPDX_IS_OSI_APPROVED_KEY) is True + else "no" + if spdx_details.get(SPDX_IS_OSI_APPROVED_KEY) is False + else None + ), + "summary": choosealicense.summary if choosealicense else None, + "permissions": choosealicense.permissions if choosealicense else (), + "conditions": choosealicense.conditions if choosealicense else (), + "limitations": choosealicense.limitations if choosealicense else (), + "osadl_obligations": osadl.splitlines() if osadl else None, + "is_proprietary": False, + } + ) + + +def _normalise_licence_text_for_comparison(text: str) -> str: + return re.sub(r"\s+", " ", normalise_licence_text(text)).strip() + + +def _log_licence_text_diff(actual_text: str, expected_text: str, licence_file: Path, source: str) -> None: + diff = "\n".join( + unified_diff( + normalise_licence_text(actual_text).splitlines(), + normalise_licence_text(expected_text).splitlines(), + fromfile=str(licence_file), + tofile=f"canonical ({source})", + lineterm="", + ) + ) + if diff: + logger.warning("Project licence file differs from canonical text:\n%s", diff) + + +def _fetch_scancode_licence_text(identifier: str) -> Optional[RetrievedProjectLicenceText]: + info = SCANCODE_DB.find(identifier) + if not info: + return None + data = _read_json(info.url) + text = data.get(SCANCODE_TEXT_KEY) + if not isinstance(text, str) or not text.strip(): + return None + return RetrievedProjectLicenceText( + identifier=identifier, + text=normalise_licence_text(text), + source="ScanCode LicenseDB", + url=info.url, + ) + + +def _fetch_spdx_licence_text(identifier: str) -> Optional[RetrievedProjectLicenceText]: + url = f"https://spdx.org/licenses/{identifier}.json" + data = _read_json(url) + text = data.get(SPDX_LICENSE_TEXT_KEY) + if not isinstance(text, str) or not text.strip(): + return None + return RetrievedProjectLicenceText( + identifier=identifier, + text=normalise_licence_text(text), + source="SPDX.org", + url=url, + ) + + +def _fetch_github_licence_text(identifier: str) -> Optional[RetrievedProjectLicenceText]: + for slug in _github_api_slugs(identifier): + url = f"https://api.github.com/licenses/{slug}" + data = _read_json(url, headers={"Accept": "application/vnd.github+json"}) + text = data.get(GITHUB_BODY_KEY) + if not isinstance(text, str) or not text.strip(): + continue + return RetrievedProjectLicenceText( + identifier=identifier, + text=normalise_licence_text(text), + source="GitHub API", + url=url, + ) + return None + + +def _fetch_choosealicense_licence_text(identifier: str) -> Optional[RetrievedProjectLicenceText]: + metadata = fetch_choosealicense_metadata(identifier) + if not metadata or not metadata.text or not metadata.url: + return None + return RetrievedProjectLicenceText( + identifier=identifier, + text=normalise_licence_text(metadata.text), + source="choosealicense.com", + url=metadata.url, + ) + + +def _github_api_slugs(identifier: str) -> Iterable[str]: + yield identifier.lower() + simplified = re.sub(r"-(?:only|or-later)$", "", identifier, flags=re.IGNORECASE).lower() + if simplified != identifier.lower(): + yield simplified + + +def _choosealicense_slugs(identifier: str) -> Iterable[str]: + yield from _github_api_slugs(identifier) + + +def _read_text(url: str, headers: Optional[Dict[str, str]] = None) -> str: + request_headers = {"User-Agent": USER_AGENT, **(headers or {})} + request = Request(url, headers=request_headers) + with urlopen(request, timeout=REQUEST_TIMEOUT) as response: + payload = response.read(MAX_RESPONSE_SIZE + 1) + if len(payload) > MAX_RESPONSE_SIZE: + raise ValueError(f"Response from {url} exceeds the maximum supported size") + return payload.decode("utf8") + + +def _read_json(url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str, object]: + try: + data = json.loads(_read_text(url, headers=headers)) + except (HTTPError, URLError, TimeoutError, OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"Could not retrieve JSON from {url}: {error}") + if not isinstance(data, dict): + raise ValueError(f"JSON payload from {url} is not an object") + return data + + +def _extract_choosealicense_summary(page: str) -> Optional[str]: + match = re.search(r'
.*?

\s*(.*?)\s*

', page, flags=re.DOTALL) + return _strip_html(match.group(1)) if match else None + + +def _extract_choosealicense_list(page: str, class_name: str) -> list[str]: + block = re.search(rf'', page, flags=re.DOTALL) + if not block: + return [] + items = [] + for entry in re.findall(r"]*>(.*?)", block.group(1), flags=re.DOTALL): + text = _strip_html(entry) + if text: + items.append(text) + return items + + +def _extract_choosealicense_text(page: str) -> Optional[str]: + match = re.search(r'
]*>(.*?)
', page, flags=re.DOTALL) + return normalise_licence_text(match.group(1)) if match else None + + +def _strip_html(value: str) -> str: + text = re.sub(r"<[^>]+>", " ", html.unescape(value)) + return re.sub(r"\s+", " ", text).strip() + + +def _get_report_jinja2_env() -> jinja2.Environment: + return jinja2.Environment( + loader=jinja2.FileSystemLoader(str(Path(__file__).resolve().parent.joinpath("templates"))), + autoescape=lambda name: bool(name and name.endswith(".html.jinja2")), + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + ) + + +def _render_licence_obligations_report(arguments: Dict[str, object]) -> str: + template = _get_report_jinja2_env().get_template(JINJA_TEMPLATE_LICENCE_OBLIGATIONS_TEXT) + return template.render(**arguments) diff --git a/continuous_delivery_scripts/spdx_report/spdx_helpers.py b/continuous_delivery_scripts/spdx_report/spdx_helpers.py index a3aee70..ec08338 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_helpers.py +++ b/continuous_delivery_scripts/spdx_report/spdx_helpers.py @@ -14,9 +14,8 @@ - https://github.com/spdx/tools/blob/master/Examples/SPDXTagExample-v2.1.spdx """ -import re - import logging +import re import toml from dataclasses import dataclass from pathlib import Path @@ -89,8 +88,6 @@ def __str__(self) -> str: "**/__pycache__/**", "**/*.egg-info/**", ] - - def determine_file_licence(path: Path) -> Optional[str]: """Determines the licence of a file based on the SPDX identifier.""" licence = None diff --git a/continuous_delivery_scripts/spdx_report/templates/licence_obligations_report.txt.jinja2 b/continuous_delivery_scripts/spdx_report/templates/licence_obligations_report.txt.jinja2 new file mode 100644 index 0000000..411b1b1 --- /dev/null +++ b/continuous_delivery_scripts/spdx_report/templates/licence_obligations_report.txt.jinja2 @@ -0,0 +1,24 @@ +SPDX identifier: {{ identifier }} +{% if is_proprietary %} + +This project uses a proprietary licence. +The author must provide the full licence text, and redistribution, modification, notice, source, and patent obligations must be reviewed against the custom terms. +{% else %} +{% if name %}Name: {{ name }} +{% endif %}{% if is_osi_approved %}OSI approved: {{ is_osi_approved }} +{% endif %}{% if summary %} +{{ summary }} +{% endif %}{% if permissions %} +Permissions: +{% for item in permissions %}- {{ item }} +{% endfor %}{% endif %}{% if conditions %} +Conditions: +{% for item in conditions %}- {{ item }} +{% endfor %}{% endif %}{% if limitations %} +Limitations: +{% for item in limitations %}- {{ item }} +{% endfor %}{% endif %} +OSADL obligations: +{% if osadl_obligations %}{% for line in osadl_obligations %}{{ line }} +{% endfor %}{% else %}OSADL checklist unavailable for this SPDX identifier. +{% endif %}{% endif %} diff --git a/continuous_delivery_scripts/sync_project_licence.py b/continuous_delivery_scripts/sync_project_licence.py new file mode 100644 index 0000000..5b9b881 --- /dev/null +++ b/continuous_delivery_scripts/sync_project_licence.py @@ -0,0 +1,38 @@ +# +# Copyright (C) 2020-2026 Arm Limited or its affiliates and Contributors. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Ensure the configured top-level licence file exists and matches project configuration.""" + +import argparse +import logging +import sys + +from continuous_delivery_scripts.spdx_report.project_licensing import ( + sync_project_licence_file, +) +from continuous_delivery_scripts.utils.logging import log_exception, set_log_level + +logger = logging.getLogger(__name__) + + +def main() -> int: + """Create a CLI for syncing the project's top-level licence file.""" + parser = argparse.ArgumentParser( + description="Create the configured project licence file when it is missing and check it matches the configured SPDX identifier." + ) + parser.add_argument("-v", "--verbose", action="count", default=0, help="Increase logging verbosity.") + args = parser.parse_args() + set_log_level(args.verbose) + try: + licence_file = sync_project_licence_file() + if licence_file.exists(): + print(f"Project licence file matches configured SPDX identifier: {licence_file}") + return 0 + except Exception as error: + log_exception(logger, error) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/continuous_delivery_scripts/utils/configuration.py b/continuous_delivery_scripts/utils/configuration.py index ab5dd95..f57403b 100644 --- a/continuous_delivery_scripts/utils/configuration.py +++ b/continuous_delivery_scripts/utils/configuration.py @@ -113,6 +113,8 @@ class ConfigurationVariable(enum.Enum): """Skip automatic dependency downloads before licence and SPDX dependency analysis.""" DOCUMENTATION_SITE_URL = 47 """Optional public base URL of the generated documentation website.""" + LICENCE_FILE_PATH = 48 + """Relative or absolute path to the project's top-level licence file.""" @staticmethod def choices() -> List[str]: @@ -227,6 +229,7 @@ class StaticConfig(GenericConfig): DEPENDENCY_UPDATE_NEWS_MESSAGE = "Dependency upgrade: {message}" DEPENDENCY_UPDATE_NEWS_TYPE = NewsType.bugfix DEPENDENCY_UPDATE_BRANCH_PATTERN = r"^\s*[Dd]ependabot\/.+\/(?P.+)" + LICENCE_FILE_PATH = "License.txt" ACCEPTED_THIRD_PARTY_LICENCES = [ "Apache-2.0", "BSD*", diff --git a/continuous_delivery_scripts/utils/third_party_licences.py b/continuous_delivery_scripts/utils/third_party_licences.py index 7732f38..c0cad55 100644 --- a/continuous_delivery_scripts/utils/third_party_licences.py +++ b/continuous_delivery_scripts/utils/third_party_licences.py @@ -312,6 +312,11 @@ def normalise_proprietary_licence(licence_expression: str) -> str: ) +def is_proprietary_licence(licence_expression: str) -> bool: + """Return whether a licence declaration resolves to the proprietary SPDX reference.""" + return normalise_proprietary_licence(licence_expression.strip()) == LICENSE_REF_PROPRIETARY + + def cleanse_licence_expression(licence_expression: str) -> str: """Cleanses a licence expression by using SPDX identifiers when possible. @@ -334,7 +339,7 @@ def cleanse_licence_expression(licence_expression: str) -> str: def _iter_matching_licences(desc: str) -> Iterable[Licence]: - if normalise_proprietary_licence(desc.strip()) == LICENSE_REF_PROPRIETARY: + if is_proprietary_licence(desc): yield PROPRIETARY_LICENCE return licence = OPENSOURCE_LICENCES.get_licence(desc) diff --git a/news/202610141745000000001.feature b/news/202610141745000000001.feature new file mode 100644 index 0000000..793fa12 --- /dev/null +++ b/news/202610141745000000001.feature @@ -0,0 +1 @@ +:page_facing_up: Add `cd-sync-project-licence` to fetch, create, and validate a configured single project licence file. diff --git a/news/202610141745000000002.feature b/news/202610141745000000002.feature new file mode 100644 index 0000000..b0953c4 --- /dev/null +++ b/news/202610141745000000002.feature @@ -0,0 +1 @@ +:books: Add `cd-describe-licence-obligations` to report obligations for a configured single SPDX licence using SPDX, choosealicense.com, and OSADL data. diff --git a/news/202610141745000000003.bugfix b/news/202610141745000000003.bugfix new file mode 100644 index 0000000..815654f --- /dev/null +++ b/news/202610141745000000003.bugfix @@ -0,0 +1 @@ +:hammer_and_wrench: Make `cd-check-licence-compliance` validate a configured single project licence file before dependency compliance checks. diff --git a/setup.py b/setup.py index 1c5a212..99f0600 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,8 @@ f"cd-tag-and-release={SOURCE_DIR}.tag_and_release:main", f"cd-get-config={SOURCE_DIR}.get_config:main", f"cd-license-files={SOURCE_DIR}.license_files:main", + f"cd-sync-project-licence={SOURCE_DIR}.sync_project_licence:main", + f"cd-describe-licence-obligations={SOURCE_DIR}.describe_licence_obligations:main", f"cd-generate-spdx={SOURCE_DIR}.report_third_party_ip:main", f"cd-check-licence-compliance={SOURCE_DIR}.check_licence_compliance:main", f"cd-detect-secrets={SOURCE_DIR}.detect_secrets:main", diff --git a/tests/licensing/test_describe_licence_obligations.py b/tests/licensing/test_describe_licence_obligations.py new file mode 100644 index 0000000..4ae803d --- /dev/null +++ b/tests/licensing/test_describe_licence_obligations.py @@ -0,0 +1,48 @@ +# +# Copyright (C) 2020-2026 Arm Limited or its affiliates and Contributors. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +import unittest +from unittest.mock import patch + +from continuous_delivery_scripts.spdx_report.project_licensing import ChooseALicenseMetadata +from continuous_delivery_scripts.spdx_report.project_licensing import describe_licence_obligations + + +class TestDescribeLicenceObligations(unittest.TestCase): + def test_rejects_compound_licence_expression(self): + with self.assertRaisesRegex(ValueError, "does not cover compound licence expressions"): + describe_licence_obligations("MIT OR Apache-2.0") + + def test_describes_proprietary_licence(self): + description = describe_licence_obligations("LicenseRef-Proprietary") + self.assertIn("This project uses a proprietary licence.", description) + self.assertIn("author must provide the full licence text", description) + + @patch( + "continuous_delivery_scripts.spdx_report.project_licensing.fetch_choosealicense_metadata", + return_value=ChooseALicenseMetadata( + summary="A permissive licence with notice obligations.", + permissions=("Commercial use", "Modification"), + conditions=("License and copyright notice", "State changes"), + limitations=("Liability",), + text=None, + url="https://choosealicense.com/licenses/apache-2.0/", + ), + ) + @patch( + "continuous_delivery_scripts.spdx_report.project_licensing.fetch_osadl_obligations", + return_value="USE CASE Source code delivery\n\tYOU MUST Provide License text", + ) + @patch( + "continuous_delivery_scripts.spdx_report.project_licensing.fetch_spdx_licence_details", + return_value={"name": "Apache License 2.0", "isOsiApproved": True}, + ) + def test_combines_spdx_choosealicense_and_osadl_data(self, *_args): + description = describe_licence_obligations("Apache-2.0") + self.assertIn("SPDX identifier: Apache-2.0", description) + self.assertIn("Name: Apache License 2.0", description) + self.assertIn("OSI approved: yes", description) + self.assertIn("Permissions:\n- Commercial use\n- Modification", description) + self.assertIn("Conditions:\n- License and copyright notice\n- State changes", description) + self.assertIn("OSADL obligations:\nUSE CASE Source code delivery", description) diff --git a/tests/licensing/test_describe_licence_obligations_cli.py b/tests/licensing/test_describe_licence_obligations_cli.py new file mode 100644 index 0000000..b8daf95 --- /dev/null +++ b/tests/licensing/test_describe_licence_obligations_cli.py @@ -0,0 +1,21 @@ +# +# Copyright (C) 2020-2026 Arm Limited or its affiliates and Contributors. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +import unittest +from io import StringIO +from unittest.mock import patch + +from continuous_delivery_scripts.describe_licence_obligations import main + + +class TestDescribeLicenceObligationsCli(unittest.TestCase): + def test_uses_project_configuration_identifier(self): + with patch("sys.argv", ["cd-describe-licence-obligations"]), patch( + "continuous_delivery_scripts.describe_licence_obligations.describe_licence_obligations", + return_value="SPDX identifier: Apache-2.0\n", + ) as describe, patch("sys.stdout", new_callable=StringIO) as output: + self.assertEqual(main(), 0) + + describe.assert_called_once_with() + self.assertEqual("SPDX identifier: Apache-2.0\n", output.getvalue()) diff --git a/tests/licensing/test_opensource_licences.py b/tests/licensing/test_opensource_licences.py index df6196a..830c385 100644 --- a/tests/licensing/test_opensource_licences.py +++ b/tests/licensing/test_opensource_licences.py @@ -19,6 +19,7 @@ determine_allowed_opensource_licences_from_string, cleanse_licence_expression, cleanse_licence_descriptor, + is_proprietary_licence, normalise_proprietary_licence, LICENSE_REF_PROPRIETARY, ) @@ -129,6 +130,9 @@ def test_cleanse_licence_expression(self): ) def test_proprietary_licence_is_not_fuzzily_matched_to_an_open_source_licence(self): + self.assertTrue(is_proprietary_licence("proprietary")) + self.assertTrue(is_proprietary_licence("LicenseRef-Proprietary")) + self.assertFalse(is_proprietary_licence("MIT")) self.assertEqual(normalise_proprietary_licence("proprietary"), LICENSE_REF_PROPRIETARY) self.assertEqual(normalise_proprietary_licence("Proprietary Licence AND MIT"), "LicenseRef-Proprietary AND MIT") self.assertEqual(cleanse_licence_expression("Proprietary"), LICENSE_REF_PROPRIETARY) diff --git a/tests/licensing/test_sync_project_licence.py b/tests/licensing/test_sync_project_licence.py new file mode 100644 index 0000000..8b47887 --- /dev/null +++ b/tests/licensing/test_sync_project_licence.py @@ -0,0 +1,174 @@ +# +# Copyright (C) 2020-2026 Arm Limited or its affiliates and Contributors. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +import unittest +from unittest.mock import patch + +from continuous_delivery_scripts.spdx_report.project_licensing import ensure_project_licence_file_conforms +from continuous_delivery_scripts.spdx_report.project_licensing import RetrievedProjectLicenceText +from continuous_delivery_scripts.spdx_report.project_licensing import sync_project_licence_file +from continuous_delivery_scripts.utils.configuration import ConfigurationVariable, configuration +from continuous_delivery_scripts.utils.filesystem_helpers import TemporaryDirectory + + +class TestSyncProjectLicence(unittest.TestCase): + def test_writes_missing_project_licence_file(self): + get_value = configuration.get_value + with TemporaryDirectory() as directory, patch.object( + configuration, + "get_value", + side_effect=lambda key: { + ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "Apache-2.0", + ConfigurationVariable.PROJECT_ROOT: str(directory), + ConfigurationVariable.LICENCE_FILE_PATH: "License.txt", + }.get(key, get_value(key)), + ), patch( + "continuous_delivery_scripts.spdx_report.project_licensing.fetch_project_licence_text", + return_value=RetrievedProjectLicenceText( + identifier="Apache-2.0", + text="Apache License 2.0", + source="SPDX.org", + url="https://spdx.org/licenses/Apache-2.0.json", + ), + ): + licence_file = sync_project_licence_file() + + self.assertEqual((directory / "License.txt"), licence_file) + self.assertTrue(licence_file.exists()) + self.assertEqual("Apache License 2.0\n", licence_file.read_text(encoding="utf8")) + + def test_existing_project_licence_file_is_not_rewritten(self): + get_value = configuration.get_value + with TemporaryDirectory() as directory: + licence_file = directory / "docs" / "License.txt" + licence_file.parent.mkdir() + licence_file.write_text("Apache License 2.0\n", encoding="utf8") + with patch.object( + configuration, + "get_value", + side_effect=lambda key: { + ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "Apache-2.0", + ConfigurationVariable.PROJECT_ROOT: str(directory), + ConfigurationVariable.LICENCE_FILE_PATH: "docs/License.txt", + }.get(key, get_value(key)), + ), patch( + "continuous_delivery_scripts.spdx_report.project_licensing.fetch_project_licence_text", + return_value=RetrievedProjectLicenceText( + identifier="Apache-2.0", + text="Apache License 2.0", + source="SPDX.org", + url="https://spdx.org/licenses/Apache-2.0.json", + ), + ) as fetch: + resolved = sync_project_licence_file() + + self.assertEqual(licence_file, resolved) + self.assertEqual("Apache License 2.0\n", licence_file.read_text(encoding="utf8")) + + fetch.assert_called_once_with("Apache-2.0") + + def test_existing_project_licence_file_must_match_configured_identifier(self): + get_value = configuration.get_value + with TemporaryDirectory() as directory: + licence_file = directory / "License.txt" + licence_file.write_text("Different licence text\n", encoding="utf8") + with patch.object( + configuration, + "get_value", + side_effect=lambda key: { + ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "Apache-2.0", + ConfigurationVariable.PROJECT_ROOT: str(directory), + ConfigurationVariable.LICENCE_FILE_PATH: "License.txt", + }.get(key, get_value(key)), + ), patch( + "continuous_delivery_scripts.spdx_report.project_licensing.fetch_project_licence_text", + return_value=RetrievedProjectLicenceText( + identifier="Apache-2.0", + text="Apache License 2.0", + source="SPDX.org", + url="https://spdx.org/licenses/Apache-2.0.json", + ), + ): + with self.assertRaisesRegex(ValueError, "does not match the contents"): + sync_project_licence_file() + + def test_verbose_mismatch_logs_licence_diff(self): + get_value = configuration.get_value + with TemporaryDirectory() as directory: + licence_file = directory / "License.txt" + licence_file.write_text("Line one\nDifferent line\n", encoding="utf8") + with patch.object( + configuration, + "get_value", + side_effect=lambda key: { + ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "Apache-2.0", + ConfigurationVariable.PROJECT_ROOT: str(directory), + ConfigurationVariable.LICENCE_FILE_PATH: "License.txt", + }.get(key, get_value(key)), + ), patch( + "continuous_delivery_scripts.spdx_report.project_licensing.fetch_project_licence_text", + return_value=RetrievedProjectLicenceText( + identifier="Apache-2.0", + text="Line one\nCanonical line\n", + source="SPDX.org", + url="https://spdx.org/licenses/Apache-2.0.json", + ), + ), self.assertLogs("continuous_delivery_scripts.spdx_report.project_licensing", level="WARNING") as logs: + with self.assertRaisesRegex(ValueError, "does not match the contents"): + sync_project_licence_file() + + rendered = "\n".join(logs.output) + self.assertIn("Project licence file differs from canonical text:", rendered) + self.assertIn("---", rendered) + self.assertIn("+++ canonical (SPDX.org)", rendered) + + def test_validation_mode_fails_when_licence_file_is_missing(self): + get_value = configuration.get_value + with TemporaryDirectory() as directory, patch.object( + configuration, + "get_value", + side_effect=lambda key: { + ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "Apache-2.0", + ConfigurationVariable.PROJECT_ROOT: str(directory), + ConfigurationVariable.LICENCE_FILE_PATH: "License.txt", + }.get(key, get_value(key)), + ), patch( + "continuous_delivery_scripts.spdx_report.project_licensing.fetch_project_licence_text", + return_value=RetrievedProjectLicenceText( + identifier="Apache-2.0", + text="Apache License 2.0", + source="SPDX.org", + url="https://spdx.org/licenses/Apache-2.0.json", + ), + ): + with self.assertRaisesRegex(ValueError, "requires a project licence file"): + ensure_project_licence_file_conforms(create_if_missing=False) + + def test_sync_rejects_compound_licence_expression(self): + get_value = configuration.get_value + with TemporaryDirectory() as directory, patch.object( + configuration, + "get_value", + side_effect=lambda key: { + ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "MIT OR Apache-2.0", + ConfigurationVariable.PROJECT_ROOT: str(directory), + ConfigurationVariable.LICENCE_FILE_PATH: "License.txt", + }.get(key, get_value(key)), + ): + with self.assertRaisesRegex(ValueError, "does not cover compound licence expressions"): + sync_project_licence_file() + + def test_proprietary_licence_requires_manual_file(self): + get_value = configuration.get_value + with TemporaryDirectory() as directory, patch.object( + configuration, + "get_value", + side_effect=lambda key: { + ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "proprietary", + ConfigurationVariable.PROJECT_ROOT: str(directory), + ConfigurationVariable.LICENCE_FILE_PATH: "License.txt", + }.get(key, get_value(key)), + ): + with self.assertRaisesRegex(ValueError, "author must provide"): + sync_project_licence_file() diff --git a/tests/spdx/test_check_licence_compliance.py b/tests/spdx/test_check_licence_compliance.py index 7b38a8d..2ec5863 100644 --- a/tests/spdx/test_check_licence_compliance.py +++ b/tests/spdx/test_check_licence_compliance.py @@ -4,7 +4,7 @@ # """Command-line licence checks, with and without report output.""" -from contextlib import redirect_stderr, redirect_stdout +from contextlib import nullcontext, redirect_stderr, redirect_stdout import csv from io import BytesIO, StringIO import json @@ -32,8 +32,9 @@ def setUp(self): self.parser.project_metadata = self.metadata self.plugin = Mock() self.plugin.can_get_project_metadata.return_value = True + self.licence_file = self.root / "License.txt" - def _run_command(self, *args): + def _run_command(self, *args, patch_project_licence_check=True, config_overrides=None): get_value = configuration.get_value overrides = { ConfigurationVariable.PROJECT_ROOT: self.root, @@ -41,15 +42,24 @@ def _run_command(self, *args): ConfigurationVariable.SOURCE_DIR: "source", ConfigurationVariable.PROJECT_UUID: "example-id", } + if config_overrides: + overrides.update(config_overrides) with patch.object( configuration, "get_value", side_effect=lambda key: overrides.get(key, get_value(key)) ), patch( "continuous_delivery_scripts.check_licence_compliance.get_language_specifics", return_value=self.plugin - ), patch( - "sys.argv", ["cd-check-licence-compliance", *args] - ): - self.plugin.get_current_spdx_project.return_value = SpdxProject(self.parser) - return main() + ), patch("sys.argv", ["cd-check-licence-compliance", *args]): + project_licence_context = ( + patch( + "continuous_delivery_scripts.check_licence_compliance.ensure_project_licence_file_conforms", + return_value=self.licence_file, + ) + if patch_project_licence_check + else nullcontext() + ) + with project_licence_context: + self.plugin.get_current_spdx_project.return_value = SpdxProject(self.parser) + return main() def test_checks_compliance_without_writing_reports(self): self.metadata.add_dependency_metadata(PackageMetadata({"Name": "dependency", "License": "MIT"})) @@ -72,6 +82,27 @@ def test_checks_compliance_without_writing_reports(self): self.assertIn("Compliance with configured licence policy: PASS", rendered) self.assertEqual(list(self.root.iterdir()), [self.root / "source"]) + def test_project_licence_file_is_checked_before_dependency_compliance(self): + with patch( + "continuous_delivery_scripts.check_licence_compliance.ensure_project_licence_file_conforms", + side_effect=ValueError("Project licence file mismatch"), + ) as ensure: + self.assertEqual(self._run_command(patch_project_licence_check=False), 1) + + ensure.assert_called_once_with(create_if_missing=False) + self.plugin.get_current_spdx_project.assert_not_called() + + def test_compound_project_licence_expression_skips_project_licence_file_check(self): + with patch( + "continuous_delivery_scripts.check_licence_compliance.ensure_project_licence_file_conforms" + ) as ensure: + self.assertEqual( + self._run_command(config_overrides={ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "MIT OR Apache-2.0"}), + 0, + ) + + ensure.assert_not_called() + def test_output_directory_writes_summaries_but_no_spdx_documents(self): self.assertEqual(self._run_command("--output-dir", str(self.root)), 0) From 2bacec89a8e308b5dfac695914527ffbea07517a Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Wed, 14 Oct 2026 13:37:13 +0100 Subject: [PATCH 2/3] :green_heart: fixing type issues --- .../spdx_report/project_licensing.py | 26 +++++++++---------- .../spdx_report/spdx_helpers.py | 2 ++ .../sync_project_licence.py | 5 +++- .../utils/configuration.py | 2 +- tests/licensing/test_sync_project_licence.py | 4 ++- tests/spdx/test_check_licence_compliance.py | 8 ++++-- 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/continuous_delivery_scripts/spdx_report/project_licensing.py b/continuous_delivery_scripts/spdx_report/project_licensing.py index 8288444..eb1da9d 100644 --- a/continuous_delivery_scripts/spdx_report/project_licensing.py +++ b/continuous_delivery_scripts/spdx_report/project_licensing.py @@ -22,7 +22,7 @@ from difflib import unified_diff from dataclasses import dataclass from pathlib import Path -from typing import Dict, Iterable, Optional +from typing import Dict, Iterable, Optional, cast from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen @@ -89,13 +89,14 @@ def get_project_licence_identifier(identifier: Optional[str] = None) -> str: - https://spdx.org/licenses/ - https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/ """ - raw_identifier = identifier or str(configuration.get_value(ConfigurationVariable.FILE_LICENCE_IDENTIFIER)) - normalised_identifier = normalise_proprietary_licence(raw_identifier.strip()) + raw_identifier: str = identifier or str(configuration.get_value(ConfigurationVariable.FILE_LICENCE_IDENTIFIER)) + normalised_identifier: str = normalise_proprietary_licence(raw_identifier.strip()) if is_proprietary_licence(raw_identifier): return normalised_identifier if not has_single_project_licence_identifier(raw_identifier): raise ValueError( - "This command only covers one exact SPDX licence identifier and does not cover compound licence expressions such as AND, OR, or WITH." + "This command only covers one exact SPDX licence identifier and does not cover " + "compound licence expressions such as AND, OR, or WITH." ) licence = OPENSOURCE_LICENCES.get_licence(normalised_identifier, allow_fuzzy=False) if not licence or licence.identifier != normalised_identifier: @@ -110,8 +111,8 @@ def has_single_project_licence_identifier(identifier: Optional[str] = None) -> b ``WITH`` because the project-level licence-file and obligations workflows in this module only cover one canonical licence at a time. """ - raw_identifier = identifier or str(configuration.get_value(ConfigurationVariable.FILE_LICENCE_IDENTIFIER)) - normalised_identifier = normalise_proprietary_licence(raw_identifier.strip()) + raw_identifier: str = identifier or str(configuration.get_value(ConfigurationVariable.FILE_LICENCE_IDENTIFIER)) + normalised_identifier: str = normalise_proprietary_licence(raw_identifier.strip()) if is_proprietary_licence(raw_identifier): return True return not any(token in normalised_identifier for token in (" AND ", " OR ", " WITH ", "(", ")")) @@ -120,7 +121,7 @@ def has_single_project_licence_identifier(identifier: Optional[str] = None) -> b def get_project_licence_file_path() -> Path: """Resolve the configured path of the project licence file. - The default configuration is ``License.txt`` at the project root. + The default configuration is ``LICENSE`` at the project root. """ configured_path = Path(str(configuration.get_value(ConfigurationVariable.LICENCE_FILE_PATH)).strip()) if configured_path.is_absolute(): @@ -238,7 +239,8 @@ def ensure_project_licence_file_conforms(create_if_missing: bool) -> Path: fetched = fetch_project_licence_text(identifier) if not fetched: raise ValueError( - f"Could not fetch licence text for {identifier} from ScanCode LicenseDB, SPDX.org, GitHub API, or choosealicense.com." + f"Could not fetch licence text for {identifier} from ScanCode LicenseDB, SPDX.org, " + "GitHub API, or choosealicense.com." ) if licence_file.exists(): if not licence_file.is_file(): @@ -302,9 +304,7 @@ def describe_licence_obligations(identifier: Optional[str] = None) -> str: "is_osi_approved": ( "yes" if spdx_details.get(SPDX_IS_OSI_APPROVED_KEY) is True - else "no" - if spdx_details.get(SPDX_IS_OSI_APPROVED_KEY) is False - else None + else "no" if spdx_details.get(SPDX_IS_OSI_APPROVED_KEY) is False else None ), "summary": choosealicense.summary if choosealicense else None, "permissions": choosealicense.permissions if choosealicense else (), @@ -410,7 +410,7 @@ def _read_text(url: str, headers: Optional[Dict[str, str]] = None) -> str: payload = response.read(MAX_RESPONSE_SIZE + 1) if len(payload) > MAX_RESPONSE_SIZE: raise ValueError(f"Response from {url} exceeds the maximum supported size") - return payload.decode("utf8") + return cast(str, payload.decode("utf8")) def _read_json(url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str, object]: @@ -462,4 +462,4 @@ def _get_report_jinja2_env() -> jinja2.Environment: def _render_licence_obligations_report(arguments: Dict[str, object]) -> str: template = _get_report_jinja2_env().get_template(JINJA_TEMPLATE_LICENCE_OBLIGATIONS_TEXT) - return template.render(**arguments) + return cast(str, template.render(**arguments)) diff --git a/continuous_delivery_scripts/spdx_report/spdx_helpers.py b/continuous_delivery_scripts/spdx_report/spdx_helpers.py index ec08338..6020d63 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_helpers.py +++ b/continuous_delivery_scripts/spdx_report/spdx_helpers.py @@ -88,6 +88,8 @@ def __str__(self) -> str: "**/__pycache__/**", "**/*.egg-info/**", ] + + def determine_file_licence(path: Path) -> Optional[str]: """Determines the licence of a file based on the SPDX identifier.""" licence = None diff --git a/continuous_delivery_scripts/sync_project_licence.py b/continuous_delivery_scripts/sync_project_licence.py index 5b9b881..66f0c83 100644 --- a/continuous_delivery_scripts/sync_project_licence.py +++ b/continuous_delivery_scripts/sync_project_licence.py @@ -19,7 +19,10 @@ def main() -> int: """Create a CLI for syncing the project's top-level licence file.""" parser = argparse.ArgumentParser( - description="Create the configured project licence file when it is missing and check it matches the configured SPDX identifier." + description=( + "Create the configured project licence file when it is missing and check it matches " + "the configured SPDX identifier." + ) ) parser.add_argument("-v", "--verbose", action="count", default=0, help="Increase logging verbosity.") args = parser.parse_args() diff --git a/continuous_delivery_scripts/utils/configuration.py b/continuous_delivery_scripts/utils/configuration.py index f57403b..572f85e 100644 --- a/continuous_delivery_scripts/utils/configuration.py +++ b/continuous_delivery_scripts/utils/configuration.py @@ -229,7 +229,7 @@ class StaticConfig(GenericConfig): DEPENDENCY_UPDATE_NEWS_MESSAGE = "Dependency upgrade: {message}" DEPENDENCY_UPDATE_NEWS_TYPE = NewsType.bugfix DEPENDENCY_UPDATE_BRANCH_PATTERN = r"^\s*[Dd]ependabot\/.+\/(?P.+)" - LICENCE_FILE_PATH = "License.txt" + LICENCE_FILE_PATH = "LICENSE" ACCEPTED_THIRD_PARTY_LICENCES = [ "Apache-2.0", "BSD*", diff --git a/tests/licensing/test_sync_project_licence.py b/tests/licensing/test_sync_project_licence.py index 8b47887..ef8d6a4 100644 --- a/tests/licensing/test_sync_project_licence.py +++ b/tests/licensing/test_sync_project_licence.py @@ -114,7 +114,9 @@ def test_verbose_mismatch_logs_licence_diff(self): source="SPDX.org", url="https://spdx.org/licenses/Apache-2.0.json", ), - ), self.assertLogs("continuous_delivery_scripts.spdx_report.project_licensing", level="WARNING") as logs: + ), self.assertLogs( + "continuous_delivery_scripts.spdx_report.project_licensing", level="WARNING" + ) as logs: with self.assertRaisesRegex(ValueError, "does not match the contents"): sync_project_licence_file() diff --git a/tests/spdx/test_check_licence_compliance.py b/tests/spdx/test_check_licence_compliance.py index 2ec5863..9517db7 100644 --- a/tests/spdx/test_check_licence_compliance.py +++ b/tests/spdx/test_check_licence_compliance.py @@ -48,7 +48,9 @@ def _run_command(self, *args, patch_project_licence_check=True, config_overrides configuration, "get_value", side_effect=lambda key: overrides.get(key, get_value(key)) ), patch( "continuous_delivery_scripts.check_licence_compliance.get_language_specifics", return_value=self.plugin - ), patch("sys.argv", ["cd-check-licence-compliance", *args]): + ), patch( + "sys.argv", ["cd-check-licence-compliance", *args] + ): project_licence_context = ( patch( "continuous_delivery_scripts.check_licence_compliance.ensure_project_licence_file_conforms", @@ -97,7 +99,9 @@ def test_compound_project_licence_expression_skips_project_licence_file_check(se "continuous_delivery_scripts.check_licence_compliance.ensure_project_licence_file_conforms" ) as ensure: self.assertEqual( - self._run_command(config_overrides={ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "MIT OR Apache-2.0"}), + self._run_command( + config_overrides={ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "MIT OR Apache-2.0"} + ), 0, ) From db5655b439faa8d397b82560587f1f00f107ab52 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Wed, 14 Oct 2026 13:57:01 +0100 Subject: [PATCH 3/3] :book: Update License file --- LICENSE | 27 +++++++++++ .../spdx_report/project_licensing.py | 8 +++- tests/licensing/test_sync_project_licence.py | 48 +++++++++++++++---- 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/LICENSE b/LICENSE index dd5b3a5..261eeb9 100644 --- a/LICENSE +++ b/LICENSE @@ -172,3 +172,30 @@ defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/continuous_delivery_scripts/spdx_report/project_licensing.py b/continuous_delivery_scripts/spdx_report/project_licensing.py index eb1da9d..617729e 100644 --- a/continuous_delivery_scripts/spdx_report/project_licensing.py +++ b/continuous_delivery_scripts/spdx_report/project_licensing.py @@ -222,7 +222,7 @@ def fetch_choosealicense_metadata(identifier: str) -> Optional[ChooseALicenseMet return None -def ensure_project_licence_file_conforms(create_if_missing: bool) -> Path: +def ensure_project_licence_file_conforms(create_if_missing: bool, update_if_mismatched: bool = False) -> Path: """Ensure the configured project licence file exists and matches the configured SPDX identifier. For non-proprietary single-identifier projects, the file content is checked @@ -248,6 +248,10 @@ def ensure_project_licence_file_conforms(create_if_missing: bool) -> Path: actual_text = licence_file.read_text(encoding="utf8") if _normalise_licence_text_for_comparison(actual_text) != _normalise_licence_text_for_comparison(fetched.text): _log_licence_text_diff(actual_text, fetched.text, licence_file, fetched.source) + if update_if_mismatched: + licence_file.write_text(fetched.text.rstrip() + "\n", encoding="utf8") + print(f"Updated {licence_file} from {fetched.source}: {fetched.url}") + return licence_file raise ValueError(f"Configured SPDX identifier {identifier} does not match the contents of {licence_file}.") return licence_file if not create_if_missing: @@ -264,7 +268,7 @@ def sync_project_licence_file() -> Path: This workflow only supports projects configured with one exact SPDX licence identifier. """ - return ensure_project_licence_file_conforms(create_if_missing=True) + return ensure_project_licence_file_conforms(create_if_missing=True, update_if_mismatched=True) def describe_licence_obligations(identifier: Optional[str] = None) -> str: diff --git a/tests/licensing/test_sync_project_licence.py b/tests/licensing/test_sync_project_licence.py index ef8d6a4..d908da6 100644 --- a/tests/licensing/test_sync_project_licence.py +++ b/tests/licensing/test_sync_project_licence.py @@ -68,10 +68,10 @@ def test_existing_project_licence_file_is_not_rewritten(self): fetch.assert_called_once_with("Apache-2.0") - def test_existing_project_licence_file_must_match_configured_identifier(self): + def test_existing_project_licence_file_is_resynced_when_mismatched(self): get_value = configuration.get_value with TemporaryDirectory() as directory: - licence_file = directory / "License.txt" + licence_file = directory / "LICENSE" licence_file.write_text("Different licence text\n", encoding="utf8") with patch.object( configuration, @@ -79,7 +79,7 @@ def test_existing_project_licence_file_must_match_configured_identifier(self): side_effect=lambda key: { ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "Apache-2.0", ConfigurationVariable.PROJECT_ROOT: str(directory), - ConfigurationVariable.LICENCE_FILE_PATH: "License.txt", + ConfigurationVariable.LICENCE_FILE_PATH: "LICENSE", }.get(key, get_value(key)), ), patch( "continuous_delivery_scripts.spdx_report.project_licensing.fetch_project_licence_text", @@ -90,13 +90,16 @@ def test_existing_project_licence_file_must_match_configured_identifier(self): url="https://spdx.org/licenses/Apache-2.0.json", ), ): - with self.assertRaisesRegex(ValueError, "does not match the contents"): - sync_project_licence_file() + resolved = sync_project_licence_file() + contents = licence_file.read_text(encoding="utf8") + + self.assertEqual(licence_file, resolved) + self.assertEqual("Apache License 2.0\n", contents) def test_verbose_mismatch_logs_licence_diff(self): get_value = configuration.get_value with TemporaryDirectory() as directory: - licence_file = directory / "License.txt" + licence_file = directory / "LICENSE" licence_file.write_text("Line one\nDifferent line\n", encoding="utf8") with patch.object( configuration, @@ -104,7 +107,7 @@ def test_verbose_mismatch_logs_licence_diff(self): side_effect=lambda key: { ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "Apache-2.0", ConfigurationVariable.PROJECT_ROOT: str(directory), - ConfigurationVariable.LICENCE_FILE_PATH: "License.txt", + ConfigurationVariable.LICENCE_FILE_PATH: "LICENSE", }.get(key, get_value(key)), ), patch( "continuous_delivery_scripts.spdx_report.project_licensing.fetch_project_licence_text", @@ -117,14 +120,41 @@ def test_verbose_mismatch_logs_licence_diff(self): ), self.assertLogs( "continuous_delivery_scripts.spdx_report.project_licensing", level="WARNING" ) as logs: - with self.assertRaisesRegex(ValueError, "does not match the contents"): - sync_project_licence_file() + resolved = sync_project_licence_file() + contents = licence_file.read_text(encoding="utf8") + self.assertEqual(licence_file, resolved) + self.assertEqual("Line one\nCanonical line\n", contents) rendered = "\n".join(logs.output) self.assertIn("Project licence file differs from canonical text:", rendered) self.assertIn("---", rendered) self.assertIn("+++ canonical (SPDX.org)", rendered) + def test_validation_mode_fails_when_project_licence_file_does_not_match_identifier(self): + get_value = configuration.get_value + with TemporaryDirectory() as directory: + licence_file = directory / "LICENSE" + licence_file.write_text("Different licence text\n", encoding="utf8") + with patch.object( + configuration, + "get_value", + side_effect=lambda key: { + ConfigurationVariable.FILE_LICENCE_IDENTIFIER: "Apache-2.0", + ConfigurationVariable.PROJECT_ROOT: str(directory), + ConfigurationVariable.LICENCE_FILE_PATH: "LICENSE", + }.get(key, get_value(key)), + ), patch( + "continuous_delivery_scripts.spdx_report.project_licensing.fetch_project_licence_text", + return_value=RetrievedProjectLicenceText( + identifier="Apache-2.0", + text="Apache License 2.0", + source="SPDX.org", + url="https://spdx.org/licenses/Apache-2.0.json", + ), + ): + with self.assertRaisesRegex(ValueError, "does not match the contents"): + ensure_project_licence_file_conforms(create_if_missing=False) + def test_validation_mode_fails_when_licence_file_is_missing(self): get_value = configuration.get_value with TemporaryDirectory() as directory, patch.object(