diff --git a/.github/workflows/scripts/check_licenses.py b/.github/workflows/scripts/check_licenses.py index c4a14f3757..4c27d2d0b3 100755 --- a/.github/workflows/scripts/check_licenses.py +++ b/.github/workflows/scripts/check_licenses.py @@ -1,174 +1,308 @@ #!/usr/bin/env python3 -from pathlib import Path -import tomli +import argparse +import os import sys +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + import requests +import tomli import urllib3 -from typing import Dict, List, Optional, Set -# Define allowed licenses and exceptions directly in the script -ALLOWED_LICENSES = { - "MIT", - "BSD-3-Clause", - "Apache-2.0", - "Apache Software License", - "Python Software Foundation License", - "BSD License", - "ISC" -} -# Package-specific exceptions -EXCEPTIONS = { - "ai-exchange": True, # Local workspace package - "tiktoken": True, # Known MIT license with non-standard format -} +class Color(str, Enum): + """ANSI color codes with fallback for non-color terminals""" + + @staticmethod + def supports_color() -> bool: + """Check if the terminal supports color output.""" + if not hasattr(sys.stdout, "isatty"): + return False + if not sys.stdout.isatty(): + return False + + if "NO_COLOR" in os.environ: + return False + + term = os.environ.get("TERM", "") + if term == "dumb": + return False + + return True + + has_color = supports_color() + + RED = "\033[91m" if has_color else "" + GREEN = "\033[92m" if has_color else "" + RESET = "\033[0m" if has_color else "" + BOLD = "\033[1m" if has_color else "" + + +@dataclass(frozen=True) +class LicenseConfig: + allowed_licenses: frozenset[str] = frozenset( + { + "MIT", + "BSD-3-Clause", + "Apache-2.0", + "Apache Software License", + "Python Software Foundation License", + "BSD License", + "ISC", + } + ) + exceptions: frozenset[str] = frozenset( + { + "ai-exchange", + "tiktoken", + } + ) + + +@dataclass(frozen=True) +class LicenseInfo: + license: str | None + allowed: bool = False + + def __str__(self) -> str: + status = "✓" if self.allowed else "✗" + color = Color.GREEN if self.allowed else Color.RED + return f"{color}{status}{Color.RESET} {self.license}" + class LicenseChecker: - def __init__(self): - self.session = requests.Session() - # Configure session for robust SSL handling - self.session.verify = True - adapter = requests.adapters.HTTPAdapter( - max_retries=urllib3.util.Retry( - total=3, - backoff_factor=0.5, - status_forcelist=[500, 502, 503, 504] - ) + def __init__(self, config: LicenseConfig = LicenseConfig()) -> None: + self.config = config + self.session = self._setup_session() + + def _setup_session(self) -> requests.Session: + session = requests.Session() + session.verify = True + max_retries = urllib3.util.Retry( + total=3, + backoff_factor=0.5, + status_forcelist=[ + 500, + 502, + 503, + 504, + ], ) - self.session.mount('https://', adapter) - - def normalize_license(self, license_str: Optional[str]) -> Optional[str]: - """Normalize license string for comparison.""" + adapter = requests.adapters.HTTPAdapter(max_retries=max_retries) + session.mount("https://", adapter) + return session + + def normalize_license(self, license_str: str | None) -> str | None: + """ + Normalize license string for comparison. + + This method takes a license string and normalizes it by: + 1. Converting to uppercase + 2. Removing 'LICENSE' or 'LICENCE' suffixes + 3. Stripping whitespace + 4. Replacing common variations with standardized forms + + Args: + license_str (str | None): The original license string to normalize. + + Returns: + str | None: The normalized license string, or None if the input was None. + """ if not license_str: return None - - # Convert to uppercase and remove common words and punctuation - normalized = license_str.upper().replace(' LICENSE', '').replace(' LICENCE', '').strip() - - # Common substitutions - replacements = { - 'APACHE 2.0': 'APACHE-2.0', - 'APACHE SOFTWARE LICENSE': 'APACHE-2.0', - 'BSD': 'BSD-3-CLAUSE', - 'MIT LICENSE': 'MIT', - 'PYTHON SOFTWARE FOUNDATION': 'PSF', - } - - return replacements.get(normalized, normalized) - - def get_package_license(self, package_name: str) -> Optional[str]: - """Fetch license information from PyPI.""" - if package_name in EXCEPTIONS: - return "APPROVED-EXCEPTION" + # fmt: off + normalized = ( + license_str.upper() + .replace(" LICENSE", "") + .replace(" LICENCE", "") + .strip() + ) + # fmt: on + + replacements = { + "APACHE 2.0": "APACHE-2.0", + "APACHE SOFTWARE LICENSE": "APACHE-2.0", + "BSD": "BSD-3-CLAUSE", + "MIT LICENSE": "MIT", + "PYTHON SOFTWARE FOUNDATION": "PSF", + } + + return replacements.get(normalized, normalized) + + def get_package_license(self, package_name: str) -> str | None: + """Fetch license information from PyPI. + + Args: + package_name (str): The name of the package to fetch the license for. + + Returns: + str | None: The license of the package, or None if not found. + """ try: - response = self.session.get(f"https://pypi.org/pypi/{package_name}/json") + response = self.session.get( + f"https://pypi.org/pypi/{package_name}/json", + timeout=10, + ) response.raise_for_status() data = response.json() - + + # fmt: off license_info = ( - data['info'].get('license') or - data['info'].get('classifiers', []) + data["info"].get("license") or + data["info"].get("classifiers", []) ) - + # fmt: on + if isinstance(license_info, list): for classifier in license_info: - if classifier.startswith('License :: '): - parts = classifier.split(' :: ') + if classifier.startswith("License :: "): + parts = classifier.split(" :: ") return parts[-1] - + return license_info if isinstance(license_info, str) else None - + except requests.exceptions.SSLError as e: print(f"SSL Error fetching license for {package_name}: {e}", file=sys.stderr) - return None except Exception as e: print(f"Warning: Could not fetch license for {package_name}: {e}", file=sys.stderr) - return None + return None - def extract_dependencies(self, toml_file: Path) -> List[str]: + def extract_dependencies(self, toml_file: Path) -> list[str]: """Extract all dependencies from a TOML file.""" - with open(toml_file, 'rb') as f: + with open(toml_file, "rb") as f: data = tomli.load(f) - + dependencies = [] - + # Get direct dependencies - project_deps = data.get('project', {}).get('dependencies', []) + project_deps = data.get("project", {}).get("dependencies", []) dependencies.extend(self._parse_dependency_strings(project_deps)) - + # Get dev dependencies - tool_deps = data.get('tool', {}).get('uv', {}).get('dev-dependencies', []) + tool_deps = data.get("tool", {}).get("uv", {}).get("dev-dependencies", []) dependencies.extend(self._parse_dependency_strings(tool_deps)) - + return list(set(dependencies)) - - def _parse_dependency_strings(self, deps: List[str]) -> List[str]: - """Parse dependency strings to extract package names.""" + + def _parse_dependency_strings(self, deps: list[str]) -> list[str]: + """ + Parse dependency strings to extract package names. + + Args: + deps (list[str]): A list of dependency strings to parse. + + Returns: + list[str]: A list of extracted package names. + """ packages = [] for dep in deps: - # Skip workspace references - if dep.endswith('workspace = true}'): + if "workspace = true" in dep: continue - + + # fmt: off # Handle basic package specifiers - package = dep.split('>=')[0].split('==')[0].split('<')[0].split('>')[0].strip() - package = package.split('{')[0].strip() - packages.append(package) + package = ( + dep.split(">=")[0] + .split("==")[0] + .split("<")[0] + .split(">")[0] + .strip() + ) + package = package.split("{")[0].strip() + # fmt: on + if package: + packages.append(package) return packages - - def check_licenses(self, toml_file: Path) -> Dict[str, Dict[str, bool]]: - """Check licenses for all dependencies in the TOML file.""" + + def check_licenses(self, toml_file: Path) -> dict[str, LicenseInfo]: + """ + Check licenses for all dependencies in the TOML file. + + Args: + toml_file (Path): The path to the TOML file containing the dependencies. + + Returns: + dict[str, LicenseInfo]: A dictionary where the keys are package names and the values are LicenseInfo objects + containing the license information and whether it's allowed.""" dependencies = self.extract_dependencies(toml_file) - results = {} - checked = set() - + results: dict[str, LicenseInfo] = {} + checked: set[str] = set() + for package in dependencies: if package in checked: continue - + checked.add(package) - - if package in EXCEPTIONS: - results[package] = { - 'license': 'Approved Exception', - 'allowed': True - } - continue - - license_info = self.get_package_license(package) - normalized_license = self.normalize_license(license_info) - allowed = False - - if normalized_license: - allowed = (normalized_license in {self.normalize_license(l) for l in ALLOWED_LICENSES} or - package in EXCEPTIONS) - - results[package] = { - 'license': license_info, - 'allowed': allowed - } - + results[package] = self._check_package(package) + return results -def main(): - if len(sys.argv) < 2: - print("Usage: check_licenses.py ", file=sys.stderr) - sys.exit(1) - - toml_file = Path(sys.argv[1]) + def _check_package(self, package: str) -> LicenseInfo: + """ + Check license for a single package. + + Args: + package (str): The name of the package to check. + + Returns: + LicenseInfo: A LicenseInfo object containing the license + information and whether it's allowed. + """ + + if package in self.config.exceptions: + return LicenseInfo("Approved Exception", True) + + license_info = self.get_package_license(package) + normalized_license = self.normalize_license(license_info) + allowed = False + + # fmt: off + if normalized_license: + allowed = normalized_license in { + self.normalize_license(x) + for x in self.config.allowed_licenses + } + # fmt: on + return LicenseInfo(license_info, allowed) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Check package licenses in TOML files") + parser.add_argument("toml_files", type=Path, nargs="+", help="Paths to TOML files") + args = parser.parse_args() + checker = LicenseChecker() - results = checker.check_licenses(toml_file) - + all_results: dict[str, LicenseInfo] = {} + + for toml_file in args.toml_files: + results = checker.check_licenses(toml_file) + for package, info in results.items(): + if package in all_results and all_results[package] != info: + print(f"Warning: Package {package} has conflicting license info:", file=sys.stderr) + print(f" {toml_file}: {info}", file=sys.stderr) + print(f" Previous: {all_results[package]}", file=sys.stderr) + all_results[package] = info + + max_package_length = max(len(package) for package in all_results.keys()) any_disallowed = False - for package, info in sorted(results.items()): - status = "✓" if info['allowed'] else "✗" - print(f"{status} {package}: {info['license']}") - if not info['allowed']: + + for package, info in sorted(all_results.items()): + if Color.has_color: + package_name = f"{Color.BOLD}{package}{Color.RESET}" + padding = len(Color.BOLD) + len(Color.RESET) + else: + package_name = package + padding = 0 + + print(f"{package_name:<{max_package_length + padding}} {info}") + if not info.allowed: any_disallowed = True - + sys.exit(1 if any_disallowed else 0) -if __name__ == '__main__': - main() \ No newline at end of file + +if __name__ == "__main__": + main() diff --git a/.github/workflows/scripts/test_check_licenses.py b/.github/workflows/scripts/test_check_licenses.py new file mode 100644 index 0000000000..3ec951deb5 --- /dev/null +++ b/.github/workflows/scripts/test_check_licenses.py @@ -0,0 +1,248 @@ +from pathlib import Path +from typing import Optional +from unittest.mock import Mock, patch + +import pytest +import tomli +from check_licenses import Color, LicenseChecker, LicenseConfig, LicenseInfo, main + + +@pytest.fixture +def checker() -> LicenseChecker: + return LicenseChecker() + + +@pytest.fixture +def mock_pypi_response() -> Mock: + response = Mock() + response.status_code = 200 + response.raise_for_status = Mock() + response.ok = True + response.json.return_value = { + "info": { + "license": "Apache-2.0", + } + } + return response + + +@pytest.fixture +def mock_toml_content() -> str: + return """ +[project] +dependencies = [ + "requests>=2.28.0", + "tomli==2.0.1", + "urllib3<2.0.0", + "package-with-workspace{workspace = true}", +] + +[tool.uv] +dev-dependencies = [ + "pytest>=7.0.0", + "black==23.3.0", +] +""" + + +@pytest.fixture +def mock_toml_files(tmp_path: Path) -> list[Path]: + """Create mock TOML files with different dependencies.""" + file1 = tmp_path / "pyproject1.toml" + + file1.write_text(""" +[project] +dependencies = [ + "requests>=2.28.0", + "tomli==2.0.1", +] +""") + + file2 = tmp_path / "pyproject2.toml" + file2.write_text(""" +[project] +dependencies = [ + "urllib3<2.0.0", + "requests>=2.27.0", # Different version but same package +] +""") + + return [file1, file2] + + +def test_normalize_license_variations(checker: LicenseChecker) -> None: + assert checker.normalize_license("MIT License") == "MIT" + assert checker.normalize_license("Apache 2.0") == "APACHE-2.0" + assert checker.normalize_license("BSD") == "BSD-3-CLAUSE" + assert checker.normalize_license(None) is None + assert checker.normalize_license("") is None + assert checker.normalize_license("MIT License") == "MIT" + assert checker.normalize_license("Apache 2.0") == "APACHE-2.0" + assert checker.normalize_license("BSD") == "BSD-3-CLAUSE" + assert checker.normalize_license(None) is None + assert checker.normalize_license("") is None + + +@patch.object(LicenseChecker, "get_package_license") +def test_package_license_verification(mock_get_license: Mock, checker: LicenseChecker) -> None: + def check_package_license( + package: str, license: Optional[str], expected_allowed: bool, expected_license: str + ) -> None: + if license: + mock_get_license.return_value = license + result = checker._check_package(package=package) + assert result.allowed is expected_allowed + assert result.license == expected_license + + check_package_license( + package="tiktoken", + license=None, + expected_allowed=True, + expected_license="Approved Exception", + ) + check_package_license( + package="requests", + license="Apache-2.0", + expected_allowed=True, + expected_license="Apache-2.0", + ) + check_package_license( + package="gpl-package", + license="GPL", + expected_allowed=False, + expected_license="GPL", + ) + + +def test_color_support_with_environment_variables(monkeypatch: pytest.MonkeyPatch) -> None: + def verify_color_support_disabled(*, env_var: str, value: str) -> None: + monkeypatch.setenv(env_var, value) + assert not Color.supports_color() + monkeypatch.undo() + + verify_color_support_disabled(env_var="NO_COLOR", value="1") + verify_color_support_disabled(env_var="TERM", value="dumb") + + +@patch("tomli.load") +@patch("builtins.open") +def test_extract_dependencies( + mock_open: Mock, mock_tomli_load: Mock, checker: LicenseChecker, mock_toml_content: str +) -> None: + mock_tomli_load.return_value = tomli.loads(mock_toml_content) + mock_file = Mock() + mock_open.return_value.__enter__.return_value = mock_file + + dependencies = checker.extract_dependencies(Path("mock_pyproject.toml")) + expected = ["requests", "tomli", "urllib3", "pytest", "black"] + assert sorted(dependencies) == sorted(expected) + + +@patch("requests.Session") +def test_get_package_license(mock_session: Mock, checker: LicenseChecker, mock_pypi_response: Mock) -> None: + mock_session.return_value.get.return_value = mock_pypi_response + assert checker.get_package_license("requests") == "Apache-2.0" + + # test exception handling + mock_session.return_value.get.side_effect = Exception("error") + assert checker.get_package_license("nonexistent-package") is None + + +def test_license_info_string_representation() -> None: + def assert_license_info_str(info: LicenseInfo, no_color_expected: str, color_expected: str) -> None: + expected = color_expected if Color.has_color else no_color_expected + assert str(info) == expected + + assert_license_info_str(LicenseInfo("MIT", True), "✓ MIT", f"{Color.GREEN}✓{Color.RESET} MIT") + assert_license_info_str(LicenseInfo("GPL", False), "✗ GPL", f"{Color.RED}✗{Color.RESET} GPL") + + +def test_custom_license_config() -> None: + custom_config = LicenseConfig( + allowed_licenses=frozenset({"MIT", "Apache-2.0"}), exceptions=frozenset({"special-package"}) + ) + checker = LicenseChecker(config=custom_config) + + assert "special-package" in checker.config.exceptions + assert len(checker.config.allowed_licenses) == 2 + + +def test_dependency_parsing_scenarios() -> None: + """Test various dependency parsing scenarios.""" + + def parse_dependencies(toml_string: str) -> list[str]: + checker = LicenseChecker() + return checker._parse_dependency_strings(tomli.loads(toml_string)["project"]["dependencies"]) + + # basic version specifiers + parsed = parse_dependencies(""" + [project] + dependencies = [ + "requests>=2.28.0", + "tomli==2.0.1", + "urllib3<2.0.0", + ] + """) + assert sorted(parsed) == sorted(["requests", "tomli", "urllib3"]) + + # workspace dependencies + parsed = parse_dependencies(""" + [project] + dependencies = [ + "package-with-workspace{workspace = true}", + ] + """) + assert parsed == [] + + # mixed dependencies + parsed = parse_dependencies(""" + [project] + dependencies = [ + "requests>=2.28.0", + "package-with-workspace{workspace = true}", + "urllib3<2.0.0", + ] + """) + assert sorted(parsed) == sorted(["requests", "urllib3"]) + + # multiple version constraints + parsed = parse_dependencies(""" + [project] + dependencies = [ + "urllib3>=1.25.4,<2.0.0", + "requests>=2.28.0,<3.0.0", + ] + """) + assert sorted(parsed) == sorted(["urllib3", "requests"]) + + # empty dependencies + parsed = parse_dependencies(""" + [project] + dependencies = [] + """) + assert parsed == [] + + +@patch.object(LicenseChecker, "get_package_license") +def test_multiple_toml_files( + mock_get_license: Mock, + mock_toml_files: list[Path], + capsys: pytest.CaptureFixture, +) -> None: + def get_license(package: str) -> Optional[str]: + licenses = {"requests": "Apache-2.0", "tomli": "MIT", "urllib3": "MIT"} + return licenses.get(package) + + mock_get_license.side_effect = get_license + + with patch("sys.argv", ["check_licenses.py"] + [str(f) for f in mock_toml_files]): + try: + main() + except SystemExit as e: + assert e.code == 0 + + captured = capsys.readouterr() + assert "requests" in captured.out + assert "tomli" in captured.out + assert "urllib3" in captured.out + assert "✗" not in captured.out diff --git a/justfile b/justfile index 65b8e424c1..f18f5ecf26 100644 --- a/justfile +++ b/justfile @@ -4,12 +4,23 @@ default: # run tests test *FLAGS: + #! /usr/bin/env bash uv run pytest tests -m "not integration" {{ FLAGS }} + if ! git diff --quiet pyproject.toml; then + uv run pytest .github/workflows {{ FLAGS }} + fi # run integration tests integration *FLAGS: uv run pytest tests -m integration {{ FLAGS }} +# check licenses +check-licenses: + #!/usr/bin/env bash + if ! git diff --quiet pyproject.toml; then + uv run .github/workflows/scripts/check_licenses.py pyproject.toml + fi + # format code format: #!/usr/bin/env bash @@ -47,6 +58,7 @@ install-hooks: #!/usr/bin/env bash just format + just check-licenses EOF if [ ! -f "$HOOKS_DIR/pre-commit" ]; then