feat(extensions,presets): authenticate GitHub-hosted catalog and download requests with GITHUB_TOKEN/GH_TOKEN (#2331)

* feat(extensions,presets): authenticate GitHub-hosted catalog and download requests with GITHUB_TOKEN/GH_TOKEN

Squashed from #2087 (original author: @anasseth).

Adds GitHub-token authentication to extension and preset catalog fetching
and ZIP downloads so private GitHub repos work when GITHUB_TOKEN/GH_TOKEN
is set, while preventing credential leakage to non-GitHub hosts.

- Introduces shared _github_http module with build_github_request() and
  open_github_url() helpers
- Routes ExtensionCatalog and PresetCatalog network calls through
  GitHub-auth-aware opener
- Adds comprehensive unit/integration tests for auth header behavior
- Updates user docs for both extensions and presets

Co-authored-by: anasseth <16745089+anasseth@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix(auth): address review feedback from #2087

- Fix redirect handler to preserve Authorization on GitHub-to-GitHub
  redirects (e.g. github.com → codeload.github.com). The previous
  implementation relied on super().redirect_request() which strips
  auth on cross-host redirects, breaking private repo archive downloads.
- Add codeload.github.com to documented host lists in both
  EXTENSION-USER-GUIDE.md and presets/README.md
- Add redirect auth-preservation and auth-stripping tests

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix(auth): use Bearer scheme instead of token for consistency

Aligns with the rest of the codebase (e.g. __init__.py:1721) and
GitHub's current API guidance. Updates all test assertions accordingly.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: address second round of Copilot review feedback

- Fix docstring to say Bearer instead of token (matches implementation)
- Remove unused imports/fixtures from redirect tests (GITHUB_HOSTS,
  MagicMock, temp_dir, monkeypatch)
- Replace __import__('io').BytesIO() with normal import io pattern
  in test_presets.py

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: anasseth <16745089+anasseth@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Taylor Mulder
2026-04-24 15:17:40 -04:00
committed by GitHub
parent ca51d739fb
commit 232c19cb04
7 changed files with 522 additions and 20 deletions

View File

@@ -2416,6 +2416,215 @@ class TestExtensionCatalog:
assert not catalog.cache_file.exists()
assert not catalog.cache_metadata_file.exists()
# --- _make_request / GitHub auth ---
def _make_catalog(self, temp_dir):
project_dir = temp_dir / "project"
project_dir.mkdir()
(project_dir / ".specify").mkdir()
return ExtensionCatalog(project_dir)
def test_make_request_no_token_no_auth_header(self, temp_dir, monkeypatch):
"""Without a token, requests carry no Authorization header."""
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
monkeypatch.delenv("GH_TOKEN", raising=False)
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json")
assert "Authorization" not in req.headers
def test_make_request_whitespace_only_github_token_ignored(self, temp_dir, monkeypatch):
"""A whitespace-only GITHUB_TOKEN is treated as unset."""
monkeypatch.setenv("GITHUB_TOKEN", " ")
monkeypatch.delenv("GH_TOKEN", raising=False)
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json")
assert "Authorization" not in req.headers
def test_make_request_whitespace_github_token_falls_back_to_gh_token(self, temp_dir, monkeypatch):
"""When GITHUB_TOKEN is whitespace-only, GH_TOKEN is used as fallback."""
monkeypatch.setenv("GITHUB_TOKEN", " ")
monkeypatch.setenv("GH_TOKEN", "ghp_fallback")
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json")
assert req.get_header("Authorization") == "Bearer ghp_fallback"
def test_make_request_github_token_added_for_raw_githubusercontent(self, temp_dir, monkeypatch):
"""GITHUB_TOKEN is attached for raw.githubusercontent.com URLs."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
monkeypatch.delenv("GH_TOKEN", raising=False)
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json")
assert req.get_header("Authorization") == "Bearer ghp_testtoken"
def test_make_request_gh_token_fallback(self, temp_dir, monkeypatch):
"""GH_TOKEN is used when GITHUB_TOKEN is absent."""
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
monkeypatch.setenv("GH_TOKEN", "ghp_ghtoken")
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://github.com/org/repo/releases/download/v1/ext.zip")
assert req.get_header("Authorization") == "Bearer ghp_ghtoken"
def test_make_request_github_token_takes_precedence_over_gh_token(self, temp_dir, monkeypatch):
"""GITHUB_TOKEN takes precedence over GH_TOKEN when both are set."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_primary")
monkeypatch.setenv("GH_TOKEN", "ghp_secondary")
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://api.github.com/repos/org/repo")
assert req.get_header("Authorization") == "Bearer ghp_primary"
def test_make_request_token_not_added_for_non_github_url(self, temp_dir, monkeypatch):
"""Auth header is never attached to non-GitHub URLs to prevent credential leakage."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://internal.example.com/catalog.json")
assert "Authorization" not in req.headers
def test_make_request_token_not_added_for_github_lookalike_host(self, temp_dir, monkeypatch):
"""Auth header is not attached to hosts that include github.com as a suffix."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://github.com.evil.com/org/repo/releases/download/v1/ext.zip")
assert "Authorization" not in req.headers
def test_make_request_token_not_added_for_github_in_path(self, temp_dir, monkeypatch):
"""Auth header is not attached when github.com appears only in the URL path."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://evil.example.com/github.com/org/repo/releases/download/v1/ext.zip")
assert "Authorization" not in req.headers
def test_make_request_token_not_added_for_github_in_query(self, temp_dir, monkeypatch):
"""Auth header is not attached when github.com appears only in the query string."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://evil.example.com/download?source=https://github.com/org/repo/v1/ext.zip")
assert "Authorization" not in req.headers
def test_make_request_token_added_for_api_github_com(self, temp_dir, monkeypatch):
"""GITHUB_TOKEN is attached for api.github.com URLs."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://api.github.com/repos/org/repo/releases/assets/1")
assert req.get_header("Authorization") == "Bearer ghp_testtoken"
def test_make_request_token_added_for_codeload_github_com(self, temp_dir, monkeypatch):
"""GITHUB_TOKEN is attached for codeload.github.com URLs (GitHub archive redirects)."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = self._make_catalog(temp_dir)
req = catalog._make_request("https://codeload.github.com/org/repo/zip/refs/tags/v1.0.0")
assert req.get_header("Authorization") == "Bearer ghp_testtoken"
def test_redirect_preserves_auth_for_github_to_codeload(self):
"""Auth header is preserved when GitHub redirects to codeload.github.com."""
from specify_cli._github_http import _StripAuthOnRedirect
from urllib.request import Request
import io
handler = _StripAuthOnRedirect()
original_url = "https://github.com/org/repo/archive/refs/tags/v1.zip"
redirect_url = "https://codeload.github.com/org/repo/zip/refs/tags/v1"
req = Request(original_url, headers={"Authorization": "Bearer ghp_test"})
fp = io.BytesIO(b"")
new_req = handler.redirect_request(req, fp, 302, "Found", {}, redirect_url)
assert new_req is not None
auth = new_req.get_header("Authorization") or new_req.unredirected_hdrs.get("Authorization")
assert auth == "Bearer ghp_test"
def test_redirect_strips_auth_for_github_to_external(self):
"""Auth header is stripped when GitHub redirects to a non-GitHub host."""
from specify_cli._github_http import _StripAuthOnRedirect
from urllib.request import Request
import io
handler = _StripAuthOnRedirect()
original_url = "https://github.com/org/repo/releases/download/v1/asset.zip"
redirect_url = "https://objects.githubusercontent.com/github-production-release-asset/12345"
req = Request(original_url, headers={"Authorization": "Bearer ghp_test"})
fp = io.BytesIO(b"")
new_req = handler.redirect_request(req, fp, 302, "Found", {}, redirect_url)
assert new_req is not None
auth_header = new_req.headers.get("Authorization")
auth_unredirected = new_req.unredirected_hdrs.get("Authorization")
assert auth_header is None
assert auth_unredirected is None
def test_fetch_single_catalog_sends_auth_header(self, temp_dir, monkeypatch):
"""_fetch_single_catalog passes Authorization header via opener for GitHub URLs."""
from unittest.mock import patch, MagicMock
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = self._make_catalog(temp_dir)
catalog_data = {"schema_version": "1.0", "extensions": {}}
mock_response = MagicMock()
mock_response.read.return_value = json.dumps(catalog_data).encode()
mock_response.__enter__ = lambda s: s
mock_response.__exit__ = MagicMock(return_value=False)
captured = {}
mock_opener = MagicMock()
def fake_open(req, timeout=None):
captured["req"] = req
return mock_response
mock_opener.open.side_effect = fake_open
entry = CatalogEntry(
url="https://raw.githubusercontent.com/org/repo/main/catalog.json",
name="private",
priority=1,
install_allowed=True,
)
with patch("urllib.request.build_opener", return_value=mock_opener):
catalog._fetch_single_catalog(entry, force_refresh=True)
assert captured["req"].get_header("Authorization") == "Bearer ghp_testtoken"
def test_download_extension_sends_auth_header(self, temp_dir, monkeypatch):
"""download_extension passes Authorization header via opener for GitHub URLs."""
from unittest.mock import patch, MagicMock
import zipfile, io
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = self._make_catalog(temp_dir)
# Build a minimal valid ZIP in memory
zip_buf = io.BytesIO()
with zipfile.ZipFile(zip_buf, "w") as zf:
zf.writestr("extension.yml", "id: test-ext\nname: Test\nversion: 1.0.0\n")
zip_bytes = zip_buf.getvalue()
mock_response = MagicMock()
mock_response.read.return_value = zip_bytes
mock_response.__enter__ = lambda s: s
mock_response.__exit__ = MagicMock(return_value=False)
captured = {}
mock_opener = MagicMock()
def fake_open(req, timeout=None):
captured["req"] = req
return mock_response
mock_opener.open.side_effect = fake_open
ext_info = {
"id": "test-ext",
"name": "Test Extension",
"version": "1.0.0",
"download_url": "https://github.com/org/repo/releases/download/v1/test-ext.zip",
}
with patch.object(catalog, "get_extension_info", return_value=ext_info), \
patch("urllib.request.build_opener", return_value=mock_opener):
catalog.download_extension("test-ext", target_dir=temp_dir)
assert captured["req"].get_header("Authorization") == "Bearer ghp_testtoken"
# ===== CatalogEntry Tests =====

View File

@@ -1363,6 +1363,166 @@ class TestPresetCatalog:
catalog = PresetCatalog(project_dir)
assert catalog.get_catalog_url() == "https://custom.example.com/catalog.json"
# --- _make_request / GitHub auth ---
def test_make_request_no_token_no_auth_header(self, project_dir, monkeypatch):
"""Without a token, requests carry no Authorization header."""
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
monkeypatch.delenv("GH_TOKEN", raising=False)
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json")
assert "Authorization" not in req.headers
def test_make_request_whitespace_only_github_token_ignored(self, project_dir, monkeypatch):
"""A whitespace-only GITHUB_TOKEN is treated as unset."""
monkeypatch.setenv("GITHUB_TOKEN", " ")
monkeypatch.delenv("GH_TOKEN", raising=False)
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json")
assert "Authorization" not in req.headers
def test_make_request_whitespace_github_token_falls_back_to_gh_token(self, project_dir, monkeypatch):
"""When GITHUB_TOKEN is whitespace-only, GH_TOKEN is used as fallback."""
monkeypatch.setenv("GITHUB_TOKEN", " ")
monkeypatch.setenv("GH_TOKEN", "ghp_fallback")
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json")
assert req.get_header("Authorization") == "Bearer ghp_fallback"
def test_make_request_github_token_added_for_github_url(self, project_dir, monkeypatch):
"""GITHUB_TOKEN is attached for raw.githubusercontent.com URLs."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
monkeypatch.delenv("GH_TOKEN", raising=False)
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json")
assert req.get_header("Authorization") == "Bearer ghp_testtoken"
def test_make_request_gh_token_fallback(self, project_dir, monkeypatch):
"""GH_TOKEN is used when GITHUB_TOKEN is absent."""
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
monkeypatch.setenv("GH_TOKEN", "ghp_ghtoken")
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://github.com/org/repo/releases/download/v1/pack.zip")
assert req.get_header("Authorization") == "Bearer ghp_ghtoken"
def test_make_request_github_token_takes_precedence(self, project_dir, monkeypatch):
"""GITHUB_TOKEN takes precedence over GH_TOKEN when both are set."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_primary")
monkeypatch.setenv("GH_TOKEN", "ghp_secondary")
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://api.github.com/repos/org/repo")
assert req.get_header("Authorization") == "Bearer ghp_primary"
def test_make_request_token_added_for_codeload_github_com(self, project_dir, monkeypatch):
"""GITHUB_TOKEN is attached for codeload.github.com URLs (GitHub archive redirects)."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://codeload.github.com/org/repo/zip/refs/tags/v1.0.0")
assert req.get_header("Authorization") == "Bearer ghp_testtoken"
def test_make_request_token_not_added_for_non_github_url(self, project_dir, monkeypatch):
"""Auth header is never attached to non-GitHub URLs to prevent credential leakage."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://internal.example.com/catalog.json")
assert "Authorization" not in req.headers
def test_make_request_token_not_added_for_github_lookalike_host(self, project_dir, monkeypatch):
"""Auth header is not attached to hosts that include github.com as a suffix."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://github.com.evil.com/org/repo/releases/download/v1/pack.zip")
assert "Authorization" not in req.headers
def test_make_request_token_not_added_for_github_in_path(self, project_dir, monkeypatch):
"""Auth header is not attached when github.com appears only in the URL path."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://evil.example.com/github.com/org/repo/releases/download/v1/pack.zip")
assert "Authorization" not in req.headers
def test_make_request_token_not_added_for_github_in_query(self, project_dir, monkeypatch):
"""Auth header is not attached when github.com appears only in the query string."""
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = PresetCatalog(project_dir)
req = catalog._make_request("https://evil.example.com/download?source=https://github.com/org/repo/v1/pack.zip")
assert "Authorization" not in req.headers
def test_fetch_single_catalog_sends_auth_header(self, project_dir, monkeypatch):
"""_fetch_single_catalog passes Authorization header via opener for GitHub URLs."""
from unittest.mock import patch, MagicMock
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = PresetCatalog(project_dir)
catalog_data = {"schema_version": "1.0", "presets": {}}
mock_response = MagicMock()
mock_response.read.return_value = json.dumps(catalog_data).encode()
mock_response.__enter__ = lambda s: s
mock_response.__exit__ = MagicMock(return_value=False)
captured = {}
mock_opener = MagicMock()
def fake_open(req, timeout=None):
captured["req"] = req
return mock_response
mock_opener.open.side_effect = fake_open
entry = PresetCatalogEntry(
url="https://raw.githubusercontent.com/org/repo/main/presets/catalog.json",
name="private",
priority=1,
install_allowed=True,
)
with patch("urllib.request.build_opener", return_value=mock_opener):
catalog._fetch_single_catalog(entry, force_refresh=True)
assert captured["req"].get_header("Authorization") == "Bearer ghp_testtoken"
def test_download_pack_sends_auth_header(self, project_dir, monkeypatch):
"""download_pack passes Authorization header via opener for GitHub URLs."""
from unittest.mock import patch, MagicMock
monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken")
catalog = PresetCatalog(project_dir)
import io
zip_buf = io.BytesIO()
with zipfile.ZipFile(zip_buf, "w") as zf:
zf.writestr("preset.yml", "id: test-pack\nname: Test\nversion: 1.0.0\n")
zip_bytes = zip_buf.getvalue()
mock_response = MagicMock()
mock_response.read.return_value = zip_bytes
mock_response.__enter__ = lambda s: s
mock_response.__exit__ = MagicMock(return_value=False)
captured = {}
mock_opener = MagicMock()
def fake_open(req, timeout=None):
captured["req"] = req
return mock_response
mock_opener.open.side_effect = fake_open
pack_info = {
"id": "test-pack",
"name": "Test Pack",
"version": "1.0.0",
"download_url": "https://github.com/org/repo/releases/download/v1/test-pack.zip",
"_install_allowed": True,
}
with patch.object(catalog, "get_pack_info", return_value=pack_info), \
patch("urllib.request.build_opener", return_value=mock_opener):
catalog.download_pack("test-pack", target_dir=project_dir)
assert captured["req"].get_header("Authorization") == "Bearer ghp_testtoken"
# ===== Integration Tests =====