"""Pluggable, credential-free image source registry and its HTTPS layer. Nothing in this module reads an API key, token, registration or secret from arguments, environment variables or configuration files: Wikimedia Commons and Openverse are queried anonymously. The module provides * :class:`HTTPSClient` -- a small HTTPS client that identifies itself with a stable ``User-Agent``, refuses every URL outside the approved hosts (Wikimedia Commons and Openverse) on port 443, retries HTTP 429/5xx a bounded number of times with capped exponential backoff, and raises :class:`~tools.errors.SourceAccessError` instead of returning an empty result when the host is unreachable or keeps failing; * :class:`SourceAdapter` -- the interface every image source implements; * :class:`WikimediaCommonsAdapter` and :class:`OpenverseAdapter` -- two working credential-free sources behind that interface; * :class:`SourceRegistry` -- the plugin list that reports available source ids and turns an unknown id into :class:`~tools.errors.UnknownSourceError` listing them. """ from __future__ import annotations import json import time import urllib.error import urllib.parse import urllib.request from dataclasses import dataclass, field from typing import Any, Callable, Iterable, Mapping, Protocol, runtime_checkable from .errors import ( DuplicateSourceError, HostNotAllowedError, SearchRequestError, SourceAccessError, SourceHTTPError, SourceError, SourcePayloadError, SourceUnreachableError, UnknownSourceError, ) __all__ = [ "APPROVED_HOSTS", "DEFAULT_LIMIT", "DEFAULT_MAX_ATTEMPTS", "HTTPSClient", "HTTPResponse", "MAX_LIMIT", "OPENVERSE_API_HOST", "OpenverseAdapter", "SourceAdapter", "SourceCandidate", "SourceRegistry", "USER_AGENT", "WIKIMEDIA_COMMONS_API_HOST", "WIKIMEDIA_COMMONS_MEDIA_HOST", "build_registry", "default_registry", "is_approved_url", "search", "validate_source_url", ] PRODUCT_NAME = "product-54354138-1" USER_AGENT = f"{PRODUCT_NAME}-image-toolkit/0.1.0 (credential-free image source client)" WIKIMEDIA_COMMONS_API_HOST = "commons.wikimedia.org" WIKIMEDIA_COMMONS_MEDIA_HOST = "upload.wikimedia.org" OPENVERSE_API_HOST = "api.openverse.org" # Wikimedia Commons is reachable through its API host plus the media host that # serves the original files, Openverse through its API host (and the thumbnail # endpoint on the same API host). APPROVED_HOSTS = frozenset( {WIKIMEDIA_COMMONS_API_HOST, WIKIMEDIA_COMMONS_MEDIA_HOST, OPENVERSE_API_HOST} ) APPROVED_SCHEME = "https" APPROVED_PORT = 443 DEFAULT_TIMEOUT = 10.0 DEFAULT_MAX_ATTEMPTS = 3 DEFAULT_BACKOFF_SECONDS = 0.5 MAX_BACKOFF_SECONDS = 8.0 RETRY_STATUS_CODES = frozenset({429}) | frozenset(range(500, 600)) @dataclass(frozen=True) class HTTPResponse: """A transport-independent HTTP response.""" status: int body: bytes headers: Mapping[str, str] = field(default_factory=dict) def json(self, *, url: str = "", source_id: str | None = None) -> Any: try: return json.loads(self.body) except (ValueError, UnicodeDecodeError) as exc: # pragma: no cover - defensive raise SourcePayloadError(url or "", source_id=source_id, reason=str(exc)) from exc class Transport(Protocol): """Callable performing one HTTP GET; injectable so tests never hit the net.""" def __call__(self, url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse: ... def validate_source_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> str: """Return *url* when it is an approved HTTPS URL, else raise a typed error.""" allowed_set = frozenset(allowed) if not isinstance(url, str) or not url: raise HostNotAllowedError(str(url), allowed=allowed_set) parts = urllib.parse.urlsplit(url) if parts.scheme != APPROVED_SCHEME: raise HostNotAllowedError(url, allowed=allowed_set) host = (parts.hostname or "").lower() if host not in allowed_set: raise HostNotAllowedError(url, allowed=allowed_set) try: port = parts.port except ValueError as exc: raise HostNotAllowedError(url, allowed=allowed_set) from exc if port not in (None, APPROVED_PORT): raise HostNotAllowedError(url, allowed=allowed_set) return url def is_approved_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> bool: """Non-raising variant of :func:`validate_source_url` used by tests/callers.""" try: validate_source_url(url, allowed=allowed) except HostNotAllowedError: return False return True def urllib_transport(url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse: """Default transport: a plain stdlib GET returning status, body and headers.""" request = urllib.request.Request(url, headers=dict(headers), method="GET") try: with urllib.request.urlopen(request, timeout=timeout) as response: return HTTPResponse( status=int(getattr(response, "status", 200)), body=response.read(), headers=dict(response.headers.items()), ) except urllib.error.HTTPError as exc: body = b"" try: body = exc.read() except Exception: # pragma: no cover - body optional body = b"" return HTTPResponse( status=int(exc.code), body=body, headers=dict(getattr(exc, "headers", {}) or {}), ) class HTTPSClient: """Fetch a URL only when it is approved, with bounded retry/backoff. The client never returns an empty result for a broken source: unreachable hosts raise :class:`SourceUnreachableError` and exhausted/non-retryable statuses raise :class:`SourceHTTPError`. Both derive from :class:`SourceAccessError`. """ def __init__( self, *, user_agent: str = USER_AGENT, allowed_hosts: Iterable[str] = APPROVED_HOSTS, timeout: float = DEFAULT_TIMEOUT, max_attempts: int = DEFAULT_MAX_ATTEMPTS, backoff_seconds: float = DEFAULT_BACKOFF_SECONDS, max_backoff_seconds: float = MAX_BACKOFF_SECONDS, transport: Transport | None = None, sleep: Callable[[float], None] = time.sleep, ) -> None: if max_attempts < 1: raise ValueError("max_attempts must be >= 1") self.user_agent = user_agent self.allowed_hosts = frozenset(allowed_hosts) self.timeout = float(timeout) self.max_attempts = int(max_attempts) self.backoff_seconds = float(backoff_seconds) self.max_backoff_seconds = float(max_backoff_seconds) self._transport: Transport = transport or urllib_transport self._sleep = sleep # -- public API ----------------------------------------------------- def headers(self) -> dict[str, str]: """Request headers, always including the identifying User-Agent.""" return { "User-Agent": self.user_agent, "Accept": "application/json, image/*;q=0.8, */*;q=0.5", } def backoff_delay(self, attempt: int) -> float: """Capped exponential backoff delay for 1-based *attempt*.""" delay = self.backoff_seconds * (2 ** max(0, attempt - 1)) return min(delay, self.max_backoff_seconds) def get(self, url: str, *, source_id: str | None = None) -> HTTPResponse: """GET *url*, retrying transient statuses, then raise typed errors.""" validate_source_url(url, allowed=self.allowed_hosts) response: HTTPResponse | None = None for attempt in range(1, self.max_attempts + 1): try: response = self._transport(url, self.headers(), self.timeout) except SourceAccessError: raise except Exception as exc: # transport-level failure if attempt >= self.max_attempts: raise SourceUnreachableError( url, source_id=source_id, reason=f"{type(exc).__name__}: {exc}" ) from exc self._sleep(self.backoff_delay(attempt)) continue if response.status in RETRY_STATUS_CODES and attempt < self.max_attempts: self._sleep(self.backoff_delay(attempt)) continue if 200 <= response.status < 300: return response raise SourceHTTPError(url, response.status, source_id=source_id) # Defensive: the loop either returned or raised above. status = response.status if response is not None else 0 raise SourceHTTPError(url, status, source_id=source_id) def get_json(self, url: str, *, source_id: str | None = None) -> Any: response = self.get(url, source_id=source_id) try: return json.loads(response.body) except (ValueError, UnicodeDecodeError) as exc: raise SourcePayloadError(url, source_id=source_id, reason=str(exc)) from exc DEFAULT_LIMIT = 10 MAX_LIMIT = 50 @dataclass(frozen=True) class SourceCandidate: """One normalized search hit. ``source_id`` identifies the adapter, ``file_url`` is the direct HTTPS location of the original image, ``mime_type`` is the type the source reports (may be ``None`` when unstated) and ``fetch_bytes`` is a zero-argument callable returning the image bytes through the approved HTTPS client. """ source_id: str file_url: str mime_type: str | None fetch_bytes: Callable[[], bytes] title: str | None = None page_url: str | None = None width: int | None = None height: int | None = None extra: Mapping[str, Any] = field(default_factory=dict) def fetch(self) -> bytes: """Convenience passthrough to :attr:`fetch_bytes`.""" return self.fetch_bytes() def to_dict(self) -> dict[str, Any]: """Metadata-only projection (the callable is intentionally omitted).""" payload: dict[str, Any] = { "source_id": self.source_id, "file_url": self.file_url, "mime_type": self.mime_type, } for key in ("title", "page_url", "width", "height"): value = getattr(self, key) if value is not None: payload[key] = value return payload @runtime_checkable class SourceAdapter(Protocol): """Interface implemented by every image source.""" source_id: str def search(self, topic: str, limit: int) -> list[SourceCandidate]: ... def normalize_search_args(topic: str, limit: int) -> tuple[str, int]: """Validate the caller-supplied topic and result limit.""" if not isinstance(topic, str): raise SearchRequestError( f"topic must be a string, got {type(topic).__name__}", details={"topic": repr(topic)}, ) topic = topic.strip() if not topic: raise SearchRequestError("topic must not be empty", details={"topic": topic}) if isinstance(limit, bool) or not isinstance(limit, int): raise SearchRequestError( f"limit must be an integer, got {type(limit).__name__}", details={"limit": repr(limit)}, ) if limit < 1: raise SearchRequestError(f"limit must be >= 1, got {limit}", details={"limit": limit}) if limit > MAX_LIMIT: raise SearchRequestError( f"limit must be <= {MAX_LIMIT}, got {limit}", details={"limit": limit} ) return topic, limit class _BaseAdapter: """Shared normalization helpers for concrete adapters.""" source_id = "" def __init__(self, client: HTTPSClient | None = None) -> None: self.client = client or HTTPSClient() def _candidate( self, *, file_url: str, mime_type: str | None, title: str | None = None, page_url: str | None = None, width: int | None = None, height: int | None = None, extra: Mapping[str, Any] | None = None, validate: bool = True, ) -> SourceCandidate: if validate: validate_source_url(file_url, allowed=self.client.allowed_hosts) source_id = self.source_id client = self.client return SourceCandidate( source_id=source_id, file_url=file_url, mime_type=mime_type, fetch_bytes=lambda url=file_url: client.get(url, source_id=source_id).body, title=title, page_url=page_url, width=width, height=height, extra=dict(extra or {}), ) def __repr__(self) -> str: # pragma: no cover - trivial return f"<{type(self).__name__} source_id={self.source_id!r}>" class WikimediaCommonsAdapter(_BaseAdapter): """Wikimedia Commons search through the anonymous MediaWiki action API.""" source_id = "wikimedia_commons" api_host = WIKIMEDIA_COMMONS_API_HOST def build_search_url(self, topic: str, limit: int) -> str: """Construct the generator search URL for *topic* and *limit*.""" topic, limit = normalize_search_args(topic, limit) params = { "action": "query", "format": "json", "formatversion": "2", "generator": "search", "gsrsearch": topic, "gsrnamespace": "6", "gsrlimit": str(limit), "prop": "imageinfo", "iiprop": "url|mime|size", "iiurlwidth": "1024", } query = urllib.parse.urlencode(params) return f"https://{self.api_host}/w/api.php?{query}" def search(self, topic: str, limit: int) -> list[SourceCandidate]: url = self.build_search_url(topic, limit) payload = self.client.get_json(url, source_id=self.source_id) if not isinstance(payload, Mapping): raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object") if "error" in payload: raise SourceAccessError( f"Wikimedia Commons API error: {payload['error']}", url=url, source_id=self.source_id, ) pages = (payload.get("query") or {}).get("pages") or [] candidates: list[SourceCandidate] = [] for page in pages: infos = page.get("imageinfo") or [] if not infos: continue info = infos[0] file_url = info.get("url") if not file_url: continue candidates.append( self._candidate( file_url=file_url, mime_type=info.get("mime"), title=page.get("title"), page_url=info.get("descriptionurl"), width=info.get("width"), height=info.get("height"), extra={"thumb_url": info.get("thumburl")} if info.get("thumburl") else None, ) ) if len(candidates) >= limit: break return candidates[:limit] class OpenverseAdapter(_BaseAdapter): """Openverse search through its anonymous public images endpoint.""" source_id = "openverse" api_host = OPENVERSE_API_HOST def build_search_url(self, topic: str, limit: int) -> str: """Construct the Openverse image search URL for *topic* and *limit*.""" topic, limit = normalize_search_args(topic, limit) query = urllib.parse.urlencode({"q": topic, "page_size": str(limit)}) return f"https://{self.api_host}/v1/images/?{query}" def search(self, topic: str, limit: int) -> list[SourceCandidate]: url = self.build_search_url(topic, limit) payload = self.client.get_json(url, source_id=self.source_id) if not isinstance(payload, Mapping): raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object") results = payload.get("results") if results is None: raise SourcePayloadError(url, source_id=self.source_id, reason="missing 'results'") candidates: list[SourceCandidate] = [] for item in results: if not isinstance(item, Mapping): continue original_url = item.get("url") file_url = item.get("thumbnail") or original_url if not file_url: continue candidates.append( self._candidate( file_url=file_url, # Origin files may live on third-party hosts; the client # still refuses them at fetch time with a typed error. validate=is_approved_url(file_url, allowed=self.client.allowed_hosts), mime_type=item.get("filetype"), title=item.get("title"), page_url=item.get("foreign_landing_url") or item.get("detail_url"), width=item.get("width"), height=item.get("height"), extra={ "license": item.get("license"), "creator": item.get("creator"), "original_url": original_url, }, ) ) if len(candidates) >= limit: break return candidates[:limit] class SourceRegistry: """Ordered plugin registry of credential-free image sources.""" def __init__(self, adapters: Iterable[SourceAdapter] | None = None) -> None: self._adapters: dict[str, SourceAdapter] = {} for adapter in adapters or (): self.register(adapter) # -- registration --------------------------------------------------- def register(self, adapter: SourceAdapter, *, replace: bool = False) -> SourceAdapter: """Add *adapter*; a duplicate id raises :class:`DuplicateSourceError`.""" source_id = getattr(adapter, "source_id", None) if not isinstance(source_id, str) or not source_id: raise SourceError("adapter must expose a non-empty string 'source_id'") if source_id in self._adapters and not replace: raise DuplicateSourceError(source_id) self._adapters[source_id] = adapter return adapter def unregister(self, source_id: str) -> SourceAdapter: if source_id not in self._adapters: raise UnknownSourceError(source_id, self.available_sources()) return self._adapters.pop(source_id) # -- lookup --------------------------------------------------------- def available_sources(self) -> list[str]: """Sorted ids of every registered source.""" return sorted(self._adapters) def __contains__(self, source_id: object) -> bool: return isinstance(source_id, str) and source_id in self._adapters def __len__(self) -> int: return len(self._adapters) def __iter__(self): return iter(self.available_sources()) def get(self, source_id: str) -> SourceAdapter: """Return the adapter for *source_id* or raise a listing error.""" try: return self._adapters[source_id] except (KeyError, TypeError): raise UnknownSourceError(source_id, self.available_sources()) from None def create(self, source_id: str) -> SourceAdapter: """Explicit alias of :meth:`get` used by the CLI layer.""" return self.get(source_id) # -- search --------------------------------------------------------- def search(self, source_id: str, topic: str, limit: int = DEFAULT_LIMIT) -> list[SourceCandidate]: """Search *source_id* for *topic*, returning at most *limit* candidates.""" adapter = self.get(source_id) topic, limit = normalize_search_args(topic, limit) return list(adapter.search(topic, limit)) def __repr__(self) -> str: # pragma: no cover - trivial return f"SourceRegistry({self.available_sources()!r})" def build_registry( *, client: HTTPSClient | None = None, adapters: Iterable[SourceAdapter] | None = None, ) -> SourceRegistry: """Build the default registry (Wikimedia Commons + Openverse).""" shared_client = client or HTTPSClient() if adapters is None: adapters = ( WikimediaCommonsAdapter(shared_client), OpenverseAdapter(shared_client), ) return SourceRegistry(adapters) def default_registry(client: HTTPSClient | None = None) -> SourceRegistry: """Convenience wrapper returning a fresh default registry.""" return build_registry(client=client) def search( topic: str, *, source: str = "wikimedia_commons", limit: int = DEFAULT_LIMIT, registry: SourceRegistry | None = None, client: HTTPSClient | None = None, ) -> list[SourceCandidate]: """Search *source* for *topic* using *registry* (default registry by default).""" active = registry if registry is not None else build_registry(client=client) return active.search(source, topic, limit)