ai-tester
44473a1055
Repository Validate / repository-checks (push) Successful in 21s
Co-authored-by: AI Tester <ai-tester@git.p2prod.ru>
579 lines
21 KiB
Python
579 lines
21 KiB
Python
"""Pluggable, credential-free image source registry and its HTTPS layer.
|
|
|
|
Nothing in this module reads an API key, token, registration or secret from
|
|
arguments, environment variables or configuration files: Wikimedia Commons and
|
|
Openverse are queried anonymously. The module provides
|
|
|
|
* :class:`HTTPSClient` -- a small HTTPS client that identifies itself with a
|
|
stable ``User-Agent``, refuses every URL outside the approved hosts
|
|
(Wikimedia Commons and Openverse) on port 443, retries HTTP 429/5xx a bounded
|
|
number of times with capped exponential backoff, and raises
|
|
:class:`~tools.errors.SourceAccessError` instead of returning an empty result
|
|
when the host is unreachable or keeps failing;
|
|
* :class:`SourceAdapter` -- the interface every image source implements;
|
|
* :class:`WikimediaCommonsAdapter` and :class:`OpenverseAdapter` -- two working
|
|
credential-free sources behind that interface;
|
|
* :class:`SourceRegistry` -- the plugin list that reports available source ids
|
|
and turns an unknown id into :class:`~tools.errors.UnknownSourceError` listing
|
|
them.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Callable, Iterable, Mapping, Protocol, runtime_checkable
|
|
|
|
from .errors import (
|
|
DuplicateSourceError,
|
|
HostNotAllowedError,
|
|
SearchRequestError,
|
|
SourceAccessError,
|
|
SourceHTTPError,
|
|
SourceError,
|
|
SourcePayloadError,
|
|
SourceUnreachableError,
|
|
UnknownSourceError,
|
|
)
|
|
|
|
__all__ = [
|
|
"APPROVED_HOSTS",
|
|
"DEFAULT_LIMIT",
|
|
"DEFAULT_MAX_ATTEMPTS",
|
|
"HTTPSClient",
|
|
"HTTPResponse",
|
|
"MAX_LIMIT",
|
|
"OPENVERSE_API_HOST",
|
|
"OpenverseAdapter",
|
|
"SourceAdapter",
|
|
"SourceCandidate",
|
|
"SourceRegistry",
|
|
"USER_AGENT",
|
|
"WIKIMEDIA_COMMONS_API_HOST",
|
|
"WIKIMEDIA_COMMONS_MEDIA_HOST",
|
|
"build_registry",
|
|
"default_registry",
|
|
"is_approved_url",
|
|
"search",
|
|
"validate_source_url",
|
|
]
|
|
|
|
PRODUCT_NAME = "product-54354138-1"
|
|
USER_AGENT = f"{PRODUCT_NAME}-image-toolkit/0.1.0 (credential-free image source client)"
|
|
|
|
WIKIMEDIA_COMMONS_API_HOST = "commons.wikimedia.org"
|
|
WIKIMEDIA_COMMONS_MEDIA_HOST = "upload.wikimedia.org"
|
|
OPENVERSE_API_HOST = "api.openverse.org"
|
|
|
|
# Wikimedia Commons is reachable through its API host plus the media host that
|
|
# serves the original files, Openverse through its API host (and the thumbnail
|
|
# endpoint on the same API host).
|
|
APPROVED_HOSTS = frozenset(
|
|
{WIKIMEDIA_COMMONS_API_HOST, WIKIMEDIA_COMMONS_MEDIA_HOST, OPENVERSE_API_HOST}
|
|
)
|
|
APPROVED_SCHEME = "https"
|
|
APPROVED_PORT = 443
|
|
|
|
DEFAULT_TIMEOUT = 10.0
|
|
DEFAULT_MAX_ATTEMPTS = 3
|
|
DEFAULT_BACKOFF_SECONDS = 0.5
|
|
MAX_BACKOFF_SECONDS = 8.0
|
|
RETRY_STATUS_CODES = frozenset({429}) | frozenset(range(500, 600))
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class HTTPResponse:
|
|
"""A transport-independent HTTP response."""
|
|
|
|
status: int
|
|
body: bytes
|
|
headers: Mapping[str, str] = field(default_factory=dict)
|
|
|
|
def json(self, *, url: str = "", source_id: str | None = None) -> Any:
|
|
try:
|
|
return json.loads(self.body)
|
|
except (ValueError, UnicodeDecodeError) as exc: # pragma: no cover - defensive
|
|
raise SourcePayloadError(url or "<response>", source_id=source_id, reason=str(exc)) from exc
|
|
|
|
|
|
class Transport(Protocol):
|
|
"""Callable performing one HTTP GET; injectable so tests never hit the net."""
|
|
|
|
def __call__(self, url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse: ...
|
|
|
|
|
|
def validate_source_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> str:
|
|
"""Return *url* when it is an approved HTTPS URL, else raise a typed error."""
|
|
|
|
allowed_set = frozenset(allowed)
|
|
if not isinstance(url, str) or not url:
|
|
raise HostNotAllowedError(str(url), allowed=allowed_set)
|
|
parts = urllib.parse.urlsplit(url)
|
|
if parts.scheme != APPROVED_SCHEME:
|
|
raise HostNotAllowedError(url, allowed=allowed_set)
|
|
host = (parts.hostname or "").lower()
|
|
if host not in allowed_set:
|
|
raise HostNotAllowedError(url, allowed=allowed_set)
|
|
try:
|
|
port = parts.port
|
|
except ValueError as exc:
|
|
raise HostNotAllowedError(url, allowed=allowed_set) from exc
|
|
if port not in (None, APPROVED_PORT):
|
|
raise HostNotAllowedError(url, allowed=allowed_set)
|
|
return url
|
|
|
|
|
|
def is_approved_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> bool:
|
|
"""Non-raising variant of :func:`validate_source_url` used by tests/callers."""
|
|
|
|
try:
|
|
validate_source_url(url, allowed=allowed)
|
|
except HostNotAllowedError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def urllib_transport(url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse:
|
|
"""Default transport: a plain stdlib GET returning status, body and headers."""
|
|
|
|
request = urllib.request.Request(url, headers=dict(headers), method="GET")
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
return HTTPResponse(
|
|
status=int(getattr(response, "status", 200)),
|
|
body=response.read(),
|
|
headers=dict(response.headers.items()),
|
|
)
|
|
except urllib.error.HTTPError as exc:
|
|
body = b""
|
|
try:
|
|
body = exc.read()
|
|
except Exception: # pragma: no cover - body optional
|
|
body = b""
|
|
return HTTPResponse(
|
|
status=int(exc.code),
|
|
body=body,
|
|
headers=dict(getattr(exc, "headers", {}) or {}),
|
|
)
|
|
|
|
|
|
class HTTPSClient:
|
|
"""Fetch a URL only when it is approved, with bounded retry/backoff.
|
|
|
|
The client never returns an empty result for a broken source: unreachable
|
|
hosts raise :class:`SourceUnreachableError` and exhausted/non-retryable
|
|
statuses raise :class:`SourceHTTPError`. Both derive from
|
|
:class:`SourceAccessError`.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
user_agent: str = USER_AGENT,
|
|
allowed_hosts: Iterable[str] = APPROVED_HOSTS,
|
|
timeout: float = DEFAULT_TIMEOUT,
|
|
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
|
backoff_seconds: float = DEFAULT_BACKOFF_SECONDS,
|
|
max_backoff_seconds: float = MAX_BACKOFF_SECONDS,
|
|
transport: Transport | None = None,
|
|
sleep: Callable[[float], None] = time.sleep,
|
|
) -> None:
|
|
if max_attempts < 1:
|
|
raise ValueError("max_attempts must be >= 1")
|
|
self.user_agent = user_agent
|
|
self.allowed_hosts = frozenset(allowed_hosts)
|
|
self.timeout = float(timeout)
|
|
self.max_attempts = int(max_attempts)
|
|
self.backoff_seconds = float(backoff_seconds)
|
|
self.max_backoff_seconds = float(max_backoff_seconds)
|
|
self._transport: Transport = transport or urllib_transport
|
|
self._sleep = sleep
|
|
|
|
# -- public API -----------------------------------------------------
|
|
def headers(self) -> dict[str, str]:
|
|
"""Request headers, always including the identifying User-Agent."""
|
|
|
|
return {
|
|
"User-Agent": self.user_agent,
|
|
"Accept": "application/json, image/*;q=0.8, */*;q=0.5",
|
|
}
|
|
|
|
def backoff_delay(self, attempt: int) -> float:
|
|
"""Capped exponential backoff delay for 1-based *attempt*."""
|
|
|
|
delay = self.backoff_seconds * (2 ** max(0, attempt - 1))
|
|
return min(delay, self.max_backoff_seconds)
|
|
|
|
def get(self, url: str, *, source_id: str | None = None) -> HTTPResponse:
|
|
"""GET *url*, retrying transient statuses, then raise typed errors."""
|
|
|
|
validate_source_url(url, allowed=self.allowed_hosts)
|
|
response: HTTPResponse | None = None
|
|
for attempt in range(1, self.max_attempts + 1):
|
|
try:
|
|
response = self._transport(url, self.headers(), self.timeout)
|
|
except SourceAccessError:
|
|
raise
|
|
except Exception as exc: # transport-level failure
|
|
if attempt >= self.max_attempts:
|
|
raise SourceUnreachableError(
|
|
url, source_id=source_id, reason=f"{type(exc).__name__}: {exc}"
|
|
) from exc
|
|
self._sleep(self.backoff_delay(attempt))
|
|
continue
|
|
if response.status in RETRY_STATUS_CODES and attempt < self.max_attempts:
|
|
self._sleep(self.backoff_delay(attempt))
|
|
continue
|
|
if 200 <= response.status < 300:
|
|
return response
|
|
raise SourceHTTPError(url, response.status, source_id=source_id)
|
|
# Defensive: the loop either returned or raised above.
|
|
status = response.status if response is not None else 0
|
|
raise SourceHTTPError(url, status, source_id=source_id)
|
|
|
|
def get_json(self, url: str, *, source_id: str | None = None) -> Any:
|
|
response = self.get(url, source_id=source_id)
|
|
try:
|
|
return json.loads(response.body)
|
|
except (ValueError, UnicodeDecodeError) as exc:
|
|
raise SourcePayloadError(url, source_id=source_id, reason=str(exc)) from exc
|
|
|
|
|
|
DEFAULT_LIMIT = 10
|
|
MAX_LIMIT = 50
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SourceCandidate:
|
|
"""One normalized search hit.
|
|
|
|
``source_id`` identifies the adapter, ``file_url`` is the direct HTTPS
|
|
location of the original image, ``mime_type`` is the type the source
|
|
reports (may be ``None`` when unstated) and ``fetch_bytes`` is a zero-argument
|
|
callable returning the image bytes through the approved HTTPS client.
|
|
"""
|
|
|
|
source_id: str
|
|
file_url: str
|
|
mime_type: str | None
|
|
fetch_bytes: Callable[[], bytes]
|
|
title: str | None = None
|
|
page_url: str | None = None
|
|
width: int | None = None
|
|
height: int | None = None
|
|
extra: Mapping[str, Any] = field(default_factory=dict)
|
|
|
|
def fetch(self) -> bytes:
|
|
"""Convenience passthrough to :attr:`fetch_bytes`."""
|
|
|
|
return self.fetch_bytes()
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
"""Metadata-only projection (the callable is intentionally omitted)."""
|
|
|
|
payload: dict[str, Any] = {
|
|
"source_id": self.source_id,
|
|
"file_url": self.file_url,
|
|
"mime_type": self.mime_type,
|
|
}
|
|
for key in ("title", "page_url", "width", "height"):
|
|
value = getattr(self, key)
|
|
if value is not None:
|
|
payload[key] = value
|
|
return payload
|
|
|
|
|
|
@runtime_checkable
|
|
class SourceAdapter(Protocol):
|
|
"""Interface implemented by every image source."""
|
|
|
|
source_id: str
|
|
|
|
def search(self, topic: str, limit: int) -> list[SourceCandidate]: ...
|
|
|
|
|
|
def normalize_search_args(topic: str, limit: int) -> tuple[str, int]:
|
|
"""Validate the caller-supplied topic and result limit."""
|
|
|
|
if not isinstance(topic, str):
|
|
raise SearchRequestError(
|
|
f"topic must be a string, got {type(topic).__name__}",
|
|
details={"topic": repr(topic)},
|
|
)
|
|
topic = topic.strip()
|
|
if not topic:
|
|
raise SearchRequestError("topic must not be empty", details={"topic": topic})
|
|
if isinstance(limit, bool) or not isinstance(limit, int):
|
|
raise SearchRequestError(
|
|
f"limit must be an integer, got {type(limit).__name__}",
|
|
details={"limit": repr(limit)},
|
|
)
|
|
if limit < 1:
|
|
raise SearchRequestError(f"limit must be >= 1, got {limit}", details={"limit": limit})
|
|
if limit > MAX_LIMIT:
|
|
raise SearchRequestError(
|
|
f"limit must be <= {MAX_LIMIT}, got {limit}", details={"limit": limit}
|
|
)
|
|
return topic, limit
|
|
|
|
|
|
class _BaseAdapter:
|
|
"""Shared normalization helpers for concrete adapters."""
|
|
|
|
source_id = ""
|
|
|
|
def __init__(self, client: HTTPSClient | None = None) -> None:
|
|
self.client = client or HTTPSClient()
|
|
|
|
def _candidate(
|
|
self,
|
|
*,
|
|
file_url: str,
|
|
mime_type: str | None,
|
|
title: str | None = None,
|
|
page_url: str | None = None,
|
|
width: int | None = None,
|
|
height: int | None = None,
|
|
extra: Mapping[str, Any] | None = None,
|
|
validate: bool = True,
|
|
) -> SourceCandidate:
|
|
if validate:
|
|
validate_source_url(file_url, allowed=self.client.allowed_hosts)
|
|
source_id = self.source_id
|
|
client = self.client
|
|
return SourceCandidate(
|
|
source_id=source_id,
|
|
file_url=file_url,
|
|
mime_type=mime_type,
|
|
fetch_bytes=lambda url=file_url: client.get(url, source_id=source_id).body,
|
|
title=title,
|
|
page_url=page_url,
|
|
width=width,
|
|
height=height,
|
|
extra=dict(extra or {}),
|
|
)
|
|
|
|
def __repr__(self) -> str: # pragma: no cover - trivial
|
|
return f"<{type(self).__name__} source_id={self.source_id!r}>"
|
|
|
|
|
|
class WikimediaCommonsAdapter(_BaseAdapter):
|
|
"""Wikimedia Commons search through the anonymous MediaWiki action API."""
|
|
|
|
source_id = "wikimedia_commons"
|
|
api_host = WIKIMEDIA_COMMONS_API_HOST
|
|
|
|
def build_search_url(self, topic: str, limit: int) -> str:
|
|
"""Construct the generator search URL for *topic* and *limit*."""
|
|
|
|
topic, limit = normalize_search_args(topic, limit)
|
|
params = {
|
|
"action": "query",
|
|
"format": "json",
|
|
"formatversion": "2",
|
|
"generator": "search",
|
|
"gsrsearch": topic,
|
|
"gsrnamespace": "6",
|
|
"gsrlimit": str(limit),
|
|
"prop": "imageinfo",
|
|
"iiprop": "url|mime|size",
|
|
"iiurlwidth": "1024",
|
|
}
|
|
query = urllib.parse.urlencode(params)
|
|
return f"https://{self.api_host}/w/api.php?{query}"
|
|
|
|
def search(self, topic: str, limit: int) -> list[SourceCandidate]:
|
|
url = self.build_search_url(topic, limit)
|
|
payload = self.client.get_json(url, source_id=self.source_id)
|
|
if not isinstance(payload, Mapping):
|
|
raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object")
|
|
if "error" in payload:
|
|
raise SourceAccessError(
|
|
f"Wikimedia Commons API error: {payload['error']}",
|
|
url=url,
|
|
source_id=self.source_id,
|
|
)
|
|
pages = (payload.get("query") or {}).get("pages") or []
|
|
candidates: list[SourceCandidate] = []
|
|
for page in pages:
|
|
infos = page.get("imageinfo") or []
|
|
if not infos:
|
|
continue
|
|
info = infos[0]
|
|
file_url = info.get("url")
|
|
if not file_url:
|
|
continue
|
|
candidates.append(
|
|
self._candidate(
|
|
file_url=file_url,
|
|
mime_type=info.get("mime"),
|
|
title=page.get("title"),
|
|
page_url=info.get("descriptionurl"),
|
|
width=info.get("width"),
|
|
height=info.get("height"),
|
|
extra={"thumb_url": info.get("thumburl")} if info.get("thumburl") else None,
|
|
)
|
|
)
|
|
if len(candidates) >= limit:
|
|
break
|
|
return candidates[:limit]
|
|
|
|
|
|
class OpenverseAdapter(_BaseAdapter):
|
|
"""Openverse search through its anonymous public images endpoint."""
|
|
|
|
source_id = "openverse"
|
|
api_host = OPENVERSE_API_HOST
|
|
|
|
def build_search_url(self, topic: str, limit: int) -> str:
|
|
"""Construct the Openverse image search URL for *topic* and *limit*."""
|
|
|
|
topic, limit = normalize_search_args(topic, limit)
|
|
query = urllib.parse.urlencode({"q": topic, "page_size": str(limit)})
|
|
return f"https://{self.api_host}/v1/images/?{query}"
|
|
|
|
def search(self, topic: str, limit: int) -> list[SourceCandidate]:
|
|
url = self.build_search_url(topic, limit)
|
|
payload = self.client.get_json(url, source_id=self.source_id)
|
|
if not isinstance(payload, Mapping):
|
|
raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object")
|
|
results = payload.get("results")
|
|
if results is None:
|
|
raise SourcePayloadError(url, source_id=self.source_id, reason="missing 'results'")
|
|
candidates: list[SourceCandidate] = []
|
|
for item in results:
|
|
if not isinstance(item, Mapping):
|
|
continue
|
|
original_url = item.get("url")
|
|
file_url = item.get("thumbnail") or original_url
|
|
if not file_url:
|
|
continue
|
|
candidates.append(
|
|
self._candidate(
|
|
file_url=file_url,
|
|
# Origin files may live on third-party hosts; the client
|
|
# still refuses them at fetch time with a typed error.
|
|
validate=is_approved_url(file_url, allowed=self.client.allowed_hosts),
|
|
mime_type=item.get("filetype"),
|
|
title=item.get("title"),
|
|
page_url=item.get("foreign_landing_url") or item.get("detail_url"),
|
|
width=item.get("width"),
|
|
height=item.get("height"),
|
|
extra={
|
|
"license": item.get("license"),
|
|
"creator": item.get("creator"),
|
|
"original_url": original_url,
|
|
},
|
|
)
|
|
)
|
|
if len(candidates) >= limit:
|
|
break
|
|
return candidates[:limit]
|
|
|
|
|
|
class SourceRegistry:
|
|
"""Ordered plugin registry of credential-free image sources."""
|
|
|
|
def __init__(self, adapters: Iterable[SourceAdapter] | None = None) -> None:
|
|
self._adapters: dict[str, SourceAdapter] = {}
|
|
for adapter in adapters or ():
|
|
self.register(adapter)
|
|
|
|
# -- registration ---------------------------------------------------
|
|
def register(self, adapter: SourceAdapter, *, replace: bool = False) -> SourceAdapter:
|
|
"""Add *adapter*; a duplicate id raises :class:`DuplicateSourceError`."""
|
|
|
|
source_id = getattr(adapter, "source_id", None)
|
|
if not isinstance(source_id, str) or not source_id:
|
|
raise SourceError("adapter must expose a non-empty string 'source_id'")
|
|
if source_id in self._adapters and not replace:
|
|
raise DuplicateSourceError(source_id)
|
|
self._adapters[source_id] = adapter
|
|
return adapter
|
|
|
|
def unregister(self, source_id: str) -> SourceAdapter:
|
|
if source_id not in self._adapters:
|
|
raise UnknownSourceError(source_id, self.available_sources())
|
|
return self._adapters.pop(source_id)
|
|
|
|
# -- lookup ---------------------------------------------------------
|
|
def available_sources(self) -> list[str]:
|
|
"""Sorted ids of every registered source."""
|
|
|
|
return sorted(self._adapters)
|
|
|
|
def __contains__(self, source_id: object) -> bool:
|
|
return isinstance(source_id, str) and source_id in self._adapters
|
|
|
|
def __len__(self) -> int:
|
|
return len(self._adapters)
|
|
|
|
def __iter__(self):
|
|
return iter(self.available_sources())
|
|
|
|
def get(self, source_id: str) -> SourceAdapter:
|
|
"""Return the adapter for *source_id* or raise a listing error."""
|
|
|
|
try:
|
|
return self._adapters[source_id]
|
|
except (KeyError, TypeError):
|
|
raise UnknownSourceError(source_id, self.available_sources()) from None
|
|
|
|
def create(self, source_id: str) -> SourceAdapter:
|
|
"""Explicit alias of :meth:`get` used by the CLI layer."""
|
|
|
|
return self.get(source_id)
|
|
|
|
# -- search ---------------------------------------------------------
|
|
def search(self, source_id: str, topic: str, limit: int = DEFAULT_LIMIT) -> list[SourceCandidate]:
|
|
"""Search *source_id* for *topic*, returning at most *limit* candidates."""
|
|
|
|
adapter = self.get(source_id)
|
|
topic, limit = normalize_search_args(topic, limit)
|
|
return list(adapter.search(topic, limit))
|
|
|
|
def __repr__(self) -> str: # pragma: no cover - trivial
|
|
return f"SourceRegistry({self.available_sources()!r})"
|
|
|
|
|
|
|
|
|
|
def build_registry(
|
|
*,
|
|
client: HTTPSClient | None = None,
|
|
adapters: Iterable[SourceAdapter] | None = None,
|
|
) -> SourceRegistry:
|
|
"""Build the default registry (Wikimedia Commons + Openverse)."""
|
|
|
|
shared_client = client or HTTPSClient()
|
|
if adapters is None:
|
|
adapters = (
|
|
WikimediaCommonsAdapter(shared_client),
|
|
OpenverseAdapter(shared_client),
|
|
)
|
|
return SourceRegistry(adapters)
|
|
|
|
|
|
def default_registry(client: HTTPSClient | None = None) -> SourceRegistry:
|
|
"""Convenience wrapper returning a fresh default registry."""
|
|
|
|
return build_registry(client=client)
|
|
|
|
|
|
def search(
|
|
topic: str,
|
|
*,
|
|
source: str = "wikimedia_commons",
|
|
limit: int = DEFAULT_LIMIT,
|
|
registry: SourceRegistry | None = None,
|
|
client: HTTPSClient | None = None,
|
|
) -> list[SourceCandidate]:
|
|
"""Search *source* for *topic* using *registry* (default registry by default)."""
|
|
|
|
active = registry if registry is not None else build_registry(client=client)
|
|
return active.search(source, topic, limit)
|