Final acceptance: Консольная утилита поиска и скачивания изображений для вызова ИИ-агентом (#18)

Co-authored-by: AI Tester <ai-tester@git.p2prod.ru>

tools/sources.py

@@ -0,0 +1,578 @@
+"""Pluggable, credential-free image source registry and its HTTPS layer.
+
+Nothing in this module reads an API key, token, registration or secret from
+arguments, environment variables or configuration files: Wikimedia Commons and
+Openverse are queried anonymously.  The module provides
+
+* :class:`HTTPSClient` -- a small HTTPS client that identifies itself with a
+  stable ``User-Agent``, refuses every URL outside the approved hosts
+  (Wikimedia Commons and Openverse) on port 443, retries HTTP 429/5xx a bounded
+  number of times with capped exponential backoff, and raises
+  :class:`~tools.errors.SourceAccessError` instead of returning an empty result
+  when the host is unreachable or keeps failing;
+* :class:`SourceAdapter` -- the interface every image source implements;
+* :class:`WikimediaCommonsAdapter` and :class:`OpenverseAdapter` -- two working
+  credential-free sources behind that interface;
+* :class:`SourceRegistry` -- the plugin list that reports available source ids
+  and turns an unknown id into :class:`~tools.errors.UnknownSourceError` listing
+  them.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass, field
+from typing import Any, Callable, Iterable, Mapping, Protocol, runtime_checkable
+
+from .errors import (
+    DuplicateSourceError,
+    HostNotAllowedError,
+    SearchRequestError,
+    SourceAccessError,
+    SourceHTTPError,
+    SourceError,
+    SourcePayloadError,
+    SourceUnreachableError,
+    UnknownSourceError,
+)
+
+__all__ = [
+    "APPROVED_HOSTS",
+    "DEFAULT_LIMIT",
+    "DEFAULT_MAX_ATTEMPTS",
+    "HTTPSClient",
+    "HTTPResponse",
+    "MAX_LIMIT",
+    "OPENVERSE_API_HOST",
+    "OpenverseAdapter",
+    "SourceAdapter",
+    "SourceCandidate",
+    "SourceRegistry",
+    "USER_AGENT",
+    "WIKIMEDIA_COMMONS_API_HOST",
+    "WIKIMEDIA_COMMONS_MEDIA_HOST",
+    "build_registry",
+    "default_registry",
+    "is_approved_url",
+    "search",
+    "validate_source_url",
+]
+
+PRODUCT_NAME = "product-54354138-1"
+USER_AGENT = f"{PRODUCT_NAME}-image-toolkit/0.1.0 (credential-free image source client)"
+
+WIKIMEDIA_COMMONS_API_HOST = "commons.wikimedia.org"
+WIKIMEDIA_COMMONS_MEDIA_HOST = "upload.wikimedia.org"
+OPENVERSE_API_HOST = "api.openverse.org"
+
+# Wikimedia Commons is reachable through its API host plus the media host that
+# serves the original files, Openverse through its API host (and the thumbnail
+# endpoint on the same API host).
+APPROVED_HOSTS = frozenset(
+    {WIKIMEDIA_COMMONS_API_HOST, WIKIMEDIA_COMMONS_MEDIA_HOST, OPENVERSE_API_HOST}
+)
+APPROVED_SCHEME = "https"
+APPROVED_PORT = 443
+
+DEFAULT_TIMEOUT = 10.0
+DEFAULT_MAX_ATTEMPTS = 3
+DEFAULT_BACKOFF_SECONDS = 0.5
+MAX_BACKOFF_SECONDS = 8.0
+RETRY_STATUS_CODES = frozenset({429}) | frozenset(range(500, 600))
+
+
+@dataclass(frozen=True)
+class HTTPResponse:
+    """A transport-independent HTTP response."""
+
+    status: int
+    body: bytes
+    headers: Mapping[str, str] = field(default_factory=dict)
+
+    def json(self, *, url: str = "", source_id: str | None = None) -> Any:
+        try:
+            return json.loads(self.body)
+        except (ValueError, UnicodeDecodeError) as exc:  # pragma: no cover - defensive
+            raise SourcePayloadError(url or "<response>", source_id=source_id, reason=str(exc)) from exc
+
+
+class Transport(Protocol):
+    """Callable performing one HTTP GET; injectable so tests never hit the net."""
+
+    def __call__(self, url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse: ...
+
+
+def validate_source_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> str:
+    """Return *url* when it is an approved HTTPS URL, else raise a typed error."""
+
+    allowed_set = frozenset(allowed)
+    if not isinstance(url, str) or not url:
+        raise HostNotAllowedError(str(url), allowed=allowed_set)
+    parts = urllib.parse.urlsplit(url)
+    if parts.scheme != APPROVED_SCHEME:
+        raise HostNotAllowedError(url, allowed=allowed_set)
+    host = (parts.hostname or "").lower()
+    if host not in allowed_set:
+        raise HostNotAllowedError(url, allowed=allowed_set)
+    try:
+        port = parts.port
+    except ValueError as exc:
+        raise HostNotAllowedError(url, allowed=allowed_set) from exc
+    if port not in (None, APPROVED_PORT):
+        raise HostNotAllowedError(url, allowed=allowed_set)
+    return url
+
+
+def is_approved_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> bool:
+    """Non-raising variant of :func:`validate_source_url` used by tests/callers."""
+
+    try:
+        validate_source_url(url, allowed=allowed)
+    except HostNotAllowedError:
+        return False
+    return True
+
+
+def urllib_transport(url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse:
+    """Default transport: a plain stdlib GET returning status, body and headers."""
+
+    request = urllib.request.Request(url, headers=dict(headers), method="GET")
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            return HTTPResponse(
+                status=int(getattr(response, "status", 200)),
+                body=response.read(),
+                headers=dict(response.headers.items()),
+            )
+    except urllib.error.HTTPError as exc:
+        body = b""
+        try:
+            body = exc.read()
+        except Exception:  # pragma: no cover - body optional
+            body = b""
+        return HTTPResponse(
+            status=int(exc.code),
+            body=body,
+            headers=dict(getattr(exc, "headers", {}) or {}),
+        )
+
+
+class HTTPSClient:
+    """Fetch a URL only when it is approved, with bounded retry/backoff.
+
+    The client never returns an empty result for a broken source: unreachable
+    hosts raise :class:`SourceUnreachableError` and exhausted/non-retryable
+    statuses raise :class:`SourceHTTPError`.  Both derive from
+    :class:`SourceAccessError`.
+    """
+
+    def __init__(
+        self,
+        *,
+        user_agent: str = USER_AGENT,
+        allowed_hosts: Iterable[str] = APPROVED_HOSTS,
+        timeout: float = DEFAULT_TIMEOUT,
+        max_attempts: int = DEFAULT_MAX_ATTEMPTS,
+        backoff_seconds: float = DEFAULT_BACKOFF_SECONDS,
+        max_backoff_seconds: float = MAX_BACKOFF_SECONDS,
+        transport: Transport | None = None,
+        sleep: Callable[[float], None] = time.sleep,
+    ) -> None:
+        if max_attempts < 1:
+            raise ValueError("max_attempts must be >= 1")
+        self.user_agent = user_agent
+        self.allowed_hosts = frozenset(allowed_hosts)
+        self.timeout = float(timeout)
+        self.max_attempts = int(max_attempts)
+        self.backoff_seconds = float(backoff_seconds)
+        self.max_backoff_seconds = float(max_backoff_seconds)
+        self._transport: Transport = transport or urllib_transport
+        self._sleep = sleep
+
+    # -- public API -----------------------------------------------------
+    def headers(self) -> dict[str, str]:
+        """Request headers, always including the identifying User-Agent."""
+
+        return {
+            "User-Agent": self.user_agent,
+            "Accept": "application/json, image/*;q=0.8, */*;q=0.5",
+        }
+
+    def backoff_delay(self, attempt: int) -> float:
+        """Capped exponential backoff delay for 1-based *attempt*."""
+
+        delay = self.backoff_seconds * (2 ** max(0, attempt - 1))
+        return min(delay, self.max_backoff_seconds)
+
+    def get(self, url: str, *, source_id: str | None = None) -> HTTPResponse:
+        """GET *url*, retrying transient statuses, then raise typed errors."""
+
+        validate_source_url(url, allowed=self.allowed_hosts)
+        response: HTTPResponse | None = None
+        for attempt in range(1, self.max_attempts + 1):
+            try:
+                response = self._transport(url, self.headers(), self.timeout)
+            except SourceAccessError:
+                raise
+            except Exception as exc:  # transport-level failure
+                if attempt >= self.max_attempts:
+                    raise SourceUnreachableError(
+                        url, source_id=source_id, reason=f"{type(exc).__name__}: {exc}"
+                    ) from exc
+                self._sleep(self.backoff_delay(attempt))
+                continue
+            if response.status in RETRY_STATUS_CODES and attempt < self.max_attempts:
+                self._sleep(self.backoff_delay(attempt))
+                continue
+            if 200 <= response.status < 300:
+                return response
+            raise SourceHTTPError(url, response.status, source_id=source_id)
+        # Defensive: the loop either returned or raised above.
+        status = response.status if response is not None else 0
+        raise SourceHTTPError(url, status, source_id=source_id)
+
+    def get_json(self, url: str, *, source_id: str | None = None) -> Any:
+        response = self.get(url, source_id=source_id)
+        try:
+            return json.loads(response.body)
+        except (ValueError, UnicodeDecodeError) as exc:
+            raise SourcePayloadError(url, source_id=source_id, reason=str(exc)) from exc
+
+
+DEFAULT_LIMIT = 10
+MAX_LIMIT = 50
+
+
+@dataclass(frozen=True)
+class SourceCandidate:
+    """One normalized search hit.
+
+    ``source_id`` identifies the adapter, ``file_url`` is the direct HTTPS
+    location of the original image, ``mime_type`` is the type the source
+    reports (may be ``None`` when unstated) and ``fetch_bytes`` is a zero-argument
+    callable returning the image bytes through the approved HTTPS client.
+    """
+
+    source_id: str
+    file_url: str
+    mime_type: str | None
+    fetch_bytes: Callable[[], bytes]
+    title: str | None = None
+    page_url: str | None = None
+    width: int | None = None
+    height: int | None = None
+    extra: Mapping[str, Any] = field(default_factory=dict)
+
+    def fetch(self) -> bytes:
+        """Convenience passthrough to :attr:`fetch_bytes`."""
+
+        return self.fetch_bytes()
+
+    def to_dict(self) -> dict[str, Any]:
+        """Metadata-only projection (the callable is intentionally omitted)."""
+
+        payload: dict[str, Any] = {
+            "source_id": self.source_id,
+            "file_url": self.file_url,
+            "mime_type": self.mime_type,
+        }
+        for key in ("title", "page_url", "width", "height"):
+            value = getattr(self, key)
+            if value is not None:
+                payload[key] = value
+        return payload
+
+
+@runtime_checkable
+class SourceAdapter(Protocol):
+    """Interface implemented by every image source."""
+
+    source_id: str
+
+    def search(self, topic: str, limit: int) -> list[SourceCandidate]: ...
+
+
+def normalize_search_args(topic: str, limit: int) -> tuple[str, int]:
+    """Validate the caller-supplied topic and result limit."""
+
+    if not isinstance(topic, str):
+        raise SearchRequestError(
+            f"topic must be a string, got {type(topic).__name__}",
+            details={"topic": repr(topic)},
+        )
+    topic = topic.strip()
+    if not topic:
+        raise SearchRequestError("topic must not be empty", details={"topic": topic})
+    if isinstance(limit, bool) or not isinstance(limit, int):
+        raise SearchRequestError(
+            f"limit must be an integer, got {type(limit).__name__}",
+            details={"limit": repr(limit)},
+        )
+    if limit < 1:
+        raise SearchRequestError(f"limit must be >= 1, got {limit}", details={"limit": limit})
+    if limit > MAX_LIMIT:
+        raise SearchRequestError(
+            f"limit must be <= {MAX_LIMIT}, got {limit}", details={"limit": limit}
+        )
+    return topic, limit
+
+
+class _BaseAdapter:
+    """Shared normalization helpers for concrete adapters."""
+
+    source_id = ""
+
+    def __init__(self, client: HTTPSClient | None = None) -> None:
+        self.client = client or HTTPSClient()
+
+    def _candidate(
+        self,
+        *,
+        file_url: str,
+        mime_type: str | None,
+        title: str | None = None,
+        page_url: str | None = None,
+        width: int | None = None,
+        height: int | None = None,
+        extra: Mapping[str, Any] | None = None,
+        validate: bool = True,
+    ) -> SourceCandidate:
+        if validate:
+            validate_source_url(file_url, allowed=self.client.allowed_hosts)
+        source_id = self.source_id
+        client = self.client
+        return SourceCandidate(
+            source_id=source_id,
+            file_url=file_url,
+            mime_type=mime_type,
+            fetch_bytes=lambda url=file_url: client.get(url, source_id=source_id).body,
+            title=title,
+            page_url=page_url,
+            width=width,
+            height=height,
+            extra=dict(extra or {}),
+        )
+
+    def __repr__(self) -> str:  # pragma: no cover - trivial
+        return f"<{type(self).__name__} source_id={self.source_id!r}>"
+
+
+class WikimediaCommonsAdapter(_BaseAdapter):
+    """Wikimedia Commons search through the anonymous MediaWiki action API."""
+
+    source_id = "wikimedia_commons"
+    api_host = WIKIMEDIA_COMMONS_API_HOST
+
+    def build_search_url(self, topic: str, limit: int) -> str:
+        """Construct the generator search URL for *topic* and *limit*."""
+
+        topic, limit = normalize_search_args(topic, limit)
+        params = {
+            "action": "query",
+            "format": "json",
+            "formatversion": "2",
+            "generator": "search",
+            "gsrsearch": topic,
+            "gsrnamespace": "6",
+            "gsrlimit": str(limit),
+            "prop": "imageinfo",
+            "iiprop": "url|mime|size",
+            "iiurlwidth": "1024",
+        }
+        query = urllib.parse.urlencode(params)
+        return f"https://{self.api_host}/w/api.php?{query}"
+
+    def search(self, topic: str, limit: int) -> list[SourceCandidate]:
+        url = self.build_search_url(topic, limit)
+        payload = self.client.get_json(url, source_id=self.source_id)
+        if not isinstance(payload, Mapping):
+            raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object")
+        if "error" in payload:
+            raise SourceAccessError(
+                f"Wikimedia Commons API error: {payload['error']}",
+                url=url,
+                source_id=self.source_id,
+            )
+        pages = (payload.get("query") or {}).get("pages") or []
+        candidates: list[SourceCandidate] = []
+        for page in pages:
+            infos = page.get("imageinfo") or []
+            if not infos:
+                continue
+            info = infos[0]
+            file_url = info.get("url")
+            if not file_url:
+                continue
+            candidates.append(
+                self._candidate(
+                    file_url=file_url,
+                    mime_type=info.get("mime"),
+                    title=page.get("title"),
+                    page_url=info.get("descriptionurl"),
+                    width=info.get("width"),
+                    height=info.get("height"),
+                    extra={"thumb_url": info.get("thumburl")} if info.get("thumburl") else None,
+                )
+            )
+            if len(candidates) >= limit:
+                break
+        return candidates[:limit]
+
+
+class OpenverseAdapter(_BaseAdapter):
+    """Openverse search through its anonymous public images endpoint."""
+
+    source_id = "openverse"
+    api_host = OPENVERSE_API_HOST
+
+    def build_search_url(self, topic: str, limit: int) -> str:
+        """Construct the Openverse image search URL for *topic* and *limit*."""
+
+        topic, limit = normalize_search_args(topic, limit)
+        query = urllib.parse.urlencode({"q": topic, "page_size": str(limit)})
+        return f"https://{self.api_host}/v1/images/?{query}"
+
+    def search(self, topic: str, limit: int) -> list[SourceCandidate]:
+        url = self.build_search_url(topic, limit)
+        payload = self.client.get_json(url, source_id=self.source_id)
+        if not isinstance(payload, Mapping):
+            raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object")
+        results = payload.get("results")
+        if results is None:
+            raise SourcePayloadError(url, source_id=self.source_id, reason="missing 'results'")
+        candidates: list[SourceCandidate] = []
+        for item in results:
+            if not isinstance(item, Mapping):
+                continue
+            original_url = item.get("url")
+            file_url = item.get("thumbnail") or original_url
+            if not file_url:
+                continue
+            candidates.append(
+                self._candidate(
+                    file_url=file_url,
+                    # Origin files may live on third-party hosts; the client
+                    # still refuses them at fetch time with a typed error.
+                    validate=is_approved_url(file_url, allowed=self.client.allowed_hosts),
+                    mime_type=item.get("filetype"),
+                    title=item.get("title"),
+                    page_url=item.get("foreign_landing_url") or item.get("detail_url"),
+                    width=item.get("width"),
+                    height=item.get("height"),
+                    extra={
+                        "license": item.get("license"),
+                        "creator": item.get("creator"),
+                        "original_url": original_url,
+                    },
+                )
+            )
+            if len(candidates) >= limit:
+                break
+        return candidates[:limit]
+
+
+class SourceRegistry:
+    """Ordered plugin registry of credential-free image sources."""
+
+    def __init__(self, adapters: Iterable[SourceAdapter] | None = None) -> None:
+        self._adapters: dict[str, SourceAdapter] = {}
+        for adapter in adapters or ():
+            self.register(adapter)
+
+    # -- registration ---------------------------------------------------
+    def register(self, adapter: SourceAdapter, *, replace: bool = False) -> SourceAdapter:
+        """Add *adapter*; a duplicate id raises :class:`DuplicateSourceError`."""
+
+        source_id = getattr(adapter, "source_id", None)
+        if not isinstance(source_id, str) or not source_id:
+            raise SourceError("adapter must expose a non-empty string 'source_id'")
+        if source_id in self._adapters and not replace:
+            raise DuplicateSourceError(source_id)
+        self._adapters[source_id] = adapter
+        return adapter
+
+    def unregister(self, source_id: str) -> SourceAdapter:
+        if source_id not in self._adapters:
+            raise UnknownSourceError(source_id, self.available_sources())
+        return self._adapters.pop(source_id)
+
+    # -- lookup ---------------------------------------------------------
+    def available_sources(self) -> list[str]:
+        """Sorted ids of every registered source."""
+
+        return sorted(self._adapters)
+
+    def __contains__(self, source_id: object) -> bool:
+        return isinstance(source_id, str) and source_id in self._adapters
+
+    def __len__(self) -> int:
+        return len(self._adapters)
+
+    def __iter__(self):
+        return iter(self.available_sources())
+
+    def get(self, source_id: str) -> SourceAdapter:
+        """Return the adapter for *source_id* or raise a listing error."""
+
+        try:
+            return self._adapters[source_id]
+        except (KeyError, TypeError):
+            raise UnknownSourceError(source_id, self.available_sources()) from None
+
+    def create(self, source_id: str) -> SourceAdapter:
+        """Explicit alias of :meth:`get` used by the CLI layer."""
+
+        return self.get(source_id)
+
+    # -- search ---------------------------------------------------------
+    def search(self, source_id: str, topic: str, limit: int = DEFAULT_LIMIT) -> list[SourceCandidate]:
+        """Search *source_id* for *topic*, returning at most *limit* candidates."""
+
+        adapter = self.get(source_id)
+        topic, limit = normalize_search_args(topic, limit)
+        return list(adapter.search(topic, limit))
+
+    def __repr__(self) -> str:  # pragma: no cover - trivial
+        return f"SourceRegistry({self.available_sources()!r})"
+
+
+
+
+def build_registry(
+    *,
+    client: HTTPSClient | None = None,
+    adapters: Iterable[SourceAdapter] | None = None,
+) -> SourceRegistry:
+    """Build the default registry (Wikimedia Commons + Openverse)."""
+
+    shared_client = client or HTTPSClient()
+    if adapters is None:
+        adapters = (
+            WikimediaCommonsAdapter(shared_client),
+            OpenverseAdapter(shared_client),
+        )
+    return SourceRegistry(adapters)
+
+
+def default_registry(client: HTTPSClient | None = None) -> SourceRegistry:
+    """Convenience wrapper returning a fresh default registry."""
+
+    return build_registry(client=client)
+
+
+def search(
+    topic: str,
+    *,
+    source: str = "wikimedia_commons",
+    limit: int = DEFAULT_LIMIT,
+    registry: SourceRegistry | None = None,
+    client: HTTPSClient | None = None,
+) -> list[SourceCandidate]:
+    """Search *source* for *topic* using *registry* (default registry by default)."""
+
+    active = registry if registry is not None else build_registry(client=client)
+    return active.search(source, topic, limit)