Final acceptance: Консольная утилита поиска и скачивания изображений для вызова ИИ-агентом (#18)
Repository Validate / repository-checks (push) Successful in 21s

Co-authored-by: AI Tester <ai-tester@git.p2prod.ru>
This commit was merged in pull request #18.
This commit is contained in:
2026-11-03 02:51:14 +00:00
committed by ai-reviewer
parent 1b21e0ba82
commit 44473a1055
23 changed files with 6643 additions and 2 deletions
+12
View File
@@ -0,0 +1,12 @@
"""Importable toolkit package for the request-document CLI.
The package intentionally exposes only its version metadata here; the
executable behaviour lives in :mod:`tools.example` (and in the modules that
later increments add).
"""
from __future__ import annotations
__all__ = ["__version__"]
__version__ = "0.1.0"
+263
View File
@@ -0,0 +1,263 @@
"""Status vocabulary, exit codes and the JSON response envelope.
Every invocation of ``python -m tools.example`` writes exactly one JSON
document to stdout. The document always carries a ``status`` value from the
fixed vocabulary below and the process exit code is derived from that value, so
a calling AI agent can tell a successful search apart from "nothing matched",
a broken request and an unusable source without parsing prose.
"""
from __future__ import annotations
import json
import sys
from dataclasses import dataclass
from typing import Any, Iterable, Mapping, TextIO
from .errors import (
RequestError,
SearchRequestError,
SourceAccessError,
ToolkitError,
UnknownSourceError,
)
__all__ = [
"EXIT_ACCESS_ERROR",
"EXIT_INVALID_REQUEST",
"EXIT_NO_FORMAT",
"EXIT_NO_SIZE",
"EXIT_OK",
"EXIT_PARTIAL_DOWNLOAD",
"Response",
"ResponseBuilder",
"STATUS_ACCESS_ERROR",
"STATUS_DRY_RUN",
"STATUS_EXIT_CODES",
"STATUS_INVALID_REQUEST",
"STATUS_NO_FORMAT",
"STATUS_NO_SIZE",
"STATUS_OK",
"STATUS_PARTIAL_DOWNLOAD",
"STATUS_VOCABULARY",
"error_fragment",
"exit_code_for",
"is_success",
"status_for_error",
]
#: Successful search/download.
STATUS_OK = "ok"
#: The request was validated and echoed without contacting a source.
STATUS_DRY_RUN = "dry_run"
#: The request itself is unusable (bad flags, bad document, unknown source or format).
STATUS_INVALID_REQUEST = "invalid_request"
#: The source answered but offers nothing in the requested format.
STATUS_NO_FORMAT = "no_format"
#: The source answered but offers nothing in the requested size.
STATUS_NO_SIZE = "no_size"
#: The source could not be reached, refused to answer or returned unusable data.
STATUS_ACCESS_ERROR = "access_error"
#: Some downloads succeeded and some failed.
STATUS_PARTIAL_DOWNLOAD = "partial_download"
#: The single source of truth for the status -> exit code mapping.
STATUS_EXIT_CODES: dict[str, int] = {
STATUS_OK: 0,
STATUS_DRY_RUN: 0,
STATUS_INVALID_REQUEST: 2,
STATUS_NO_FORMAT: 3,
STATUS_NO_SIZE: 4,
STATUS_ACCESS_ERROR: 5,
STATUS_PARTIAL_DOWNLOAD: 6,
}
#: Stable ordering used in help text and documentation.
STATUS_VOCABULARY: tuple[str, ...] = tuple(STATUS_EXIT_CODES)
EXIT_OK = STATUS_EXIT_CODES[STATUS_OK]
EXIT_INVALID_REQUEST = STATUS_EXIT_CODES[STATUS_INVALID_REQUEST]
EXIT_NO_FORMAT = STATUS_EXIT_CODES[STATUS_NO_FORMAT]
EXIT_NO_SIZE = STATUS_EXIT_CODES[STATUS_NO_SIZE]
EXIT_ACCESS_ERROR = STATUS_EXIT_CODES[STATUS_ACCESS_ERROR]
EXIT_PARTIAL_DOWNLOAD = STATUS_EXIT_CODES[STATUS_PARTIAL_DOWNLOAD]
def exit_code_for(status: str) -> int:
"""Return the process exit code for ``status``.
Raises :class:`ValueError` for anything outside the fixed vocabulary, so a
typo can never silently produce an unknown exit code.
"""
try:
return STATUS_EXIT_CODES[status]
except KeyError:
raise ValueError(
f"unknown status {status!r}; expected one of: {', '.join(STATUS_VOCABULARY)}"
) from None
def is_success(status: str) -> bool:
"""True when ``status`` maps onto a zero exit code."""
return exit_code_for(status) == 0
def error_fragment(
code: str,
message: str,
*,
details: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Build the machine-readable ``error`` fragment of a response."""
payload: dict[str, Any] = {"code": str(code), "message": str(message)}
if details:
payload["details"] = dict(details)
return payload
def status_for_error(error: BaseException) -> str:
"""Map a toolkit error onto the fixed status vocabulary."""
if isinstance(error, (RequestError, UnknownSourceError, SearchRequestError)):
return STATUS_INVALID_REQUEST
if isinstance(error, SourceAccessError):
return STATUS_ACCESS_ERROR
if isinstance(error, ToolkitError):
return STATUS_ACCESS_ERROR
return STATUS_ACCESS_ERROR
@dataclass(frozen=True)
class Response:
"""One complete machine-readable answer of the toolkit."""
status: str
source: str | None = None
request: Mapping[str, Any] | None = None
items: tuple[Mapping[str, Any], ...] = ()
error: Mapping[str, Any] | None = None
def __post_init__(self) -> None:
if self.status not in STATUS_EXIT_CODES:
raise ValueError(
f"unknown status {self.status!r}; expected one of: "
f"{', '.join(STATUS_VOCABULARY)}"
)
if self.error is not None and "code" not in self.error:
raise ValueError("error fragment must carry a 'code'")
@property
def exit_code(self) -> int:
"""Process exit code implied by :attr:`status`."""
return exit_code_for(self.status)
@property
def succeeded(self) -> bool:
"""True when the status is one of the zero-exit statuses."""
return self.exit_code == 0
def to_dict(self) -> dict[str, Any]:
"""Return the JSON-ready document with a fixed set of top-level keys."""
return {
"status": self.status,
"source": self.source,
"request": dict(self.request) if self.request is not None else None,
"items": [dict(item) for item in self.items],
"error": dict(self.error) if self.error is not None else None,
}
def to_json(self) -> str:
"""Serialise the response as one deterministic JSON line."""
return json.dumps(self.to_dict(), sort_keys=True, ensure_ascii=True)
def emit(self, stream: TextIO | None = None) -> None:
"""Write exactly one JSON document to ``stream`` (stdout by default)."""
print(self.to_json(), file=stream if stream is not None else sys.stdout)
class ResponseBuilder:
"""Build :class:`Response` objects sharing one source/request context."""
def __init__(
self,
*,
source: str | None = None,
request: Mapping[str, Any] | None = None,
) -> None:
self.source = source
self.request = request
def with_context(
self,
*,
source: str | None = None,
request: Mapping[str, Any] | None = None,
) -> "ResponseBuilder":
"""Return a copy carrying the supplied context."""
return ResponseBuilder(
source=self.source if source is None else source,
request=self.request if request is None else request,
)
def build(
self,
status: str,
*,
items: Iterable[Mapping[str, Any]] = (),
error: Mapping[str, Any] | None = None,
) -> Response:
"""Assemble a response for an explicit ``status``."""
return Response(
status=status,
source=self.source,
request=self.request,
items=tuple(items),
error=error,
)
# -- convenience constructors --------------------------------------
def ok(self, *, items: Iterable[Mapping[str, Any]] = ()) -> Response:
return self.build(STATUS_OK, items=items)
def dry_run(self, *, items: Iterable[Mapping[str, Any]] = ()) -> Response:
return self.build(STATUS_DRY_RUN, items=items)
def invalid_request(self, *, error: Mapping[str, Any] | None = None) -> Response:
return self.build(STATUS_INVALID_REQUEST, error=error)
def no_format(self, *, error: Mapping[str, Any] | None = None) -> Response:
return self.build(STATUS_NO_FORMAT, error=error)
def no_size(self, *, error: Mapping[str, Any] | None = None) -> Response:
return self.build(STATUS_NO_SIZE, error=error)
def access_error(
self,
*,
items: Iterable[Mapping[str, Any]] = (),
error: Mapping[str, Any] | None = None,
) -> Response:
return self.build(STATUS_ACCESS_ERROR, items=items, error=error)
def partial_download(
self,
*,
items: Iterable[Mapping[str, Any]] = (),
error: Mapping[str, Any] | None = None,
) -> Response:
return self.build(STATUS_PARTIAL_DOWNLOAD, items=items, error=error)
def from_error(self, error: ToolkitError) -> Response:
"""Build a response for ``error`` using :func:`status_for_error`."""
return self.build(status_for_error(error), error=error.to_dict())
+273
View File
@@ -0,0 +1,273 @@
"""Typed errors raised deliberately by the toolkit.
The image-source layer never degrades a failed or unknown source into a silent,
empty result. Every such situation raises a :class:`ToolkitError` subclass that
carries a stable ``code`` string plus machine-readable ``details`` so the CLI can
map it onto an exit status and a structured JSON envelope.
Hierarchy::
ToolkitError
+-- SourceError
+-- UnknownSourceError (unknown source id, lists available ids)
+-- DuplicateSourceError (registry already holds that id)
+-- SearchRequestError (topic/limit rejected before any request)
+-- SourceAccessError (the source could not be used)
+-- SourceUnreachableError (transport/DNS/timeout failure)
+-- SourceHTTPError (non-retryable or exhausted HTTP status)
+-- SourcePayloadError (body is not the expected JSON shape)
+-- HostNotAllowedError (URL outside the approved hosts)
"""
from __future__ import annotations
from typing import Any, Iterable, Mapping
__all__ = [
"DuplicateSourceError",
"HostNotAllowedError",
"SearchRequestError",
"SourceAccessError",
"SourceError",
"SourceHTTPError",
"SourcePayloadError",
"SourceUnreachableError",
"ToolkitError",
"UnknownSourceError",
]
class ToolkitError(Exception):
"""Base class for errors the toolkit raises on purpose."""
code = "toolkit_error"
def __init__(
self,
message: str,
*,
code: str | None = None,
details: Mapping[str, Any] | None = None,
) -> None:
super().__init__(message)
self.message = message
if code is not None:
self.code = code
self.details: dict[str, Any] = dict(details or {})
def to_dict(self) -> dict[str, Any]:
"""Return the JSON-safe envelope fragment describing this error."""
payload: dict[str, Any] = {"code": self.code, "message": self.message}
if self.details:
payload["details"] = dict(self.details)
return payload
def __str__(self) -> str: # pragma: no cover - trivial
return self.message
class SourceError(ToolkitError):
"""Base class for every image-source problem."""
code = "source_error"
class UnknownSourceError(SourceError):
"""Raised when the caller asks for a source id the registry does not know."""
code = "source_unknown"
def __init__(self, source_id: str, available: Iterable[str]) -> None:
self.source_id = str(source_id)
self.available: tuple[str, ...] = tuple(sorted(str(item) for item in available))
listing = ", ".join(self.available) if self.available else "<none>"
super().__init__(
f"unknown image source {self.source_id!r}; available sources: {listing}",
details={"source_id": self.source_id, "available_sources": list(self.available)},
)
class DuplicateSourceError(SourceError):
"""Raised when an adapter id is registered twice."""
code = "source_duplicate"
def __init__(self, source_id: str) -> None:
self.source_id = str(source_id)
super().__init__(
f"image source {self.source_id!r} is already registered",
details={"source_id": self.source_id},
)
class SearchRequestError(SourceError):
"""Raised when the caller supplies a topic/limit a source cannot honour."""
code = "search_invalid_request"
def __init__(self, message: str, *, details: Mapping[str, Any] | None = None) -> None:
super().__init__(message, details=details)
class SourceAccessError(SourceError):
"""Raised when a known source cannot be reached or refuses to answer."""
code = "source_access"
def __init__(
self,
message: str,
*,
url: str | None = None,
source_id: str | None = None,
details: Mapping[str, Any] | None = None,
) -> None:
merged: dict[str, Any] = dict(details or {})
if url is not None:
merged["url"] = url
if source_id is not None:
merged["source_id"] = source_id
super().__init__(message, details=merged)
self.url = url
self.source_id = source_id
class SourceUnreachableError(SourceAccessError):
"""Raised when the transport cannot deliver a request to the host at all."""
code = "source_unreachable"
def __init__(self, url: str, *, source_id: str | None = None, reason: str | None = None) -> None:
message = f"image source host is unreachable: {url}"
if reason:
message = f"{message} ({reason})"
super().__init__(message, url=url, source_id=source_id, details={"reason": reason} if reason else None)
class SourceHTTPError(SourceAccessError):
"""Raised for an HTTP status the client cannot recover from."""
code = "source_http_error"
def __init__(self, url: str, status: int, *, source_id: str | None = None) -> None:
super().__init__(
f"image source returned HTTP status {status}: {url}",
url=url,
source_id=source_id,
details={"status": status},
)
self.status = int(status)
class SourcePayloadError(SourceAccessError):
"""Raised when a source answers with something the adapter cannot read."""
code = "source_invalid_response"
def __init__(self, url: str, *, source_id: str | None = None, reason: str | None = None) -> None:
message = f"image source returned an unusable response: {url}"
if reason:
message = f"{message} ({reason})"
super().__init__(message, url=url, source_id=source_id, details={"reason": reason} if reason else None)
class HostNotAllowedError(SourceAccessError):
"""Raised when a URL leaves the approved HTTPS host allowlist."""
code = "source_host_not_allowed"
def __init__(self, url: str, *, allowed: Iterable[str] = ()) -> None:
self.allowed: tuple[str, ...] = tuple(sorted(str(item) for item in allowed))
super().__init__(
f"refusing to contact {url!r}: only approved HTTPS hosts are allowed "
f"({', '.join(self.allowed) if self.allowed else 'none configured'})",
url=str(url),
details={"allowed_hosts": list(self.allowed)},
)
# ---------------------------------------------------------------------------
# Request errors: invalid command lines and JSON request documents
# ---------------------------------------------------------------------------
#: Image formats the CLI accepts.
ACCEPTED_FORMATS: tuple[str, ...] = ("png", "ico")
class RequestError(ToolkitError):
"""Base class for every unusable request (CLI flags or request document)."""
code = "request_invalid"
def __init__(
self,
message: str,
*,
code: str | None = None,
details: Mapping[str, Any] | None = None,
field: str | None = None,
accepted_fields: Iterable[str] | None = None,
accepted_formats: Iterable[str] | None = None,
) -> None:
merged: dict[str, Any] = dict(details or {})
if field is not None:
merged["field"] = field
if accepted_fields is not None:
merged["accepted_fields"] = list(accepted_fields)
if accepted_formats is not None:
merged["accepted_formats"] = list(accepted_formats)
super().__init__(message, code=code, details=merged)
self.field = field
self.accepted_fields: tuple[str, ...] = tuple(accepted_fields or ())
self.accepted_formats: tuple[str, ...] = tuple(accepted_formats or ())
class RequestDocumentError(RequestError):
"""Raised when the JSON request document cannot be read or decoded."""
code = "request_document"
class RequestFieldError(RequestError):
"""Raised when a single request field is missing, unknown or malformed."""
code = "request_field"
class UnsupportedFormatError(RequestError):
"""Raised for a format outside :data:`ACCEPTED_FORMATS`."""
code = "request_format_unsupported"
def __init__(self, value: Any, accepted: Iterable[str] = ACCEPTED_FORMATS) -> None:
accepted = tuple(accepted)
self.value = str(value)
super().__init__(
f"unsupported image format {self.value!r}; "
f"accepted formats: {', '.join(accepted)}",
field="format",
accepted_formats=accepted,
details={"requested_format": self.value},
)
class ImageResizeError(ToolkitError):
"""Raised when a downloaded image cannot be downscaled to the target size."""
code = "resize_failed"
def __init__(self, reason: str, *, details: Mapping[str, Any] | None = None) -> None:
merged: dict[str, Any] = dict(details or {})
merged["reason"] = reason
super().__init__(f"cannot resize image: {reason}", details=merged)
__all__ += [
"ACCEPTED_FORMATS",
"ImageResizeError",
"RequestDocumentError",
"RequestError",
"RequestFieldError",
"UnsupportedFormatError",
]
+838
View File
@@ -0,0 +1,838 @@
"""Plain native-process CLI for the image toolkit.
Usage::
python -m tools.example examples/input.json
python -m tools.example --topic "red fox" --width 640 --height 480 --format png
Exactly one machine-readable JSON document is written to stdout. The process
exit code is derived from the fixed status vocabulary in :mod:`tools.envelope`.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Mapping, Sequence
from . import images
from .envelope import (
STATUS_ACCESS_ERROR,
STATUS_NO_FORMAT,
STATUS_NO_SIZE,
STATUS_OK,
STATUS_PARTIAL_DOWNLOAD,
Response,
ResponseBuilder,
error_fragment,
)
from .errors import (
ACCEPTED_FORMATS,
ImageResizeError,
RequestDocumentError,
RequestError,
RequestFieldError,
SourceAccessError,
ToolkitError,
UnknownSourceError,
UnsupportedFormatError,
)
from .images import ImageFormatError, ImageInfo
from .sources import DEFAULT_LIMIT, SourceCandidate, default_registry
PROG = "python -m tools.example"
#: Keys accepted in a JSON request document (``version`` is optional metadata).
REQUEST_FIELDS: tuple[str, ...] = (
"topic",
"width",
"height",
"format",
"source",
"count",
"download",
"output_dir",
"resize",
"dry_run",
)
OPTIONAL_DOCUMENT_FIELDS: tuple[str, ...] = ("version",)
DEFAULT_SOURCE = "wikimedia_commons"
MIN_COUNT = 1
MAX_COUNT = 50
#: How many candidates are fetched per wanted image. Nothing a source reports
#: about a hit is trusted, so a candidate is only known to be usable once its
#: bytes have been downloaded and parsed; the source is therefore asked for a
#: window of ``count * SAMPLE_FACTOR`` hits and the window is cut short as soon
#: as ``count`` usable images have been found.
SAMPLE_FACTOR = 4
#: Hard upper bound on the number of candidates inspected in one invocation.
MAX_CANDIDATE_CAP = MAX_COUNT
def candidate_cap(count: int) -> int:
"""Return the bounded number of candidates to fetch for ``count`` images."""
wanted = max(MIN_COUNT, int(count))
return min(MAX_CANDIDATE_CAP, wanted * SAMPLE_FACTOR)
class _Parser(argparse.ArgumentParser):
"""Argument parser that reports bad flags as request errors."""
def error(self, message: str) -> Any: # pragma: no cover - argparse hook
raise RequestError(message, accepted_fields=REQUEST_FIELDS)
def build_parser() -> argparse.ArgumentParser:
"""Return the CLI parser; ``--help`` documents every supported parameter."""
parser = _Parser(
prog=PROG,
description=(
"Search an image source and report the matching items as one JSON "
"document. Either pass a JSON request document or use the flags."
),
epilog=(
"Every run writes exactly one JSON document to stdout with the keys "
"status, source, request, items and error."
),
)
parser.add_argument(
"request",
nargs="?",
metavar="REQUEST.json",
help=(
"path to a JSON request document holding the same fields as the "
"flags; it cannot be combined with any of the flags"
),
)
parser.add_argument(
"-t",
"--topic",
metavar="TEXT",
help="search topic (required unless a request document is given)",
)
parser.add_argument(
"--width",
type=int,
metavar="PX",
help="target width in pixels; only images at least this wide are accepted",
)
parser.add_argument(
"--height",
type=int,
metavar="PX",
help="target height in pixels; only images at least this tall are accepted",
)
parser.add_argument(
"-f",
"--format",
metavar="FMT",
help="image format: " + "|".join(ACCEPTED_FORMATS),
)
parser.add_argument(
"-s",
"--source",
metavar="ID",
help=f"image source id (default: {DEFAULT_SOURCE}); unknown ids are rejected",
)
parser.add_argument(
"-c",
"--count",
type=int,
metavar="N",
help=f"how many candidates to request, {MIN_COUNT}-{MAX_COUNT} (default: {DEFAULT_LIMIT})",
)
parser.add_argument(
"--download",
action="store_true",
help="download every matching image into the output directory",
)
parser.add_argument(
"-o",
"--output-dir",
metavar="DIR",
help="caller-specified download directory; required with --download",
)
parser.add_argument(
"--resize",
action="store_true",
help=(
"downscale every downloaded image that is larger than the request "
"in either dimension to exactly the requested width and height, "
"re-encoded in the requested format; images are never scaled up; "
"without it the original bytes and dimensions are written"
),
)
parser.add_argument(
"--dry-run",
action="store_true",
help="validate the request and emit the envelope without any network access",
)
return parser
def _load_document(path: str) -> Mapping[str, Any]:
"""Read a JSON request document from ``path``."""
target = Path(path)
try:
text = target.read_text(encoding="utf-8")
except OSError as exc:
raise RequestDocumentError(
f"cannot read request document {path!r}: {exc.strerror or exc}",
details={"path": str(path)},
) from exc
try:
payload = json.loads(text)
except json.JSONDecodeError as exc:
raise RequestDocumentError(
f"request document {path!r} is not valid JSON: {exc.msg}",
details={"path": str(path), "line": exc.lineno, "column": exc.colno},
) from exc
if not isinstance(payload, Mapping):
raise RequestDocumentError(
f"request document {path!r} must contain a JSON object",
details={"path": str(path), "type": type(payload).__name__},
)
return payload
def _document_to_mapping(payload: Mapping[str, Any]) -> dict[str, Any]:
"""Reject unknown keys and return the accepted request fields."""
accepted = REQUEST_FIELDS + OPTIONAL_DOCUMENT_FIELDS
unknown = sorted(str(key) for key in payload if key not in accepted)
if unknown:
raise RequestFieldError(
"request document contains unsupported fields: " + ", ".join(unknown),
field=unknown[0],
details={"unknown_fields": unknown, "accepted_fields": list(REQUEST_FIELDS)},
)
return {key: payload[key] for key in REQUEST_FIELDS if key in payload}
def _flags_to_mapping(namespace: argparse.Namespace) -> dict[str, Any]:
"""Collect the explicitly supplied flags into a request mapping."""
mapping: dict[str, Any] = {}
for field in ("topic", "width", "height", "format", "source", "count", "output_dir"):
value = getattr(namespace, field, None)
if value is not None:
mapping[field] = value
for field in ("download", "resize", "dry_run"):
if getattr(namespace, field, False):
mapping[field] = True
return mapping
def _as_text(value: Any, field: str) -> str:
if not isinstance(value, str):
raise RequestFieldError(
f"{field} must be a string, got {type(value).__name__}", field=field
)
text = value.strip()
if not text:
raise RequestFieldError(f"{field} must not be empty", field=field)
return text
def _as_int(value: Any, field: str) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise RequestFieldError(
f"{field} must be an integer, got {type(value).__name__}", field=field
)
return int(value)
def _as_bool(value: Any, field: str) -> bool:
if not isinstance(value, bool):
raise RequestFieldError(
f"{field} must be a boolean, got {type(value).__name__}", field=field
)
return bool(value)
def normalize_request(raw: Mapping[str, Any]) -> dict[str, Any]:
"""Validate ``raw`` and return the complete, typed request mapping."""
if "topic" not in raw:
raise RequestFieldError(
"topic is required",
field="topic",
details={"required_fields": ["topic"], "accepted_fields": list(REQUEST_FIELDS)},
)
topic = _as_text(raw["topic"], "topic")
size: dict[str, int | None] = {"width": None, "height": None}
for field in ("width", "height"):
if raw.get(field) is None:
continue
number = _as_int(raw[field], field)
if number <= 0:
raise RequestFieldError(f"{field} must be positive", field=field)
size[field] = number
if "format" not in raw or raw["format"] is None:
raise RequestFieldError(
"format is required",
field="format",
details={"accepted_formats": list(ACCEPTED_FORMATS)},
)
image_format = _as_text(raw["format"], "format").lower()
if image_format not in ACCEPTED_FORMATS:
raise UnsupportedFormatError(image_format)
source = DEFAULT_SOURCE
if raw.get("source") is not None:
source = _as_text(raw["source"], "source")
count = DEFAULT_LIMIT
if raw.get("count") is not None:
count = _as_int(raw["count"], "count")
if not MIN_COUNT <= count <= MAX_COUNT:
raise RequestFieldError(
f"count must be between {MIN_COUNT} and {MAX_COUNT}",
field="count",
details={"min": MIN_COUNT, "max": MAX_COUNT},
)
output_dir: str | None = None
if raw.get("output_dir") is not None:
output_dir = _as_text(raw["output_dir"], "output_dir")
download = _as_bool(raw.get("download", False), "download")
if download and output_dir is None:
# There is no hidden default: a download run without a caller-supplied
# directory could only guess at where the files should go.
raise RequestFieldError(
"output_dir is required when download is enabled; pass --output-dir <dir>",
field="output_dir",
details={"required_with": "download", "accepted_fields": list(REQUEST_FIELDS)},
)
return {
"topic": topic,
"width": size["width"],
"height": size["height"],
"format": image_format,
"source": source,
"count": count,
"download": download,
"output_dir": output_dir,
"resize": _as_bool(raw.get("resize", False), "resize"),
"dry_run": _as_bool(raw.get("dry_run", False), "dry_run"),
}
def _meets_minimum_size(info: ImageInfo, request: Mapping[str, Any]) -> bool:
"""True when the *real* pixel size of an image is usable for the request.
The requested width and height are a minimum, not an exact match: any image
that is at least that large is usable because it can always be downscaled,
while a smaller image would have to be scaled up and is therefore rejected.
``--resize`` only decides whether the downloaded bytes are downscaled; it
never widens or narrows this filter.
"""
wanted_width = request.get("width")
wanted_height = request.get("height")
if wanted_width is not None and info.width < wanted_width:
return False
if wanted_height is not None and info.height < wanted_height:
return False
return True
def _failure(candidate: SourceCandidate, fragment: Mapping[str, Any]) -> dict[str, Any]:
"""Describe one candidate whose bytes could not be turned into an image."""
return {
"source": str(getattr(candidate, "source_id", "") or ""),
"url": str(getattr(candidate, "file_url", "") or ""),
"error": dict(fragment),
}
@dataclass(frozen=True)
class KeptImage:
"""One candidate whose downloaded bytes really match the request."""
candidate: SourceCandidate
info: ImageInfo
payload: bytes
def to_item(self) -> dict[str, Any]:
"""Report ``source``, ``url``, ``format`` and the *actual* dimensions."""
source = str(getattr(self.candidate, "source_id", "") or "")
url = str(getattr(self.candidate, "file_url", "") or "")
item: dict[str, Any] = {
"source": source,
"url": url,
# Kept as an alias so existing consumers keep working unchanged.
"file_url": url,
"format": self.info.format,
"width": self.info.width,
"height": self.info.height,
"size_bytes": len(self.payload),
}
for key in ("title", "page_url", "mime_type"):
value = getattr(self.candidate, key, None)
if value is not None:
item[key] = value
return item
@dataclass(frozen=True)
class Selection:
"""Outcome of inspecting a bounded window of downloaded candidates.
``format_rejects`` counts candidates whose real payload is *not* the
requested format -- including payloads that are not a recognised image at
all. ``size_rejects`` counts candidates of the requested format that are
too small. ``failures`` only holds candidates whose bytes could not be
fetched at all, because that is the one situation worth an access error.
"""
kept: tuple[KeptImage, ...] = ()
format_rejects: int = 0
size_rejects: int = 0
other_formats: Mapping[str, int] = field(default_factory=dict)
failures: tuple[dict[str, Any], ...] = ()
@property
def inspected(self) -> int:
"""Number of candidates whose bytes were read successfully."""
return len(self.kept) + self.format_rejects + self.size_rejects
#: Bucket name for a downloaded payload that is not a recognised image at all.
UNKNOWN_FORMAT = "unknown"
def _count_other_format(other_formats: dict[str, int], detected: str | None) -> None:
"""Record which other format a rejected candidate really was."""
key = detected or UNKNOWN_FORMAT
other_formats[key] = other_formats.get(key, 0) + 1
def _select(candidates: Sequence[SourceCandidate], request: Mapping[str, Any]) -> Selection:
"""Fetch and inspect candidates, keeping only what the real bytes justify.
Every kept item has been downloaded and parsed: its format comes from the
container signature and its width/height come from the PNG ``IHDR`` or the
ICO directory, never from what the source claimed. A candidate that serves
other-format bytes -- a JPEG, an SVG, a TIFF, an HTML error page, anything
that is not a real ``png``/``ico`` payload -- counts as a *format rejection*
so the caller gets ``no_format`` instead of an access error, and a candidate
of the wanted format that is too small counts as a size rejection. Only a
candidate whose bytes cannot be fetched at all is recorded as a failure.
"""
wanted_format = str(request["format"])
limit = int(request["count"])
kept: list[KeptImage] = []
format_rejects = 0
size_rejects = 0
other_formats: dict[str, int] = {}
failures: list[dict[str, Any]] = []
for candidate in candidates:
if len(kept) >= limit:
break
try:
payload = candidate.fetch_bytes()
except ToolkitError as error:
failures.append(_failure(candidate, error.to_dict()))
continue
except OSError as error:
failures.append(
_failure(candidate, error_fragment("fetch_failed", str(error)))
)
continue
detected = images.detect_format(payload)
if detected != wanted_format:
# Whatever these bytes are (another image format, or no recognisable
# image at all), they are not the requested format: that is a format
# verdict about the topic, never an access problem.
format_rejects += 1
_count_other_format(other_formats, detected)
continue
try:
info = images.inspect_image(payload)
except ImageFormatError as error:
# The signature claims the requested format but the header is
# corrupt, so the payload itself is unusable rather than merely the
# wrong format.
failures.append(_failure(candidate, error.to_dict()))
continue
if not _meets_minimum_size(info, request):
size_rejects += 1
continue
kept.append(KeptImage(candidate=candidate, info=info, payload=payload))
return Selection(
kept=tuple(kept),
format_rejects=format_rejects,
size_rejects=size_rejects,
other_formats=dict(other_formats),
failures=tuple(failures),
)
def _slug(candidate: SourceCandidate) -> str:
raw = str(getattr(candidate, "title", None) or getattr(candidate, "source_id", "") or "image").lower()
slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")
return (slug or "image")[:60]
def _target_size(width: int, height: int, request: Mapping[str, Any]) -> tuple[int, int]:
"""Return the exact size to store: the requested size, never enlarged.
``--resize`` means "give me the requested size" rather than "give me
something that fits in it": a requested width **and** height are hit
exactly, and a dimension the caller left open keeps the source value. A
source dimension that is already at or below the requested one is never
scaled up, so ``--resize`` can only ever shrink an image.
"""
wanted_width = request.get("width")
wanted_height = request.get("height")
target_width = width if wanted_width is None else min(width, int(wanted_width))
target_height = height if wanted_height is None else min(height, int(wanted_height))
return max(1, target_width), max(1, target_height)
def resize_image(payload: bytes, request: Mapping[str, Any]) -> tuple[bytes, dict[str, Any]]:
"""Downscale ``payload`` to the requested size and re-encode it.
The stored image is re-encoded in the requested container format (``png``
or ``ico``) so the saved bytes really open as that format. An image that
is not larger than the request in either dimension is returned untouched:
``--resize`` never enlarges and never rewrites pixels it does not have to.
"""
decoded = images.decode_image(payload)
wanted_format = str(request.get("format") or decoded.format)
if decoded.format != wanted_format:
raise ImageResizeError(
f"cannot re-encode a {decoded.format} image as {wanted_format}",
details={"from_format": decoded.format, "to_format": wanted_format},
)
target_width, target_height = _target_size(decoded.width, decoded.height, request)
fragment: dict[str, Any] = {
"requested": {"width": request.get("width"), "height": request.get("height")},
"from": {"width": decoded.width, "height": decoded.height},
"format": wanted_format,
}
if (target_width, target_height) == (decoded.width, decoded.height):
fragment.update(
{
"applied": False,
"reason": "already_fits",
"to": {"width": decoded.width, "height": decoded.height},
}
)
return payload, fragment
rows = images.resample_rows(
decoded.rows, decoded.width, decoded.height, target_width, target_height
)
output = images.encode_image(wanted_format, target_width, target_height, rows)
fragment.update({"applied": True, "to": {"width": target_width, "height": target_height}})
return output, fragment
def _discard(path: Path) -> None:
"""Remove a file whose stored bytes failed verification."""
try:
path.unlink()
except OSError:
# Never chase a path that is not a regular file we just wrote: the run
# must not delete anything else in the caller's directory.
pass
def _verify_saved(
path: Path, request: Mapping[str, Any], expected: tuple[int, int]
) -> ImageInfo:
"""Read a stored file back and prove it opens as the requested format.
A reported path is only trustworthy when the file really exists and really
holds bytes of the requested format at the expected size, so the stored
payload is decoded again instead of holding the writer to its word.
"""
wanted_format = str(request["format"])
decoded = images.decode_image(path.read_bytes())
if decoded.format != wanted_format:
raise ImageFormatError(
"stored file does not open as the requested format",
details={
"path": str(path),
"expected_format": wanted_format,
"actual_format": decoded.format,
},
)
if (decoded.width, decoded.height) != expected:
raise ImageFormatError(
"stored file does not have the expected pixel size",
details={
"path": str(path),
"expected": {"width": expected[0], "height": expected[1]},
"actual": {"width": decoded.width, "height": decoded.height},
},
)
return ImageInfo(format=decoded.format, width=decoded.width, height=decoded.height)
def _download(
selected: Sequence[KeptImage], request: Mapping[str, Any]
) -> tuple[list[dict[str, Any]], int]:
"""Write the already-fetched payloads; return ``(items, failure_count)``.
Every selected image is stored inside the caller-specified directory. The
bytes verified by :func:`_select` are written as they are, or downscaled
first when ``resize`` is enabled, so an item can never be downloaded twice
or stored in a format that was not verified. Each stored file is read back
and decoded before its path is reported, and a file that does not open as
the requested format at the expected size is deleted and reported as a
per-item save failure instead of being left behind as a broken path.
"""
directory = Path(request["output_dir"]).expanduser()
try:
directory.mkdir(parents=True, exist_ok=True)
except OSError as exc:
raise SourceAccessError(
f"cannot create output directory {str(directory)!r}: {exc.strerror or exc}",
details={"output_dir": str(directory)},
) from exc
suffix = "." + str(request["format"])
items: list[dict[str, Any]] = []
failures = 0
for index, entry in enumerate(selected, start=1):
record = entry.to_item()
destination = directory / f"{index:03d}-{_slug(entry.candidate)}{suffix}"
payload = entry.payload
resize_fragment: dict[str, Any] | None = None
expected = (entry.info.width, entry.info.height)
try:
if request["resize"]:
payload, resize_fragment = resize_image(payload, request)
expected = (
int(resize_fragment["to"]["width"]),
int(resize_fragment["to"]["height"]),
)
destination.write_bytes(payload)
stored = _verify_saved(destination, request, expected)
except (ToolkitError, OSError) as exc:
_discard(destination)
failures += 1
error = error_fragment(getattr(exc, "code", "download_failed"), str(exc))
if isinstance(exc, ToolkitError) and exc.details:
error["details"] = dict(exc.details)
# A failed save reports no path: only files that really exist and
# really open are ever named as saved.
record["download"] = {"ok": False, "target": str(destination), "error": error}
else:
download: dict[str, Any] = {
"ok": True,
"path": str(destination),
"bytes": len(payload),
"format": stored.format,
"width": stored.width,
"height": stored.height,
}
if resize_fragment is not None:
download["resize"] = resize_fragment
record["download"] = download
items.append(record)
return items, failures
def _guess_source(argv: Sequence[str]) -> str | None:
"""Best-effort source id extraction for error envelopes."""
for index, token in enumerate(argv):
if token == "--source" or token == "-s":
if index + 1 < len(argv):
return argv[index + 1]
elif token.startswith("--source="):
return token.split("=", 1)[1]
return None
def _request_from(argv: Sequence[str]) -> tuple[dict[str, Any], str, Any]:
"""Turn CLI arguments into ``(normalized_request, source_id, registry)``."""
parser = build_parser()
namespace = parser.parse_args(list(argv))
flags = _flags_to_mapping(namespace)
document = namespace.request
if document is not None:
if flags:
raise RequestError(
"a request document cannot be combined with flags: "
+ ", ".join(sorted(flags)),
details={"conflicting_flags": sorted(flags)},
accepted_fields=REQUEST_FIELDS,
)
raw = _document_to_mapping(_load_document(document))
elif flags:
raw = flags
else:
raise RequestError(
"no request given; pass a JSON request document or at least one flag",
accepted_fields=REQUEST_FIELDS,
)
request = normalize_request(raw)
registry = default_registry()
available = list(registry.available_sources())
if request["source"] not in available:
raise UnknownSourceError(request["source"], available)
return request, request["source"], registry
def _execute(argv: Sequence[str]) -> Response:
"""Run one request and return the response that must be printed.
The search path downloads a bounded window of candidates and filters that
window on the parsed bytes, so the answer distinguishes three different
situations through both the status and the exit code:
* ``ok`` -- at least one image really is the requested format and at least
the requested width and height;
* ``no_format`` -- the source had hits, but none of them really are the
requested format;
* ``no_size`` -- images of the requested format existed, but all of them
are smaller than requested;
* ``access_error`` -- the source could not be searched, or not a single
candidate could be fetched, which is never reported as a "no match".
A candidate whose downloaded bytes are a different format (a JPEG, an SVG,
an HTML error page, ...) is a format mismatch about the topic, not an access
problem: it feeds ``no_format`` instead of turning a working search into an
error, and a stray unfetchable candidate never demotes a usable answer to
``partial_download``.
"""
request, source, registry = _request_from(argv)
builder = ResponseBuilder(source=source, request=request)
if request["dry_run"]:
return builder.dry_run()
cap = candidate_cap(request["count"])
candidates = registry.search(source, request["topic"], cap)
selection = _select(candidates, request)
unreadable = list(selection.failures)
if not selection.kept:
details = {
"requested_format": request["format"],
"requested_size": {"width": request["width"], "height": request["height"]},
"minimum_size": {"width": request["width"], "height": request["height"]},
"candidate_count": len(candidates),
"candidates_inspected": selection.inspected,
"format_matches": selection.size_rejects,
"other_format_candidates": selection.format_rejects,
"observed_formats": dict(selection.other_formats),
"size_matches": 0,
"unreadable_candidates": len(unreadable),
}
if selection.size_rejects:
return builder.build(
STATUS_NO_SIZE,
error=error_fragment(
"no_size_match",
"the source returned no images at least the requested width and height",
details=details,
),
)
if selection.format_rejects or not unreadable:
# Either the source really had no images in the requested format, or
# it had no hits at all; both answer the topic, not an outage.
return builder.build(
STATUS_NO_FORMAT,
error=error_fragment(
"no_format_match",
f"the source returned no {request['format']} images for {request['topic']!r}",
details=details,
),
)
return builder.access_error(
error=error_fragment(
"candidate_unreadable",
"none of the candidates could be downloaded",
details={**details, "failures": unreadable},
)
)
if not request["download"]:
# Every reported item was fetched and verified against its real bytes, so
# an unfetchable candidate is simply skipped: it must not demote an
# otherwise usable answer to a partial-download exit code.
return builder.build(STATUS_OK, items=[entry.to_item() for entry in selection.kept])
items, save_failures = _download(selection.kept, request)
saved_paths = [
item["download"]["path"] for item in items if item["download"].get("ok") is True
]
failed = save_failures + len(unreadable)
if failed:
# The run still reports every saved path, but a per-item save (or fetch)
# failure is never hidden behind ``ok``: it has its own status and exit
# code so the caller can tell a complete run from an incomplete one.
return builder.build(
STATUS_PARTIAL_DOWNLOAD,
items=items,
error=error_fragment(
"download_incomplete",
f"{failed} of {len(items) + len(unreadable)} images could not be saved",
details={
"failed": failed,
"saved": len(saved_paths),
"total": len(items) + len(unreadable),
"saved_paths": saved_paths,
"failures": unreadable,
},
),
)
return builder.build(STATUS_OK, items=items)
def main(argv: Sequence[str] | None = None) -> int:
"""Entry point: emit exactly one JSON document and return the exit code."""
arguments = list(sys.argv[1:] if argv is None else argv)
try:
response = _execute(arguments)
except ToolkitError as error:
builder = ResponseBuilder(source=_guess_source(arguments))
response = builder.from_error(error)
except OSError as error:
builder = ResponseBuilder(source=_guess_source(arguments))
response = builder.build(
STATUS_ACCESS_ERROR,
error=error_fragment("access_error", str(error)),
)
response.emit()
return response.exit_code
if __name__ == "__main__": # pragma: no cover - module entry point
sys.exit(main())
+771
View File
@@ -0,0 +1,771 @@
"""Stdlib-only inspection and construction of the image formats we accept.
The CLI decides what to keep from the **downloaded bytes** and never from the
metadata a source reports about a file: a search hit that claims to be a
``image/png`` of 800x600 pixels is kept only when the bytes really are a PNG of
at least the requested size. This module is the single place that knows how to
read the real format and the real pixel dimensions out of those bytes.
It deliberately depends on nothing but the standard library:
* :func:`detect_format` recognises the container signatures of the two accepted
formats (``png``, ``ico``) plus the formats a real image source commonly
serves instead (``jpeg``, ``gif``, ``bmp``, ``webp``, ``svg``, ``tiff``,
``pdf``). Every recognised-but-different format, and every payload that is
not a recognised image at all, counts as *another format* rather than an
access problem: "the source has images, but none of them are the format you
asked for" is a different answer from "the source could not be used".
* :func:`read_png_dimensions` reads the ``IHDR`` header of a PNG.
* :func:`read_ico_dimensions` reads the ICONDIR/ICONDIRENTRY directory of an
ICO file (the directory stores the real sizes, including the ``0``-means-256
encoding).
* :func:`inspect_image` combines both into an :class:`ImageInfo`.
* :func:`png_bytes` and :func:`ico_bytes` *build* the same formats so tests (and
callers) can produce real payloads without Pillow or any other dependency.
* :func:`decode_png`, :func:`encode_png`, :func:`ico_wrap` and
:func:`ico_unwrap` decode and re-encode those containers, and
:func:`decode_image`, :func:`resample_rows` and :func:`encode_image` are the
building blocks the CLI uses to downscale a downloaded image and store it in
the requested format -- still with nothing but the standard library.
:class:`ImageFormatError` -- a :class:`~tools.errors.ToolkitError` subclass with
a stable ``code`` -- is reserved for bytes that are genuinely unusable: an
unknown container, or a ``png``/``ico`` payload whose header is truncated or
corrupt. A readable payload of another format is never an error, so the CLI
keeps a working search out of the ``access_error`` status.
"""
from __future__ import annotations
import struct
import zlib
from dataclasses import dataclass
from typing import Any, Iterable, Sequence
from .errors import ToolkitError
__all__ = [
"BMP",
"GIF",
"ICO",
"ICO_ENTRY_SIZE",
"ICO_HEADER_SIZE",
"ICO_MAX_DIMENSION",
"ImageFormatError",
"ImageInfo",
"JPEG",
"KNOWN_FORMATS",
"PDF",
"PNG",
"PNG_SIGNATURE",
"SVG",
"TIFF",
"WEBP",
"detect_format",
"ico_bytes",
"ico_entries",
"inspect_image",
"png_bytes",
"read_ico_dimensions",
"read_png_dimensions",
"DecodedImage",
"decode_image",
"decode_png",
"encode_image",
"encode_png",
"ico_unwrap",
"ico_wrap",
"png_chunks",
"resample_rows",
]
#: The two formats the CLI accepts as search requests.
PNG = "png"
ICO = "ico"
#: Formats that are recognised (so they can be reported as "another format")
#: but never accepted by the CLI.
JPEG = "jpeg"
GIF = "gif"
BMP = "bmp"
WEBP = "webp"
#: Vector and document formats a live image source serves instead of a bitmap.
#: Naming them lets a search say *which* other formats the topic offered instead
#: of degrading into an access problem.
SVG = "svg"
TIFF = "tiff"
PDF = "pdf"
#: Every format :func:`detect_format` can name.
KNOWN_FORMATS: tuple[str, ...] = (PNG, ICO, JPEG, GIF, BMP, WEBP, SVG, TIFF, PDF)
#: The 8-byte signature every PNG file starts with.
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
#: ICONDIR: reserved (0), type (1 = icon) and the entry count.
ICO_HEADER_SIZE = 6
#: One ICONDIRENTRY: 16 bytes of width/height/colours/planes/bpp/size/offset.
ICO_ENTRY_SIZE = 16
#: A single ICO dimension byte of 0 encodes 256 pixels.
ICO_MAX_DIMENSION = 256
#: Magic prefixes of the recognised but unsupported formats.
_JPEG_SIGNATURE = b"\xff\xd8\xff"
_GIF_SIGNATURES = (b"GIF87a", b"GIF89a")
_BMP_SIGNATURE = b"BM"
_RIFF_SIGNATURE = b"RIFF"
_WEBP_SIGNATURE = b"WEBP"
_TIFF_SIGNATURES = (b"II*\x00", b"MM\x00*")
_PDF_SIGNATURE = b"%PDF-"
#: How many leading bytes are searched for an ``<svg`` root element.
_SVG_PROBE = 1024
#: Number of payload bytes worth quoting in an error detail.
_SIGNATURE_PROBE = 16
class ImageFormatError(ToolkitError):
"""Raised when downloaded bytes are not a readable image of a known format.
The bytes could not be attributed to any format the toolkit understands, or
they claim to be an accepted format but their header is truncated/corrupt.
It derives from :class:`~tools.errors.ToolkitError`, so the CLI reports it
as an access problem (the source handed us something unusable) rather than
as a "no such format" / "no such size" verdict about the topic.
"""
code = "image_unreadable"
def __init__(self, message: str, *, details: dict[str, Any] | None = None) -> None:
super().__init__(message, details=details)
@dataclass(frozen=True)
class ImageInfo:
"""The real format and real pixel size of one payload."""
format: str
width: int
height: int
def to_dict(self) -> dict[str, Any]:
"""JSON-ready projection used in CLI items."""
return {"format": self.format, "width": self.width, "height": self.height}
@property
def area(self) -> int:
return self.width * self.height
def _probe(payload: bytes) -> bytes:
return bytes(payload[:_SIGNATURE_PROBE])
def _looks_like_svg(data: bytes) -> bool:
"""True when *data* starts with an SVG root element (or its XML prologue)."""
head = data[:_SVG_PROBE]
if head.startswith(b"\xef\xbb\xbf"):
head = head[3:]
head = head.lstrip()
if head.lower().startswith(b"<svg"):
return True
return head.startswith(b"<?xml") and b"<svg" in data[:_SVG_PROBE].lower()
def detect_format(payload: bytes) -> str | None:
"""Return the real container format of *payload*, or ``None``.
A recognised format other than ``png``/``ico`` is a perfectly good answer:
the source does have images, just not in the requested format. ``None``
means the bytes are not a recognised image at all -- still "not the requested
format" from the caller's point of view, and never an access problem on its
own.
"""
if not isinstance(payload, (bytes, bytearray, memoryview)):
return None
data = bytes(payload)
if data.startswith(PNG_SIGNATURE):
return PNG
if data[:4] == b"\x00\x00\x01\x00":
return ICO
if data.startswith(_JPEG_SIGNATURE):
return JPEG
if data.startswith(_GIF_SIGNATURES):
return GIF
if data.startswith(_BMP_SIGNATURE) and len(data) >= 14:
return BMP
if data.startswith(_RIFF_SIGNATURE) and data[8:12] == _WEBP_SIGNATURE:
return WEBP
if data.startswith(_TIFF_SIGNATURES):
return TIFF
if data.startswith(_PDF_SIGNATURE):
return PDF
if _looks_like_svg(data):
return SVG
return None
def read_png_dimensions(payload: bytes) -> tuple[int, int]:
"""Return ``(width, height)`` from the ``IHDR`` chunk of a PNG payload."""
data = bytes(payload)
if not data.startswith(PNG_SIGNATURE):
raise ImageFormatError(
"payload does not start with the PNG signature",
details={"signature": _probe(data).hex()},
)
offset = len(PNG_SIGNATURE)
if len(data) < offset + 8:
raise ImageFormatError("PNG header is truncated")
(length,) = struct.unpack(">I", data[offset : offset + 4])
chunk_type = data[offset + 4 : offset + 8]
if chunk_type != b"IHDR":
raise ImageFormatError(
"PNG payload does not start with an IHDR chunk",
details={"first_chunk": chunk_type.decode("latin-1")},
)
if length != 13:
raise ImageFormatError(
"PNG IHDR chunk has an unexpected length",
details={"length": length},
)
header = data[offset + 8 : offset + 8 + length]
if len(header) != length:
raise ImageFormatError("PNG IHDR chunk is truncated")
width, height = struct.unpack(">II", header[:8])
if width < 1 or height < 1:
raise ImageFormatError(
"PNG image declares no pixels",
details={"width": width, "height": height},
)
return int(width), int(height)
def ico_entries(payload: bytes) -> tuple[tuple[int, int, int, int], ...]:
"""Return ``(width, height, size, offset)`` for every ICO directory entry."""
data = bytes(payload)
if len(data) < ICO_HEADER_SIZE:
raise ImageFormatError("ICO header is truncated")
reserved, kind, count = struct.unpack("<HHH", data[:ICO_HEADER_SIZE])
if reserved != 0 or kind != 1 or count < 1:
raise ImageFormatError(
"payload is not an ICO image",
details={"reserved": reserved, "type": kind, "count": count},
)
directory_end = ICO_HEADER_SIZE + count * ICO_ENTRY_SIZE
if len(data) < directory_end:
raise ImageFormatError(
"ICO directory is truncated",
details={"count": count, "bytes": len(data)},
)
entries: list[tuple[int, int, int, int]] = []
for index in range(count):
start = ICO_HEADER_SIZE + index * ICO_ENTRY_SIZE
raw_width, raw_height, _colours, _reserved, _planes, _bits, size, offset = struct.unpack(
"<BBBBHHII", data[start : start + ICO_ENTRY_SIZE]
)
width = raw_width or ICO_MAX_DIMENSION
height = raw_height or ICO_MAX_DIMENSION
if size < 1 or offset < directory_end or offset + size > len(data):
raise ImageFormatError(
"ICO directory entry points outside the payload",
details={"index": index, "size": size, "offset": offset, "bytes": len(data)},
)
entries.append((width, height, size, offset))
return tuple(entries)
def read_ico_dimensions(payload: bytes) -> tuple[int, int]:
"""Return the dimensions of the largest image stored in an ICO payload.
The ICONDIRENTRY directory is authoritative: it carries one width/height pair
per stored image, so no decoding of the embedded bitmap or PNG is needed.
"""
entries = ico_entries(payload)
width, height, _size, _offset = max(entries, key=lambda entry: entry[0] * entry[1])
return int(width), int(height)
def inspect_image(payload: bytes) -> ImageInfo:
"""Return the real format and pixel size of *payload*.
Raises :class:`ImageFormatError` when the bytes are not an image at all, or
when they are not one of the two formats whose dimensions can be read.
"""
image_format = detect_format(payload)
if image_format is None:
raise ImageFormatError(
"downloaded payload is not a recognised image format",
details={"signature": _probe(bytes(payload)).hex()},
)
if image_format == PNG:
width, height = read_png_dimensions(payload)
return ImageInfo(format=PNG, width=width, height=height)
if image_format == ICO:
width, height = read_ico_dimensions(payload)
return ImageInfo(format=ICO, width=width, height=height)
raise ImageFormatError(
f"pixel dimensions are only readable for png and ico, not {image_format}",
details={"format": image_format},
)
# ---------------------------------------------------------------------------
# Builders: the same formats, produced with the standard library only
# ---------------------------------------------------------------------------
def _png_chunk(chunk_type: bytes, chunk_data: bytes) -> bytes:
crc = zlib.crc32(chunk_type + chunk_data) & 0xFFFFFFFF
return struct.pack(">I", len(chunk_data)) + chunk_type + chunk_data + struct.pack(">I", crc)
def png_bytes(
width: int,
height: int,
*,
rgba: tuple[int, int, int, int] = (255, 255, 255, 255),
rows: Sequence[bytes] | None = None,
level: int = 6,
) -> bytes:
"""Build a valid 8-bit RGBA PNG of exactly ``width`` x ``height`` pixels.
``rows`` may supply raw (filter-0) RGBA scanlines; otherwise every pixel is
filled with ``rgba``. The result is a real PNG file: a correct signature,
a 13-byte ``IHDR``, a zlib-compressed ``IDAT`` and an empty ``IEND``.
"""
if isinstance(width, bool) or isinstance(height, bool):
raise ValueError("width and height must be integers")
if width < 1 or height < 1:
raise ValueError("width and height must be >= 1")
if rows is None:
pixel = bytes(rgba)
line = pixel * width
raw = bytearray()
for _ in range(height):
raw.append(0)
raw += line
raw_bytes = bytes(raw)
else:
if len(rows) != height:
raise ValueError("rows must contain exactly height scanlines")
raw = bytearray()
for row in rows:
if len(row) != width * 4:
raise ValueError("each scanline must hold width * 4 bytes")
raw.append(0)
raw += row
raw_bytes = bytes(raw)
header = struct.pack(">IIBBBBB", width, height, 8, 6, 0, 0, 0)
return (
PNG_SIGNATURE
+ _png_chunk(b"IHDR", header)
+ _png_chunk(b"IDAT", zlib.compress(raw_bytes, level))
+ _png_chunk(b"IEND", b"")
)
def ico_bytes(
width: int,
height: int,
*,
rgba: tuple[int, int, int, int] = (255, 255, 255, 255),
frames: Iterable[tuple[int, int]] | None = None,
) -> bytes:
"""Build an ICO payload whose directory lists the requested sizes.
By default the ICO stores one embedded PNG of ``width`` x ``height`` pixels
and the directory entry repeats those dimensions (``0`` encodes 256). Pass
``frames`` to describe several directory entries instead, for example
``ico_bytes(48, 48, frames=[(16, 16), (48, 48)])``: the dimensions of the
*largest* entry then become the reported size, which is exactly what
:func:`read_ico_dimensions` returns.
"""
sizes = list(frames) if frames is not None else [(width, height)]
if not sizes:
raise ValueError("an ICO payload needs at least one frame")
payloads: list[bytes] = []
for frame_width, frame_height in sizes:
if not 1 <= frame_width <= ICO_MAX_DIMENSION:
raise ValueError("ICO widths must be between 1 and 256 pixels")
if not 1 <= frame_height <= ICO_MAX_DIMENSION:
raise ValueError("ICO heights must be between 1 and 256 pixels")
payloads.append(png_bytes(frame_width, frame_height, rgba=rgba))
header = struct.pack("<HHH", 0, 1, len(payloads))
offset = ICO_HEADER_SIZE + ICO_ENTRY_SIZE * len(payloads)
entries = bytearray()
for (frame_width, frame_height), payload in zip(sizes, payloads):
entries += struct.pack(
"<BBBBHHII",
frame_width % ICO_MAX_DIMENSION,
frame_height % ICO_MAX_DIMENSION,
0,
0,
1,
32,
len(payload),
offset,
)
offset += len(payload)
return header + bytes(entries) + b"".join(payloads)
# ---------------------------------------------------------------------------
# Decoding and re-encoding: PNG pixels and ICO containers
# ---------------------------------------------------------------------------
#: Channels per PNG colour type (0 grey, 2 rgb, 3 palette, 4 grey+alpha, 6 rgba).
_PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
@dataclass(frozen=True)
class DecodedImage:
"""The container format, pixel size and 8-bit RGBA scanlines of a payload."""
format: str
width: int
height: int
rows: tuple[bytes, ...]
def to_dict(self) -> dict[str, Any]:
"""JSON-ready projection: format plus the real pixel size."""
return {"format": self.format, "width": self.width, "height": self.height}
def png_chunks(payload: bytes) -> tuple[tuple[bytes, bytes], ...]:
"""Split a PNG payload into ``(chunk_type, chunk_data)`` pairs.
Raises :class:`ImageFormatError` when the signature is missing, a chunk is
truncated or the stream never reaches ``IEND``: those bytes cannot be
decoded, so stitching a half image would be worse than reporting it.
"""
data = bytes(payload)
if not data.startswith(PNG_SIGNATURE):
raise ImageFormatError(
"payload does not start with the PNG signature",
details={"signature": _probe(data).hex()},
)
chunks: list[tuple[bytes, bytes]] = []
offset = len(PNG_SIGNATURE)
while offset + 8 <= len(data):
(length,) = struct.unpack(">I", data[offset : offset + 4])
chunk_type = data[offset + 4 : offset + 8]
chunk_data = data[offset + 8 : offset + 8 + length]
if len(chunk_data) != length:
raise ImageFormatError(
"PNG chunk is truncated",
details={"chunk": chunk_type.decode("latin-1"), "length": length},
)
chunks.append((chunk_type, chunk_data))
offset += 12 + length
if chunk_type == b"IEND":
return tuple(chunks)
raise ImageFormatError("PNG image has no IEND chunk")
def _unfilter_rows(raw: bytes, width: int, height: int, bpp: int) -> list[bytearray]:
"""Undo the per-scanline PNG filters, returning one row per pixel line."""
stride = width * bpp
if len(raw) < height * (stride + 1):
raise ImageFormatError(
"PNG image data is truncated",
details={"width": width, "height": height, "bytes": len(raw)},
)
rows: list[bytearray] = []
previous = bytearray(stride)
position = 0
for _ in range(height):
filter_type = raw[position]
position += 1
line = bytearray(raw[position : position + stride])
position += stride
if filter_type == 1:
for index in range(bpp, stride):
line[index] = (line[index] + line[index - bpp]) & 0xFF
elif filter_type == 2:
for index in range(stride):
line[index] = (line[index] + previous[index]) & 0xFF
elif filter_type == 3:
for index in range(stride):
left = line[index - bpp] if index >= bpp else 0
line[index] = (line[index] + ((left + previous[index]) >> 1)) & 0xFF
elif filter_type == 4:
for index in range(stride):
left = line[index - bpp] if index >= bpp else 0
up = previous[index]
up_left = previous[index - bpp] if index >= bpp else 0
estimate = left + up - up_left
to_left = abs(estimate - left)
to_up = abs(estimate - up)
to_up_left = abs(estimate - up_left)
if to_left <= to_up and to_left <= to_up_left:
predictor = left
elif to_up <= to_up_left:
predictor = up
else:
predictor = up_left
line[index] = (line[index] + predictor) & 0xFF
elif filter_type != 0:
raise ImageFormatError(
"unsupported PNG filter type",
details={"filter_type": int(filter_type)},
)
previous = line
rows.append(line)
return rows
def _row_to_rgba(
row: bytearray,
width: int,
color_type: int,
depth: int,
palette: bytes,
transparency: bytes | None,
) -> bytes:
"""Convert one unfiltered scanline into 8-bit RGBA samples."""
step = depth // 8
# Bytes per pixel: one sample per channel, so the channel offsets inside a
# pixel are ``step`` apart while pixels are ``channels * step`` apart.
pixel_stride = _PNG_CHANNELS[color_type] * step
out = bytearray()
if color_type == 3:
for x in range(width):
index = row[x]
base = index * 3
if base + 2 >= len(palette):
raise ImageFormatError(
"PNG palette index is out of range",
details={"index": int(index), "palette_bytes": len(palette)},
)
alpha = transparency[index] if transparency and index < len(transparency) else 255
out += bytes((palette[base], palette[base + 1], palette[base + 2], alpha))
return bytes(out)
for x in range(width):
offset = x * pixel_stride
if color_type == 0:
value = row[offset]
out += bytes((value, value, value, 255))
elif color_type == 4:
value = row[offset]
out += bytes((value, value, value, row[offset + step]))
elif color_type == 2:
out += bytes((row[offset], row[offset + step], row[offset + 2 * step], 255))
else:
out += bytes(
(
row[offset],
row[offset + step],
row[offset + 2 * step],
row[offset + 3 * step],
)
)
return bytes(out)
def decode_png(payload: bytes) -> tuple[int, int, tuple[bytes, ...]]:
"""Decode a PNG payload into ``(width, height, rgba_rows)``.
Supports the non-interlaced 8-bit and 16-bit PNG colour types ImageMagick,
GIMP and the Wikimedia thumbnailers produce; anything else raises
:class:`ImageFormatError` instead of guessing at pixels.
"""
header: tuple[int, ...] | None = None
palette = b""
transparency: bytes | None = None
data = bytearray()
for chunk_type, chunk_data in png_chunks(payload):
if chunk_type == b"IHDR":
if len(chunk_data) < 13:
raise ImageFormatError("PNG header is truncated")
header = struct.unpack(">IIBBBBB", chunk_data[:13])
elif chunk_type == b"PLTE":
palette = chunk_data
elif chunk_type == b"tRNS":
transparency = chunk_data
elif chunk_type == b"IDAT":
data += chunk_data
if header is None:
raise ImageFormatError("PNG image has no IHDR chunk")
width, height, depth, color_type, compression, filter_method, interlace = header
if width < 1 or height < 1:
raise ImageFormatError("PNG image declares no pixels")
if compression != 0 or filter_method != 0:
raise ImageFormatError(
"unsupported PNG compression or filter method",
details={"compression": compression, "filter_method": filter_method},
)
if interlace != 0:
raise ImageFormatError(
"interlaced PNG images are not supported",
details={"interlace": interlace},
)
channels = _PNG_CHANNELS.get(color_type)
if channels is None:
raise ImageFormatError(
"unsupported PNG colour type",
details={"color_type": color_type},
)
if depth not in (8, 16):
raise ImageFormatError(
"unsupported PNG bit depth",
details={"bit_depth": depth},
)
if color_type == 3 and depth > 8:
raise ImageFormatError(
"unsupported PNG palette bit depth",
details={"bit_depth": depth},
)
if color_type == 3 and not palette:
raise ImageFormatError("PNG palette image has no PLTE chunk")
try:
raw = zlib.decompress(bytes(data))
except zlib.error as exc:
raise ImageFormatError(
"PNG image data is corrupt",
details={"reason": str(exc)},
) from exc
rows = _unfilter_rows(raw, width, height, channels * (depth // 8))
rgba_rows = tuple(
_row_to_rgba(row, width, color_type, depth, palette, transparency) for row in rows
)
return int(width), int(height), rgba_rows
def encode_png(width: int, height: int, rows: Sequence[bytes]) -> bytes:
"""Encode 8-bit RGBA ``rows`` as a real PNG with correct chunk CRCs."""
return png_bytes(width, height, rows=list(rows))
def ico_wrap(embedded_png: bytes, width: int, height: int) -> bytes:
"""Wrap a PNG payload in a single-entry ICO container of that size."""
if not 1 <= width <= ICO_MAX_DIMENSION:
raise ImageFormatError(
"ICO widths must be between 1 and 256 pixels",
details={"width": width},
)
if not 1 <= height <= ICO_MAX_DIMENSION:
raise ImageFormatError(
"ICO heights must be between 1 and 256 pixels",
details={"height": height},
)
png_payload = bytes(embedded_png)
if not png_payload.startswith(PNG_SIGNATURE):
raise ImageFormatError("an ICO container can only wrap a PNG payload here")
header = struct.pack("<HHH", 0, 1, 1)
entry = struct.pack(
"<BBBBHHII",
width % ICO_MAX_DIMENSION,
height % ICO_MAX_DIMENSION,
0,
0,
1,
32,
len(png_payload),
ICO_HEADER_SIZE + ICO_ENTRY_SIZE,
)
return header + entry + png_payload
def ico_unwrap(payload: bytes) -> tuple[int, int, bytes]:
"""Return the ``(width, height, embedded_png)`` of an ICO's largest frame."""
entries = ico_entries(payload)
width, height, size, offset = max(entries, key=lambda entry: entry[0] * entry[1])
embedded = bytes(payload)[offset : offset + size]
if not embedded.startswith(PNG_SIGNATURE):
raise ImageFormatError(
"ICO frames stored as raw bitmaps are not supported",
details={"width": width, "height": height},
)
return int(width), int(height), embedded
def decode_image(payload: bytes) -> DecodedImage:
"""Decode a PNG or ICO payload into :class:`DecodedImage` RGBA scanlines."""
data = bytes(payload)
image_format = detect_format(data)
if image_format == PNG:
width, height, rows = decode_png(data)
return DecodedImage(format=PNG, width=width, height=height, rows=rows)
if image_format == ICO:
width, height, embedded = ico_unwrap(data)
inner_width, inner_height, rows = decode_png(embedded)
if (inner_width, inner_height) != (width, height):
# The directory is authoritative for the reported size, but the
# pixels must agree or the container is inconsistent.
raise ImageFormatError(
"ICO directory and embedded PNG disagree about the size",
details={
"directory": {"width": width, "height": height},
"embedded": {"width": inner_width, "height": inner_height},
},
)
return DecodedImage(format=ICO, width=width, height=height, rows=rows)
raise ImageFormatError(
"pixel data is only decodable for png and ico",
details={"format": image_format},
)
def resample_rows(
rows: Sequence[bytes],
width: int,
height: int,
target_width: int,
target_height: int,
) -> tuple[bytes, ...]:
"""Nearest-neighbour resample of RGBA ``rows`` to exactly the target size."""
if target_width < 1 or target_height < 1:
raise ImageFormatError("resampled dimensions must be at least 1x1")
if len(rows) != height:
raise ImageFormatError(
"row count does not match the source height",
details={"rows": len(rows), "height": height},
)
if width < 1 or height < 1:
raise ImageFormatError("source image has no pixels")
columns = [min(width - 1, (x * width) // target_width) * 4 for x in range(target_width)]
resized: list[bytes] = []
for y in range(target_height):
source = rows[min(height - 1, (y * height) // target_height)]
line = bytearray()
for offset in columns:
line += source[offset : offset + 4]
resized.append(bytes(line))
return tuple(resized)
def encode_image(
image_format: str,
width: int,
height: int,
rows: Sequence[bytes],
) -> bytes:
"""Encode RGBA ``rows`` in ``image_format`` (``png`` or ``ico``)."""
png_payload = encode_png(width, height, rows)
if image_format == PNG:
return png_payload
if image_format == ICO:
return ico_wrap(png_payload, width, height)
raise ImageFormatError(
"only png and ico can be encoded",
details={"format": image_format},
)
+578
View File
@@ -0,0 +1,578 @@
"""Pluggable, credential-free image source registry and its HTTPS layer.
Nothing in this module reads an API key, token, registration or secret from
arguments, environment variables or configuration files: Wikimedia Commons and
Openverse are queried anonymously. The module provides
* :class:`HTTPSClient` -- a small HTTPS client that identifies itself with a
stable ``User-Agent``, refuses every URL outside the approved hosts
(Wikimedia Commons and Openverse) on port 443, retries HTTP 429/5xx a bounded
number of times with capped exponential backoff, and raises
:class:`~tools.errors.SourceAccessError` instead of returning an empty result
when the host is unreachable or keeps failing;
* :class:`SourceAdapter` -- the interface every image source implements;
* :class:`WikimediaCommonsAdapter` and :class:`OpenverseAdapter` -- two working
credential-free sources behind that interface;
* :class:`SourceRegistry` -- the plugin list that reports available source ids
and turns an unknown id into :class:`~tools.errors.UnknownSourceError` listing
them.
"""
from __future__ import annotations
import json
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass, field
from typing import Any, Callable, Iterable, Mapping, Protocol, runtime_checkable
from .errors import (
DuplicateSourceError,
HostNotAllowedError,
SearchRequestError,
SourceAccessError,
SourceHTTPError,
SourceError,
SourcePayloadError,
SourceUnreachableError,
UnknownSourceError,
)
__all__ = [
"APPROVED_HOSTS",
"DEFAULT_LIMIT",
"DEFAULT_MAX_ATTEMPTS",
"HTTPSClient",
"HTTPResponse",
"MAX_LIMIT",
"OPENVERSE_API_HOST",
"OpenverseAdapter",
"SourceAdapter",
"SourceCandidate",
"SourceRegistry",
"USER_AGENT",
"WIKIMEDIA_COMMONS_API_HOST",
"WIKIMEDIA_COMMONS_MEDIA_HOST",
"build_registry",
"default_registry",
"is_approved_url",
"search",
"validate_source_url",
]
PRODUCT_NAME = "product-54354138-1"
USER_AGENT = f"{PRODUCT_NAME}-image-toolkit/0.1.0 (credential-free image source client)"
WIKIMEDIA_COMMONS_API_HOST = "commons.wikimedia.org"
WIKIMEDIA_COMMONS_MEDIA_HOST = "upload.wikimedia.org"
OPENVERSE_API_HOST = "api.openverse.org"
# Wikimedia Commons is reachable through its API host plus the media host that
# serves the original files, Openverse through its API host (and the thumbnail
# endpoint on the same API host).
APPROVED_HOSTS = frozenset(
{WIKIMEDIA_COMMONS_API_HOST, WIKIMEDIA_COMMONS_MEDIA_HOST, OPENVERSE_API_HOST}
)
APPROVED_SCHEME = "https"
APPROVED_PORT = 443
DEFAULT_TIMEOUT = 10.0
DEFAULT_MAX_ATTEMPTS = 3
DEFAULT_BACKOFF_SECONDS = 0.5
MAX_BACKOFF_SECONDS = 8.0
RETRY_STATUS_CODES = frozenset({429}) | frozenset(range(500, 600))
@dataclass(frozen=True)
class HTTPResponse:
"""A transport-independent HTTP response."""
status: int
body: bytes
headers: Mapping[str, str] = field(default_factory=dict)
def json(self, *, url: str = "", source_id: str | None = None) -> Any:
try:
return json.loads(self.body)
except (ValueError, UnicodeDecodeError) as exc: # pragma: no cover - defensive
raise SourcePayloadError(url or "<response>", source_id=source_id, reason=str(exc)) from exc
class Transport(Protocol):
"""Callable performing one HTTP GET; injectable so tests never hit the net."""
def __call__(self, url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse: ...
def validate_source_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> str:
"""Return *url* when it is an approved HTTPS URL, else raise a typed error."""
allowed_set = frozenset(allowed)
if not isinstance(url, str) or not url:
raise HostNotAllowedError(str(url), allowed=allowed_set)
parts = urllib.parse.urlsplit(url)
if parts.scheme != APPROVED_SCHEME:
raise HostNotAllowedError(url, allowed=allowed_set)
host = (parts.hostname or "").lower()
if host not in allowed_set:
raise HostNotAllowedError(url, allowed=allowed_set)
try:
port = parts.port
except ValueError as exc:
raise HostNotAllowedError(url, allowed=allowed_set) from exc
if port not in (None, APPROVED_PORT):
raise HostNotAllowedError(url, allowed=allowed_set)
return url
def is_approved_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> bool:
"""Non-raising variant of :func:`validate_source_url` used by tests/callers."""
try:
validate_source_url(url, allowed=allowed)
except HostNotAllowedError:
return False
return True
def urllib_transport(url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse:
"""Default transport: a plain stdlib GET returning status, body and headers."""
request = urllib.request.Request(url, headers=dict(headers), method="GET")
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
return HTTPResponse(
status=int(getattr(response, "status", 200)),
body=response.read(),
headers=dict(response.headers.items()),
)
except urllib.error.HTTPError as exc:
body = b""
try:
body = exc.read()
except Exception: # pragma: no cover - body optional
body = b""
return HTTPResponse(
status=int(exc.code),
body=body,
headers=dict(getattr(exc, "headers", {}) or {}),
)
class HTTPSClient:
"""Fetch a URL only when it is approved, with bounded retry/backoff.
The client never returns an empty result for a broken source: unreachable
hosts raise :class:`SourceUnreachableError` and exhausted/non-retryable
statuses raise :class:`SourceHTTPError`. Both derive from
:class:`SourceAccessError`.
"""
def __init__(
self,
*,
user_agent: str = USER_AGENT,
allowed_hosts: Iterable[str] = APPROVED_HOSTS,
timeout: float = DEFAULT_TIMEOUT,
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
backoff_seconds: float = DEFAULT_BACKOFF_SECONDS,
max_backoff_seconds: float = MAX_BACKOFF_SECONDS,
transport: Transport | None = None,
sleep: Callable[[float], None] = time.sleep,
) -> None:
if max_attempts < 1:
raise ValueError("max_attempts must be >= 1")
self.user_agent = user_agent
self.allowed_hosts = frozenset(allowed_hosts)
self.timeout = float(timeout)
self.max_attempts = int(max_attempts)
self.backoff_seconds = float(backoff_seconds)
self.max_backoff_seconds = float(max_backoff_seconds)
self._transport: Transport = transport or urllib_transport
self._sleep = sleep
# -- public API -----------------------------------------------------
def headers(self) -> dict[str, str]:
"""Request headers, always including the identifying User-Agent."""
return {
"User-Agent": self.user_agent,
"Accept": "application/json, image/*;q=0.8, */*;q=0.5",
}
def backoff_delay(self, attempt: int) -> float:
"""Capped exponential backoff delay for 1-based *attempt*."""
delay = self.backoff_seconds * (2 ** max(0, attempt - 1))
return min(delay, self.max_backoff_seconds)
def get(self, url: str, *, source_id: str | None = None) -> HTTPResponse:
"""GET *url*, retrying transient statuses, then raise typed errors."""
validate_source_url(url, allowed=self.allowed_hosts)
response: HTTPResponse | None = None
for attempt in range(1, self.max_attempts + 1):
try:
response = self._transport(url, self.headers(), self.timeout)
except SourceAccessError:
raise
except Exception as exc: # transport-level failure
if attempt >= self.max_attempts:
raise SourceUnreachableError(
url, source_id=source_id, reason=f"{type(exc).__name__}: {exc}"
) from exc
self._sleep(self.backoff_delay(attempt))
continue
if response.status in RETRY_STATUS_CODES and attempt < self.max_attempts:
self._sleep(self.backoff_delay(attempt))
continue
if 200 <= response.status < 300:
return response
raise SourceHTTPError(url, response.status, source_id=source_id)
# Defensive: the loop either returned or raised above.
status = response.status if response is not None else 0
raise SourceHTTPError(url, status, source_id=source_id)
def get_json(self, url: str, *, source_id: str | None = None) -> Any:
response = self.get(url, source_id=source_id)
try:
return json.loads(response.body)
except (ValueError, UnicodeDecodeError) as exc:
raise SourcePayloadError(url, source_id=source_id, reason=str(exc)) from exc
DEFAULT_LIMIT = 10
MAX_LIMIT = 50
@dataclass(frozen=True)
class SourceCandidate:
"""One normalized search hit.
``source_id`` identifies the adapter, ``file_url`` is the direct HTTPS
location of the original image, ``mime_type`` is the type the source
reports (may be ``None`` when unstated) and ``fetch_bytes`` is a zero-argument
callable returning the image bytes through the approved HTTPS client.
"""
source_id: str
file_url: str
mime_type: str | None
fetch_bytes: Callable[[], bytes]
title: str | None = None
page_url: str | None = None
width: int | None = None
height: int | None = None
extra: Mapping[str, Any] = field(default_factory=dict)
def fetch(self) -> bytes:
"""Convenience passthrough to :attr:`fetch_bytes`."""
return self.fetch_bytes()
def to_dict(self) -> dict[str, Any]:
"""Metadata-only projection (the callable is intentionally omitted)."""
payload: dict[str, Any] = {
"source_id": self.source_id,
"file_url": self.file_url,
"mime_type": self.mime_type,
}
for key in ("title", "page_url", "width", "height"):
value = getattr(self, key)
if value is not None:
payload[key] = value
return payload
@runtime_checkable
class SourceAdapter(Protocol):
"""Interface implemented by every image source."""
source_id: str
def search(self, topic: str, limit: int) -> list[SourceCandidate]: ...
def normalize_search_args(topic: str, limit: int) -> tuple[str, int]:
"""Validate the caller-supplied topic and result limit."""
if not isinstance(topic, str):
raise SearchRequestError(
f"topic must be a string, got {type(topic).__name__}",
details={"topic": repr(topic)},
)
topic = topic.strip()
if not topic:
raise SearchRequestError("topic must not be empty", details={"topic": topic})
if isinstance(limit, bool) or not isinstance(limit, int):
raise SearchRequestError(
f"limit must be an integer, got {type(limit).__name__}",
details={"limit": repr(limit)},
)
if limit < 1:
raise SearchRequestError(f"limit must be >= 1, got {limit}", details={"limit": limit})
if limit > MAX_LIMIT:
raise SearchRequestError(
f"limit must be <= {MAX_LIMIT}, got {limit}", details={"limit": limit}
)
return topic, limit
class _BaseAdapter:
"""Shared normalization helpers for concrete adapters."""
source_id = ""
def __init__(self, client: HTTPSClient | None = None) -> None:
self.client = client or HTTPSClient()
def _candidate(
self,
*,
file_url: str,
mime_type: str | None,
title: str | None = None,
page_url: str | None = None,
width: int | None = None,
height: int | None = None,
extra: Mapping[str, Any] | None = None,
validate: bool = True,
) -> SourceCandidate:
if validate:
validate_source_url(file_url, allowed=self.client.allowed_hosts)
source_id = self.source_id
client = self.client
return SourceCandidate(
source_id=source_id,
file_url=file_url,
mime_type=mime_type,
fetch_bytes=lambda url=file_url: client.get(url, source_id=source_id).body,
title=title,
page_url=page_url,
width=width,
height=height,
extra=dict(extra or {}),
)
def __repr__(self) -> str: # pragma: no cover - trivial
return f"<{type(self).__name__} source_id={self.source_id!r}>"
class WikimediaCommonsAdapter(_BaseAdapter):
"""Wikimedia Commons search through the anonymous MediaWiki action API."""
source_id = "wikimedia_commons"
api_host = WIKIMEDIA_COMMONS_API_HOST
def build_search_url(self, topic: str, limit: int) -> str:
"""Construct the generator search URL for *topic* and *limit*."""
topic, limit = normalize_search_args(topic, limit)
params = {
"action": "query",
"format": "json",
"formatversion": "2",
"generator": "search",
"gsrsearch": topic,
"gsrnamespace": "6",
"gsrlimit": str(limit),
"prop": "imageinfo",
"iiprop": "url|mime|size",
"iiurlwidth": "1024",
}
query = urllib.parse.urlencode(params)
return f"https://{self.api_host}/w/api.php?{query}"
def search(self, topic: str, limit: int) -> list[SourceCandidate]:
url = self.build_search_url(topic, limit)
payload = self.client.get_json(url, source_id=self.source_id)
if not isinstance(payload, Mapping):
raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object")
if "error" in payload:
raise SourceAccessError(
f"Wikimedia Commons API error: {payload['error']}",
url=url,
source_id=self.source_id,
)
pages = (payload.get("query") or {}).get("pages") or []
candidates: list[SourceCandidate] = []
for page in pages:
infos = page.get("imageinfo") or []
if not infos:
continue
info = infos[0]
file_url = info.get("url")
if not file_url:
continue
candidates.append(
self._candidate(
file_url=file_url,
mime_type=info.get("mime"),
title=page.get("title"),
page_url=info.get("descriptionurl"),
width=info.get("width"),
height=info.get("height"),
extra={"thumb_url": info.get("thumburl")} if info.get("thumburl") else None,
)
)
if len(candidates) >= limit:
break
return candidates[:limit]
class OpenverseAdapter(_BaseAdapter):
"""Openverse search through its anonymous public images endpoint."""
source_id = "openverse"
api_host = OPENVERSE_API_HOST
def build_search_url(self, topic: str, limit: int) -> str:
"""Construct the Openverse image search URL for *topic* and *limit*."""
topic, limit = normalize_search_args(topic, limit)
query = urllib.parse.urlencode({"q": topic, "page_size": str(limit)})
return f"https://{self.api_host}/v1/images/?{query}"
def search(self, topic: str, limit: int) -> list[SourceCandidate]:
url = self.build_search_url(topic, limit)
payload = self.client.get_json(url, source_id=self.source_id)
if not isinstance(payload, Mapping):
raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object")
results = payload.get("results")
if results is None:
raise SourcePayloadError(url, source_id=self.source_id, reason="missing 'results'")
candidates: list[SourceCandidate] = []
for item in results:
if not isinstance(item, Mapping):
continue
original_url = item.get("url")
file_url = item.get("thumbnail") or original_url
if not file_url:
continue
candidates.append(
self._candidate(
file_url=file_url,
# Origin files may live on third-party hosts; the client
# still refuses them at fetch time with a typed error.
validate=is_approved_url(file_url, allowed=self.client.allowed_hosts),
mime_type=item.get("filetype"),
title=item.get("title"),
page_url=item.get("foreign_landing_url") or item.get("detail_url"),
width=item.get("width"),
height=item.get("height"),
extra={
"license": item.get("license"),
"creator": item.get("creator"),
"original_url": original_url,
},
)
)
if len(candidates) >= limit:
break
return candidates[:limit]
class SourceRegistry:
"""Ordered plugin registry of credential-free image sources."""
def __init__(self, adapters: Iterable[SourceAdapter] | None = None) -> None:
self._adapters: dict[str, SourceAdapter] = {}
for adapter in adapters or ():
self.register(adapter)
# -- registration ---------------------------------------------------
def register(self, adapter: SourceAdapter, *, replace: bool = False) -> SourceAdapter:
"""Add *adapter*; a duplicate id raises :class:`DuplicateSourceError`."""
source_id = getattr(adapter, "source_id", None)
if not isinstance(source_id, str) or not source_id:
raise SourceError("adapter must expose a non-empty string 'source_id'")
if source_id in self._adapters and not replace:
raise DuplicateSourceError(source_id)
self._adapters[source_id] = adapter
return adapter
def unregister(self, source_id: str) -> SourceAdapter:
if source_id not in self._adapters:
raise UnknownSourceError(source_id, self.available_sources())
return self._adapters.pop(source_id)
# -- lookup ---------------------------------------------------------
def available_sources(self) -> list[str]:
"""Sorted ids of every registered source."""
return sorted(self._adapters)
def __contains__(self, source_id: object) -> bool:
return isinstance(source_id, str) and source_id in self._adapters
def __len__(self) -> int:
return len(self._adapters)
def __iter__(self):
return iter(self.available_sources())
def get(self, source_id: str) -> SourceAdapter:
"""Return the adapter for *source_id* or raise a listing error."""
try:
return self._adapters[source_id]
except (KeyError, TypeError):
raise UnknownSourceError(source_id, self.available_sources()) from None
def create(self, source_id: str) -> SourceAdapter:
"""Explicit alias of :meth:`get` used by the CLI layer."""
return self.get(source_id)
# -- search ---------------------------------------------------------
def search(self, source_id: str, topic: str, limit: int = DEFAULT_LIMIT) -> list[SourceCandidate]:
"""Search *source_id* for *topic*, returning at most *limit* candidates."""
adapter = self.get(source_id)
topic, limit = normalize_search_args(topic, limit)
return list(adapter.search(topic, limit))
def __repr__(self) -> str: # pragma: no cover - trivial
return f"SourceRegistry({self.available_sources()!r})"
def build_registry(
*,
client: HTTPSClient | None = None,
adapters: Iterable[SourceAdapter] | None = None,
) -> SourceRegistry:
"""Build the default registry (Wikimedia Commons + Openverse)."""
shared_client = client or HTTPSClient()
if adapters is None:
adapters = (
WikimediaCommonsAdapter(shared_client),
OpenverseAdapter(shared_client),
)
return SourceRegistry(adapters)
def default_registry(client: HTTPSClient | None = None) -> SourceRegistry:
"""Convenience wrapper returning a fresh default registry."""
return build_registry(client=client)
def search(
topic: str,
*,
source: str = "wikimedia_commons",
limit: int = DEFAULT_LIMIT,
registry: SourceRegistry | None = None,
client: HTTPSClient | None = None,
) -> list[SourceCandidate]:
"""Search *source* for *topic* using *registry* (default registry by default)."""
active = registry if registry is not None else build_registry(client=client)
return active.search(source, topic, limit)