Final acceptance: Консольная утилита поиска и скачивания изображений для вызова ИИ-агентом (#18)
Repository Validate / repository-checks (push) Successful in 21s
Repository Validate / repository-checks (push) Successful in 21s
Co-authored-by: AI Tester <ai-tester@git.p2prod.ru>
This commit was merged in pull request #18.
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
"""Importable toolkit package for the request-document CLI.
|
||||
|
||||
The package intentionally exposes only its version metadata here; the
|
||||
executable behaviour lives in :mod:`tools.example` (and in the modules that
|
||||
later increments add).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["__version__"]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Status vocabulary, exit codes and the JSON response envelope.
|
||||
|
||||
Every invocation of ``python -m tools.example`` writes exactly one JSON
|
||||
document to stdout. The document always carries a ``status`` value from the
|
||||
fixed vocabulary below and the process exit code is derived from that value, so
|
||||
a calling AI agent can tell a successful search apart from "nothing matched",
|
||||
a broken request and an unusable source without parsing prose.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable, Mapping, TextIO
|
||||
|
||||
from .errors import (
|
||||
RequestError,
|
||||
SearchRequestError,
|
||||
SourceAccessError,
|
||||
ToolkitError,
|
||||
UnknownSourceError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"EXIT_ACCESS_ERROR",
|
||||
"EXIT_INVALID_REQUEST",
|
||||
"EXIT_NO_FORMAT",
|
||||
"EXIT_NO_SIZE",
|
||||
"EXIT_OK",
|
||||
"EXIT_PARTIAL_DOWNLOAD",
|
||||
"Response",
|
||||
"ResponseBuilder",
|
||||
"STATUS_ACCESS_ERROR",
|
||||
"STATUS_DRY_RUN",
|
||||
"STATUS_EXIT_CODES",
|
||||
"STATUS_INVALID_REQUEST",
|
||||
"STATUS_NO_FORMAT",
|
||||
"STATUS_NO_SIZE",
|
||||
"STATUS_OK",
|
||||
"STATUS_PARTIAL_DOWNLOAD",
|
||||
"STATUS_VOCABULARY",
|
||||
"error_fragment",
|
||||
"exit_code_for",
|
||||
"is_success",
|
||||
"status_for_error",
|
||||
]
|
||||
|
||||
#: Successful search/download.
|
||||
STATUS_OK = "ok"
|
||||
#: The request was validated and echoed without contacting a source.
|
||||
STATUS_DRY_RUN = "dry_run"
|
||||
#: The request itself is unusable (bad flags, bad document, unknown source or format).
|
||||
STATUS_INVALID_REQUEST = "invalid_request"
|
||||
#: The source answered but offers nothing in the requested format.
|
||||
STATUS_NO_FORMAT = "no_format"
|
||||
#: The source answered but offers nothing in the requested size.
|
||||
STATUS_NO_SIZE = "no_size"
|
||||
#: The source could not be reached, refused to answer or returned unusable data.
|
||||
STATUS_ACCESS_ERROR = "access_error"
|
||||
#: Some downloads succeeded and some failed.
|
||||
STATUS_PARTIAL_DOWNLOAD = "partial_download"
|
||||
|
||||
#: The single source of truth for the status -> exit code mapping.
|
||||
STATUS_EXIT_CODES: dict[str, int] = {
|
||||
STATUS_OK: 0,
|
||||
STATUS_DRY_RUN: 0,
|
||||
STATUS_INVALID_REQUEST: 2,
|
||||
STATUS_NO_FORMAT: 3,
|
||||
STATUS_NO_SIZE: 4,
|
||||
STATUS_ACCESS_ERROR: 5,
|
||||
STATUS_PARTIAL_DOWNLOAD: 6,
|
||||
}
|
||||
|
||||
#: Stable ordering used in help text and documentation.
|
||||
STATUS_VOCABULARY: tuple[str, ...] = tuple(STATUS_EXIT_CODES)
|
||||
|
||||
EXIT_OK = STATUS_EXIT_CODES[STATUS_OK]
|
||||
EXIT_INVALID_REQUEST = STATUS_EXIT_CODES[STATUS_INVALID_REQUEST]
|
||||
EXIT_NO_FORMAT = STATUS_EXIT_CODES[STATUS_NO_FORMAT]
|
||||
EXIT_NO_SIZE = STATUS_EXIT_CODES[STATUS_NO_SIZE]
|
||||
EXIT_ACCESS_ERROR = STATUS_EXIT_CODES[STATUS_ACCESS_ERROR]
|
||||
EXIT_PARTIAL_DOWNLOAD = STATUS_EXIT_CODES[STATUS_PARTIAL_DOWNLOAD]
|
||||
|
||||
|
||||
def exit_code_for(status: str) -> int:
|
||||
"""Return the process exit code for ``status``.
|
||||
|
||||
Raises :class:`ValueError` for anything outside the fixed vocabulary, so a
|
||||
typo can never silently produce an unknown exit code.
|
||||
"""
|
||||
|
||||
try:
|
||||
return STATUS_EXIT_CODES[status]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
f"unknown status {status!r}; expected one of: {', '.join(STATUS_VOCABULARY)}"
|
||||
) from None
|
||||
|
||||
|
||||
def is_success(status: str) -> bool:
|
||||
"""True when ``status`` maps onto a zero exit code."""
|
||||
|
||||
return exit_code_for(status) == 0
|
||||
|
||||
|
||||
def error_fragment(
|
||||
code: str,
|
||||
message: str,
|
||||
*,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the machine-readable ``error`` fragment of a response."""
|
||||
|
||||
payload: dict[str, Any] = {"code": str(code), "message": str(message)}
|
||||
if details:
|
||||
payload["details"] = dict(details)
|
||||
return payload
|
||||
|
||||
|
||||
def status_for_error(error: BaseException) -> str:
|
||||
"""Map a toolkit error onto the fixed status vocabulary."""
|
||||
|
||||
if isinstance(error, (RequestError, UnknownSourceError, SearchRequestError)):
|
||||
return STATUS_INVALID_REQUEST
|
||||
if isinstance(error, SourceAccessError):
|
||||
return STATUS_ACCESS_ERROR
|
||||
if isinstance(error, ToolkitError):
|
||||
return STATUS_ACCESS_ERROR
|
||||
return STATUS_ACCESS_ERROR
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Response:
|
||||
"""One complete machine-readable answer of the toolkit."""
|
||||
|
||||
status: str
|
||||
source: str | None = None
|
||||
request: Mapping[str, Any] | None = None
|
||||
items: tuple[Mapping[str, Any], ...] = ()
|
||||
error: Mapping[str, Any] | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.status not in STATUS_EXIT_CODES:
|
||||
raise ValueError(
|
||||
f"unknown status {self.status!r}; expected one of: "
|
||||
f"{', '.join(STATUS_VOCABULARY)}"
|
||||
)
|
||||
if self.error is not None and "code" not in self.error:
|
||||
raise ValueError("error fragment must carry a 'code'")
|
||||
|
||||
@property
|
||||
def exit_code(self) -> int:
|
||||
"""Process exit code implied by :attr:`status`."""
|
||||
|
||||
return exit_code_for(self.status)
|
||||
|
||||
@property
|
||||
def succeeded(self) -> bool:
|
||||
"""True when the status is one of the zero-exit statuses."""
|
||||
|
||||
return self.exit_code == 0
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Return the JSON-ready document with a fixed set of top-level keys."""
|
||||
|
||||
return {
|
||||
"status": self.status,
|
||||
"source": self.source,
|
||||
"request": dict(self.request) if self.request is not None else None,
|
||||
"items": [dict(item) for item in self.items],
|
||||
"error": dict(self.error) if self.error is not None else None,
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Serialise the response as one deterministic JSON line."""
|
||||
|
||||
return json.dumps(self.to_dict(), sort_keys=True, ensure_ascii=True)
|
||||
|
||||
def emit(self, stream: TextIO | None = None) -> None:
|
||||
"""Write exactly one JSON document to ``stream`` (stdout by default)."""
|
||||
|
||||
print(self.to_json(), file=stream if stream is not None else sys.stdout)
|
||||
|
||||
|
||||
class ResponseBuilder:
|
||||
"""Build :class:`Response` objects sharing one source/request context."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
source: str | None = None,
|
||||
request: Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
self.source = source
|
||||
self.request = request
|
||||
|
||||
def with_context(
|
||||
self,
|
||||
*,
|
||||
source: str | None = None,
|
||||
request: Mapping[str, Any] | None = None,
|
||||
) -> "ResponseBuilder":
|
||||
"""Return a copy carrying the supplied context."""
|
||||
|
||||
return ResponseBuilder(
|
||||
source=self.source if source is None else source,
|
||||
request=self.request if request is None else request,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
status: str,
|
||||
*,
|
||||
items: Iterable[Mapping[str, Any]] = (),
|
||||
error: Mapping[str, Any] | None = None,
|
||||
) -> Response:
|
||||
"""Assemble a response for an explicit ``status``."""
|
||||
|
||||
return Response(
|
||||
status=status,
|
||||
source=self.source,
|
||||
request=self.request,
|
||||
items=tuple(items),
|
||||
error=error,
|
||||
)
|
||||
|
||||
# -- convenience constructors --------------------------------------
|
||||
def ok(self, *, items: Iterable[Mapping[str, Any]] = ()) -> Response:
|
||||
return self.build(STATUS_OK, items=items)
|
||||
|
||||
def dry_run(self, *, items: Iterable[Mapping[str, Any]] = ()) -> Response:
|
||||
return self.build(STATUS_DRY_RUN, items=items)
|
||||
|
||||
def invalid_request(self, *, error: Mapping[str, Any] | None = None) -> Response:
|
||||
return self.build(STATUS_INVALID_REQUEST, error=error)
|
||||
|
||||
def no_format(self, *, error: Mapping[str, Any] | None = None) -> Response:
|
||||
return self.build(STATUS_NO_FORMAT, error=error)
|
||||
|
||||
def no_size(self, *, error: Mapping[str, Any] | None = None) -> Response:
|
||||
return self.build(STATUS_NO_SIZE, error=error)
|
||||
|
||||
def access_error(
|
||||
self,
|
||||
*,
|
||||
items: Iterable[Mapping[str, Any]] = (),
|
||||
error: Mapping[str, Any] | None = None,
|
||||
) -> Response:
|
||||
return self.build(STATUS_ACCESS_ERROR, items=items, error=error)
|
||||
|
||||
def partial_download(
|
||||
self,
|
||||
*,
|
||||
items: Iterable[Mapping[str, Any]] = (),
|
||||
error: Mapping[str, Any] | None = None,
|
||||
) -> Response:
|
||||
return self.build(STATUS_PARTIAL_DOWNLOAD, items=items, error=error)
|
||||
|
||||
def from_error(self, error: ToolkitError) -> Response:
|
||||
"""Build a response for ``error`` using :func:`status_for_error`."""
|
||||
|
||||
return self.build(status_for_error(error), error=error.to_dict())
|
||||
+273
@@ -0,0 +1,273 @@
|
||||
"""Typed errors raised deliberately by the toolkit.
|
||||
|
||||
The image-source layer never degrades a failed or unknown source into a silent,
|
||||
empty result. Every such situation raises a :class:`ToolkitError` subclass that
|
||||
carries a stable ``code`` string plus machine-readable ``details`` so the CLI can
|
||||
map it onto an exit status and a structured JSON envelope.
|
||||
|
||||
Hierarchy::
|
||||
|
||||
ToolkitError
|
||||
+-- SourceError
|
||||
+-- UnknownSourceError (unknown source id, lists available ids)
|
||||
+-- DuplicateSourceError (registry already holds that id)
|
||||
+-- SearchRequestError (topic/limit rejected before any request)
|
||||
+-- SourceAccessError (the source could not be used)
|
||||
+-- SourceUnreachableError (transport/DNS/timeout failure)
|
||||
+-- SourceHTTPError (non-retryable or exhausted HTTP status)
|
||||
+-- SourcePayloadError (body is not the expected JSON shape)
|
||||
+-- HostNotAllowedError (URL outside the approved hosts)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
__all__ = [
|
||||
"DuplicateSourceError",
|
||||
"HostNotAllowedError",
|
||||
"SearchRequestError",
|
||||
"SourceAccessError",
|
||||
"SourceError",
|
||||
"SourceHTTPError",
|
||||
"SourcePayloadError",
|
||||
"SourceUnreachableError",
|
||||
"ToolkitError",
|
||||
"UnknownSourceError",
|
||||
]
|
||||
|
||||
|
||||
class ToolkitError(Exception):
|
||||
"""Base class for errors the toolkit raises on purpose."""
|
||||
|
||||
code = "toolkit_error"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
code: str | None = None,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
if code is not None:
|
||||
self.code = code
|
||||
self.details: dict[str, Any] = dict(details or {})
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Return the JSON-safe envelope fragment describing this error."""
|
||||
|
||||
payload: dict[str, Any] = {"code": self.code, "message": self.message}
|
||||
if self.details:
|
||||
payload["details"] = dict(self.details)
|
||||
return payload
|
||||
|
||||
def __str__(self) -> str: # pragma: no cover - trivial
|
||||
return self.message
|
||||
|
||||
|
||||
class SourceError(ToolkitError):
|
||||
"""Base class for every image-source problem."""
|
||||
|
||||
code = "source_error"
|
||||
|
||||
|
||||
class UnknownSourceError(SourceError):
|
||||
"""Raised when the caller asks for a source id the registry does not know."""
|
||||
|
||||
code = "source_unknown"
|
||||
|
||||
def __init__(self, source_id: str, available: Iterable[str]) -> None:
|
||||
self.source_id = str(source_id)
|
||||
self.available: tuple[str, ...] = tuple(sorted(str(item) for item in available))
|
||||
listing = ", ".join(self.available) if self.available else "<none>"
|
||||
super().__init__(
|
||||
f"unknown image source {self.source_id!r}; available sources: {listing}",
|
||||
details={"source_id": self.source_id, "available_sources": list(self.available)},
|
||||
)
|
||||
|
||||
|
||||
class DuplicateSourceError(SourceError):
|
||||
"""Raised when an adapter id is registered twice."""
|
||||
|
||||
code = "source_duplicate"
|
||||
|
||||
def __init__(self, source_id: str) -> None:
|
||||
self.source_id = str(source_id)
|
||||
super().__init__(
|
||||
f"image source {self.source_id!r} is already registered",
|
||||
details={"source_id": self.source_id},
|
||||
)
|
||||
|
||||
|
||||
class SearchRequestError(SourceError):
|
||||
"""Raised when the caller supplies a topic/limit a source cannot honour."""
|
||||
|
||||
code = "search_invalid_request"
|
||||
|
||||
def __init__(self, message: str, *, details: Mapping[str, Any] | None = None) -> None:
|
||||
super().__init__(message, details=details)
|
||||
|
||||
|
||||
class SourceAccessError(SourceError):
|
||||
"""Raised when a known source cannot be reached or refuses to answer."""
|
||||
|
||||
code = "source_access"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
url: str | None = None,
|
||||
source_id: str | None = None,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
merged: dict[str, Any] = dict(details or {})
|
||||
if url is not None:
|
||||
merged["url"] = url
|
||||
if source_id is not None:
|
||||
merged["source_id"] = source_id
|
||||
super().__init__(message, details=merged)
|
||||
self.url = url
|
||||
self.source_id = source_id
|
||||
|
||||
|
||||
class SourceUnreachableError(SourceAccessError):
|
||||
"""Raised when the transport cannot deliver a request to the host at all."""
|
||||
|
||||
code = "source_unreachable"
|
||||
|
||||
def __init__(self, url: str, *, source_id: str | None = None, reason: str | None = None) -> None:
|
||||
message = f"image source host is unreachable: {url}"
|
||||
if reason:
|
||||
message = f"{message} ({reason})"
|
||||
super().__init__(message, url=url, source_id=source_id, details={"reason": reason} if reason else None)
|
||||
|
||||
|
||||
class SourceHTTPError(SourceAccessError):
|
||||
"""Raised for an HTTP status the client cannot recover from."""
|
||||
|
||||
code = "source_http_error"
|
||||
|
||||
def __init__(self, url: str, status: int, *, source_id: str | None = None) -> None:
|
||||
super().__init__(
|
||||
f"image source returned HTTP status {status}: {url}",
|
||||
url=url,
|
||||
source_id=source_id,
|
||||
details={"status": status},
|
||||
)
|
||||
self.status = int(status)
|
||||
|
||||
|
||||
class SourcePayloadError(SourceAccessError):
|
||||
"""Raised when a source answers with something the adapter cannot read."""
|
||||
|
||||
code = "source_invalid_response"
|
||||
|
||||
def __init__(self, url: str, *, source_id: str | None = None, reason: str | None = None) -> None:
|
||||
message = f"image source returned an unusable response: {url}"
|
||||
if reason:
|
||||
message = f"{message} ({reason})"
|
||||
super().__init__(message, url=url, source_id=source_id, details={"reason": reason} if reason else None)
|
||||
|
||||
|
||||
class HostNotAllowedError(SourceAccessError):
|
||||
"""Raised when a URL leaves the approved HTTPS host allowlist."""
|
||||
|
||||
code = "source_host_not_allowed"
|
||||
|
||||
def __init__(self, url: str, *, allowed: Iterable[str] = ()) -> None:
|
||||
self.allowed: tuple[str, ...] = tuple(sorted(str(item) for item in allowed))
|
||||
super().__init__(
|
||||
f"refusing to contact {url!r}: only approved HTTPS hosts are allowed "
|
||||
f"({', '.join(self.allowed) if self.allowed else 'none configured'})",
|
||||
url=str(url),
|
||||
details={"allowed_hosts": list(self.allowed)},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request errors: invalid command lines and JSON request documents
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Image formats the CLI accepts.
|
||||
ACCEPTED_FORMATS: tuple[str, ...] = ("png", "ico")
|
||||
|
||||
|
||||
class RequestError(ToolkitError):
|
||||
"""Base class for every unusable request (CLI flags or request document)."""
|
||||
|
||||
code = "request_invalid"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
code: str | None = None,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
field: str | None = None,
|
||||
accepted_fields: Iterable[str] | None = None,
|
||||
accepted_formats: Iterable[str] | None = None,
|
||||
) -> None:
|
||||
merged: dict[str, Any] = dict(details or {})
|
||||
if field is not None:
|
||||
merged["field"] = field
|
||||
if accepted_fields is not None:
|
||||
merged["accepted_fields"] = list(accepted_fields)
|
||||
if accepted_formats is not None:
|
||||
merged["accepted_formats"] = list(accepted_formats)
|
||||
super().__init__(message, code=code, details=merged)
|
||||
self.field = field
|
||||
self.accepted_fields: tuple[str, ...] = tuple(accepted_fields or ())
|
||||
self.accepted_formats: tuple[str, ...] = tuple(accepted_formats or ())
|
||||
|
||||
|
||||
class RequestDocumentError(RequestError):
|
||||
"""Raised when the JSON request document cannot be read or decoded."""
|
||||
|
||||
code = "request_document"
|
||||
|
||||
|
||||
class RequestFieldError(RequestError):
|
||||
"""Raised when a single request field is missing, unknown or malformed."""
|
||||
|
||||
code = "request_field"
|
||||
|
||||
|
||||
class UnsupportedFormatError(RequestError):
|
||||
"""Raised for a format outside :data:`ACCEPTED_FORMATS`."""
|
||||
|
||||
code = "request_format_unsupported"
|
||||
|
||||
def __init__(self, value: Any, accepted: Iterable[str] = ACCEPTED_FORMATS) -> None:
|
||||
accepted = tuple(accepted)
|
||||
self.value = str(value)
|
||||
super().__init__(
|
||||
f"unsupported image format {self.value!r}; "
|
||||
f"accepted formats: {', '.join(accepted)}",
|
||||
field="format",
|
||||
accepted_formats=accepted,
|
||||
details={"requested_format": self.value},
|
||||
)
|
||||
|
||||
|
||||
class ImageResizeError(ToolkitError):
|
||||
"""Raised when a downloaded image cannot be downscaled to the target size."""
|
||||
|
||||
code = "resize_failed"
|
||||
|
||||
def __init__(self, reason: str, *, details: Mapping[str, Any] | None = None) -> None:
|
||||
merged: dict[str, Any] = dict(details or {})
|
||||
merged["reason"] = reason
|
||||
super().__init__(f"cannot resize image: {reason}", details=merged)
|
||||
|
||||
|
||||
__all__ += [
|
||||
"ACCEPTED_FORMATS",
|
||||
"ImageResizeError",
|
||||
"RequestDocumentError",
|
||||
"RequestError",
|
||||
"RequestFieldError",
|
||||
"UnsupportedFormatError",
|
||||
]
|
||||
@@ -0,0 +1,838 @@
|
||||
"""Plain native-process CLI for the image toolkit.
|
||||
|
||||
Usage::
|
||||
|
||||
python -m tools.example examples/input.json
|
||||
python -m tools.example --topic "red fox" --width 640 --height 480 --format png
|
||||
|
||||
Exactly one machine-readable JSON document is written to stdout. The process
|
||||
exit code is derived from the fixed status vocabulary in :mod:`tools.envelope`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from . import images
|
||||
from .envelope import (
|
||||
STATUS_ACCESS_ERROR,
|
||||
STATUS_NO_FORMAT,
|
||||
STATUS_NO_SIZE,
|
||||
STATUS_OK,
|
||||
STATUS_PARTIAL_DOWNLOAD,
|
||||
Response,
|
||||
ResponseBuilder,
|
||||
error_fragment,
|
||||
)
|
||||
from .errors import (
|
||||
ACCEPTED_FORMATS,
|
||||
ImageResizeError,
|
||||
RequestDocumentError,
|
||||
RequestError,
|
||||
RequestFieldError,
|
||||
SourceAccessError,
|
||||
ToolkitError,
|
||||
UnknownSourceError,
|
||||
UnsupportedFormatError,
|
||||
)
|
||||
from .images import ImageFormatError, ImageInfo
|
||||
from .sources import DEFAULT_LIMIT, SourceCandidate, default_registry
|
||||
|
||||
PROG = "python -m tools.example"
|
||||
|
||||
#: Keys accepted in a JSON request document (``version`` is optional metadata).
|
||||
REQUEST_FIELDS: tuple[str, ...] = (
|
||||
"topic",
|
||||
"width",
|
||||
"height",
|
||||
"format",
|
||||
"source",
|
||||
"count",
|
||||
"download",
|
||||
"output_dir",
|
||||
"resize",
|
||||
"dry_run",
|
||||
)
|
||||
|
||||
OPTIONAL_DOCUMENT_FIELDS: tuple[str, ...] = ("version",)
|
||||
|
||||
DEFAULT_SOURCE = "wikimedia_commons"
|
||||
MIN_COUNT = 1
|
||||
MAX_COUNT = 50
|
||||
|
||||
#: How many candidates are fetched per wanted image. Nothing a source reports
|
||||
#: about a hit is trusted, so a candidate is only known to be usable once its
|
||||
#: bytes have been downloaded and parsed; the source is therefore asked for a
|
||||
#: window of ``count * SAMPLE_FACTOR`` hits and the window is cut short as soon
|
||||
#: as ``count`` usable images have been found.
|
||||
SAMPLE_FACTOR = 4
|
||||
#: Hard upper bound on the number of candidates inspected in one invocation.
|
||||
MAX_CANDIDATE_CAP = MAX_COUNT
|
||||
|
||||
|
||||
def candidate_cap(count: int) -> int:
|
||||
"""Return the bounded number of candidates to fetch for ``count`` images."""
|
||||
|
||||
wanted = max(MIN_COUNT, int(count))
|
||||
return min(MAX_CANDIDATE_CAP, wanted * SAMPLE_FACTOR)
|
||||
|
||||
|
||||
class _Parser(argparse.ArgumentParser):
|
||||
"""Argument parser that reports bad flags as request errors."""
|
||||
|
||||
def error(self, message: str) -> Any: # pragma: no cover - argparse hook
|
||||
raise RequestError(message, accepted_fields=REQUEST_FIELDS)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""Return the CLI parser; ``--help`` documents every supported parameter."""
|
||||
|
||||
parser = _Parser(
|
||||
prog=PROG,
|
||||
description=(
|
||||
"Search an image source and report the matching items as one JSON "
|
||||
"document. Either pass a JSON request document or use the flags."
|
||||
),
|
||||
epilog=(
|
||||
"Every run writes exactly one JSON document to stdout with the keys "
|
||||
"status, source, request, items and error."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"request",
|
||||
nargs="?",
|
||||
metavar="REQUEST.json",
|
||||
help=(
|
||||
"path to a JSON request document holding the same fields as the "
|
||||
"flags; it cannot be combined with any of the flags"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--topic",
|
||||
metavar="TEXT",
|
||||
help="search topic (required unless a request document is given)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--width",
|
||||
type=int,
|
||||
metavar="PX",
|
||||
help="target width in pixels; only images at least this wide are accepted",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--height",
|
||||
type=int,
|
||||
metavar="PX",
|
||||
help="target height in pixels; only images at least this tall are accepted",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--format",
|
||||
metavar="FMT",
|
||||
help="image format: " + "|".join(ACCEPTED_FORMATS),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s",
|
||||
"--source",
|
||||
metavar="ID",
|
||||
help=f"image source id (default: {DEFAULT_SOURCE}); unknown ids are rejected",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
"--count",
|
||||
type=int,
|
||||
metavar="N",
|
||||
help=f"how many candidates to request, {MIN_COUNT}-{MAX_COUNT} (default: {DEFAULT_LIMIT})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--download",
|
||||
action="store_true",
|
||||
help="download every matching image into the output directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
metavar="DIR",
|
||||
help="caller-specified download directory; required with --download",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--resize",
|
||||
action="store_true",
|
||||
help=(
|
||||
"downscale every downloaded image that is larger than the request "
|
||||
"in either dimension to exactly the requested width and height, "
|
||||
"re-encoded in the requested format; images are never scaled up; "
|
||||
"without it the original bytes and dimensions are written"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="validate the request and emit the envelope without any network access",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _load_document(path: str) -> Mapping[str, Any]:
|
||||
"""Read a JSON request document from ``path``."""
|
||||
|
||||
target = Path(path)
|
||||
try:
|
||||
text = target.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise RequestDocumentError(
|
||||
f"cannot read request document {path!r}: {exc.strerror or exc}",
|
||||
details={"path": str(path)},
|
||||
) from exc
|
||||
try:
|
||||
payload = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RequestDocumentError(
|
||||
f"request document {path!r} is not valid JSON: {exc.msg}",
|
||||
details={"path": str(path), "line": exc.lineno, "column": exc.colno},
|
||||
) from exc
|
||||
if not isinstance(payload, Mapping):
|
||||
raise RequestDocumentError(
|
||||
f"request document {path!r} must contain a JSON object",
|
||||
details={"path": str(path), "type": type(payload).__name__},
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _document_to_mapping(payload: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Reject unknown keys and return the accepted request fields."""
|
||||
|
||||
accepted = REQUEST_FIELDS + OPTIONAL_DOCUMENT_FIELDS
|
||||
unknown = sorted(str(key) for key in payload if key not in accepted)
|
||||
if unknown:
|
||||
raise RequestFieldError(
|
||||
"request document contains unsupported fields: " + ", ".join(unknown),
|
||||
field=unknown[0],
|
||||
details={"unknown_fields": unknown, "accepted_fields": list(REQUEST_FIELDS)},
|
||||
)
|
||||
return {key: payload[key] for key in REQUEST_FIELDS if key in payload}
|
||||
|
||||
|
||||
def _flags_to_mapping(namespace: argparse.Namespace) -> dict[str, Any]:
|
||||
"""Collect the explicitly supplied flags into a request mapping."""
|
||||
|
||||
mapping: dict[str, Any] = {}
|
||||
for field in ("topic", "width", "height", "format", "source", "count", "output_dir"):
|
||||
value = getattr(namespace, field, None)
|
||||
if value is not None:
|
||||
mapping[field] = value
|
||||
for field in ("download", "resize", "dry_run"):
|
||||
if getattr(namespace, field, False):
|
||||
mapping[field] = True
|
||||
return mapping
|
||||
|
||||
|
||||
def _as_text(value: Any, field: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise RequestFieldError(
|
||||
f"{field} must be a string, got {type(value).__name__}", field=field
|
||||
)
|
||||
text = value.strip()
|
||||
if not text:
|
||||
raise RequestFieldError(f"{field} must not be empty", field=field)
|
||||
return text
|
||||
|
||||
|
||||
def _as_int(value: Any, field: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise RequestFieldError(
|
||||
f"{field} must be an integer, got {type(value).__name__}", field=field
|
||||
)
|
||||
return int(value)
|
||||
|
||||
|
||||
def _as_bool(value: Any, field: str) -> bool:
|
||||
if not isinstance(value, bool):
|
||||
raise RequestFieldError(
|
||||
f"{field} must be a boolean, got {type(value).__name__}", field=field
|
||||
)
|
||||
return bool(value)
|
||||
|
||||
|
||||
def normalize_request(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Validate ``raw`` and return the complete, typed request mapping."""
|
||||
|
||||
if "topic" not in raw:
|
||||
raise RequestFieldError(
|
||||
"topic is required",
|
||||
field="topic",
|
||||
details={"required_fields": ["topic"], "accepted_fields": list(REQUEST_FIELDS)},
|
||||
)
|
||||
topic = _as_text(raw["topic"], "topic")
|
||||
|
||||
size: dict[str, int | None] = {"width": None, "height": None}
|
||||
for field in ("width", "height"):
|
||||
if raw.get(field) is None:
|
||||
continue
|
||||
number = _as_int(raw[field], field)
|
||||
if number <= 0:
|
||||
raise RequestFieldError(f"{field} must be positive", field=field)
|
||||
size[field] = number
|
||||
|
||||
if "format" not in raw or raw["format"] is None:
|
||||
raise RequestFieldError(
|
||||
"format is required",
|
||||
field="format",
|
||||
details={"accepted_formats": list(ACCEPTED_FORMATS)},
|
||||
)
|
||||
image_format = _as_text(raw["format"], "format").lower()
|
||||
if image_format not in ACCEPTED_FORMATS:
|
||||
raise UnsupportedFormatError(image_format)
|
||||
|
||||
source = DEFAULT_SOURCE
|
||||
if raw.get("source") is not None:
|
||||
source = _as_text(raw["source"], "source")
|
||||
|
||||
count = DEFAULT_LIMIT
|
||||
if raw.get("count") is not None:
|
||||
count = _as_int(raw["count"], "count")
|
||||
if not MIN_COUNT <= count <= MAX_COUNT:
|
||||
raise RequestFieldError(
|
||||
f"count must be between {MIN_COUNT} and {MAX_COUNT}",
|
||||
field="count",
|
||||
details={"min": MIN_COUNT, "max": MAX_COUNT},
|
||||
)
|
||||
|
||||
output_dir: str | None = None
|
||||
if raw.get("output_dir") is not None:
|
||||
output_dir = _as_text(raw["output_dir"], "output_dir")
|
||||
|
||||
download = _as_bool(raw.get("download", False), "download")
|
||||
if download and output_dir is None:
|
||||
# There is no hidden default: a download run without a caller-supplied
|
||||
# directory could only guess at where the files should go.
|
||||
raise RequestFieldError(
|
||||
"output_dir is required when download is enabled; pass --output-dir <dir>",
|
||||
field="output_dir",
|
||||
details={"required_with": "download", "accepted_fields": list(REQUEST_FIELDS)},
|
||||
)
|
||||
|
||||
return {
|
||||
"topic": topic,
|
||||
"width": size["width"],
|
||||
"height": size["height"],
|
||||
"format": image_format,
|
||||
"source": source,
|
||||
"count": count,
|
||||
"download": download,
|
||||
"output_dir": output_dir,
|
||||
"resize": _as_bool(raw.get("resize", False), "resize"),
|
||||
"dry_run": _as_bool(raw.get("dry_run", False), "dry_run"),
|
||||
}
|
||||
|
||||
|
||||
def _meets_minimum_size(info: ImageInfo, request: Mapping[str, Any]) -> bool:
|
||||
"""True when the *real* pixel size of an image is usable for the request.
|
||||
|
||||
The requested width and height are a minimum, not an exact match: any image
|
||||
that is at least that large is usable because it can always be downscaled,
|
||||
while a smaller image would have to be scaled up and is therefore rejected.
|
||||
``--resize`` only decides whether the downloaded bytes are downscaled; it
|
||||
never widens or narrows this filter.
|
||||
"""
|
||||
|
||||
wanted_width = request.get("width")
|
||||
wanted_height = request.get("height")
|
||||
if wanted_width is not None and info.width < wanted_width:
|
||||
return False
|
||||
if wanted_height is not None and info.height < wanted_height:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _failure(candidate: SourceCandidate, fragment: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Describe one candidate whose bytes could not be turned into an image."""
|
||||
|
||||
return {
|
||||
"source": str(getattr(candidate, "source_id", "") or ""),
|
||||
"url": str(getattr(candidate, "file_url", "") or ""),
|
||||
"error": dict(fragment),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KeptImage:
|
||||
"""One candidate whose downloaded bytes really match the request."""
|
||||
|
||||
candidate: SourceCandidate
|
||||
info: ImageInfo
|
||||
payload: bytes
|
||||
|
||||
def to_item(self) -> dict[str, Any]:
|
||||
"""Report ``source``, ``url``, ``format`` and the *actual* dimensions."""
|
||||
|
||||
source = str(getattr(self.candidate, "source_id", "") or "")
|
||||
url = str(getattr(self.candidate, "file_url", "") or "")
|
||||
item: dict[str, Any] = {
|
||||
"source": source,
|
||||
"url": url,
|
||||
# Kept as an alias so existing consumers keep working unchanged.
|
||||
"file_url": url,
|
||||
"format": self.info.format,
|
||||
"width": self.info.width,
|
||||
"height": self.info.height,
|
||||
"size_bytes": len(self.payload),
|
||||
}
|
||||
for key in ("title", "page_url", "mime_type"):
|
||||
value = getattr(self.candidate, key, None)
|
||||
if value is not None:
|
||||
item[key] = value
|
||||
return item
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Selection:
|
||||
"""Outcome of inspecting a bounded window of downloaded candidates.
|
||||
|
||||
``format_rejects`` counts candidates whose real payload is *not* the
|
||||
requested format -- including payloads that are not a recognised image at
|
||||
all. ``size_rejects`` counts candidates of the requested format that are
|
||||
too small. ``failures`` only holds candidates whose bytes could not be
|
||||
fetched at all, because that is the one situation worth an access error.
|
||||
"""
|
||||
|
||||
kept: tuple[KeptImage, ...] = ()
|
||||
format_rejects: int = 0
|
||||
size_rejects: int = 0
|
||||
other_formats: Mapping[str, int] = field(default_factory=dict)
|
||||
failures: tuple[dict[str, Any], ...] = ()
|
||||
|
||||
@property
|
||||
def inspected(self) -> int:
|
||||
"""Number of candidates whose bytes were read successfully."""
|
||||
|
||||
return len(self.kept) + self.format_rejects + self.size_rejects
|
||||
|
||||
|
||||
#: Bucket name for a downloaded payload that is not a recognised image at all.
|
||||
UNKNOWN_FORMAT = "unknown"
|
||||
|
||||
|
||||
def _count_other_format(other_formats: dict[str, int], detected: str | None) -> None:
|
||||
"""Record which other format a rejected candidate really was."""
|
||||
|
||||
key = detected or UNKNOWN_FORMAT
|
||||
other_formats[key] = other_formats.get(key, 0) + 1
|
||||
|
||||
|
||||
def _select(candidates: Sequence[SourceCandidate], request: Mapping[str, Any]) -> Selection:
|
||||
"""Fetch and inspect candidates, keeping only what the real bytes justify.
|
||||
|
||||
Every kept item has been downloaded and parsed: its format comes from the
|
||||
container signature and its width/height come from the PNG ``IHDR`` or the
|
||||
ICO directory, never from what the source claimed. A candidate that serves
|
||||
other-format bytes -- a JPEG, an SVG, a TIFF, an HTML error page, anything
|
||||
that is not a real ``png``/``ico`` payload -- counts as a *format rejection*
|
||||
so the caller gets ``no_format`` instead of an access error, and a candidate
|
||||
of the wanted format that is too small counts as a size rejection. Only a
|
||||
candidate whose bytes cannot be fetched at all is recorded as a failure.
|
||||
"""
|
||||
|
||||
wanted_format = str(request["format"])
|
||||
limit = int(request["count"])
|
||||
kept: list[KeptImage] = []
|
||||
format_rejects = 0
|
||||
size_rejects = 0
|
||||
other_formats: dict[str, int] = {}
|
||||
failures: list[dict[str, Any]] = []
|
||||
|
||||
for candidate in candidates:
|
||||
if len(kept) >= limit:
|
||||
break
|
||||
try:
|
||||
payload = candidate.fetch_bytes()
|
||||
except ToolkitError as error:
|
||||
failures.append(_failure(candidate, error.to_dict()))
|
||||
continue
|
||||
except OSError as error:
|
||||
failures.append(
|
||||
_failure(candidate, error_fragment("fetch_failed", str(error)))
|
||||
)
|
||||
continue
|
||||
detected = images.detect_format(payload)
|
||||
if detected != wanted_format:
|
||||
# Whatever these bytes are (another image format, or no recognisable
|
||||
# image at all), they are not the requested format: that is a format
|
||||
# verdict about the topic, never an access problem.
|
||||
format_rejects += 1
|
||||
_count_other_format(other_formats, detected)
|
||||
continue
|
||||
try:
|
||||
info = images.inspect_image(payload)
|
||||
except ImageFormatError as error:
|
||||
# The signature claims the requested format but the header is
|
||||
# corrupt, so the payload itself is unusable rather than merely the
|
||||
# wrong format.
|
||||
failures.append(_failure(candidate, error.to_dict()))
|
||||
continue
|
||||
if not _meets_minimum_size(info, request):
|
||||
size_rejects += 1
|
||||
continue
|
||||
kept.append(KeptImage(candidate=candidate, info=info, payload=payload))
|
||||
|
||||
return Selection(
|
||||
kept=tuple(kept),
|
||||
format_rejects=format_rejects,
|
||||
size_rejects=size_rejects,
|
||||
other_formats=dict(other_formats),
|
||||
failures=tuple(failures),
|
||||
)
|
||||
|
||||
|
||||
def _slug(candidate: SourceCandidate) -> str:
|
||||
raw = str(getattr(candidate, "title", None) or getattr(candidate, "source_id", "") or "image").lower()
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")
|
||||
return (slug or "image")[:60]
|
||||
|
||||
|
||||
def _target_size(width: int, height: int, request: Mapping[str, Any]) -> tuple[int, int]:
|
||||
"""Return the exact size to store: the requested size, never enlarged.
|
||||
|
||||
``--resize`` means "give me the requested size" rather than "give me
|
||||
something that fits in it": a requested width **and** height are hit
|
||||
exactly, and a dimension the caller left open keeps the source value. A
|
||||
source dimension that is already at or below the requested one is never
|
||||
scaled up, so ``--resize`` can only ever shrink an image.
|
||||
"""
|
||||
|
||||
wanted_width = request.get("width")
|
||||
wanted_height = request.get("height")
|
||||
target_width = width if wanted_width is None else min(width, int(wanted_width))
|
||||
target_height = height if wanted_height is None else min(height, int(wanted_height))
|
||||
return max(1, target_width), max(1, target_height)
|
||||
|
||||
|
||||
def resize_image(payload: bytes, request: Mapping[str, Any]) -> tuple[bytes, dict[str, Any]]:
|
||||
"""Downscale ``payload`` to the requested size and re-encode it.
|
||||
|
||||
The stored image is re-encoded in the requested container format (``png``
|
||||
or ``ico``) so the saved bytes really open as that format. An image that
|
||||
is not larger than the request in either dimension is returned untouched:
|
||||
``--resize`` never enlarges and never rewrites pixels it does not have to.
|
||||
"""
|
||||
|
||||
decoded = images.decode_image(payload)
|
||||
wanted_format = str(request.get("format") or decoded.format)
|
||||
if decoded.format != wanted_format:
|
||||
raise ImageResizeError(
|
||||
f"cannot re-encode a {decoded.format} image as {wanted_format}",
|
||||
details={"from_format": decoded.format, "to_format": wanted_format},
|
||||
)
|
||||
target_width, target_height = _target_size(decoded.width, decoded.height, request)
|
||||
fragment: dict[str, Any] = {
|
||||
"requested": {"width": request.get("width"), "height": request.get("height")},
|
||||
"from": {"width": decoded.width, "height": decoded.height},
|
||||
"format": wanted_format,
|
||||
}
|
||||
if (target_width, target_height) == (decoded.width, decoded.height):
|
||||
fragment.update(
|
||||
{
|
||||
"applied": False,
|
||||
"reason": "already_fits",
|
||||
"to": {"width": decoded.width, "height": decoded.height},
|
||||
}
|
||||
)
|
||||
return payload, fragment
|
||||
rows = images.resample_rows(
|
||||
decoded.rows, decoded.width, decoded.height, target_width, target_height
|
||||
)
|
||||
output = images.encode_image(wanted_format, target_width, target_height, rows)
|
||||
fragment.update({"applied": True, "to": {"width": target_width, "height": target_height}})
|
||||
return output, fragment
|
||||
|
||||
|
||||
def _discard(path: Path) -> None:
|
||||
"""Remove a file whose stored bytes failed verification."""
|
||||
|
||||
try:
|
||||
path.unlink()
|
||||
except OSError:
|
||||
# Never chase a path that is not a regular file we just wrote: the run
|
||||
# must not delete anything else in the caller's directory.
|
||||
pass
|
||||
|
||||
|
||||
def _verify_saved(
|
||||
path: Path, request: Mapping[str, Any], expected: tuple[int, int]
|
||||
) -> ImageInfo:
|
||||
"""Read a stored file back and prove it opens as the requested format.
|
||||
|
||||
A reported path is only trustworthy when the file really exists and really
|
||||
holds bytes of the requested format at the expected size, so the stored
|
||||
payload is decoded again instead of holding the writer to its word.
|
||||
"""
|
||||
|
||||
wanted_format = str(request["format"])
|
||||
decoded = images.decode_image(path.read_bytes())
|
||||
if decoded.format != wanted_format:
|
||||
raise ImageFormatError(
|
||||
"stored file does not open as the requested format",
|
||||
details={
|
||||
"path": str(path),
|
||||
"expected_format": wanted_format,
|
||||
"actual_format": decoded.format,
|
||||
},
|
||||
)
|
||||
if (decoded.width, decoded.height) != expected:
|
||||
raise ImageFormatError(
|
||||
"stored file does not have the expected pixel size",
|
||||
details={
|
||||
"path": str(path),
|
||||
"expected": {"width": expected[0], "height": expected[1]},
|
||||
"actual": {"width": decoded.width, "height": decoded.height},
|
||||
},
|
||||
)
|
||||
return ImageInfo(format=decoded.format, width=decoded.width, height=decoded.height)
|
||||
|
||||
|
||||
def _download(
|
||||
selected: Sequence[KeptImage], request: Mapping[str, Any]
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Write the already-fetched payloads; return ``(items, failure_count)``.
|
||||
|
||||
Every selected image is stored inside the caller-specified directory. The
|
||||
bytes verified by :func:`_select` are written as they are, or downscaled
|
||||
first when ``resize`` is enabled, so an item can never be downloaded twice
|
||||
or stored in a format that was not verified. Each stored file is read back
|
||||
and decoded before its path is reported, and a file that does not open as
|
||||
the requested format at the expected size is deleted and reported as a
|
||||
per-item save failure instead of being left behind as a broken path.
|
||||
"""
|
||||
|
||||
directory = Path(request["output_dir"]).expanduser()
|
||||
try:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as exc:
|
||||
raise SourceAccessError(
|
||||
f"cannot create output directory {str(directory)!r}: {exc.strerror or exc}",
|
||||
details={"output_dir": str(directory)},
|
||||
) from exc
|
||||
|
||||
suffix = "." + str(request["format"])
|
||||
items: list[dict[str, Any]] = []
|
||||
failures = 0
|
||||
for index, entry in enumerate(selected, start=1):
|
||||
record = entry.to_item()
|
||||
destination = directory / f"{index:03d}-{_slug(entry.candidate)}{suffix}"
|
||||
payload = entry.payload
|
||||
resize_fragment: dict[str, Any] | None = None
|
||||
expected = (entry.info.width, entry.info.height)
|
||||
try:
|
||||
if request["resize"]:
|
||||
payload, resize_fragment = resize_image(payload, request)
|
||||
expected = (
|
||||
int(resize_fragment["to"]["width"]),
|
||||
int(resize_fragment["to"]["height"]),
|
||||
)
|
||||
destination.write_bytes(payload)
|
||||
stored = _verify_saved(destination, request, expected)
|
||||
except (ToolkitError, OSError) as exc:
|
||||
_discard(destination)
|
||||
failures += 1
|
||||
error = error_fragment(getattr(exc, "code", "download_failed"), str(exc))
|
||||
if isinstance(exc, ToolkitError) and exc.details:
|
||||
error["details"] = dict(exc.details)
|
||||
# A failed save reports no path: only files that really exist and
|
||||
# really open are ever named as saved.
|
||||
record["download"] = {"ok": False, "target": str(destination), "error": error}
|
||||
else:
|
||||
download: dict[str, Any] = {
|
||||
"ok": True,
|
||||
"path": str(destination),
|
||||
"bytes": len(payload),
|
||||
"format": stored.format,
|
||||
"width": stored.width,
|
||||
"height": stored.height,
|
||||
}
|
||||
if resize_fragment is not None:
|
||||
download["resize"] = resize_fragment
|
||||
record["download"] = download
|
||||
items.append(record)
|
||||
return items, failures
|
||||
|
||||
|
||||
def _guess_source(argv: Sequence[str]) -> str | None:
|
||||
"""Best-effort source id extraction for error envelopes."""
|
||||
|
||||
for index, token in enumerate(argv):
|
||||
if token == "--source" or token == "-s":
|
||||
if index + 1 < len(argv):
|
||||
return argv[index + 1]
|
||||
elif token.startswith("--source="):
|
||||
return token.split("=", 1)[1]
|
||||
return None
|
||||
|
||||
|
||||
def _request_from(argv: Sequence[str]) -> tuple[dict[str, Any], str, Any]:
|
||||
"""Turn CLI arguments into ``(normalized_request, source_id, registry)``."""
|
||||
|
||||
parser = build_parser()
|
||||
namespace = parser.parse_args(list(argv))
|
||||
flags = _flags_to_mapping(namespace)
|
||||
document = namespace.request
|
||||
|
||||
if document is not None:
|
||||
if flags:
|
||||
raise RequestError(
|
||||
"a request document cannot be combined with flags: "
|
||||
+ ", ".join(sorted(flags)),
|
||||
details={"conflicting_flags": sorted(flags)},
|
||||
accepted_fields=REQUEST_FIELDS,
|
||||
)
|
||||
raw = _document_to_mapping(_load_document(document))
|
||||
elif flags:
|
||||
raw = flags
|
||||
else:
|
||||
raise RequestError(
|
||||
"no request given; pass a JSON request document or at least one flag",
|
||||
accepted_fields=REQUEST_FIELDS,
|
||||
)
|
||||
|
||||
request = normalize_request(raw)
|
||||
registry = default_registry()
|
||||
available = list(registry.available_sources())
|
||||
if request["source"] not in available:
|
||||
raise UnknownSourceError(request["source"], available)
|
||||
return request, request["source"], registry
|
||||
|
||||
|
||||
def _execute(argv: Sequence[str]) -> Response:
|
||||
"""Run one request and return the response that must be printed.
|
||||
|
||||
The search path downloads a bounded window of candidates and filters that
|
||||
window on the parsed bytes, so the answer distinguishes three different
|
||||
situations through both the status and the exit code:
|
||||
|
||||
* ``ok`` -- at least one image really is the requested format and at least
|
||||
the requested width and height;
|
||||
* ``no_format`` -- the source had hits, but none of them really are the
|
||||
requested format;
|
||||
* ``no_size`` -- images of the requested format existed, but all of them
|
||||
are smaller than requested;
|
||||
* ``access_error`` -- the source could not be searched, or not a single
|
||||
candidate could be fetched, which is never reported as a "no match".
|
||||
|
||||
A candidate whose downloaded bytes are a different format (a JPEG, an SVG,
|
||||
an HTML error page, ...) is a format mismatch about the topic, not an access
|
||||
problem: it feeds ``no_format`` instead of turning a working search into an
|
||||
error, and a stray unfetchable candidate never demotes a usable answer to
|
||||
``partial_download``.
|
||||
"""
|
||||
|
||||
request, source, registry = _request_from(argv)
|
||||
builder = ResponseBuilder(source=source, request=request)
|
||||
|
||||
if request["dry_run"]:
|
||||
return builder.dry_run()
|
||||
|
||||
cap = candidate_cap(request["count"])
|
||||
candidates = registry.search(source, request["topic"], cap)
|
||||
selection = _select(candidates, request)
|
||||
unreadable = list(selection.failures)
|
||||
|
||||
if not selection.kept:
|
||||
details = {
|
||||
"requested_format": request["format"],
|
||||
"requested_size": {"width": request["width"], "height": request["height"]},
|
||||
"minimum_size": {"width": request["width"], "height": request["height"]},
|
||||
"candidate_count": len(candidates),
|
||||
"candidates_inspected": selection.inspected,
|
||||
"format_matches": selection.size_rejects,
|
||||
"other_format_candidates": selection.format_rejects,
|
||||
"observed_formats": dict(selection.other_formats),
|
||||
"size_matches": 0,
|
||||
"unreadable_candidates": len(unreadable),
|
||||
}
|
||||
if selection.size_rejects:
|
||||
return builder.build(
|
||||
STATUS_NO_SIZE,
|
||||
error=error_fragment(
|
||||
"no_size_match",
|
||||
"the source returned no images at least the requested width and height",
|
||||
details=details,
|
||||
),
|
||||
)
|
||||
if selection.format_rejects or not unreadable:
|
||||
# Either the source really had no images in the requested format, or
|
||||
# it had no hits at all; both answer the topic, not an outage.
|
||||
return builder.build(
|
||||
STATUS_NO_FORMAT,
|
||||
error=error_fragment(
|
||||
"no_format_match",
|
||||
f"the source returned no {request['format']} images for {request['topic']!r}",
|
||||
details=details,
|
||||
),
|
||||
)
|
||||
return builder.access_error(
|
||||
error=error_fragment(
|
||||
"candidate_unreadable",
|
||||
"none of the candidates could be downloaded",
|
||||
details={**details, "failures": unreadable},
|
||||
)
|
||||
)
|
||||
|
||||
if not request["download"]:
|
||||
# Every reported item was fetched and verified against its real bytes, so
|
||||
# an unfetchable candidate is simply skipped: it must not demote an
|
||||
# otherwise usable answer to a partial-download exit code.
|
||||
return builder.build(STATUS_OK, items=[entry.to_item() for entry in selection.kept])
|
||||
|
||||
items, save_failures = _download(selection.kept, request)
|
||||
saved_paths = [
|
||||
item["download"]["path"] for item in items if item["download"].get("ok") is True
|
||||
]
|
||||
failed = save_failures + len(unreadable)
|
||||
if failed:
|
||||
# The run still reports every saved path, but a per-item save (or fetch)
|
||||
# failure is never hidden behind ``ok``: it has its own status and exit
|
||||
# code so the caller can tell a complete run from an incomplete one.
|
||||
return builder.build(
|
||||
STATUS_PARTIAL_DOWNLOAD,
|
||||
items=items,
|
||||
error=error_fragment(
|
||||
"download_incomplete",
|
||||
f"{failed} of {len(items) + len(unreadable)} images could not be saved",
|
||||
details={
|
||||
"failed": failed,
|
||||
"saved": len(saved_paths),
|
||||
"total": len(items) + len(unreadable),
|
||||
"saved_paths": saved_paths,
|
||||
"failures": unreadable,
|
||||
},
|
||||
),
|
||||
)
|
||||
return builder.build(STATUS_OK, items=items)
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
"""Entry point: emit exactly one JSON document and return the exit code."""
|
||||
|
||||
arguments = list(sys.argv[1:] if argv is None else argv)
|
||||
try:
|
||||
response = _execute(arguments)
|
||||
except ToolkitError as error:
|
||||
builder = ResponseBuilder(source=_guess_source(arguments))
|
||||
response = builder.from_error(error)
|
||||
except OSError as error:
|
||||
builder = ResponseBuilder(source=_guess_source(arguments))
|
||||
response = builder.build(
|
||||
STATUS_ACCESS_ERROR,
|
||||
error=error_fragment("access_error", str(error)),
|
||||
)
|
||||
response.emit()
|
||||
return response.exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - module entry point
|
||||
sys.exit(main())
|
||||
+771
@@ -0,0 +1,771 @@
|
||||
"""Stdlib-only inspection and construction of the image formats we accept.
|
||||
|
||||
The CLI decides what to keep from the **downloaded bytes** and never from the
|
||||
metadata a source reports about a file: a search hit that claims to be a
|
||||
``image/png`` of 800x600 pixels is kept only when the bytes really are a PNG of
|
||||
at least the requested size. This module is the single place that knows how to
|
||||
read the real format and the real pixel dimensions out of those bytes.
|
||||
|
||||
It deliberately depends on nothing but the standard library:
|
||||
|
||||
* :func:`detect_format` recognises the container signatures of the two accepted
|
||||
formats (``png``, ``ico``) plus the formats a real image source commonly
|
||||
serves instead (``jpeg``, ``gif``, ``bmp``, ``webp``, ``svg``, ``tiff``,
|
||||
``pdf``). Every recognised-but-different format, and every payload that is
|
||||
not a recognised image at all, counts as *another format* rather than an
|
||||
access problem: "the source has images, but none of them are the format you
|
||||
asked for" is a different answer from "the source could not be used".
|
||||
* :func:`read_png_dimensions` reads the ``IHDR`` header of a PNG.
|
||||
* :func:`read_ico_dimensions` reads the ICONDIR/ICONDIRENTRY directory of an
|
||||
ICO file (the directory stores the real sizes, including the ``0``-means-256
|
||||
encoding).
|
||||
* :func:`inspect_image` combines both into an :class:`ImageInfo`.
|
||||
* :func:`png_bytes` and :func:`ico_bytes` *build* the same formats so tests (and
|
||||
callers) can produce real payloads without Pillow or any other dependency.
|
||||
* :func:`decode_png`, :func:`encode_png`, :func:`ico_wrap` and
|
||||
:func:`ico_unwrap` decode and re-encode those containers, and
|
||||
:func:`decode_image`, :func:`resample_rows` and :func:`encode_image` are the
|
||||
building blocks the CLI uses to downscale a downloaded image and store it in
|
||||
the requested format -- still with nothing but the standard library.
|
||||
|
||||
:class:`ImageFormatError` -- a :class:`~tools.errors.ToolkitError` subclass with
|
||||
a stable ``code`` -- is reserved for bytes that are genuinely unusable: an
|
||||
unknown container, or a ``png``/``ico`` payload whose header is truncated or
|
||||
corrupt. A readable payload of another format is never an error, so the CLI
|
||||
keeps a working search out of the ``access_error`` status.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable, Sequence
|
||||
|
||||
from .errors import ToolkitError
|
||||
|
||||
__all__ = [
|
||||
"BMP",
|
||||
"GIF",
|
||||
"ICO",
|
||||
"ICO_ENTRY_SIZE",
|
||||
"ICO_HEADER_SIZE",
|
||||
"ICO_MAX_DIMENSION",
|
||||
"ImageFormatError",
|
||||
"ImageInfo",
|
||||
"JPEG",
|
||||
"KNOWN_FORMATS",
|
||||
"PDF",
|
||||
"PNG",
|
||||
"PNG_SIGNATURE",
|
||||
"SVG",
|
||||
"TIFF",
|
||||
"WEBP",
|
||||
"detect_format",
|
||||
"ico_bytes",
|
||||
"ico_entries",
|
||||
"inspect_image",
|
||||
"png_bytes",
|
||||
"read_ico_dimensions",
|
||||
"read_png_dimensions",
|
||||
"DecodedImage",
|
||||
"decode_image",
|
||||
"decode_png",
|
||||
"encode_image",
|
||||
"encode_png",
|
||||
"ico_unwrap",
|
||||
"ico_wrap",
|
||||
"png_chunks",
|
||||
"resample_rows",
|
||||
]
|
||||
|
||||
#: The two formats the CLI accepts as search requests.
|
||||
PNG = "png"
|
||||
ICO = "ico"
|
||||
#: Formats that are recognised (so they can be reported as "another format")
|
||||
#: but never accepted by the CLI.
|
||||
JPEG = "jpeg"
|
||||
GIF = "gif"
|
||||
BMP = "bmp"
|
||||
WEBP = "webp"
|
||||
#: Vector and document formats a live image source serves instead of a bitmap.
|
||||
#: Naming them lets a search say *which* other formats the topic offered instead
|
||||
#: of degrading into an access problem.
|
||||
SVG = "svg"
|
||||
TIFF = "tiff"
|
||||
PDF = "pdf"
|
||||
|
||||
#: Every format :func:`detect_format` can name.
|
||||
KNOWN_FORMATS: tuple[str, ...] = (PNG, ICO, JPEG, GIF, BMP, WEBP, SVG, TIFF, PDF)
|
||||
|
||||
#: The 8-byte signature every PNG file starts with.
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
#: ICONDIR: reserved (0), type (1 = icon) and the entry count.
|
||||
ICO_HEADER_SIZE = 6
|
||||
#: One ICONDIRENTRY: 16 bytes of width/height/colours/planes/bpp/size/offset.
|
||||
ICO_ENTRY_SIZE = 16
|
||||
#: A single ICO dimension byte of 0 encodes 256 pixels.
|
||||
ICO_MAX_DIMENSION = 256
|
||||
|
||||
#: Magic prefixes of the recognised but unsupported formats.
|
||||
_JPEG_SIGNATURE = b"\xff\xd8\xff"
|
||||
_GIF_SIGNATURES = (b"GIF87a", b"GIF89a")
|
||||
_BMP_SIGNATURE = b"BM"
|
||||
_RIFF_SIGNATURE = b"RIFF"
|
||||
_WEBP_SIGNATURE = b"WEBP"
|
||||
_TIFF_SIGNATURES = (b"II*\x00", b"MM\x00*")
|
||||
_PDF_SIGNATURE = b"%PDF-"
|
||||
#: How many leading bytes are searched for an ``<svg`` root element.
|
||||
_SVG_PROBE = 1024
|
||||
|
||||
#: Number of payload bytes worth quoting in an error detail.
|
||||
_SIGNATURE_PROBE = 16
|
||||
|
||||
|
||||
class ImageFormatError(ToolkitError):
|
||||
"""Raised when downloaded bytes are not a readable image of a known format.
|
||||
|
||||
The bytes could not be attributed to any format the toolkit understands, or
|
||||
they claim to be an accepted format but their header is truncated/corrupt.
|
||||
It derives from :class:`~tools.errors.ToolkitError`, so the CLI reports it
|
||||
as an access problem (the source handed us something unusable) rather than
|
||||
as a "no such format" / "no such size" verdict about the topic.
|
||||
"""
|
||||
|
||||
code = "image_unreadable"
|
||||
|
||||
def __init__(self, message: str, *, details: dict[str, Any] | None = None) -> None:
|
||||
super().__init__(message, details=details)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageInfo:
|
||||
"""The real format and real pixel size of one payload."""
|
||||
|
||||
format: str
|
||||
width: int
|
||||
height: int
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""JSON-ready projection used in CLI items."""
|
||||
|
||||
return {"format": self.format, "width": self.width, "height": self.height}
|
||||
|
||||
@property
|
||||
def area(self) -> int:
|
||||
return self.width * self.height
|
||||
|
||||
|
||||
def _probe(payload: bytes) -> bytes:
|
||||
return bytes(payload[:_SIGNATURE_PROBE])
|
||||
|
||||
|
||||
def _looks_like_svg(data: bytes) -> bool:
|
||||
"""True when *data* starts with an SVG root element (or its XML prologue)."""
|
||||
|
||||
head = data[:_SVG_PROBE]
|
||||
if head.startswith(b"\xef\xbb\xbf"):
|
||||
head = head[3:]
|
||||
head = head.lstrip()
|
||||
if head.lower().startswith(b"<svg"):
|
||||
return True
|
||||
return head.startswith(b"<?xml") and b"<svg" in data[:_SVG_PROBE].lower()
|
||||
|
||||
|
||||
def detect_format(payload: bytes) -> str | None:
|
||||
"""Return the real container format of *payload*, or ``None``.
|
||||
|
||||
A recognised format other than ``png``/``ico`` is a perfectly good answer:
|
||||
the source does have images, just not in the requested format. ``None``
|
||||
means the bytes are not a recognised image at all -- still "not the requested
|
||||
format" from the caller's point of view, and never an access problem on its
|
||||
own.
|
||||
"""
|
||||
|
||||
if not isinstance(payload, (bytes, bytearray, memoryview)):
|
||||
return None
|
||||
data = bytes(payload)
|
||||
if data.startswith(PNG_SIGNATURE):
|
||||
return PNG
|
||||
if data[:4] == b"\x00\x00\x01\x00":
|
||||
return ICO
|
||||
if data.startswith(_JPEG_SIGNATURE):
|
||||
return JPEG
|
||||
if data.startswith(_GIF_SIGNATURES):
|
||||
return GIF
|
||||
if data.startswith(_BMP_SIGNATURE) and len(data) >= 14:
|
||||
return BMP
|
||||
if data.startswith(_RIFF_SIGNATURE) and data[8:12] == _WEBP_SIGNATURE:
|
||||
return WEBP
|
||||
if data.startswith(_TIFF_SIGNATURES):
|
||||
return TIFF
|
||||
if data.startswith(_PDF_SIGNATURE):
|
||||
return PDF
|
||||
if _looks_like_svg(data):
|
||||
return SVG
|
||||
return None
|
||||
|
||||
|
||||
def read_png_dimensions(payload: bytes) -> tuple[int, int]:
|
||||
"""Return ``(width, height)`` from the ``IHDR`` chunk of a PNG payload."""
|
||||
|
||||
data = bytes(payload)
|
||||
if not data.startswith(PNG_SIGNATURE):
|
||||
raise ImageFormatError(
|
||||
"payload does not start with the PNG signature",
|
||||
details={"signature": _probe(data).hex()},
|
||||
)
|
||||
offset = len(PNG_SIGNATURE)
|
||||
if len(data) < offset + 8:
|
||||
raise ImageFormatError("PNG header is truncated")
|
||||
(length,) = struct.unpack(">I", data[offset : offset + 4])
|
||||
chunk_type = data[offset + 4 : offset + 8]
|
||||
if chunk_type != b"IHDR":
|
||||
raise ImageFormatError(
|
||||
"PNG payload does not start with an IHDR chunk",
|
||||
details={"first_chunk": chunk_type.decode("latin-1")},
|
||||
)
|
||||
if length != 13:
|
||||
raise ImageFormatError(
|
||||
"PNG IHDR chunk has an unexpected length",
|
||||
details={"length": length},
|
||||
)
|
||||
header = data[offset + 8 : offset + 8 + length]
|
||||
if len(header) != length:
|
||||
raise ImageFormatError("PNG IHDR chunk is truncated")
|
||||
width, height = struct.unpack(">II", header[:8])
|
||||
if width < 1 or height < 1:
|
||||
raise ImageFormatError(
|
||||
"PNG image declares no pixels",
|
||||
details={"width": width, "height": height},
|
||||
)
|
||||
return int(width), int(height)
|
||||
|
||||
|
||||
def ico_entries(payload: bytes) -> tuple[tuple[int, int, int, int], ...]:
|
||||
"""Return ``(width, height, size, offset)`` for every ICO directory entry."""
|
||||
|
||||
data = bytes(payload)
|
||||
if len(data) < ICO_HEADER_SIZE:
|
||||
raise ImageFormatError("ICO header is truncated")
|
||||
reserved, kind, count = struct.unpack("<HHH", data[:ICO_HEADER_SIZE])
|
||||
if reserved != 0 or kind != 1 or count < 1:
|
||||
raise ImageFormatError(
|
||||
"payload is not an ICO image",
|
||||
details={"reserved": reserved, "type": kind, "count": count},
|
||||
)
|
||||
directory_end = ICO_HEADER_SIZE + count * ICO_ENTRY_SIZE
|
||||
if len(data) < directory_end:
|
||||
raise ImageFormatError(
|
||||
"ICO directory is truncated",
|
||||
details={"count": count, "bytes": len(data)},
|
||||
)
|
||||
entries: list[tuple[int, int, int, int]] = []
|
||||
for index in range(count):
|
||||
start = ICO_HEADER_SIZE + index * ICO_ENTRY_SIZE
|
||||
raw_width, raw_height, _colours, _reserved, _planes, _bits, size, offset = struct.unpack(
|
||||
"<BBBBHHII", data[start : start + ICO_ENTRY_SIZE]
|
||||
)
|
||||
width = raw_width or ICO_MAX_DIMENSION
|
||||
height = raw_height or ICO_MAX_DIMENSION
|
||||
if size < 1 or offset < directory_end or offset + size > len(data):
|
||||
raise ImageFormatError(
|
||||
"ICO directory entry points outside the payload",
|
||||
details={"index": index, "size": size, "offset": offset, "bytes": len(data)},
|
||||
)
|
||||
entries.append((width, height, size, offset))
|
||||
return tuple(entries)
|
||||
|
||||
|
||||
def read_ico_dimensions(payload: bytes) -> tuple[int, int]:
|
||||
"""Return the dimensions of the largest image stored in an ICO payload.
|
||||
|
||||
The ICONDIRENTRY directory is authoritative: it carries one width/height pair
|
||||
per stored image, so no decoding of the embedded bitmap or PNG is needed.
|
||||
"""
|
||||
|
||||
entries = ico_entries(payload)
|
||||
width, height, _size, _offset = max(entries, key=lambda entry: entry[0] * entry[1])
|
||||
return int(width), int(height)
|
||||
|
||||
|
||||
def inspect_image(payload: bytes) -> ImageInfo:
|
||||
"""Return the real format and pixel size of *payload*.
|
||||
|
||||
Raises :class:`ImageFormatError` when the bytes are not an image at all, or
|
||||
when they are not one of the two formats whose dimensions can be read.
|
||||
"""
|
||||
|
||||
image_format = detect_format(payload)
|
||||
if image_format is None:
|
||||
raise ImageFormatError(
|
||||
"downloaded payload is not a recognised image format",
|
||||
details={"signature": _probe(bytes(payload)).hex()},
|
||||
)
|
||||
if image_format == PNG:
|
||||
width, height = read_png_dimensions(payload)
|
||||
return ImageInfo(format=PNG, width=width, height=height)
|
||||
if image_format == ICO:
|
||||
width, height = read_ico_dimensions(payload)
|
||||
return ImageInfo(format=ICO, width=width, height=height)
|
||||
raise ImageFormatError(
|
||||
f"pixel dimensions are only readable for png and ico, not {image_format}",
|
||||
details={"format": image_format},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Builders: the same formats, produced with the standard library only
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _png_chunk(chunk_type: bytes, chunk_data: bytes) -> bytes:
|
||||
crc = zlib.crc32(chunk_type + chunk_data) & 0xFFFFFFFF
|
||||
return struct.pack(">I", len(chunk_data)) + chunk_type + chunk_data + struct.pack(">I", crc)
|
||||
|
||||
|
||||
def png_bytes(
|
||||
width: int,
|
||||
height: int,
|
||||
*,
|
||||
rgba: tuple[int, int, int, int] = (255, 255, 255, 255),
|
||||
rows: Sequence[bytes] | None = None,
|
||||
level: int = 6,
|
||||
) -> bytes:
|
||||
"""Build a valid 8-bit RGBA PNG of exactly ``width`` x ``height`` pixels.
|
||||
|
||||
``rows`` may supply raw (filter-0) RGBA scanlines; otherwise every pixel is
|
||||
filled with ``rgba``. The result is a real PNG file: a correct signature,
|
||||
a 13-byte ``IHDR``, a zlib-compressed ``IDAT`` and an empty ``IEND``.
|
||||
"""
|
||||
|
||||
if isinstance(width, bool) or isinstance(height, bool):
|
||||
raise ValueError("width and height must be integers")
|
||||
if width < 1 or height < 1:
|
||||
raise ValueError("width and height must be >= 1")
|
||||
if rows is None:
|
||||
pixel = bytes(rgba)
|
||||
line = pixel * width
|
||||
raw = bytearray()
|
||||
for _ in range(height):
|
||||
raw.append(0)
|
||||
raw += line
|
||||
raw_bytes = bytes(raw)
|
||||
else:
|
||||
if len(rows) != height:
|
||||
raise ValueError("rows must contain exactly height scanlines")
|
||||
raw = bytearray()
|
||||
for row in rows:
|
||||
if len(row) != width * 4:
|
||||
raise ValueError("each scanline must hold width * 4 bytes")
|
||||
raw.append(0)
|
||||
raw += row
|
||||
raw_bytes = bytes(raw)
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 6, 0, 0, 0)
|
||||
return (
|
||||
PNG_SIGNATURE
|
||||
+ _png_chunk(b"IHDR", header)
|
||||
+ _png_chunk(b"IDAT", zlib.compress(raw_bytes, level))
|
||||
+ _png_chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def ico_bytes(
|
||||
width: int,
|
||||
height: int,
|
||||
*,
|
||||
rgba: tuple[int, int, int, int] = (255, 255, 255, 255),
|
||||
frames: Iterable[tuple[int, int]] | None = None,
|
||||
) -> bytes:
|
||||
"""Build an ICO payload whose directory lists the requested sizes.
|
||||
|
||||
By default the ICO stores one embedded PNG of ``width`` x ``height`` pixels
|
||||
and the directory entry repeats those dimensions (``0`` encodes 256). Pass
|
||||
``frames`` to describe several directory entries instead, for example
|
||||
``ico_bytes(48, 48, frames=[(16, 16), (48, 48)])``: the dimensions of the
|
||||
*largest* entry then become the reported size, which is exactly what
|
||||
:func:`read_ico_dimensions` returns.
|
||||
"""
|
||||
|
||||
sizes = list(frames) if frames is not None else [(width, height)]
|
||||
if not sizes:
|
||||
raise ValueError("an ICO payload needs at least one frame")
|
||||
payloads: list[bytes] = []
|
||||
for frame_width, frame_height in sizes:
|
||||
if not 1 <= frame_width <= ICO_MAX_DIMENSION:
|
||||
raise ValueError("ICO widths must be between 1 and 256 pixels")
|
||||
if not 1 <= frame_height <= ICO_MAX_DIMENSION:
|
||||
raise ValueError("ICO heights must be between 1 and 256 pixels")
|
||||
payloads.append(png_bytes(frame_width, frame_height, rgba=rgba))
|
||||
header = struct.pack("<HHH", 0, 1, len(payloads))
|
||||
offset = ICO_HEADER_SIZE + ICO_ENTRY_SIZE * len(payloads)
|
||||
entries = bytearray()
|
||||
for (frame_width, frame_height), payload in zip(sizes, payloads):
|
||||
entries += struct.pack(
|
||||
"<BBBBHHII",
|
||||
frame_width % ICO_MAX_DIMENSION,
|
||||
frame_height % ICO_MAX_DIMENSION,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
32,
|
||||
len(payload),
|
||||
offset,
|
||||
)
|
||||
offset += len(payload)
|
||||
return header + bytes(entries) + b"".join(payloads)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Decoding and re-encoding: PNG pixels and ICO containers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Channels per PNG colour type (0 grey, 2 rgb, 3 palette, 4 grey+alpha, 6 rgba).
|
||||
_PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DecodedImage:
|
||||
"""The container format, pixel size and 8-bit RGBA scanlines of a payload."""
|
||||
|
||||
format: str
|
||||
width: int
|
||||
height: int
|
||||
rows: tuple[bytes, ...]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""JSON-ready projection: format plus the real pixel size."""
|
||||
|
||||
return {"format": self.format, "width": self.width, "height": self.height}
|
||||
|
||||
|
||||
def png_chunks(payload: bytes) -> tuple[tuple[bytes, bytes], ...]:
|
||||
"""Split a PNG payload into ``(chunk_type, chunk_data)`` pairs.
|
||||
|
||||
Raises :class:`ImageFormatError` when the signature is missing, a chunk is
|
||||
truncated or the stream never reaches ``IEND``: those bytes cannot be
|
||||
decoded, so stitching a half image would be worse than reporting it.
|
||||
"""
|
||||
|
||||
data = bytes(payload)
|
||||
if not data.startswith(PNG_SIGNATURE):
|
||||
raise ImageFormatError(
|
||||
"payload does not start with the PNG signature",
|
||||
details={"signature": _probe(data).hex()},
|
||||
)
|
||||
chunks: list[tuple[bytes, bytes]] = []
|
||||
offset = len(PNG_SIGNATURE)
|
||||
while offset + 8 <= len(data):
|
||||
(length,) = struct.unpack(">I", data[offset : offset + 4])
|
||||
chunk_type = data[offset + 4 : offset + 8]
|
||||
chunk_data = data[offset + 8 : offset + 8 + length]
|
||||
if len(chunk_data) != length:
|
||||
raise ImageFormatError(
|
||||
"PNG chunk is truncated",
|
||||
details={"chunk": chunk_type.decode("latin-1"), "length": length},
|
||||
)
|
||||
chunks.append((chunk_type, chunk_data))
|
||||
offset += 12 + length
|
||||
if chunk_type == b"IEND":
|
||||
return tuple(chunks)
|
||||
raise ImageFormatError("PNG image has no IEND chunk")
|
||||
|
||||
|
||||
def _unfilter_rows(raw: bytes, width: int, height: int, bpp: int) -> list[bytearray]:
|
||||
"""Undo the per-scanline PNG filters, returning one row per pixel line."""
|
||||
|
||||
stride = width * bpp
|
||||
if len(raw) < height * (stride + 1):
|
||||
raise ImageFormatError(
|
||||
"PNG image data is truncated",
|
||||
details={"width": width, "height": height, "bytes": len(raw)},
|
||||
)
|
||||
rows: list[bytearray] = []
|
||||
previous = bytearray(stride)
|
||||
position = 0
|
||||
for _ in range(height):
|
||||
filter_type = raw[position]
|
||||
position += 1
|
||||
line = bytearray(raw[position : position + stride])
|
||||
position += stride
|
||||
if filter_type == 1:
|
||||
for index in range(bpp, stride):
|
||||
line[index] = (line[index] + line[index - bpp]) & 0xFF
|
||||
elif filter_type == 2:
|
||||
for index in range(stride):
|
||||
line[index] = (line[index] + previous[index]) & 0xFF
|
||||
elif filter_type == 3:
|
||||
for index in range(stride):
|
||||
left = line[index - bpp] if index >= bpp else 0
|
||||
line[index] = (line[index] + ((left + previous[index]) >> 1)) & 0xFF
|
||||
elif filter_type == 4:
|
||||
for index in range(stride):
|
||||
left = line[index - bpp] if index >= bpp else 0
|
||||
up = previous[index]
|
||||
up_left = previous[index - bpp] if index >= bpp else 0
|
||||
estimate = left + up - up_left
|
||||
to_left = abs(estimate - left)
|
||||
to_up = abs(estimate - up)
|
||||
to_up_left = abs(estimate - up_left)
|
||||
if to_left <= to_up and to_left <= to_up_left:
|
||||
predictor = left
|
||||
elif to_up <= to_up_left:
|
||||
predictor = up
|
||||
else:
|
||||
predictor = up_left
|
||||
line[index] = (line[index] + predictor) & 0xFF
|
||||
elif filter_type != 0:
|
||||
raise ImageFormatError(
|
||||
"unsupported PNG filter type",
|
||||
details={"filter_type": int(filter_type)},
|
||||
)
|
||||
previous = line
|
||||
rows.append(line)
|
||||
return rows
|
||||
|
||||
|
||||
def _row_to_rgba(
|
||||
row: bytearray,
|
||||
width: int,
|
||||
color_type: int,
|
||||
depth: int,
|
||||
palette: bytes,
|
||||
transparency: bytes | None,
|
||||
) -> bytes:
|
||||
"""Convert one unfiltered scanline into 8-bit RGBA samples."""
|
||||
|
||||
step = depth // 8
|
||||
# Bytes per pixel: one sample per channel, so the channel offsets inside a
|
||||
# pixel are ``step`` apart while pixels are ``channels * step`` apart.
|
||||
pixel_stride = _PNG_CHANNELS[color_type] * step
|
||||
out = bytearray()
|
||||
if color_type == 3:
|
||||
for x in range(width):
|
||||
index = row[x]
|
||||
base = index * 3
|
||||
if base + 2 >= len(palette):
|
||||
raise ImageFormatError(
|
||||
"PNG palette index is out of range",
|
||||
details={"index": int(index), "palette_bytes": len(palette)},
|
||||
)
|
||||
alpha = transparency[index] if transparency and index < len(transparency) else 255
|
||||
out += bytes((palette[base], palette[base + 1], palette[base + 2], alpha))
|
||||
return bytes(out)
|
||||
for x in range(width):
|
||||
offset = x * pixel_stride
|
||||
if color_type == 0:
|
||||
value = row[offset]
|
||||
out += bytes((value, value, value, 255))
|
||||
elif color_type == 4:
|
||||
value = row[offset]
|
||||
out += bytes((value, value, value, row[offset + step]))
|
||||
elif color_type == 2:
|
||||
out += bytes((row[offset], row[offset + step], row[offset + 2 * step], 255))
|
||||
else:
|
||||
out += bytes(
|
||||
(
|
||||
row[offset],
|
||||
row[offset + step],
|
||||
row[offset + 2 * step],
|
||||
row[offset + 3 * step],
|
||||
)
|
||||
)
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def decode_png(payload: bytes) -> tuple[int, int, tuple[bytes, ...]]:
|
||||
"""Decode a PNG payload into ``(width, height, rgba_rows)``.
|
||||
|
||||
Supports the non-interlaced 8-bit and 16-bit PNG colour types ImageMagick,
|
||||
GIMP and the Wikimedia thumbnailers produce; anything else raises
|
||||
:class:`ImageFormatError` instead of guessing at pixels.
|
||||
"""
|
||||
|
||||
header: tuple[int, ...] | None = None
|
||||
palette = b""
|
||||
transparency: bytes | None = None
|
||||
data = bytearray()
|
||||
for chunk_type, chunk_data in png_chunks(payload):
|
||||
if chunk_type == b"IHDR":
|
||||
if len(chunk_data) < 13:
|
||||
raise ImageFormatError("PNG header is truncated")
|
||||
header = struct.unpack(">IIBBBBB", chunk_data[:13])
|
||||
elif chunk_type == b"PLTE":
|
||||
palette = chunk_data
|
||||
elif chunk_type == b"tRNS":
|
||||
transparency = chunk_data
|
||||
elif chunk_type == b"IDAT":
|
||||
data += chunk_data
|
||||
if header is None:
|
||||
raise ImageFormatError("PNG image has no IHDR chunk")
|
||||
width, height, depth, color_type, compression, filter_method, interlace = header
|
||||
if width < 1 or height < 1:
|
||||
raise ImageFormatError("PNG image declares no pixels")
|
||||
if compression != 0 or filter_method != 0:
|
||||
raise ImageFormatError(
|
||||
"unsupported PNG compression or filter method",
|
||||
details={"compression": compression, "filter_method": filter_method},
|
||||
)
|
||||
if interlace != 0:
|
||||
raise ImageFormatError(
|
||||
"interlaced PNG images are not supported",
|
||||
details={"interlace": interlace},
|
||||
)
|
||||
channels = _PNG_CHANNELS.get(color_type)
|
||||
if channels is None:
|
||||
raise ImageFormatError(
|
||||
"unsupported PNG colour type",
|
||||
details={"color_type": color_type},
|
||||
)
|
||||
if depth not in (8, 16):
|
||||
raise ImageFormatError(
|
||||
"unsupported PNG bit depth",
|
||||
details={"bit_depth": depth},
|
||||
)
|
||||
if color_type == 3 and depth > 8:
|
||||
raise ImageFormatError(
|
||||
"unsupported PNG palette bit depth",
|
||||
details={"bit_depth": depth},
|
||||
)
|
||||
if color_type == 3 and not palette:
|
||||
raise ImageFormatError("PNG palette image has no PLTE chunk")
|
||||
try:
|
||||
raw = zlib.decompress(bytes(data))
|
||||
except zlib.error as exc:
|
||||
raise ImageFormatError(
|
||||
"PNG image data is corrupt",
|
||||
details={"reason": str(exc)},
|
||||
) from exc
|
||||
rows = _unfilter_rows(raw, width, height, channels * (depth // 8))
|
||||
rgba_rows = tuple(
|
||||
_row_to_rgba(row, width, color_type, depth, palette, transparency) for row in rows
|
||||
)
|
||||
return int(width), int(height), rgba_rows
|
||||
|
||||
|
||||
def encode_png(width: int, height: int, rows: Sequence[bytes]) -> bytes:
|
||||
"""Encode 8-bit RGBA ``rows`` as a real PNG with correct chunk CRCs."""
|
||||
|
||||
return png_bytes(width, height, rows=list(rows))
|
||||
|
||||
|
||||
def ico_wrap(embedded_png: bytes, width: int, height: int) -> bytes:
|
||||
"""Wrap a PNG payload in a single-entry ICO container of that size."""
|
||||
|
||||
if not 1 <= width <= ICO_MAX_DIMENSION:
|
||||
raise ImageFormatError(
|
||||
"ICO widths must be between 1 and 256 pixels",
|
||||
details={"width": width},
|
||||
)
|
||||
if not 1 <= height <= ICO_MAX_DIMENSION:
|
||||
raise ImageFormatError(
|
||||
"ICO heights must be between 1 and 256 pixels",
|
||||
details={"height": height},
|
||||
)
|
||||
png_payload = bytes(embedded_png)
|
||||
if not png_payload.startswith(PNG_SIGNATURE):
|
||||
raise ImageFormatError("an ICO container can only wrap a PNG payload here")
|
||||
header = struct.pack("<HHH", 0, 1, 1)
|
||||
entry = struct.pack(
|
||||
"<BBBBHHII",
|
||||
width % ICO_MAX_DIMENSION,
|
||||
height % ICO_MAX_DIMENSION,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
32,
|
||||
len(png_payload),
|
||||
ICO_HEADER_SIZE + ICO_ENTRY_SIZE,
|
||||
)
|
||||
return header + entry + png_payload
|
||||
|
||||
|
||||
def ico_unwrap(payload: bytes) -> tuple[int, int, bytes]:
|
||||
"""Return the ``(width, height, embedded_png)`` of an ICO's largest frame."""
|
||||
|
||||
entries = ico_entries(payload)
|
||||
width, height, size, offset = max(entries, key=lambda entry: entry[0] * entry[1])
|
||||
embedded = bytes(payload)[offset : offset + size]
|
||||
if not embedded.startswith(PNG_SIGNATURE):
|
||||
raise ImageFormatError(
|
||||
"ICO frames stored as raw bitmaps are not supported",
|
||||
details={"width": width, "height": height},
|
||||
)
|
||||
return int(width), int(height), embedded
|
||||
|
||||
|
||||
def decode_image(payload: bytes) -> DecodedImage:
|
||||
"""Decode a PNG or ICO payload into :class:`DecodedImage` RGBA scanlines."""
|
||||
|
||||
data = bytes(payload)
|
||||
image_format = detect_format(data)
|
||||
if image_format == PNG:
|
||||
width, height, rows = decode_png(data)
|
||||
return DecodedImage(format=PNG, width=width, height=height, rows=rows)
|
||||
if image_format == ICO:
|
||||
width, height, embedded = ico_unwrap(data)
|
||||
inner_width, inner_height, rows = decode_png(embedded)
|
||||
if (inner_width, inner_height) != (width, height):
|
||||
# The directory is authoritative for the reported size, but the
|
||||
# pixels must agree or the container is inconsistent.
|
||||
raise ImageFormatError(
|
||||
"ICO directory and embedded PNG disagree about the size",
|
||||
details={
|
||||
"directory": {"width": width, "height": height},
|
||||
"embedded": {"width": inner_width, "height": inner_height},
|
||||
},
|
||||
)
|
||||
return DecodedImage(format=ICO, width=width, height=height, rows=rows)
|
||||
raise ImageFormatError(
|
||||
"pixel data is only decodable for png and ico",
|
||||
details={"format": image_format},
|
||||
)
|
||||
|
||||
|
||||
def resample_rows(
|
||||
rows: Sequence[bytes],
|
||||
width: int,
|
||||
height: int,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
) -> tuple[bytes, ...]:
|
||||
"""Nearest-neighbour resample of RGBA ``rows`` to exactly the target size."""
|
||||
|
||||
if target_width < 1 or target_height < 1:
|
||||
raise ImageFormatError("resampled dimensions must be at least 1x1")
|
||||
if len(rows) != height:
|
||||
raise ImageFormatError(
|
||||
"row count does not match the source height",
|
||||
details={"rows": len(rows), "height": height},
|
||||
)
|
||||
if width < 1 or height < 1:
|
||||
raise ImageFormatError("source image has no pixels")
|
||||
columns = [min(width - 1, (x * width) // target_width) * 4 for x in range(target_width)]
|
||||
resized: list[bytes] = []
|
||||
for y in range(target_height):
|
||||
source = rows[min(height - 1, (y * height) // target_height)]
|
||||
line = bytearray()
|
||||
for offset in columns:
|
||||
line += source[offset : offset + 4]
|
||||
resized.append(bytes(line))
|
||||
return tuple(resized)
|
||||
|
||||
|
||||
def encode_image(
|
||||
image_format: str,
|
||||
width: int,
|
||||
height: int,
|
||||
rows: Sequence[bytes],
|
||||
) -> bytes:
|
||||
"""Encode RGBA ``rows`` in ``image_format`` (``png`` or ``ico``)."""
|
||||
|
||||
png_payload = encode_png(width, height, rows)
|
||||
if image_format == PNG:
|
||||
return png_payload
|
||||
if image_format == ICO:
|
||||
return ico_wrap(png_payload, width, height)
|
||||
raise ImageFormatError(
|
||||
"only png and ico can be encoded",
|
||||
details={"format": image_format},
|
||||
)
|
||||
@@ -0,0 +1,578 @@
|
||||
"""Pluggable, credential-free image source registry and its HTTPS layer.
|
||||
|
||||
Nothing in this module reads an API key, token, registration or secret from
|
||||
arguments, environment variables or configuration files: Wikimedia Commons and
|
||||
Openverse are queried anonymously. The module provides
|
||||
|
||||
* :class:`HTTPSClient` -- a small HTTPS client that identifies itself with a
|
||||
stable ``User-Agent``, refuses every URL outside the approved hosts
|
||||
(Wikimedia Commons and Openverse) on port 443, retries HTTP 429/5xx a bounded
|
||||
number of times with capped exponential backoff, and raises
|
||||
:class:`~tools.errors.SourceAccessError` instead of returning an empty result
|
||||
when the host is unreachable or keeps failing;
|
||||
* :class:`SourceAdapter` -- the interface every image source implements;
|
||||
* :class:`WikimediaCommonsAdapter` and :class:`OpenverseAdapter` -- two working
|
||||
credential-free sources behind that interface;
|
||||
* :class:`SourceRegistry` -- the plugin list that reports available source ids
|
||||
and turns an unknown id into :class:`~tools.errors.UnknownSourceError` listing
|
||||
them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Iterable, Mapping, Protocol, runtime_checkable
|
||||
|
||||
from .errors import (
|
||||
DuplicateSourceError,
|
||||
HostNotAllowedError,
|
||||
SearchRequestError,
|
||||
SourceAccessError,
|
||||
SourceHTTPError,
|
||||
SourceError,
|
||||
SourcePayloadError,
|
||||
SourceUnreachableError,
|
||||
UnknownSourceError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"APPROVED_HOSTS",
|
||||
"DEFAULT_LIMIT",
|
||||
"DEFAULT_MAX_ATTEMPTS",
|
||||
"HTTPSClient",
|
||||
"HTTPResponse",
|
||||
"MAX_LIMIT",
|
||||
"OPENVERSE_API_HOST",
|
||||
"OpenverseAdapter",
|
||||
"SourceAdapter",
|
||||
"SourceCandidate",
|
||||
"SourceRegistry",
|
||||
"USER_AGENT",
|
||||
"WIKIMEDIA_COMMONS_API_HOST",
|
||||
"WIKIMEDIA_COMMONS_MEDIA_HOST",
|
||||
"build_registry",
|
||||
"default_registry",
|
||||
"is_approved_url",
|
||||
"search",
|
||||
"validate_source_url",
|
||||
]
|
||||
|
||||
PRODUCT_NAME = "product-54354138-1"
|
||||
USER_AGENT = f"{PRODUCT_NAME}-image-toolkit/0.1.0 (credential-free image source client)"
|
||||
|
||||
WIKIMEDIA_COMMONS_API_HOST = "commons.wikimedia.org"
|
||||
WIKIMEDIA_COMMONS_MEDIA_HOST = "upload.wikimedia.org"
|
||||
OPENVERSE_API_HOST = "api.openverse.org"
|
||||
|
||||
# Wikimedia Commons is reachable through its API host plus the media host that
|
||||
# serves the original files, Openverse through its API host (and the thumbnail
|
||||
# endpoint on the same API host).
|
||||
APPROVED_HOSTS = frozenset(
|
||||
{WIKIMEDIA_COMMONS_API_HOST, WIKIMEDIA_COMMONS_MEDIA_HOST, OPENVERSE_API_HOST}
|
||||
)
|
||||
APPROVED_SCHEME = "https"
|
||||
APPROVED_PORT = 443
|
||||
|
||||
DEFAULT_TIMEOUT = 10.0
|
||||
DEFAULT_MAX_ATTEMPTS = 3
|
||||
DEFAULT_BACKOFF_SECONDS = 0.5
|
||||
MAX_BACKOFF_SECONDS = 8.0
|
||||
RETRY_STATUS_CODES = frozenset({429}) | frozenset(range(500, 600))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HTTPResponse:
|
||||
"""A transport-independent HTTP response."""
|
||||
|
||||
status: int
|
||||
body: bytes
|
||||
headers: Mapping[str, str] = field(default_factory=dict)
|
||||
|
||||
def json(self, *, url: str = "", source_id: str | None = None) -> Any:
|
||||
try:
|
||||
return json.loads(self.body)
|
||||
except (ValueError, UnicodeDecodeError) as exc: # pragma: no cover - defensive
|
||||
raise SourcePayloadError(url or "<response>", source_id=source_id, reason=str(exc)) from exc
|
||||
|
||||
|
||||
class Transport(Protocol):
|
||||
"""Callable performing one HTTP GET; injectable so tests never hit the net."""
|
||||
|
||||
def __call__(self, url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse: ...
|
||||
|
||||
|
||||
def validate_source_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> str:
|
||||
"""Return *url* when it is an approved HTTPS URL, else raise a typed error."""
|
||||
|
||||
allowed_set = frozenset(allowed)
|
||||
if not isinstance(url, str) or not url:
|
||||
raise HostNotAllowedError(str(url), allowed=allowed_set)
|
||||
parts = urllib.parse.urlsplit(url)
|
||||
if parts.scheme != APPROVED_SCHEME:
|
||||
raise HostNotAllowedError(url, allowed=allowed_set)
|
||||
host = (parts.hostname or "").lower()
|
||||
if host not in allowed_set:
|
||||
raise HostNotAllowedError(url, allowed=allowed_set)
|
||||
try:
|
||||
port = parts.port
|
||||
except ValueError as exc:
|
||||
raise HostNotAllowedError(url, allowed=allowed_set) from exc
|
||||
if port not in (None, APPROVED_PORT):
|
||||
raise HostNotAllowedError(url, allowed=allowed_set)
|
||||
return url
|
||||
|
||||
|
||||
def is_approved_url(url: str, *, allowed: Iterable[str] = APPROVED_HOSTS) -> bool:
|
||||
"""Non-raising variant of :func:`validate_source_url` used by tests/callers."""
|
||||
|
||||
try:
|
||||
validate_source_url(url, allowed=allowed)
|
||||
except HostNotAllowedError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def urllib_transport(url: str, headers: Mapping[str, str], timeout: float) -> HTTPResponse:
|
||||
"""Default transport: a plain stdlib GET returning status, body and headers."""
|
||||
|
||||
request = urllib.request.Request(url, headers=dict(headers), method="GET")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return HTTPResponse(
|
||||
status=int(getattr(response, "status", 200)),
|
||||
body=response.read(),
|
||||
headers=dict(response.headers.items()),
|
||||
)
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = b""
|
||||
try:
|
||||
body = exc.read()
|
||||
except Exception: # pragma: no cover - body optional
|
||||
body = b""
|
||||
return HTTPResponse(
|
||||
status=int(exc.code),
|
||||
body=body,
|
||||
headers=dict(getattr(exc, "headers", {}) or {}),
|
||||
)
|
||||
|
||||
|
||||
class HTTPSClient:
|
||||
"""Fetch a URL only when it is approved, with bounded retry/backoff.
|
||||
|
||||
The client never returns an empty result for a broken source: unreachable
|
||||
hosts raise :class:`SourceUnreachableError` and exhausted/non-retryable
|
||||
statuses raise :class:`SourceHTTPError`. Both derive from
|
||||
:class:`SourceAccessError`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
user_agent: str = USER_AGENT,
|
||||
allowed_hosts: Iterable[str] = APPROVED_HOSTS,
|
||||
timeout: float = DEFAULT_TIMEOUT,
|
||||
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
||||
backoff_seconds: float = DEFAULT_BACKOFF_SECONDS,
|
||||
max_backoff_seconds: float = MAX_BACKOFF_SECONDS,
|
||||
transport: Transport | None = None,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
) -> None:
|
||||
if max_attempts < 1:
|
||||
raise ValueError("max_attempts must be >= 1")
|
||||
self.user_agent = user_agent
|
||||
self.allowed_hosts = frozenset(allowed_hosts)
|
||||
self.timeout = float(timeout)
|
||||
self.max_attempts = int(max_attempts)
|
||||
self.backoff_seconds = float(backoff_seconds)
|
||||
self.max_backoff_seconds = float(max_backoff_seconds)
|
||||
self._transport: Transport = transport or urllib_transport
|
||||
self._sleep = sleep
|
||||
|
||||
# -- public API -----------------------------------------------------
|
||||
def headers(self) -> dict[str, str]:
|
||||
"""Request headers, always including the identifying User-Agent."""
|
||||
|
||||
return {
|
||||
"User-Agent": self.user_agent,
|
||||
"Accept": "application/json, image/*;q=0.8, */*;q=0.5",
|
||||
}
|
||||
|
||||
def backoff_delay(self, attempt: int) -> float:
|
||||
"""Capped exponential backoff delay for 1-based *attempt*."""
|
||||
|
||||
delay = self.backoff_seconds * (2 ** max(0, attempt - 1))
|
||||
return min(delay, self.max_backoff_seconds)
|
||||
|
||||
def get(self, url: str, *, source_id: str | None = None) -> HTTPResponse:
|
||||
"""GET *url*, retrying transient statuses, then raise typed errors."""
|
||||
|
||||
validate_source_url(url, allowed=self.allowed_hosts)
|
||||
response: HTTPResponse | None = None
|
||||
for attempt in range(1, self.max_attempts + 1):
|
||||
try:
|
||||
response = self._transport(url, self.headers(), self.timeout)
|
||||
except SourceAccessError:
|
||||
raise
|
||||
except Exception as exc: # transport-level failure
|
||||
if attempt >= self.max_attempts:
|
||||
raise SourceUnreachableError(
|
||||
url, source_id=source_id, reason=f"{type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
self._sleep(self.backoff_delay(attempt))
|
||||
continue
|
||||
if response.status in RETRY_STATUS_CODES and attempt < self.max_attempts:
|
||||
self._sleep(self.backoff_delay(attempt))
|
||||
continue
|
||||
if 200 <= response.status < 300:
|
||||
return response
|
||||
raise SourceHTTPError(url, response.status, source_id=source_id)
|
||||
# Defensive: the loop either returned or raised above.
|
||||
status = response.status if response is not None else 0
|
||||
raise SourceHTTPError(url, status, source_id=source_id)
|
||||
|
||||
def get_json(self, url: str, *, source_id: str | None = None) -> Any:
|
||||
response = self.get(url, source_id=source_id)
|
||||
try:
|
||||
return json.loads(response.body)
|
||||
except (ValueError, UnicodeDecodeError) as exc:
|
||||
raise SourcePayloadError(url, source_id=source_id, reason=str(exc)) from exc
|
||||
|
||||
|
||||
DEFAULT_LIMIT = 10
|
||||
MAX_LIMIT = 50
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceCandidate:
|
||||
"""One normalized search hit.
|
||||
|
||||
``source_id`` identifies the adapter, ``file_url`` is the direct HTTPS
|
||||
location of the original image, ``mime_type`` is the type the source
|
||||
reports (may be ``None`` when unstated) and ``fetch_bytes`` is a zero-argument
|
||||
callable returning the image bytes through the approved HTTPS client.
|
||||
"""
|
||||
|
||||
source_id: str
|
||||
file_url: str
|
||||
mime_type: str | None
|
||||
fetch_bytes: Callable[[], bytes]
|
||||
title: str | None = None
|
||||
page_url: str | None = None
|
||||
width: int | None = None
|
||||
height: int | None = None
|
||||
extra: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
def fetch(self) -> bytes:
|
||||
"""Convenience passthrough to :attr:`fetch_bytes`."""
|
||||
|
||||
return self.fetch_bytes()
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Metadata-only projection (the callable is intentionally omitted)."""
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"source_id": self.source_id,
|
||||
"file_url": self.file_url,
|
||||
"mime_type": self.mime_type,
|
||||
}
|
||||
for key in ("title", "page_url", "width", "height"):
|
||||
value = getattr(self, key)
|
||||
if value is not None:
|
||||
payload[key] = value
|
||||
return payload
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SourceAdapter(Protocol):
|
||||
"""Interface implemented by every image source."""
|
||||
|
||||
source_id: str
|
||||
|
||||
def search(self, topic: str, limit: int) -> list[SourceCandidate]: ...
|
||||
|
||||
|
||||
def normalize_search_args(topic: str, limit: int) -> tuple[str, int]:
|
||||
"""Validate the caller-supplied topic and result limit."""
|
||||
|
||||
if not isinstance(topic, str):
|
||||
raise SearchRequestError(
|
||||
f"topic must be a string, got {type(topic).__name__}",
|
||||
details={"topic": repr(topic)},
|
||||
)
|
||||
topic = topic.strip()
|
||||
if not topic:
|
||||
raise SearchRequestError("topic must not be empty", details={"topic": topic})
|
||||
if isinstance(limit, bool) or not isinstance(limit, int):
|
||||
raise SearchRequestError(
|
||||
f"limit must be an integer, got {type(limit).__name__}",
|
||||
details={"limit": repr(limit)},
|
||||
)
|
||||
if limit < 1:
|
||||
raise SearchRequestError(f"limit must be >= 1, got {limit}", details={"limit": limit})
|
||||
if limit > MAX_LIMIT:
|
||||
raise SearchRequestError(
|
||||
f"limit must be <= {MAX_LIMIT}, got {limit}", details={"limit": limit}
|
||||
)
|
||||
return topic, limit
|
||||
|
||||
|
||||
class _BaseAdapter:
|
||||
"""Shared normalization helpers for concrete adapters."""
|
||||
|
||||
source_id = ""
|
||||
|
||||
def __init__(self, client: HTTPSClient | None = None) -> None:
|
||||
self.client = client or HTTPSClient()
|
||||
|
||||
def _candidate(
|
||||
self,
|
||||
*,
|
||||
file_url: str,
|
||||
mime_type: str | None,
|
||||
title: str | None = None,
|
||||
page_url: str | None = None,
|
||||
width: int | None = None,
|
||||
height: int | None = None,
|
||||
extra: Mapping[str, Any] | None = None,
|
||||
validate: bool = True,
|
||||
) -> SourceCandidate:
|
||||
if validate:
|
||||
validate_source_url(file_url, allowed=self.client.allowed_hosts)
|
||||
source_id = self.source_id
|
||||
client = self.client
|
||||
return SourceCandidate(
|
||||
source_id=source_id,
|
||||
file_url=file_url,
|
||||
mime_type=mime_type,
|
||||
fetch_bytes=lambda url=file_url: client.get(url, source_id=source_id).body,
|
||||
title=title,
|
||||
page_url=page_url,
|
||||
width=width,
|
||||
height=height,
|
||||
extra=dict(extra or {}),
|
||||
)
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - trivial
|
||||
return f"<{type(self).__name__} source_id={self.source_id!r}>"
|
||||
|
||||
|
||||
class WikimediaCommonsAdapter(_BaseAdapter):
|
||||
"""Wikimedia Commons search through the anonymous MediaWiki action API."""
|
||||
|
||||
source_id = "wikimedia_commons"
|
||||
api_host = WIKIMEDIA_COMMONS_API_HOST
|
||||
|
||||
def build_search_url(self, topic: str, limit: int) -> str:
|
||||
"""Construct the generator search URL for *topic* and *limit*."""
|
||||
|
||||
topic, limit = normalize_search_args(topic, limit)
|
||||
params = {
|
||||
"action": "query",
|
||||
"format": "json",
|
||||
"formatversion": "2",
|
||||
"generator": "search",
|
||||
"gsrsearch": topic,
|
||||
"gsrnamespace": "6",
|
||||
"gsrlimit": str(limit),
|
||||
"prop": "imageinfo",
|
||||
"iiprop": "url|mime|size",
|
||||
"iiurlwidth": "1024",
|
||||
}
|
||||
query = urllib.parse.urlencode(params)
|
||||
return f"https://{self.api_host}/w/api.php?{query}"
|
||||
|
||||
def search(self, topic: str, limit: int) -> list[SourceCandidate]:
|
||||
url = self.build_search_url(topic, limit)
|
||||
payload = self.client.get_json(url, source_id=self.source_id)
|
||||
if not isinstance(payload, Mapping):
|
||||
raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object")
|
||||
if "error" in payload:
|
||||
raise SourceAccessError(
|
||||
f"Wikimedia Commons API error: {payload['error']}",
|
||||
url=url,
|
||||
source_id=self.source_id,
|
||||
)
|
||||
pages = (payload.get("query") or {}).get("pages") or []
|
||||
candidates: list[SourceCandidate] = []
|
||||
for page in pages:
|
||||
infos = page.get("imageinfo") or []
|
||||
if not infos:
|
||||
continue
|
||||
info = infos[0]
|
||||
file_url = info.get("url")
|
||||
if not file_url:
|
||||
continue
|
||||
candidates.append(
|
||||
self._candidate(
|
||||
file_url=file_url,
|
||||
mime_type=info.get("mime"),
|
||||
title=page.get("title"),
|
||||
page_url=info.get("descriptionurl"),
|
||||
width=info.get("width"),
|
||||
height=info.get("height"),
|
||||
extra={"thumb_url": info.get("thumburl")} if info.get("thumburl") else None,
|
||||
)
|
||||
)
|
||||
if len(candidates) >= limit:
|
||||
break
|
||||
return candidates[:limit]
|
||||
|
||||
|
||||
class OpenverseAdapter(_BaseAdapter):
|
||||
"""Openverse search through its anonymous public images endpoint."""
|
||||
|
||||
source_id = "openverse"
|
||||
api_host = OPENVERSE_API_HOST
|
||||
|
||||
def build_search_url(self, topic: str, limit: int) -> str:
|
||||
"""Construct the Openverse image search URL for *topic* and *limit*."""
|
||||
|
||||
topic, limit = normalize_search_args(topic, limit)
|
||||
query = urllib.parse.urlencode({"q": topic, "page_size": str(limit)})
|
||||
return f"https://{self.api_host}/v1/images/?{query}"
|
||||
|
||||
def search(self, topic: str, limit: int) -> list[SourceCandidate]:
|
||||
url = self.build_search_url(topic, limit)
|
||||
payload = self.client.get_json(url, source_id=self.source_id)
|
||||
if not isinstance(payload, Mapping):
|
||||
raise SourcePayloadError(url, source_id=self.source_id, reason="expected a JSON object")
|
||||
results = payload.get("results")
|
||||
if results is None:
|
||||
raise SourcePayloadError(url, source_id=self.source_id, reason="missing 'results'")
|
||||
candidates: list[SourceCandidate] = []
|
||||
for item in results:
|
||||
if not isinstance(item, Mapping):
|
||||
continue
|
||||
original_url = item.get("url")
|
||||
file_url = item.get("thumbnail") or original_url
|
||||
if not file_url:
|
||||
continue
|
||||
candidates.append(
|
||||
self._candidate(
|
||||
file_url=file_url,
|
||||
# Origin files may live on third-party hosts; the client
|
||||
# still refuses them at fetch time with a typed error.
|
||||
validate=is_approved_url(file_url, allowed=self.client.allowed_hosts),
|
||||
mime_type=item.get("filetype"),
|
||||
title=item.get("title"),
|
||||
page_url=item.get("foreign_landing_url") or item.get("detail_url"),
|
||||
width=item.get("width"),
|
||||
height=item.get("height"),
|
||||
extra={
|
||||
"license": item.get("license"),
|
||||
"creator": item.get("creator"),
|
||||
"original_url": original_url,
|
||||
},
|
||||
)
|
||||
)
|
||||
if len(candidates) >= limit:
|
||||
break
|
||||
return candidates[:limit]
|
||||
|
||||
|
||||
class SourceRegistry:
|
||||
"""Ordered plugin registry of credential-free image sources."""
|
||||
|
||||
def __init__(self, adapters: Iterable[SourceAdapter] | None = None) -> None:
|
||||
self._adapters: dict[str, SourceAdapter] = {}
|
||||
for adapter in adapters or ():
|
||||
self.register(adapter)
|
||||
|
||||
# -- registration ---------------------------------------------------
|
||||
def register(self, adapter: SourceAdapter, *, replace: bool = False) -> SourceAdapter:
|
||||
"""Add *adapter*; a duplicate id raises :class:`DuplicateSourceError`."""
|
||||
|
||||
source_id = getattr(adapter, "source_id", None)
|
||||
if not isinstance(source_id, str) or not source_id:
|
||||
raise SourceError("adapter must expose a non-empty string 'source_id'")
|
||||
if source_id in self._adapters and not replace:
|
||||
raise DuplicateSourceError(source_id)
|
||||
self._adapters[source_id] = adapter
|
||||
return adapter
|
||||
|
||||
def unregister(self, source_id: str) -> SourceAdapter:
|
||||
if source_id not in self._adapters:
|
||||
raise UnknownSourceError(source_id, self.available_sources())
|
||||
return self._adapters.pop(source_id)
|
||||
|
||||
# -- lookup ---------------------------------------------------------
|
||||
def available_sources(self) -> list[str]:
|
||||
"""Sorted ids of every registered source."""
|
||||
|
||||
return sorted(self._adapters)
|
||||
|
||||
def __contains__(self, source_id: object) -> bool:
|
||||
return isinstance(source_id, str) and source_id in self._adapters
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._adapters)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.available_sources())
|
||||
|
||||
def get(self, source_id: str) -> SourceAdapter:
|
||||
"""Return the adapter for *source_id* or raise a listing error."""
|
||||
|
||||
try:
|
||||
return self._adapters[source_id]
|
||||
except (KeyError, TypeError):
|
||||
raise UnknownSourceError(source_id, self.available_sources()) from None
|
||||
|
||||
def create(self, source_id: str) -> SourceAdapter:
|
||||
"""Explicit alias of :meth:`get` used by the CLI layer."""
|
||||
|
||||
return self.get(source_id)
|
||||
|
||||
# -- search ---------------------------------------------------------
|
||||
def search(self, source_id: str, topic: str, limit: int = DEFAULT_LIMIT) -> list[SourceCandidate]:
|
||||
"""Search *source_id* for *topic*, returning at most *limit* candidates."""
|
||||
|
||||
adapter = self.get(source_id)
|
||||
topic, limit = normalize_search_args(topic, limit)
|
||||
return list(adapter.search(topic, limit))
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - trivial
|
||||
return f"SourceRegistry({self.available_sources()!r})"
|
||||
|
||||
|
||||
|
||||
|
||||
def build_registry(
|
||||
*,
|
||||
client: HTTPSClient | None = None,
|
||||
adapters: Iterable[SourceAdapter] | None = None,
|
||||
) -> SourceRegistry:
|
||||
"""Build the default registry (Wikimedia Commons + Openverse)."""
|
||||
|
||||
shared_client = client or HTTPSClient()
|
||||
if adapters is None:
|
||||
adapters = (
|
||||
WikimediaCommonsAdapter(shared_client),
|
||||
OpenverseAdapter(shared_client),
|
||||
)
|
||||
return SourceRegistry(adapters)
|
||||
|
||||
|
||||
def default_registry(client: HTTPSClient | None = None) -> SourceRegistry:
|
||||
"""Convenience wrapper returning a fresh default registry."""
|
||||
|
||||
return build_registry(client=client)
|
||||
|
||||
|
||||
def search(
|
||||
topic: str,
|
||||
*,
|
||||
source: str = "wikimedia_commons",
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
registry: SourceRegistry | None = None,
|
||||
client: HTTPSClient | None = None,
|
||||
) -> list[SourceCandidate]:
|
||||
"""Search *source* for *topic* using *registry* (default registry by default)."""
|
||||
|
||||
active = registry if registry is not None else build_registry(client=client)
|
||||
return active.search(source, topic, limit)
|
||||
Reference in New Issue
Block a user