Final acceptance: Консольная утилита поиска и скачивания изображений для вызова ИИ-агентом (#18)
Repository Validate / repository-checks (push) Successful in 21s
Repository Validate / repository-checks (push) Successful in 21s
Co-authored-by: AI Tester <ai-tester@git.p2prod.ru>
This commit was merged in pull request #18.
This commit is contained in:
@@ -0,0 +1,838 @@
|
||||
"""Plain native-process CLI for the image toolkit.
|
||||
|
||||
Usage::
|
||||
|
||||
python -m tools.example examples/input.json
|
||||
python -m tools.example --topic "red fox" --width 640 --height 480 --format png
|
||||
|
||||
Exactly one machine-readable JSON document is written to stdout. The process
|
||||
exit code is derived from the fixed status vocabulary in :mod:`tools.envelope`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from . import images
|
||||
from .envelope import (
|
||||
STATUS_ACCESS_ERROR,
|
||||
STATUS_NO_FORMAT,
|
||||
STATUS_NO_SIZE,
|
||||
STATUS_OK,
|
||||
STATUS_PARTIAL_DOWNLOAD,
|
||||
Response,
|
||||
ResponseBuilder,
|
||||
error_fragment,
|
||||
)
|
||||
from .errors import (
|
||||
ACCEPTED_FORMATS,
|
||||
ImageResizeError,
|
||||
RequestDocumentError,
|
||||
RequestError,
|
||||
RequestFieldError,
|
||||
SourceAccessError,
|
||||
ToolkitError,
|
||||
UnknownSourceError,
|
||||
UnsupportedFormatError,
|
||||
)
|
||||
from .images import ImageFormatError, ImageInfo
|
||||
from .sources import DEFAULT_LIMIT, SourceCandidate, default_registry
|
||||
|
||||
PROG = "python -m tools.example"
|
||||
|
||||
#: Keys accepted in a JSON request document (``version`` is optional metadata).
|
||||
REQUEST_FIELDS: tuple[str, ...] = (
|
||||
"topic",
|
||||
"width",
|
||||
"height",
|
||||
"format",
|
||||
"source",
|
||||
"count",
|
||||
"download",
|
||||
"output_dir",
|
||||
"resize",
|
||||
"dry_run",
|
||||
)
|
||||
|
||||
OPTIONAL_DOCUMENT_FIELDS: tuple[str, ...] = ("version",)
|
||||
|
||||
DEFAULT_SOURCE = "wikimedia_commons"
|
||||
MIN_COUNT = 1
|
||||
MAX_COUNT = 50
|
||||
|
||||
#: How many candidates are fetched per wanted image. Nothing a source reports
|
||||
#: about a hit is trusted, so a candidate is only known to be usable once its
|
||||
#: bytes have been downloaded and parsed; the source is therefore asked for a
|
||||
#: window of ``count * SAMPLE_FACTOR`` hits and the window is cut short as soon
|
||||
#: as ``count`` usable images have been found.
|
||||
SAMPLE_FACTOR = 4
|
||||
#: Hard upper bound on the number of candidates inspected in one invocation.
|
||||
MAX_CANDIDATE_CAP = MAX_COUNT
|
||||
|
||||
|
||||
def candidate_cap(count: int) -> int:
|
||||
"""Return the bounded number of candidates to fetch for ``count`` images."""
|
||||
|
||||
wanted = max(MIN_COUNT, int(count))
|
||||
return min(MAX_CANDIDATE_CAP, wanted * SAMPLE_FACTOR)
|
||||
|
||||
|
||||
class _Parser(argparse.ArgumentParser):
|
||||
"""Argument parser that reports bad flags as request errors."""
|
||||
|
||||
def error(self, message: str) -> Any: # pragma: no cover - argparse hook
|
||||
raise RequestError(message, accepted_fields=REQUEST_FIELDS)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""Return the CLI parser; ``--help`` documents every supported parameter."""
|
||||
|
||||
parser = _Parser(
|
||||
prog=PROG,
|
||||
description=(
|
||||
"Search an image source and report the matching items as one JSON "
|
||||
"document. Either pass a JSON request document or use the flags."
|
||||
),
|
||||
epilog=(
|
||||
"Every run writes exactly one JSON document to stdout with the keys "
|
||||
"status, source, request, items and error."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"request",
|
||||
nargs="?",
|
||||
metavar="REQUEST.json",
|
||||
help=(
|
||||
"path to a JSON request document holding the same fields as the "
|
||||
"flags; it cannot be combined with any of the flags"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--topic",
|
||||
metavar="TEXT",
|
||||
help="search topic (required unless a request document is given)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--width",
|
||||
type=int,
|
||||
metavar="PX",
|
||||
help="target width in pixels; only images at least this wide are accepted",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--height",
|
||||
type=int,
|
||||
metavar="PX",
|
||||
help="target height in pixels; only images at least this tall are accepted",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--format",
|
||||
metavar="FMT",
|
||||
help="image format: " + "|".join(ACCEPTED_FORMATS),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s",
|
||||
"--source",
|
||||
metavar="ID",
|
||||
help=f"image source id (default: {DEFAULT_SOURCE}); unknown ids are rejected",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
"--count",
|
||||
type=int,
|
||||
metavar="N",
|
||||
help=f"how many candidates to request, {MIN_COUNT}-{MAX_COUNT} (default: {DEFAULT_LIMIT})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--download",
|
||||
action="store_true",
|
||||
help="download every matching image into the output directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
metavar="DIR",
|
||||
help="caller-specified download directory; required with --download",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--resize",
|
||||
action="store_true",
|
||||
help=(
|
||||
"downscale every downloaded image that is larger than the request "
|
||||
"in either dimension to exactly the requested width and height, "
|
||||
"re-encoded in the requested format; images are never scaled up; "
|
||||
"without it the original bytes and dimensions are written"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="validate the request and emit the envelope without any network access",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _load_document(path: str) -> Mapping[str, Any]:
|
||||
"""Read a JSON request document from ``path``."""
|
||||
|
||||
target = Path(path)
|
||||
try:
|
||||
text = target.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise RequestDocumentError(
|
||||
f"cannot read request document {path!r}: {exc.strerror or exc}",
|
||||
details={"path": str(path)},
|
||||
) from exc
|
||||
try:
|
||||
payload = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RequestDocumentError(
|
||||
f"request document {path!r} is not valid JSON: {exc.msg}",
|
||||
details={"path": str(path), "line": exc.lineno, "column": exc.colno},
|
||||
) from exc
|
||||
if not isinstance(payload, Mapping):
|
||||
raise RequestDocumentError(
|
||||
f"request document {path!r} must contain a JSON object",
|
||||
details={"path": str(path), "type": type(payload).__name__},
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _document_to_mapping(payload: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Reject unknown keys and return the accepted request fields."""
|
||||
|
||||
accepted = REQUEST_FIELDS + OPTIONAL_DOCUMENT_FIELDS
|
||||
unknown = sorted(str(key) for key in payload if key not in accepted)
|
||||
if unknown:
|
||||
raise RequestFieldError(
|
||||
"request document contains unsupported fields: " + ", ".join(unknown),
|
||||
field=unknown[0],
|
||||
details={"unknown_fields": unknown, "accepted_fields": list(REQUEST_FIELDS)},
|
||||
)
|
||||
return {key: payload[key] for key in REQUEST_FIELDS if key in payload}
|
||||
|
||||
|
||||
def _flags_to_mapping(namespace: argparse.Namespace) -> dict[str, Any]:
|
||||
"""Collect the explicitly supplied flags into a request mapping."""
|
||||
|
||||
mapping: dict[str, Any] = {}
|
||||
for field in ("topic", "width", "height", "format", "source", "count", "output_dir"):
|
||||
value = getattr(namespace, field, None)
|
||||
if value is not None:
|
||||
mapping[field] = value
|
||||
for field in ("download", "resize", "dry_run"):
|
||||
if getattr(namespace, field, False):
|
||||
mapping[field] = True
|
||||
return mapping
|
||||
|
||||
|
||||
def _as_text(value: Any, field: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise RequestFieldError(
|
||||
f"{field} must be a string, got {type(value).__name__}", field=field
|
||||
)
|
||||
text = value.strip()
|
||||
if not text:
|
||||
raise RequestFieldError(f"{field} must not be empty", field=field)
|
||||
return text
|
||||
|
||||
|
||||
def _as_int(value: Any, field: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise RequestFieldError(
|
||||
f"{field} must be an integer, got {type(value).__name__}", field=field
|
||||
)
|
||||
return int(value)
|
||||
|
||||
|
||||
def _as_bool(value: Any, field: str) -> bool:
|
||||
if not isinstance(value, bool):
|
||||
raise RequestFieldError(
|
||||
f"{field} must be a boolean, got {type(value).__name__}", field=field
|
||||
)
|
||||
return bool(value)
|
||||
|
||||
|
||||
def normalize_request(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Validate ``raw`` and return the complete, typed request mapping."""
|
||||
|
||||
if "topic" not in raw:
|
||||
raise RequestFieldError(
|
||||
"topic is required",
|
||||
field="topic",
|
||||
details={"required_fields": ["topic"], "accepted_fields": list(REQUEST_FIELDS)},
|
||||
)
|
||||
topic = _as_text(raw["topic"], "topic")
|
||||
|
||||
size: dict[str, int | None] = {"width": None, "height": None}
|
||||
for field in ("width", "height"):
|
||||
if raw.get(field) is None:
|
||||
continue
|
||||
number = _as_int(raw[field], field)
|
||||
if number <= 0:
|
||||
raise RequestFieldError(f"{field} must be positive", field=field)
|
||||
size[field] = number
|
||||
|
||||
if "format" not in raw or raw["format"] is None:
|
||||
raise RequestFieldError(
|
||||
"format is required",
|
||||
field="format",
|
||||
details={"accepted_formats": list(ACCEPTED_FORMATS)},
|
||||
)
|
||||
image_format = _as_text(raw["format"], "format").lower()
|
||||
if image_format not in ACCEPTED_FORMATS:
|
||||
raise UnsupportedFormatError(image_format)
|
||||
|
||||
source = DEFAULT_SOURCE
|
||||
if raw.get("source") is not None:
|
||||
source = _as_text(raw["source"], "source")
|
||||
|
||||
count = DEFAULT_LIMIT
|
||||
if raw.get("count") is not None:
|
||||
count = _as_int(raw["count"], "count")
|
||||
if not MIN_COUNT <= count <= MAX_COUNT:
|
||||
raise RequestFieldError(
|
||||
f"count must be between {MIN_COUNT} and {MAX_COUNT}",
|
||||
field="count",
|
||||
details={"min": MIN_COUNT, "max": MAX_COUNT},
|
||||
)
|
||||
|
||||
output_dir: str | None = None
|
||||
if raw.get("output_dir") is not None:
|
||||
output_dir = _as_text(raw["output_dir"], "output_dir")
|
||||
|
||||
download = _as_bool(raw.get("download", False), "download")
|
||||
if download and output_dir is None:
|
||||
# There is no hidden default: a download run without a caller-supplied
|
||||
# directory could only guess at where the files should go.
|
||||
raise RequestFieldError(
|
||||
"output_dir is required when download is enabled; pass --output-dir <dir>",
|
||||
field="output_dir",
|
||||
details={"required_with": "download", "accepted_fields": list(REQUEST_FIELDS)},
|
||||
)
|
||||
|
||||
return {
|
||||
"topic": topic,
|
||||
"width": size["width"],
|
||||
"height": size["height"],
|
||||
"format": image_format,
|
||||
"source": source,
|
||||
"count": count,
|
||||
"download": download,
|
||||
"output_dir": output_dir,
|
||||
"resize": _as_bool(raw.get("resize", False), "resize"),
|
||||
"dry_run": _as_bool(raw.get("dry_run", False), "dry_run"),
|
||||
}
|
||||
|
||||
|
||||
def _meets_minimum_size(info: ImageInfo, request: Mapping[str, Any]) -> bool:
|
||||
"""True when the *real* pixel size of an image is usable for the request.
|
||||
|
||||
The requested width and height are a minimum, not an exact match: any image
|
||||
that is at least that large is usable because it can always be downscaled,
|
||||
while a smaller image would have to be scaled up and is therefore rejected.
|
||||
``--resize`` only decides whether the downloaded bytes are downscaled; it
|
||||
never widens or narrows this filter.
|
||||
"""
|
||||
|
||||
wanted_width = request.get("width")
|
||||
wanted_height = request.get("height")
|
||||
if wanted_width is not None and info.width < wanted_width:
|
||||
return False
|
||||
if wanted_height is not None and info.height < wanted_height:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _failure(candidate: SourceCandidate, fragment: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Describe one candidate whose bytes could not be turned into an image."""
|
||||
|
||||
return {
|
||||
"source": str(getattr(candidate, "source_id", "") or ""),
|
||||
"url": str(getattr(candidate, "file_url", "") or ""),
|
||||
"error": dict(fragment),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KeptImage:
|
||||
"""One candidate whose downloaded bytes really match the request."""
|
||||
|
||||
candidate: SourceCandidate
|
||||
info: ImageInfo
|
||||
payload: bytes
|
||||
|
||||
def to_item(self) -> dict[str, Any]:
|
||||
"""Report ``source``, ``url``, ``format`` and the *actual* dimensions."""
|
||||
|
||||
source = str(getattr(self.candidate, "source_id", "") or "")
|
||||
url = str(getattr(self.candidate, "file_url", "") or "")
|
||||
item: dict[str, Any] = {
|
||||
"source": source,
|
||||
"url": url,
|
||||
# Kept as an alias so existing consumers keep working unchanged.
|
||||
"file_url": url,
|
||||
"format": self.info.format,
|
||||
"width": self.info.width,
|
||||
"height": self.info.height,
|
||||
"size_bytes": len(self.payload),
|
||||
}
|
||||
for key in ("title", "page_url", "mime_type"):
|
||||
value = getattr(self.candidate, key, None)
|
||||
if value is not None:
|
||||
item[key] = value
|
||||
return item
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Selection:
|
||||
"""Outcome of inspecting a bounded window of downloaded candidates.
|
||||
|
||||
``format_rejects`` counts candidates whose real payload is *not* the
|
||||
requested format -- including payloads that are not a recognised image at
|
||||
all. ``size_rejects`` counts candidates of the requested format that are
|
||||
too small. ``failures`` only holds candidates whose bytes could not be
|
||||
fetched at all, because that is the one situation worth an access error.
|
||||
"""
|
||||
|
||||
kept: tuple[KeptImage, ...] = ()
|
||||
format_rejects: int = 0
|
||||
size_rejects: int = 0
|
||||
other_formats: Mapping[str, int] = field(default_factory=dict)
|
||||
failures: tuple[dict[str, Any], ...] = ()
|
||||
|
||||
@property
|
||||
def inspected(self) -> int:
|
||||
"""Number of candidates whose bytes were read successfully."""
|
||||
|
||||
return len(self.kept) + self.format_rejects + self.size_rejects
|
||||
|
||||
|
||||
#: Bucket name for a downloaded payload that is not a recognised image at all.
|
||||
UNKNOWN_FORMAT = "unknown"
|
||||
|
||||
|
||||
def _count_other_format(other_formats: dict[str, int], detected: str | None) -> None:
|
||||
"""Record which other format a rejected candidate really was."""
|
||||
|
||||
key = detected or UNKNOWN_FORMAT
|
||||
other_formats[key] = other_formats.get(key, 0) + 1
|
||||
|
||||
|
||||
def _select(candidates: Sequence[SourceCandidate], request: Mapping[str, Any]) -> Selection:
|
||||
"""Fetch and inspect candidates, keeping only what the real bytes justify.
|
||||
|
||||
Every kept item has been downloaded and parsed: its format comes from the
|
||||
container signature and its width/height come from the PNG ``IHDR`` or the
|
||||
ICO directory, never from what the source claimed. A candidate that serves
|
||||
other-format bytes -- a JPEG, an SVG, a TIFF, an HTML error page, anything
|
||||
that is not a real ``png``/``ico`` payload -- counts as a *format rejection*
|
||||
so the caller gets ``no_format`` instead of an access error, and a candidate
|
||||
of the wanted format that is too small counts as a size rejection. Only a
|
||||
candidate whose bytes cannot be fetched at all is recorded as a failure.
|
||||
"""
|
||||
|
||||
wanted_format = str(request["format"])
|
||||
limit = int(request["count"])
|
||||
kept: list[KeptImage] = []
|
||||
format_rejects = 0
|
||||
size_rejects = 0
|
||||
other_formats: dict[str, int] = {}
|
||||
failures: list[dict[str, Any]] = []
|
||||
|
||||
for candidate in candidates:
|
||||
if len(kept) >= limit:
|
||||
break
|
||||
try:
|
||||
payload = candidate.fetch_bytes()
|
||||
except ToolkitError as error:
|
||||
failures.append(_failure(candidate, error.to_dict()))
|
||||
continue
|
||||
except OSError as error:
|
||||
failures.append(
|
||||
_failure(candidate, error_fragment("fetch_failed", str(error)))
|
||||
)
|
||||
continue
|
||||
detected = images.detect_format(payload)
|
||||
if detected != wanted_format:
|
||||
# Whatever these bytes are (another image format, or no recognisable
|
||||
# image at all), they are not the requested format: that is a format
|
||||
# verdict about the topic, never an access problem.
|
||||
format_rejects += 1
|
||||
_count_other_format(other_formats, detected)
|
||||
continue
|
||||
try:
|
||||
info = images.inspect_image(payload)
|
||||
except ImageFormatError as error:
|
||||
# The signature claims the requested format but the header is
|
||||
# corrupt, so the payload itself is unusable rather than merely the
|
||||
# wrong format.
|
||||
failures.append(_failure(candidate, error.to_dict()))
|
||||
continue
|
||||
if not _meets_minimum_size(info, request):
|
||||
size_rejects += 1
|
||||
continue
|
||||
kept.append(KeptImage(candidate=candidate, info=info, payload=payload))
|
||||
|
||||
return Selection(
|
||||
kept=tuple(kept),
|
||||
format_rejects=format_rejects,
|
||||
size_rejects=size_rejects,
|
||||
other_formats=dict(other_formats),
|
||||
failures=tuple(failures),
|
||||
)
|
||||
|
||||
|
||||
def _slug(candidate: SourceCandidate) -> str:
|
||||
raw = str(getattr(candidate, "title", None) or getattr(candidate, "source_id", "") or "image").lower()
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")
|
||||
return (slug or "image")[:60]
|
||||
|
||||
|
||||
def _target_size(width: int, height: int, request: Mapping[str, Any]) -> tuple[int, int]:
|
||||
"""Return the exact size to store: the requested size, never enlarged.
|
||||
|
||||
``--resize`` means "give me the requested size" rather than "give me
|
||||
something that fits in it": a requested width **and** height are hit
|
||||
exactly, and a dimension the caller left open keeps the source value. A
|
||||
source dimension that is already at or below the requested one is never
|
||||
scaled up, so ``--resize`` can only ever shrink an image.
|
||||
"""
|
||||
|
||||
wanted_width = request.get("width")
|
||||
wanted_height = request.get("height")
|
||||
target_width = width if wanted_width is None else min(width, int(wanted_width))
|
||||
target_height = height if wanted_height is None else min(height, int(wanted_height))
|
||||
return max(1, target_width), max(1, target_height)
|
||||
|
||||
|
||||
def resize_image(payload: bytes, request: Mapping[str, Any]) -> tuple[bytes, dict[str, Any]]:
|
||||
"""Downscale ``payload`` to the requested size and re-encode it.
|
||||
|
||||
The stored image is re-encoded in the requested container format (``png``
|
||||
or ``ico``) so the saved bytes really open as that format. An image that
|
||||
is not larger than the request in either dimension is returned untouched:
|
||||
``--resize`` never enlarges and never rewrites pixels it does not have to.
|
||||
"""
|
||||
|
||||
decoded = images.decode_image(payload)
|
||||
wanted_format = str(request.get("format") or decoded.format)
|
||||
if decoded.format != wanted_format:
|
||||
raise ImageResizeError(
|
||||
f"cannot re-encode a {decoded.format} image as {wanted_format}",
|
||||
details={"from_format": decoded.format, "to_format": wanted_format},
|
||||
)
|
||||
target_width, target_height = _target_size(decoded.width, decoded.height, request)
|
||||
fragment: dict[str, Any] = {
|
||||
"requested": {"width": request.get("width"), "height": request.get("height")},
|
||||
"from": {"width": decoded.width, "height": decoded.height},
|
||||
"format": wanted_format,
|
||||
}
|
||||
if (target_width, target_height) == (decoded.width, decoded.height):
|
||||
fragment.update(
|
||||
{
|
||||
"applied": False,
|
||||
"reason": "already_fits",
|
||||
"to": {"width": decoded.width, "height": decoded.height},
|
||||
}
|
||||
)
|
||||
return payload, fragment
|
||||
rows = images.resample_rows(
|
||||
decoded.rows, decoded.width, decoded.height, target_width, target_height
|
||||
)
|
||||
output = images.encode_image(wanted_format, target_width, target_height, rows)
|
||||
fragment.update({"applied": True, "to": {"width": target_width, "height": target_height}})
|
||||
return output, fragment
|
||||
|
||||
|
||||
def _discard(path: Path) -> None:
|
||||
"""Remove a file whose stored bytes failed verification."""
|
||||
|
||||
try:
|
||||
path.unlink()
|
||||
except OSError:
|
||||
# Never chase a path that is not a regular file we just wrote: the run
|
||||
# must not delete anything else in the caller's directory.
|
||||
pass
|
||||
|
||||
|
||||
def _verify_saved(
|
||||
path: Path, request: Mapping[str, Any], expected: tuple[int, int]
|
||||
) -> ImageInfo:
|
||||
"""Read a stored file back and prove it opens as the requested format.
|
||||
|
||||
A reported path is only trustworthy when the file really exists and really
|
||||
holds bytes of the requested format at the expected size, so the stored
|
||||
payload is decoded again instead of holding the writer to its word.
|
||||
"""
|
||||
|
||||
wanted_format = str(request["format"])
|
||||
decoded = images.decode_image(path.read_bytes())
|
||||
if decoded.format != wanted_format:
|
||||
raise ImageFormatError(
|
||||
"stored file does not open as the requested format",
|
||||
details={
|
||||
"path": str(path),
|
||||
"expected_format": wanted_format,
|
||||
"actual_format": decoded.format,
|
||||
},
|
||||
)
|
||||
if (decoded.width, decoded.height) != expected:
|
||||
raise ImageFormatError(
|
||||
"stored file does not have the expected pixel size",
|
||||
details={
|
||||
"path": str(path),
|
||||
"expected": {"width": expected[0], "height": expected[1]},
|
||||
"actual": {"width": decoded.width, "height": decoded.height},
|
||||
},
|
||||
)
|
||||
return ImageInfo(format=decoded.format, width=decoded.width, height=decoded.height)
|
||||
|
||||
|
||||
def _download(
|
||||
selected: Sequence[KeptImage], request: Mapping[str, Any]
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Write the already-fetched payloads; return ``(items, failure_count)``.
|
||||
|
||||
Every selected image is stored inside the caller-specified directory. The
|
||||
bytes verified by :func:`_select` are written as they are, or downscaled
|
||||
first when ``resize`` is enabled, so an item can never be downloaded twice
|
||||
or stored in a format that was not verified. Each stored file is read back
|
||||
and decoded before its path is reported, and a file that does not open as
|
||||
the requested format at the expected size is deleted and reported as a
|
||||
per-item save failure instead of being left behind as a broken path.
|
||||
"""
|
||||
|
||||
directory = Path(request["output_dir"]).expanduser()
|
||||
try:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as exc:
|
||||
raise SourceAccessError(
|
||||
f"cannot create output directory {str(directory)!r}: {exc.strerror or exc}",
|
||||
details={"output_dir": str(directory)},
|
||||
) from exc
|
||||
|
||||
suffix = "." + str(request["format"])
|
||||
items: list[dict[str, Any]] = []
|
||||
failures = 0
|
||||
for index, entry in enumerate(selected, start=1):
|
||||
record = entry.to_item()
|
||||
destination = directory / f"{index:03d}-{_slug(entry.candidate)}{suffix}"
|
||||
payload = entry.payload
|
||||
resize_fragment: dict[str, Any] | None = None
|
||||
expected = (entry.info.width, entry.info.height)
|
||||
try:
|
||||
if request["resize"]:
|
||||
payload, resize_fragment = resize_image(payload, request)
|
||||
expected = (
|
||||
int(resize_fragment["to"]["width"]),
|
||||
int(resize_fragment["to"]["height"]),
|
||||
)
|
||||
destination.write_bytes(payload)
|
||||
stored = _verify_saved(destination, request, expected)
|
||||
except (ToolkitError, OSError) as exc:
|
||||
_discard(destination)
|
||||
failures += 1
|
||||
error = error_fragment(getattr(exc, "code", "download_failed"), str(exc))
|
||||
if isinstance(exc, ToolkitError) and exc.details:
|
||||
error["details"] = dict(exc.details)
|
||||
# A failed save reports no path: only files that really exist and
|
||||
# really open are ever named as saved.
|
||||
record["download"] = {"ok": False, "target": str(destination), "error": error}
|
||||
else:
|
||||
download: dict[str, Any] = {
|
||||
"ok": True,
|
||||
"path": str(destination),
|
||||
"bytes": len(payload),
|
||||
"format": stored.format,
|
||||
"width": stored.width,
|
||||
"height": stored.height,
|
||||
}
|
||||
if resize_fragment is not None:
|
||||
download["resize"] = resize_fragment
|
||||
record["download"] = download
|
||||
items.append(record)
|
||||
return items, failures
|
||||
|
||||
|
||||
def _guess_source(argv: Sequence[str]) -> str | None:
|
||||
"""Best-effort source id extraction for error envelopes."""
|
||||
|
||||
for index, token in enumerate(argv):
|
||||
if token == "--source" or token == "-s":
|
||||
if index + 1 < len(argv):
|
||||
return argv[index + 1]
|
||||
elif token.startswith("--source="):
|
||||
return token.split("=", 1)[1]
|
||||
return None
|
||||
|
||||
|
||||
def _request_from(argv: Sequence[str]) -> tuple[dict[str, Any], str, Any]:
|
||||
"""Turn CLI arguments into ``(normalized_request, source_id, registry)``."""
|
||||
|
||||
parser = build_parser()
|
||||
namespace = parser.parse_args(list(argv))
|
||||
flags = _flags_to_mapping(namespace)
|
||||
document = namespace.request
|
||||
|
||||
if document is not None:
|
||||
if flags:
|
||||
raise RequestError(
|
||||
"a request document cannot be combined with flags: "
|
||||
+ ", ".join(sorted(flags)),
|
||||
details={"conflicting_flags": sorted(flags)},
|
||||
accepted_fields=REQUEST_FIELDS,
|
||||
)
|
||||
raw = _document_to_mapping(_load_document(document))
|
||||
elif flags:
|
||||
raw = flags
|
||||
else:
|
||||
raise RequestError(
|
||||
"no request given; pass a JSON request document or at least one flag",
|
||||
accepted_fields=REQUEST_FIELDS,
|
||||
)
|
||||
|
||||
request = normalize_request(raw)
|
||||
registry = default_registry()
|
||||
available = list(registry.available_sources())
|
||||
if request["source"] not in available:
|
||||
raise UnknownSourceError(request["source"], available)
|
||||
return request, request["source"], registry
|
||||
|
||||
|
||||
def _execute(argv: Sequence[str]) -> Response:
|
||||
"""Run one request and return the response that must be printed.
|
||||
|
||||
The search path downloads a bounded window of candidates and filters that
|
||||
window on the parsed bytes, so the answer distinguishes three different
|
||||
situations through both the status and the exit code:
|
||||
|
||||
* ``ok`` -- at least one image really is the requested format and at least
|
||||
the requested width and height;
|
||||
* ``no_format`` -- the source had hits, but none of them really are the
|
||||
requested format;
|
||||
* ``no_size`` -- images of the requested format existed, but all of them
|
||||
are smaller than requested;
|
||||
* ``access_error`` -- the source could not be searched, or not a single
|
||||
candidate could be fetched, which is never reported as a "no match".
|
||||
|
||||
A candidate whose downloaded bytes are a different format (a JPEG, an SVG,
|
||||
an HTML error page, ...) is a format mismatch about the topic, not an access
|
||||
problem: it feeds ``no_format`` instead of turning a working search into an
|
||||
error, and a stray unfetchable candidate never demotes a usable answer to
|
||||
``partial_download``.
|
||||
"""
|
||||
|
||||
request, source, registry = _request_from(argv)
|
||||
builder = ResponseBuilder(source=source, request=request)
|
||||
|
||||
if request["dry_run"]:
|
||||
return builder.dry_run()
|
||||
|
||||
cap = candidate_cap(request["count"])
|
||||
candidates = registry.search(source, request["topic"], cap)
|
||||
selection = _select(candidates, request)
|
||||
unreadable = list(selection.failures)
|
||||
|
||||
if not selection.kept:
|
||||
details = {
|
||||
"requested_format": request["format"],
|
||||
"requested_size": {"width": request["width"], "height": request["height"]},
|
||||
"minimum_size": {"width": request["width"], "height": request["height"]},
|
||||
"candidate_count": len(candidates),
|
||||
"candidates_inspected": selection.inspected,
|
||||
"format_matches": selection.size_rejects,
|
||||
"other_format_candidates": selection.format_rejects,
|
||||
"observed_formats": dict(selection.other_formats),
|
||||
"size_matches": 0,
|
||||
"unreadable_candidates": len(unreadable),
|
||||
}
|
||||
if selection.size_rejects:
|
||||
return builder.build(
|
||||
STATUS_NO_SIZE,
|
||||
error=error_fragment(
|
||||
"no_size_match",
|
||||
"the source returned no images at least the requested width and height",
|
||||
details=details,
|
||||
),
|
||||
)
|
||||
if selection.format_rejects or not unreadable:
|
||||
# Either the source really had no images in the requested format, or
|
||||
# it had no hits at all; both answer the topic, not an outage.
|
||||
return builder.build(
|
||||
STATUS_NO_FORMAT,
|
||||
error=error_fragment(
|
||||
"no_format_match",
|
||||
f"the source returned no {request['format']} images for {request['topic']!r}",
|
||||
details=details,
|
||||
),
|
||||
)
|
||||
return builder.access_error(
|
||||
error=error_fragment(
|
||||
"candidate_unreadable",
|
||||
"none of the candidates could be downloaded",
|
||||
details={**details, "failures": unreadable},
|
||||
)
|
||||
)
|
||||
|
||||
if not request["download"]:
|
||||
# Every reported item was fetched and verified against its real bytes, so
|
||||
# an unfetchable candidate is simply skipped: it must not demote an
|
||||
# otherwise usable answer to a partial-download exit code.
|
||||
return builder.build(STATUS_OK, items=[entry.to_item() for entry in selection.kept])
|
||||
|
||||
items, save_failures = _download(selection.kept, request)
|
||||
saved_paths = [
|
||||
item["download"]["path"] for item in items if item["download"].get("ok") is True
|
||||
]
|
||||
failed = save_failures + len(unreadable)
|
||||
if failed:
|
||||
# The run still reports every saved path, but a per-item save (or fetch)
|
||||
# failure is never hidden behind ``ok``: it has its own status and exit
|
||||
# code so the caller can tell a complete run from an incomplete one.
|
||||
return builder.build(
|
||||
STATUS_PARTIAL_DOWNLOAD,
|
||||
items=items,
|
||||
error=error_fragment(
|
||||
"download_incomplete",
|
||||
f"{failed} of {len(items) + len(unreadable)} images could not be saved",
|
||||
details={
|
||||
"failed": failed,
|
||||
"saved": len(saved_paths),
|
||||
"total": len(items) + len(unreadable),
|
||||
"saved_paths": saved_paths,
|
||||
"failures": unreadable,
|
||||
},
|
||||
),
|
||||
)
|
||||
return builder.build(STATUS_OK, items=items)
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
"""Entry point: emit exactly one JSON document and return the exit code."""
|
||||
|
||||
arguments = list(sys.argv[1:] if argv is None else argv)
|
||||
try:
|
||||
response = _execute(arguments)
|
||||
except ToolkitError as error:
|
||||
builder = ResponseBuilder(source=_guess_source(arguments))
|
||||
response = builder.from_error(error)
|
||||
except OSError as error:
|
||||
builder = ResponseBuilder(source=_guess_source(arguments))
|
||||
response = builder.build(
|
||||
STATUS_ACCESS_ERROR,
|
||||
error=error_fragment("access_error", str(error)),
|
||||
)
|
||||
response.emit()
|
||||
return response.exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - module entry point
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user