"""Plain native-process CLI for the image toolkit. Usage:: python -m tools.example examples/input.json python -m tools.example --topic "red fox" --width 640 --height 480 --format png Exactly one machine-readable JSON document is written to stdout. The process exit code is derived from the fixed status vocabulary in :mod:`tools.envelope`. """ from __future__ import annotations import argparse import json import re import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any, Mapping, Sequence from . import images from .envelope import ( STATUS_ACCESS_ERROR, STATUS_NO_FORMAT, STATUS_NO_SIZE, STATUS_OK, STATUS_PARTIAL_DOWNLOAD, Response, ResponseBuilder, error_fragment, ) from .errors import ( ACCEPTED_FORMATS, ImageResizeError, RequestDocumentError, RequestError, RequestFieldError, SourceAccessError, ToolkitError, UnknownSourceError, UnsupportedFormatError, ) from .images import ImageFormatError, ImageInfo from .sources import DEFAULT_LIMIT, SourceCandidate, default_registry PROG = "python -m tools.example" #: Keys accepted in a JSON request document (``version`` is optional metadata). REQUEST_FIELDS: tuple[str, ...] = ( "topic", "width", "height", "format", "source", "count", "download", "output_dir", "resize", "dry_run", ) OPTIONAL_DOCUMENT_FIELDS: tuple[str, ...] = ("version",) DEFAULT_SOURCE = "wikimedia_commons" MIN_COUNT = 1 MAX_COUNT = 50 #: How many candidates are fetched per wanted image. Nothing a source reports #: about a hit is trusted, so a candidate is only known to be usable once its #: bytes have been downloaded and parsed; the source is therefore asked for a #: window of ``count * SAMPLE_FACTOR`` hits and the window is cut short as soon #: as ``count`` usable images have been found. SAMPLE_FACTOR = 4 #: Hard upper bound on the number of candidates inspected in one invocation. MAX_CANDIDATE_CAP = MAX_COUNT def candidate_cap(count: int) -> int: """Return the bounded number of candidates to fetch for ``count`` images.""" wanted = max(MIN_COUNT, int(count)) return min(MAX_CANDIDATE_CAP, wanted * SAMPLE_FACTOR) class _Parser(argparse.ArgumentParser): """Argument parser that reports bad flags as request errors.""" def error(self, message: str) -> Any: # pragma: no cover - argparse hook raise RequestError(message, accepted_fields=REQUEST_FIELDS) def build_parser() -> argparse.ArgumentParser: """Return the CLI parser; ``--help`` documents every supported parameter.""" parser = _Parser( prog=PROG, description=( "Search an image source and report the matching items as one JSON " "document. Either pass a JSON request document or use the flags." ), epilog=( "Every run writes exactly one JSON document to stdout with the keys " "status, source, request, items and error." ), ) parser.add_argument( "request", nargs="?", metavar="REQUEST.json", help=( "path to a JSON request document holding the same fields as the " "flags; it cannot be combined with any of the flags" ), ) parser.add_argument( "-t", "--topic", metavar="TEXT", help="search topic (required unless a request document is given)", ) parser.add_argument( "--width", type=int, metavar="PX", help="target width in pixels; only images at least this wide are accepted", ) parser.add_argument( "--height", type=int, metavar="PX", help="target height in pixels; only images at least this tall are accepted", ) parser.add_argument( "-f", "--format", metavar="FMT", help="image format: " + "|".join(ACCEPTED_FORMATS), ) parser.add_argument( "-s", "--source", metavar="ID", help=f"image source id (default: {DEFAULT_SOURCE}); unknown ids are rejected", ) parser.add_argument( "-c", "--count", type=int, metavar="N", help=f"how many candidates to request, {MIN_COUNT}-{MAX_COUNT} (default: {DEFAULT_LIMIT})", ) parser.add_argument( "--download", action="store_true", help="download every matching image into the output directory", ) parser.add_argument( "-o", "--output-dir", metavar="DIR", help="caller-specified download directory; required with --download", ) parser.add_argument( "--resize", action="store_true", help=( "downscale every downloaded image that is larger than the request " "in either dimension to exactly the requested width and height, " "re-encoded in the requested format; images are never scaled up; " "without it the original bytes and dimensions are written" ), ) parser.add_argument( "--dry-run", action="store_true", help="validate the request and emit the envelope without any network access", ) return parser def _load_document(path: str) -> Mapping[str, Any]: """Read a JSON request document from ``path``.""" target = Path(path) try: text = target.read_text(encoding="utf-8") except OSError as exc: raise RequestDocumentError( f"cannot read request document {path!r}: {exc.strerror or exc}", details={"path": str(path)}, ) from exc try: payload = json.loads(text) except json.JSONDecodeError as exc: raise RequestDocumentError( f"request document {path!r} is not valid JSON: {exc.msg}", details={"path": str(path), "line": exc.lineno, "column": exc.colno}, ) from exc if not isinstance(payload, Mapping): raise RequestDocumentError( f"request document {path!r} must contain a JSON object", details={"path": str(path), "type": type(payload).__name__}, ) return payload def _document_to_mapping(payload: Mapping[str, Any]) -> dict[str, Any]: """Reject unknown keys and return the accepted request fields.""" accepted = REQUEST_FIELDS + OPTIONAL_DOCUMENT_FIELDS unknown = sorted(str(key) for key in payload if key not in accepted) if unknown: raise RequestFieldError( "request document contains unsupported fields: " + ", ".join(unknown), field=unknown[0], details={"unknown_fields": unknown, "accepted_fields": list(REQUEST_FIELDS)}, ) return {key: payload[key] for key in REQUEST_FIELDS if key in payload} def _flags_to_mapping(namespace: argparse.Namespace) -> dict[str, Any]: """Collect the explicitly supplied flags into a request mapping.""" mapping: dict[str, Any] = {} for field in ("topic", "width", "height", "format", "source", "count", "output_dir"): value = getattr(namespace, field, None) if value is not None: mapping[field] = value for field in ("download", "resize", "dry_run"): if getattr(namespace, field, False): mapping[field] = True return mapping def _as_text(value: Any, field: str) -> str: if not isinstance(value, str): raise RequestFieldError( f"{field} must be a string, got {type(value).__name__}", field=field ) text = value.strip() if not text: raise RequestFieldError(f"{field} must not be empty", field=field) return text def _as_int(value: Any, field: str) -> int: if isinstance(value, bool) or not isinstance(value, int): raise RequestFieldError( f"{field} must be an integer, got {type(value).__name__}", field=field ) return int(value) def _as_bool(value: Any, field: str) -> bool: if not isinstance(value, bool): raise RequestFieldError( f"{field} must be a boolean, got {type(value).__name__}", field=field ) return bool(value) def normalize_request(raw: Mapping[str, Any]) -> dict[str, Any]: """Validate ``raw`` and return the complete, typed request mapping.""" if "topic" not in raw: raise RequestFieldError( "topic is required", field="topic", details={"required_fields": ["topic"], "accepted_fields": list(REQUEST_FIELDS)}, ) topic = _as_text(raw["topic"], "topic") size: dict[str, int | None] = {"width": None, "height": None} for field in ("width", "height"): if raw.get(field) is None: continue number = _as_int(raw[field], field) if number <= 0: raise RequestFieldError(f"{field} must be positive", field=field) size[field] = number if "format" not in raw or raw["format"] is None: raise RequestFieldError( "format is required", field="format", details={"accepted_formats": list(ACCEPTED_FORMATS)}, ) image_format = _as_text(raw["format"], "format").lower() if image_format not in ACCEPTED_FORMATS: raise UnsupportedFormatError(image_format) source = DEFAULT_SOURCE if raw.get("source") is not None: source = _as_text(raw["source"], "source") count = DEFAULT_LIMIT if raw.get("count") is not None: count = _as_int(raw["count"], "count") if not MIN_COUNT <= count <= MAX_COUNT: raise RequestFieldError( f"count must be between {MIN_COUNT} and {MAX_COUNT}", field="count", details={"min": MIN_COUNT, "max": MAX_COUNT}, ) output_dir: str | None = None if raw.get("output_dir") is not None: output_dir = _as_text(raw["output_dir"], "output_dir") download = _as_bool(raw.get("download", False), "download") if download and output_dir is None: # There is no hidden default: a download run without a caller-supplied # directory could only guess at where the files should go. raise RequestFieldError( "output_dir is required when download is enabled; pass --output-dir ", field="output_dir", details={"required_with": "download", "accepted_fields": list(REQUEST_FIELDS)}, ) return { "topic": topic, "width": size["width"], "height": size["height"], "format": image_format, "source": source, "count": count, "download": download, "output_dir": output_dir, "resize": _as_bool(raw.get("resize", False), "resize"), "dry_run": _as_bool(raw.get("dry_run", False), "dry_run"), } def _meets_minimum_size(info: ImageInfo, request: Mapping[str, Any]) -> bool: """True when the *real* pixel size of an image is usable for the request. The requested width and height are a minimum, not an exact match: any image that is at least that large is usable because it can always be downscaled, while a smaller image would have to be scaled up and is therefore rejected. ``--resize`` only decides whether the downloaded bytes are downscaled; it never widens or narrows this filter. """ wanted_width = request.get("width") wanted_height = request.get("height") if wanted_width is not None and info.width < wanted_width: return False if wanted_height is not None and info.height < wanted_height: return False return True def _failure(candidate: SourceCandidate, fragment: Mapping[str, Any]) -> dict[str, Any]: """Describe one candidate whose bytes could not be turned into an image.""" return { "source": str(getattr(candidate, "source_id", "") or ""), "url": str(getattr(candidate, "file_url", "") or ""), "error": dict(fragment), } @dataclass(frozen=True) class KeptImage: """One candidate whose downloaded bytes really match the request.""" candidate: SourceCandidate info: ImageInfo payload: bytes def to_item(self) -> dict[str, Any]: """Report ``source``, ``url``, ``format`` and the *actual* dimensions.""" source = str(getattr(self.candidate, "source_id", "") or "") url = str(getattr(self.candidate, "file_url", "") or "") item: dict[str, Any] = { "source": source, "url": url, # Kept as an alias so existing consumers keep working unchanged. "file_url": url, "format": self.info.format, "width": self.info.width, "height": self.info.height, "size_bytes": len(self.payload), } for key in ("title", "page_url", "mime_type"): value = getattr(self.candidate, key, None) if value is not None: item[key] = value return item @dataclass(frozen=True) class Selection: """Outcome of inspecting a bounded window of downloaded candidates. ``format_rejects`` counts candidates whose real payload is *not* the requested format -- including payloads that are not a recognised image at all. ``size_rejects`` counts candidates of the requested format that are too small. ``failures`` only holds candidates whose bytes could not be fetched at all, because that is the one situation worth an access error. """ kept: tuple[KeptImage, ...] = () format_rejects: int = 0 size_rejects: int = 0 other_formats: Mapping[str, int] = field(default_factory=dict) failures: tuple[dict[str, Any], ...] = () @property def inspected(self) -> int: """Number of candidates whose bytes were read successfully.""" return len(self.kept) + self.format_rejects + self.size_rejects #: Bucket name for a downloaded payload that is not a recognised image at all. UNKNOWN_FORMAT = "unknown" def _count_other_format(other_formats: dict[str, int], detected: str | None) -> None: """Record which other format a rejected candidate really was.""" key = detected or UNKNOWN_FORMAT other_formats[key] = other_formats.get(key, 0) + 1 def _select(candidates: Sequence[SourceCandidate], request: Mapping[str, Any]) -> Selection: """Fetch and inspect candidates, keeping only what the real bytes justify. Every kept item has been downloaded and parsed: its format comes from the container signature and its width/height come from the PNG ``IHDR`` or the ICO directory, never from what the source claimed. A candidate that serves other-format bytes -- a JPEG, an SVG, a TIFF, an HTML error page, anything that is not a real ``png``/``ico`` payload -- counts as a *format rejection* so the caller gets ``no_format`` instead of an access error, and a candidate of the wanted format that is too small counts as a size rejection. Only a candidate whose bytes cannot be fetched at all is recorded as a failure. """ wanted_format = str(request["format"]) limit = int(request["count"]) kept: list[KeptImage] = [] format_rejects = 0 size_rejects = 0 other_formats: dict[str, int] = {} failures: list[dict[str, Any]] = [] for candidate in candidates: if len(kept) >= limit: break try: payload = candidate.fetch_bytes() except ToolkitError as error: failures.append(_failure(candidate, error.to_dict())) continue except OSError as error: failures.append( _failure(candidate, error_fragment("fetch_failed", str(error))) ) continue detected = images.detect_format(payload) if detected != wanted_format: # Whatever these bytes are (another image format, or no recognisable # image at all), they are not the requested format: that is a format # verdict about the topic, never an access problem. format_rejects += 1 _count_other_format(other_formats, detected) continue try: info = images.inspect_image(payload) except ImageFormatError as error: # The signature claims the requested format but the header is # corrupt, so the payload itself is unusable rather than merely the # wrong format. failures.append(_failure(candidate, error.to_dict())) continue if not _meets_minimum_size(info, request): size_rejects += 1 continue kept.append(KeptImage(candidate=candidate, info=info, payload=payload)) return Selection( kept=tuple(kept), format_rejects=format_rejects, size_rejects=size_rejects, other_formats=dict(other_formats), failures=tuple(failures), ) def _slug(candidate: SourceCandidate) -> str: raw = str(getattr(candidate, "title", None) or getattr(candidate, "source_id", "") or "image").lower() slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-") return (slug or "image")[:60] def _target_size(width: int, height: int, request: Mapping[str, Any]) -> tuple[int, int]: """Return the exact size to store: the requested size, never enlarged. ``--resize`` means "give me the requested size" rather than "give me something that fits in it": a requested width **and** height are hit exactly, and a dimension the caller left open keeps the source value. A source dimension that is already at or below the requested one is never scaled up, so ``--resize`` can only ever shrink an image. """ wanted_width = request.get("width") wanted_height = request.get("height") target_width = width if wanted_width is None else min(width, int(wanted_width)) target_height = height if wanted_height is None else min(height, int(wanted_height)) return max(1, target_width), max(1, target_height) def resize_image(payload: bytes, request: Mapping[str, Any]) -> tuple[bytes, dict[str, Any]]: """Downscale ``payload`` to the requested size and re-encode it. The stored image is re-encoded in the requested container format (``png`` or ``ico``) so the saved bytes really open as that format. An image that is not larger than the request in either dimension is returned untouched: ``--resize`` never enlarges and never rewrites pixels it does not have to. """ decoded = images.decode_image(payload) wanted_format = str(request.get("format") or decoded.format) if decoded.format != wanted_format: raise ImageResizeError( f"cannot re-encode a {decoded.format} image as {wanted_format}", details={"from_format": decoded.format, "to_format": wanted_format}, ) target_width, target_height = _target_size(decoded.width, decoded.height, request) fragment: dict[str, Any] = { "requested": {"width": request.get("width"), "height": request.get("height")}, "from": {"width": decoded.width, "height": decoded.height}, "format": wanted_format, } if (target_width, target_height) == (decoded.width, decoded.height): fragment.update( { "applied": False, "reason": "already_fits", "to": {"width": decoded.width, "height": decoded.height}, } ) return payload, fragment rows = images.resample_rows( decoded.rows, decoded.width, decoded.height, target_width, target_height ) output = images.encode_image(wanted_format, target_width, target_height, rows) fragment.update({"applied": True, "to": {"width": target_width, "height": target_height}}) return output, fragment def _discard(path: Path) -> None: """Remove a file whose stored bytes failed verification.""" try: path.unlink() except OSError: # Never chase a path that is not a regular file we just wrote: the run # must not delete anything else in the caller's directory. pass def _verify_saved( path: Path, request: Mapping[str, Any], expected: tuple[int, int] ) -> ImageInfo: """Read a stored file back and prove it opens as the requested format. A reported path is only trustworthy when the file really exists and really holds bytes of the requested format at the expected size, so the stored payload is decoded again instead of holding the writer to its word. """ wanted_format = str(request["format"]) decoded = images.decode_image(path.read_bytes()) if decoded.format != wanted_format: raise ImageFormatError( "stored file does not open as the requested format", details={ "path": str(path), "expected_format": wanted_format, "actual_format": decoded.format, }, ) if (decoded.width, decoded.height) != expected: raise ImageFormatError( "stored file does not have the expected pixel size", details={ "path": str(path), "expected": {"width": expected[0], "height": expected[1]}, "actual": {"width": decoded.width, "height": decoded.height}, }, ) return ImageInfo(format=decoded.format, width=decoded.width, height=decoded.height) def _download( selected: Sequence[KeptImage], request: Mapping[str, Any] ) -> tuple[list[dict[str, Any]], int]: """Write the already-fetched payloads; return ``(items, failure_count)``. Every selected image is stored inside the caller-specified directory. The bytes verified by :func:`_select` are written as they are, or downscaled first when ``resize`` is enabled, so an item can never be downloaded twice or stored in a format that was not verified. Each stored file is read back and decoded before its path is reported, and a file that does not open as the requested format at the expected size is deleted and reported as a per-item save failure instead of being left behind as a broken path. """ directory = Path(request["output_dir"]).expanduser() try: directory.mkdir(parents=True, exist_ok=True) except OSError as exc: raise SourceAccessError( f"cannot create output directory {str(directory)!r}: {exc.strerror or exc}", details={"output_dir": str(directory)}, ) from exc suffix = "." + str(request["format"]) items: list[dict[str, Any]] = [] failures = 0 for index, entry in enumerate(selected, start=1): record = entry.to_item() destination = directory / f"{index:03d}-{_slug(entry.candidate)}{suffix}" payload = entry.payload resize_fragment: dict[str, Any] | None = None expected = (entry.info.width, entry.info.height) try: if request["resize"]: payload, resize_fragment = resize_image(payload, request) expected = ( int(resize_fragment["to"]["width"]), int(resize_fragment["to"]["height"]), ) destination.write_bytes(payload) stored = _verify_saved(destination, request, expected) except (ToolkitError, OSError) as exc: _discard(destination) failures += 1 error = error_fragment(getattr(exc, "code", "download_failed"), str(exc)) if isinstance(exc, ToolkitError) and exc.details: error["details"] = dict(exc.details) # A failed save reports no path: only files that really exist and # really open are ever named as saved. record["download"] = {"ok": False, "target": str(destination), "error": error} else: download: dict[str, Any] = { "ok": True, "path": str(destination), "bytes": len(payload), "format": stored.format, "width": stored.width, "height": stored.height, } if resize_fragment is not None: download["resize"] = resize_fragment record["download"] = download items.append(record) return items, failures def _guess_source(argv: Sequence[str]) -> str | None: """Best-effort source id extraction for error envelopes.""" for index, token in enumerate(argv): if token == "--source" or token == "-s": if index + 1 < len(argv): return argv[index + 1] elif token.startswith("--source="): return token.split("=", 1)[1] return None def _request_from(argv: Sequence[str]) -> tuple[dict[str, Any], str, Any]: """Turn CLI arguments into ``(normalized_request, source_id, registry)``.""" parser = build_parser() namespace = parser.parse_args(list(argv)) flags = _flags_to_mapping(namespace) document = namespace.request if document is not None: if flags: raise RequestError( "a request document cannot be combined with flags: " + ", ".join(sorted(flags)), details={"conflicting_flags": sorted(flags)}, accepted_fields=REQUEST_FIELDS, ) raw = _document_to_mapping(_load_document(document)) elif flags: raw = flags else: raise RequestError( "no request given; pass a JSON request document or at least one flag", accepted_fields=REQUEST_FIELDS, ) request = normalize_request(raw) registry = default_registry() available = list(registry.available_sources()) if request["source"] not in available: raise UnknownSourceError(request["source"], available) return request, request["source"], registry def _execute(argv: Sequence[str]) -> Response: """Run one request and return the response that must be printed. The search path downloads a bounded window of candidates and filters that window on the parsed bytes, so the answer distinguishes three different situations through both the status and the exit code: * ``ok`` -- at least one image really is the requested format and at least the requested width and height; * ``no_format`` -- the source had hits, but none of them really are the requested format; * ``no_size`` -- images of the requested format existed, but all of them are smaller than requested; * ``access_error`` -- the source could not be searched, or not a single candidate could be fetched, which is never reported as a "no match". A candidate whose downloaded bytes are a different format (a JPEG, an SVG, an HTML error page, ...) is a format mismatch about the topic, not an access problem: it feeds ``no_format`` instead of turning a working search into an error, and a stray unfetchable candidate never demotes a usable answer to ``partial_download``. """ request, source, registry = _request_from(argv) builder = ResponseBuilder(source=source, request=request) if request["dry_run"]: return builder.dry_run() cap = candidate_cap(request["count"]) candidates = registry.search(source, request["topic"], cap) selection = _select(candidates, request) unreadable = list(selection.failures) if not selection.kept: details = { "requested_format": request["format"], "requested_size": {"width": request["width"], "height": request["height"]}, "minimum_size": {"width": request["width"], "height": request["height"]}, "candidate_count": len(candidates), "candidates_inspected": selection.inspected, "format_matches": selection.size_rejects, "other_format_candidates": selection.format_rejects, "observed_formats": dict(selection.other_formats), "size_matches": 0, "unreadable_candidates": len(unreadable), } if selection.size_rejects: return builder.build( STATUS_NO_SIZE, error=error_fragment( "no_size_match", "the source returned no images at least the requested width and height", details=details, ), ) if selection.format_rejects or not unreadable: # Either the source really had no images in the requested format, or # it had no hits at all; both answer the topic, not an outage. return builder.build( STATUS_NO_FORMAT, error=error_fragment( "no_format_match", f"the source returned no {request['format']} images for {request['topic']!r}", details=details, ), ) return builder.access_error( error=error_fragment( "candidate_unreadable", "none of the candidates could be downloaded", details={**details, "failures": unreadable}, ) ) if not request["download"]: # Every reported item was fetched and verified against its real bytes, so # an unfetchable candidate is simply skipped: it must not demote an # otherwise usable answer to a partial-download exit code. return builder.build(STATUS_OK, items=[entry.to_item() for entry in selection.kept]) items, save_failures = _download(selection.kept, request) saved_paths = [ item["download"]["path"] for item in items if item["download"].get("ok") is True ] failed = save_failures + len(unreadable) if failed: # The run still reports every saved path, but a per-item save (or fetch) # failure is never hidden behind ``ok``: it has its own status and exit # code so the caller can tell a complete run from an incomplete one. return builder.build( STATUS_PARTIAL_DOWNLOAD, items=items, error=error_fragment( "download_incomplete", f"{failed} of {len(items) + len(unreadable)} images could not be saved", details={ "failed": failed, "saved": len(saved_paths), "total": len(items) + len(unreadable), "saved_paths": saved_paths, "failures": unreadable, }, ), ) return builder.build(STATUS_OK, items=items) def main(argv: Sequence[str] | None = None) -> int: """Entry point: emit exactly one JSON document and return the exit code.""" arguments = list(sys.argv[1:] if argv is None else argv) try: response = _execute(arguments) except ToolkitError as error: builder = ResponseBuilder(source=_guess_source(arguments)) response = builder.from_error(error) except OSError as error: builder = ResponseBuilder(source=_guess_source(arguments)) response = builder.build( STATUS_ACCESS_ERROR, error=error_fragment("access_error", str(error)), ) response.emit() return response.exit_code if __name__ == "__main__": # pragma: no cover - module entry point sys.exit(main())