"""Plain native-process CLI for the image toolkit. Usage:: python -m tools.example examples/input.json python -m tools.example --topic "red fox" --width 640 --height 480 --format png Exactly one machine-readable JSON document is written to stdout. The process exit code is derived from the fixed status vocabulary in :mod:`tools.envelope`. """ from __future__ import annotations import argparse import json import re import struct import sys import zlib from pathlib import Path from typing import Any, Mapping, Sequence from .envelope import ( STATUS_ACCESS_ERROR, STATUS_NO_FORMAT, STATUS_NO_SIZE, STATUS_OK, STATUS_PARTIAL_DOWNLOAD, Response, ResponseBuilder, error_fragment, ) from .errors import ( ACCEPTED_FORMATS, ImageResizeError, RequestDocumentError, RequestError, RequestFieldError, SourceAccessError, ToolkitError, UnknownSourceError, UnsupportedFormatError, ) from .sources import DEFAULT_LIMIT, SourceCandidate, default_registry PROG = "python -m tools.example" #: Keys accepted in a JSON request document (``version`` is optional metadata). REQUEST_FIELDS: tuple[str, ...] = ( "topic", "width", "height", "format", "source", "count", "download", "output_dir", "resize", "dry_run", ) OPTIONAL_DOCUMENT_FIELDS: tuple[str, ...] = ("version",) DEFAULT_SOURCE = "wikimedia_commons" DEFAULT_OUTPUT_DIR = "downloads" MIN_COUNT = 1 MAX_COUNT = 50 #: MIME types (and filename suffixes) that count as a hit for each format. FORMAT_MATCHERS: dict[str, tuple[str, ...]] = { "png": ("image/png", ".png"), "ico": ("image/x-icon", "image/vnd.microsoft.icon", "image/ico", ".ico"), } class _Parser(argparse.ArgumentParser): """Argument parser that reports bad flags as request errors.""" def error(self, message: str) -> Any: # pragma: no cover - argparse hook raise RequestError(message, accepted_fields=REQUEST_FIELDS) def build_parser() -> argparse.ArgumentParser: """Return the CLI parser; ``--help`` documents every supported parameter.""" parser = _Parser( prog=PROG, description=( "Search an image source and report the matching items as one JSON " "document. Either pass a JSON request document or use the flags." ), epilog=( "Every run writes exactly one JSON document to stdout with the keys " "status, source, request, items and error." ), ) parser.add_argument( "request", nargs="?", metavar="REQUEST.json", help=( "path to a JSON request document holding the same fields as the " "flags; it cannot be combined with any of the flags" ), ) parser.add_argument( "-t", "--topic", metavar="TEXT", help="search topic (required unless a request document is given)", ) parser.add_argument( "--width", type=int, metavar="PX", help="target width in pixels; only images at least this wide are accepted", ) parser.add_argument( "--height", type=int, metavar="PX", help="target height in pixels; only images at least this tall are accepted", ) parser.add_argument( "-f", "--format", metavar="FMT", help="image format: " + "|".join(ACCEPTED_FORMATS), ) parser.add_argument( "-s", "--source", metavar="ID", help=f"image source id (default: {DEFAULT_SOURCE}); unknown ids are rejected", ) parser.add_argument( "-c", "--count", type=int, metavar="N", help=f"how many candidates to request, {MIN_COUNT}-{MAX_COUNT} (default: {DEFAULT_LIMIT})", ) parser.add_argument( "--download", action="store_true", help="download every matching image into the output directory", ) parser.add_argument( "-o", "--output-dir", metavar="DIR", help=f"download directory (default: ./{DEFAULT_OUTPUT_DIR})", ) parser.add_argument( "--resize", action="store_true", help=( "downscale every downloaded image so it fits the requested width " "and height, keeping the aspect ratio and never scaling up; " "without it the original bytes are written and the actual size is " "reported" ), ) parser.add_argument( "--dry-run", action="store_true", help="validate the request and emit the envelope without any network access", ) return parser def _load_document(path: str) -> Mapping[str, Any]: """Read a JSON request document from ``path``.""" target = Path(path) try: text = target.read_text(encoding="utf-8") except OSError as exc: raise RequestDocumentError( f"cannot read request document {path!r}: {exc.strerror or exc}", details={"path": str(path)}, ) from exc try: payload = json.loads(text) except json.JSONDecodeError as exc: raise RequestDocumentError( f"request document {path!r} is not valid JSON: {exc.msg}", details={"path": str(path), "line": exc.lineno, "column": exc.colno}, ) from exc if not isinstance(payload, Mapping): raise RequestDocumentError( f"request document {path!r} must contain a JSON object", details={"path": str(path), "type": type(payload).__name__}, ) return payload def _document_to_mapping(payload: Mapping[str, Any]) -> dict[str, Any]: """Reject unknown keys and return the accepted request fields.""" accepted = REQUEST_FIELDS + OPTIONAL_DOCUMENT_FIELDS unknown = sorted(str(key) for key in payload if key not in accepted) if unknown: raise RequestFieldError( "request document contains unsupported fields: " + ", ".join(unknown), field=unknown[0], details={"unknown_fields": unknown, "accepted_fields": list(REQUEST_FIELDS)}, ) return {key: payload[key] for key in REQUEST_FIELDS if key in payload} def _flags_to_mapping(namespace: argparse.Namespace) -> dict[str, Any]: """Collect the explicitly supplied flags into a request mapping.""" mapping: dict[str, Any] = {} for field in ("topic", "width", "height", "format", "source", "count", "output_dir"): value = getattr(namespace, field, None) if value is not None: mapping[field] = value for field in ("download", "resize", "dry_run"): if getattr(namespace, field, False): mapping[field] = True return mapping def _as_text(value: Any, field: str) -> str: if not isinstance(value, str): raise RequestFieldError( f"{field} must be a string, got {type(value).__name__}", field=field ) text = value.strip() if not text: raise RequestFieldError(f"{field} must not be empty", field=field) return text def _as_int(value: Any, field: str) -> int: if isinstance(value, bool) or not isinstance(value, int): raise RequestFieldError( f"{field} must be an integer, got {type(value).__name__}", field=field ) return int(value) def _as_bool(value: Any, field: str) -> bool: if not isinstance(value, bool): raise RequestFieldError( f"{field} must be a boolean, got {type(value).__name__}", field=field ) return bool(value) def normalize_request(raw: Mapping[str, Any]) -> dict[str, Any]: """Validate ``raw`` and return the complete, typed request mapping.""" if "topic" not in raw: raise RequestFieldError( "topic is required", field="topic", details={"required_fields": ["topic"], "accepted_fields": list(REQUEST_FIELDS)}, ) topic = _as_text(raw["topic"], "topic") size: dict[str, int | None] = {"width": None, "height": None} for field in ("width", "height"): if raw.get(field) is None: continue number = _as_int(raw[field], field) if number <= 0: raise RequestFieldError(f"{field} must be positive", field=field) size[field] = number if "format" not in raw or raw["format"] is None: raise RequestFieldError( "format is required", field="format", details={"accepted_formats": list(ACCEPTED_FORMATS)}, ) image_format = _as_text(raw["format"], "format").lower() if image_format not in ACCEPTED_FORMATS: raise UnsupportedFormatError(image_format) source = DEFAULT_SOURCE if raw.get("source") is not None: source = _as_text(raw["source"], "source") count = DEFAULT_LIMIT if raw.get("count") is not None: count = _as_int(raw["count"], "count") if not MIN_COUNT <= count <= MAX_COUNT: raise RequestFieldError( f"count must be between {MIN_COUNT} and {MAX_COUNT}", field="count", details={"min": MIN_COUNT, "max": MAX_COUNT}, ) output_dir = DEFAULT_OUTPUT_DIR if raw.get("output_dir") is not None: output_dir = _as_text(raw["output_dir"], "output_dir") return { "topic": topic, "width": size["width"], "height": size["height"], "format": image_format, "source": source, "count": count, "download": _as_bool(raw.get("download", False), "download"), "output_dir": output_dir, "resize": _as_bool(raw.get("resize", False), "resize"), "dry_run": _as_bool(raw.get("dry_run", False), "dry_run"), } def _matches_format(candidate: SourceCandidate, image_format: str) -> bool: matchers = FORMAT_MATCHERS[image_format] mime = str(getattr(candidate, "mime_type", None) or "").split(";")[0].strip().lower() if mime and mime in matchers: return True url = str(getattr(candidate, "file_url", "") or "").lower() return any(url.endswith(suffix) for suffix in matchers if suffix.startswith(".")) def _matches_size(candidate: SourceCandidate, request: Mapping[str, Any]) -> bool: """True when ``candidate`` is usable for the requested target size. The requested width and height are a target size, not an exact match: any image that is at least that large is usable because it can always be downscaled, while a smaller image would have to be scaled up and is therefore rejected. ``--resize`` only decides whether the downloaded bytes are downscaled; it never widens or narrows this filter. """ wanted_width = request.get("width") wanted_height = request.get("height") if wanted_width is None and wanted_height is None: return True actual_width = getattr(candidate, "width", None) actual_height = getattr(candidate, "height", None) if isinstance(actual_width, bool) or isinstance(actual_height, bool): return False if not isinstance(actual_width, int) or not isinstance(actual_height, int): return False if wanted_width is not None and actual_width < wanted_width: return False if wanted_height is not None and actual_height < wanted_height: return False return True def _select(candidates: Sequence[SourceCandidate], request: Mapping[str, Any]) -> tuple[list[SourceCandidate], list[SourceCandidate]]: """Return ``(matching, rejected_by_size)`` split for the request.""" by_format = [c for c in candidates if _matches_format(c, request["format"])] matching = [c for c in by_format if _matches_size(c, request)] rejected = [c for c in by_format if not _matches_size(c, request)] return matching, rejected def _slug(candidate: SourceCandidate) -> str: raw = str(getattr(candidate, "title", None) or getattr(candidate, "source_id", "") or "image").lower() slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-") return (slug or "image")[:60] PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" ICO_HEADER_SIZE = 6 ICO_ENTRY_SIZE = 16 ICO_MAX_DIMENSION = 256 _PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} def _png_chunks(payload: bytes) -> list[tuple[bytes, bytes]]: """Split a PNG payload into ``(type, data)`` chunks.""" if not payload.startswith(PNG_SIGNATURE): raise ImageResizeError("payload is not a PNG image") chunks: list[tuple[bytes, bytes]] = [] offset = len(PNG_SIGNATURE) while offset + 8 <= len(payload): (length,) = struct.unpack(">I", payload[offset : offset + 4]) chunk_type = payload[offset + 4 : offset + 8] data = payload[offset + 8 : offset + 8 + length] if len(data) != length: raise ImageResizeError("truncated PNG chunk") chunks.append((chunk_type, data)) offset += 12 + length if chunk_type == b"IEND": return chunks raise ImageResizeError("PNG image has no IEND chunk") def _unfilter_rows(raw: bytes, width: int, height: int, bpp: int) -> list[bytearray]: """Undo the per-scanline PNG filters, returning one row per pixel line.""" stride = width * bpp if len(raw) < height * (stride + 1): raise ImageResizeError("truncated PNG image data") rows: list[bytearray] = [] previous = bytearray(stride) position = 0 for _ in range(height): filter_type = raw[position] position += 1 line = bytearray(raw[position : position + stride]) position += stride if filter_type == 1: for index in range(bpp, stride): line[index] = (line[index] + line[index - bpp]) & 0xFF elif filter_type == 2: for index in range(stride): line[index] = (line[index] + previous[index]) & 0xFF elif filter_type == 3: for index in range(stride): left = line[index - bpp] if index >= bpp else 0 line[index] = (line[index] + ((left + previous[index]) >> 1)) & 0xFF elif filter_type == 4: for index in range(stride): left = line[index - bpp] if index >= bpp else 0 up = previous[index] up_left = previous[index - bpp] if index >= bpp else 0 estimate = left + up - up_left to_left = abs(estimate - left) to_up = abs(estimate - up) to_up_left = abs(estimate - up_left) if to_left <= to_up and to_left <= to_up_left: predictor = left elif to_up <= to_up_left: predictor = up else: predictor = up_left line[index] = (line[index] + predictor) & 0xFF elif filter_type != 0: raise ImageResizeError(f"unsupported PNG filter type {filter_type}") previous = line rows.append(line) return rows def _row_to_rgba( row: bytearray, width: int, color_type: int, depth: int, palette: bytes, transparency: bytes | None, ) -> bytes: """Convert one unfiltered scanline into 8-bit RGBA samples.""" step = depth // 8 out = bytearray() if color_type == 3: for x in range(width): index = row[x] base = index * 3 if base + 2 >= len(palette): raise ImageResizeError("PNG palette index is out of range") alpha = transparency[index] if transparency and index < len(transparency) else 255 out += bytes((palette[base], palette[base + 1], palette[base + 2], alpha)) return bytes(out) for x in range(width): offset = x * step if color_type == 0: value = row[offset] out += bytes((value, value, value, 255)) elif color_type == 4: value = row[offset] out += bytes((value, value, value, row[offset + step])) elif color_type == 2: out += bytes((row[offset], row[offset + step], row[offset + 2 * step], 255)) else: out += bytes( ( row[offset], row[offset + step], row[offset + 2 * step], row[offset + 3 * step], ) ) return bytes(out) def _decode_png(payload: bytes) -> tuple[int, int, list[bytes]]: """Decode a PNG payload into ``(width, height, rgba_rows)``.""" header: tuple[int, ...] | None = None palette = b"" transparency: bytes | None = None data = bytearray() for chunk_type, chunk_data in _png_chunks(payload): if chunk_type == b"IHDR": if len(chunk_data) < 13: raise ImageResizeError("truncated PNG header") header = struct.unpack(">IIBBBBB", chunk_data[:13]) elif chunk_type == b"PLTE": palette = chunk_data elif chunk_type == b"tRNS": transparency = chunk_data elif chunk_type == b"IDAT": data += chunk_data if header is None: raise ImageResizeError("PNG image has no IHDR chunk") width, height, depth, color_type, compression, filter_method, interlace = header if width < 1 or height < 1: raise ImageResizeError("PNG image has no pixels") if compression != 0 or filter_method != 0: raise ImageResizeError("unsupported PNG compression or filter method") if interlace != 0: raise ImageResizeError("interlaced PNG images are not supported") channels = _PNG_CHANNELS.get(color_type) if channels is None: raise ImageResizeError(f"unsupported PNG colour type {color_type}") if depth not in (8, 16): raise ImageResizeError(f"unsupported PNG bit depth {depth}") if color_type == 3 and depth > 8: raise ImageResizeError("unsupported PNG palette bit depth") if color_type == 3 and not palette: raise ImageResizeError("PNG palette image has no PLTE chunk") try: raw = zlib.decompress(bytes(data)) except zlib.error as exc: raise ImageResizeError(f"PNG image data is corrupt: {exc}") from exc rows = _unfilter_rows(raw, width, height, channels * (depth // 8)) return width, height, [ _row_to_rgba(row, width, color_type, depth, palette, transparency) for row in rows ] def _encode_png(width: int, height: int, rows: Sequence[bytes]) -> bytes: """Encode RGBA rows as an uncompressed-filter 8-bit PNG.""" raw = bytearray() for row in rows: raw.append(0) raw += row def chunk(chunk_type: bytes, chunk_data: bytes) -> bytes: crc = zlib.crc32(chunk_type + chunk_data) & 0xFFFFFFFF return struct.pack(">I", len(chunk_data)) + chunk_type + chunk_data + struct.pack(">I", crc) header = struct.pack(">IIBBBBB", width, height, 8, 6, 0, 0, 0) return ( PNG_SIGNATURE + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(bytes(raw), 6)) + chunk(b"IEND", b"") ) def _ico_entries(payload: bytes) -> list[tuple[int, int, int, int]]: """Return ``(width, height, size, offset)`` for every ICO directory entry.""" if len(payload) < ICO_HEADER_SIZE: raise ImageResizeError("payload is not an ICO image") reserved, kind, count = struct.unpack(" len(payload): raise ImageResizeError("truncated ICO directory") width, height, _colours, _reserved, _planes, _bits, size, offset = struct.unpack( " bytes: """Wrap a PNG payload in a single-entry ICO container.""" if width > ICO_MAX_DIMENSION or height > ICO_MAX_DIMENSION: raise ImageResizeError("ICO images cannot exceed 256x256 pixels") header = struct.pack(" tuple[str, int, int, list[bytes]]: """Decode a PNG or ICO payload into ``(container, width, height, rows)``.""" if payload.startswith(PNG_SIGNATURE): width, height, rows = _decode_png(payload) return "png", width, height, rows if len(payload) >= ICO_HEADER_SIZE and payload[:4] == b"\x00\x00\x01\x00": entries = _ico_entries(payload) _width, _height, size, offset = max(entries, key=lambda item: item[0] * item[1]) embedded = payload[offset : offset + size] if not embedded.startswith(PNG_SIGNATURE): raise ImageResizeError("ICO images stored as raw bitmaps are not supported") width, height, rows = _decode_png(embedded) return "ico", width, height, rows raise ImageResizeError("payload is neither a PNG nor an ICO image") def _target_size(width: int, height: int, request: Mapping[str, Any]) -> tuple[int, int]: """Shrink-to-fit target inside the requested box, never enlarging.""" wanted_width = request.get("width") wanted_height = request.get("height") scale = 1.0 if wanted_width is not None: scale = min(scale, wanted_width / width) if wanted_height is not None: scale = min(scale, wanted_height / height) if scale >= 1.0: return width, height return max(1, int(round(width * scale))), max(1, int(round(height * scale))) def _resample(rows: Sequence[bytes], width: int, height: int, target_width: int, target_height: int) -> list[bytes]: """Nearest-neighbour downscale of RGBA rows.""" columns = [min(width - 1, (x * width) // target_width) * 4 for x in range(target_width)] resized: list[bytes] = [] for y in range(target_height): source = rows[min(height - 1, (y * height) // target_height)] line = bytearray() for offset in columns: line += source[offset : offset + 4] resized.append(bytes(line)) return resized def resize_image(payload: bytes, request: Mapping[str, Any]) -> tuple[bytes, dict[str, Any]]: """Downscale ``payload`` so it fits the requested size. The requested width and height are a bounding box: the aspect ratio is preserved, an image that already fits is left untouched and an image is never scaled up. The returned fragment records what happened so the caller can report the real dimensions of the stored file. """ container, width, height, rows = _decode_image(payload) target_width, target_height = _target_size(width, height, request) fragment: dict[str, Any] = { "requested": {"width": request.get("width"), "height": request.get("height")}, "from": {"width": width, "height": height}, } if (target_width, target_height) == (width, height): fragment.update( {"applied": False, "reason": "already_fits", "to": {"width": width, "height": height}} ) return payload, fragment png_payload = _encode_png( target_width, target_height, _resample(rows, width, height, target_width, target_height) ) output = ( _encode_ico(png_payload, target_width, target_height) if container == "ico" else png_payload ) fragment.update({"applied": True, "to": {"width": target_width, "height": target_height}}) return output, fragment def _download(candidates: Sequence[SourceCandidate], request: Mapping[str, Any]) -> tuple[list[dict[str, Any]], int]: """Download ``candidates``; return ``(items, failure_count)``. With ``resize`` enabled every stored file is downscaled to the requested target size first. A payload that cannot be decoded or downscaled is reported as a failed item rather than silently stored at the wrong size. """ directory = Path(request["output_dir"]).expanduser() try: directory.mkdir(parents=True, exist_ok=True) except OSError as exc: raise SourceAccessError( f"cannot create output directory {str(directory)!r}: {exc.strerror or exc}", details={"output_dir": str(directory)}, ) from exc suffix = "." + str(request["format"]) items: list[dict[str, Any]] = [] failures = 0 for index, candidate in enumerate(candidates, start=1): record = dict(candidate.to_dict()) destination = directory / f"{index:03d}-{_slug(candidate)}{suffix}" try: payload = candidate.fetch_bytes() resize_fragment = resize_image(payload, request) if request["resize"] else None if resize_fragment is not None: payload, resize_fragment = resize_fragment destination.write_bytes(payload) except (ToolkitError, OSError) as exc: failures += 1 error = error_fragment(getattr(exc, "code", "download_failed"), str(exc)) if isinstance(exc, ToolkitError) and exc.details: error["details"] = dict(exc.details) record["download"] = {"ok": False, "error": error} else: record["download"] = {"ok": True, "path": str(destination), "bytes": len(payload)} if resize_fragment is not None: record["download"]["resize"] = resize_fragment record["download"]["width"] = resize_fragment["to"]["width"] record["download"]["height"] = resize_fragment["to"]["height"] items.append(record) return items, failures def _guess_source(argv: Sequence[str]) -> str | None: """Best-effort source id extraction for error envelopes.""" for index, token in enumerate(argv): if token == "--source" or token == "-s": if index + 1 < len(argv): return argv[index + 1] elif token.startswith("--source="): return token.split("=", 1)[1] return None def _request_from(argv: Sequence[str]) -> tuple[dict[str, Any], str, Any]: """Turn CLI arguments into ``(normalized_request, source_id, registry)``.""" parser = build_parser() namespace = parser.parse_args(list(argv)) flags = _flags_to_mapping(namespace) document = namespace.request if document is not None: if flags: raise RequestError( "a request document cannot be combined with flags: " + ", ".join(sorted(flags)), details={"conflicting_flags": sorted(flags)}, accepted_fields=REQUEST_FIELDS, ) raw = _document_to_mapping(_load_document(document)) elif flags: raw = flags else: raise RequestError( "no request given; pass a JSON request document or at least one flag", accepted_fields=REQUEST_FIELDS, ) request = normalize_request(raw) registry = default_registry() available = list(registry.available_sources()) if request["source"] not in available: raise UnknownSourceError(request["source"], available) return request, request["source"], registry def _execute(argv: Sequence[str]) -> Response: """Run one request and return the response that must be printed.""" request, source, registry = _request_from(argv) builder = ResponseBuilder(source=source, request=request) if request["dry_run"]: return builder.dry_run() candidates = registry.search(source, request["topic"], request["count"]) matching, rejected = _select(candidates, request) if not matching: details = { "requested_format": request["format"], "requested_size": {"width": request["width"], "height": request["height"]}, "minimum_size": {"width": request["width"], "height": request["height"]}, "candidate_count": len(candidates), "format_matches": len(matching) + len(rejected), "size_matches": 0, } if rejected: return builder.build( STATUS_NO_SIZE, error=error_fragment( "no_size_match", "no image is at least the requested width and height", details=details, ), ) return builder.build( STATUS_NO_FORMAT, error=error_fragment( "no_format_match", f"the source returned no {request['format']} images for {request['topic']!r}", details=details, ), ) if not request["download"]: return builder.build(STATUS_OK, items=[c.to_dict() for c in matching]) items, failures = _download(matching, request) if failures and failures == len(items): return builder.build( STATUS_ACCESS_ERROR, items=items, error=error_fragment( "download_failed", "every download failed", details={"failed": failures, "total": len(items)}, ), ) if failures: return builder.build( STATUS_PARTIAL_DOWNLOAD, items=items, error=error_fragment( "download_incomplete", f"{failures} of {len(items)} downloads failed", details={"failed": failures, "total": len(items)}, ), ) return builder.build(STATUS_OK, items=items) def main(argv: Sequence[str] | None = None) -> int: """Entry point: emit exactly one JSON document and return the exit code.""" arguments = list(sys.argv[1:] if argv is None else argv) try: response = _execute(arguments) except ToolkitError as error: builder = ResponseBuilder(source=_guess_source(arguments)) response = builder.from_error(error) except OSError as error: builder = ResponseBuilder(source=_guess_source(arguments)) response = builder.build( STATUS_ACCESS_ERROR, error=error_fragment("access_error", str(error)), ) response.emit() return response.exit_code if __name__ == "__main__": # pragma: no cover - module entry point sys.exit(main())