Repository Validate / repository-checks (pull_request) Successful in 3s
Added tools/images.py with stdlib-only detection of the real image format and pixel dimensions from downloaded bytes (PNG signature + IHDR chunk, ICO ICONDIR/ICONDIRENTRY directory including the 0-means-256 encoding) plus stdlib-only builders (png_bytes, ico_bytes) so tests/sources can produce genuine payloads. Wired it into the CLI search path in tools/example.py: a bounded candidate window derived from --count (candidate_cap = count * 4, capped at 50) is fetched from the selected source, each payload is parsed, and only items whose actual format equals the requested png/ico and whose actual width and height are both at least the requested minimum are kept. Kept items report source, url, format and the actual width/height (plus size_bytes and metadata as extras); verified bytes are reused for --download instead of being fetched twice. Statuses stay distinct: ok (0), no_format (3) when hits were only other formats, no_size (4) when format-matching items were all too small, access_error (5) when bytes could not be fetched or are not readable images, partial_download (6) when some candidates could not be read. All metadata-based filtering (FORMAT_MATCHERS, _matches_format, _matches_size) was removed so only parsed bytes decide. Added tests/test_images.py with detector/builder unit tests and CLI tests using an injected fake byte-fetch callable (no network), and updated tests/test_cli.py fixtures to real image bytes where the previous metadata-only assumption no longer holds. Tests: Facade commands run with .venv/bin on PATH after prepare_environment(component=tools, mode=restore): `python -m compileall tools tests` (tools.validate) succeeded; `pytest` (tools.test) => 114 passed, including the new tests/test_images.py (31 tests); `python -m tools.example examples/input.json` (tools.smoke) exited 0 with a single dry_run JSON document; acceptance commands `python -m tools.example --help` exited 0 and `import tools` resolved. Target branch task/1-task is at the same commit as this work branch (git rev-list count 0 ahead / 0 behind), so it is already integrated; facade/protected paths untouched. Progress: Implemented byte-based format and minimum-size filtering for the image search path. New tools/images.py: detect_format (png/ico plus jpeg/gif/bmp/webp so "other format" is distinguishable from "unreadable"), read_png_dimensions (IHDR), read_ico_dimensions (ICONDIR entries, 0 => 256, largest entry), inspect_image/ImageInfo, ImageFormatError (code image_unreadable), and png_bytes/ico_bytes builders. tools/example.py: candidate_cap(count) bounded window (count*4, max 50) requested from the source, _select() fetches and parses every candidate, KeptImage.to_item() reports source/url/format/actual width+height/size_bytes, _download() writes the already verified bytes, and _execute() distinguishes ok/no_format/no_size/access_error/partial_download with the existing exit codes; metadata-based filtering removed. tests/test_images.py added (31 tests) and tests/test_cli.py fixtures aligned to real image bytes. All facade commands pass: compileall, pytest (114 passed), smoke, help, package import. Published by developer-agent through Tool Backend. Closes #6
1004 lines
36 KiB
Python
1004 lines
36 KiB
Python
"""Plain native-process CLI for the image toolkit.
|
|
|
|
Usage::
|
|
|
|
python -m tools.example examples/input.json
|
|
python -m tools.example --topic "red fox" --width 640 --height 480 --format png
|
|
|
|
Exactly one machine-readable JSON document is written to stdout. The process
|
|
exit code is derived from the fixed status vocabulary in :mod:`tools.envelope`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import struct
|
|
import sys
|
|
import zlib
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
from . import images
|
|
from .envelope import (
|
|
STATUS_ACCESS_ERROR,
|
|
STATUS_NO_FORMAT,
|
|
STATUS_NO_SIZE,
|
|
STATUS_OK,
|
|
STATUS_PARTIAL_DOWNLOAD,
|
|
Response,
|
|
ResponseBuilder,
|
|
error_fragment,
|
|
)
|
|
from .errors import (
|
|
ACCEPTED_FORMATS,
|
|
ImageResizeError,
|
|
RequestDocumentError,
|
|
RequestError,
|
|
RequestFieldError,
|
|
SourceAccessError,
|
|
ToolkitError,
|
|
UnknownSourceError,
|
|
UnsupportedFormatError,
|
|
)
|
|
from .images import ImageFormatError, ImageInfo
|
|
from .sources import DEFAULT_LIMIT, SourceCandidate, default_registry
|
|
|
|
PROG = "python -m tools.example"
|
|
|
|
#: Keys accepted in a JSON request document (``version`` is optional metadata).
|
|
REQUEST_FIELDS: tuple[str, ...] = (
|
|
"topic",
|
|
"width",
|
|
"height",
|
|
"format",
|
|
"source",
|
|
"count",
|
|
"download",
|
|
"output_dir",
|
|
"resize",
|
|
"dry_run",
|
|
)
|
|
|
|
OPTIONAL_DOCUMENT_FIELDS: tuple[str, ...] = ("version",)
|
|
|
|
DEFAULT_SOURCE = "wikimedia_commons"
|
|
DEFAULT_OUTPUT_DIR = "downloads"
|
|
MIN_COUNT = 1
|
|
MAX_COUNT = 50
|
|
|
|
#: How many candidates are fetched per wanted image. Nothing a source reports
|
|
#: about a hit is trusted, so a candidate is only known to be usable once its
|
|
#: bytes have been downloaded and parsed; the source is therefore asked for a
|
|
#: window of ``count * SAMPLE_FACTOR`` hits and the window is cut short as soon
|
|
#: as ``count`` usable images have been found.
|
|
SAMPLE_FACTOR = 4
|
|
#: Hard upper bound on the number of candidates inspected in one invocation.
|
|
MAX_CANDIDATE_CAP = MAX_COUNT
|
|
|
|
|
|
def candidate_cap(count: int) -> int:
|
|
"""Return the bounded number of candidates to fetch for ``count`` images."""
|
|
|
|
wanted = max(MIN_COUNT, int(count))
|
|
return min(MAX_CANDIDATE_CAP, wanted * SAMPLE_FACTOR)
|
|
|
|
|
|
class _Parser(argparse.ArgumentParser):
|
|
"""Argument parser that reports bad flags as request errors."""
|
|
|
|
def error(self, message: str) -> Any: # pragma: no cover - argparse hook
|
|
raise RequestError(message, accepted_fields=REQUEST_FIELDS)
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
"""Return the CLI parser; ``--help`` documents every supported parameter."""
|
|
|
|
parser = _Parser(
|
|
prog=PROG,
|
|
description=(
|
|
"Search an image source and report the matching items as one JSON "
|
|
"document. Either pass a JSON request document or use the flags."
|
|
),
|
|
epilog=(
|
|
"Every run writes exactly one JSON document to stdout with the keys "
|
|
"status, source, request, items and error."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"request",
|
|
nargs="?",
|
|
metavar="REQUEST.json",
|
|
help=(
|
|
"path to a JSON request document holding the same fields as the "
|
|
"flags; it cannot be combined with any of the flags"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"-t",
|
|
"--topic",
|
|
metavar="TEXT",
|
|
help="search topic (required unless a request document is given)",
|
|
)
|
|
parser.add_argument(
|
|
"--width",
|
|
type=int,
|
|
metavar="PX",
|
|
help="target width in pixels; only images at least this wide are accepted",
|
|
)
|
|
parser.add_argument(
|
|
"--height",
|
|
type=int,
|
|
metavar="PX",
|
|
help="target height in pixels; only images at least this tall are accepted",
|
|
)
|
|
parser.add_argument(
|
|
"-f",
|
|
"--format",
|
|
metavar="FMT",
|
|
help="image format: " + "|".join(ACCEPTED_FORMATS),
|
|
)
|
|
parser.add_argument(
|
|
"-s",
|
|
"--source",
|
|
metavar="ID",
|
|
help=f"image source id (default: {DEFAULT_SOURCE}); unknown ids are rejected",
|
|
)
|
|
parser.add_argument(
|
|
"-c",
|
|
"--count",
|
|
type=int,
|
|
metavar="N",
|
|
help=f"how many candidates to request, {MIN_COUNT}-{MAX_COUNT} (default: {DEFAULT_LIMIT})",
|
|
)
|
|
parser.add_argument(
|
|
"--download",
|
|
action="store_true",
|
|
help="download every matching image into the output directory",
|
|
)
|
|
parser.add_argument(
|
|
"-o",
|
|
"--output-dir",
|
|
metavar="DIR",
|
|
help=f"download directory (default: ./{DEFAULT_OUTPUT_DIR})",
|
|
)
|
|
parser.add_argument(
|
|
"--resize",
|
|
action="store_true",
|
|
help=(
|
|
"downscale every downloaded image so it fits the requested width "
|
|
"and height, keeping the aspect ratio and never scaling up; "
|
|
"without it the original bytes are written and the actual size is "
|
|
"reported"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
action="store_true",
|
|
help="validate the request and emit the envelope without any network access",
|
|
)
|
|
return parser
|
|
|
|
|
|
def _load_document(path: str) -> Mapping[str, Any]:
|
|
"""Read a JSON request document from ``path``."""
|
|
|
|
target = Path(path)
|
|
try:
|
|
text = target.read_text(encoding="utf-8")
|
|
except OSError as exc:
|
|
raise RequestDocumentError(
|
|
f"cannot read request document {path!r}: {exc.strerror or exc}",
|
|
details={"path": str(path)},
|
|
) from exc
|
|
try:
|
|
payload = json.loads(text)
|
|
except json.JSONDecodeError as exc:
|
|
raise RequestDocumentError(
|
|
f"request document {path!r} is not valid JSON: {exc.msg}",
|
|
details={"path": str(path), "line": exc.lineno, "column": exc.colno},
|
|
) from exc
|
|
if not isinstance(payload, Mapping):
|
|
raise RequestDocumentError(
|
|
f"request document {path!r} must contain a JSON object",
|
|
details={"path": str(path), "type": type(payload).__name__},
|
|
)
|
|
return payload
|
|
|
|
|
|
def _document_to_mapping(payload: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""Reject unknown keys and return the accepted request fields."""
|
|
|
|
accepted = REQUEST_FIELDS + OPTIONAL_DOCUMENT_FIELDS
|
|
unknown = sorted(str(key) for key in payload if key not in accepted)
|
|
if unknown:
|
|
raise RequestFieldError(
|
|
"request document contains unsupported fields: " + ", ".join(unknown),
|
|
field=unknown[0],
|
|
details={"unknown_fields": unknown, "accepted_fields": list(REQUEST_FIELDS)},
|
|
)
|
|
return {key: payload[key] for key in REQUEST_FIELDS if key in payload}
|
|
|
|
|
|
def _flags_to_mapping(namespace: argparse.Namespace) -> dict[str, Any]:
|
|
"""Collect the explicitly supplied flags into a request mapping."""
|
|
|
|
mapping: dict[str, Any] = {}
|
|
for field in ("topic", "width", "height", "format", "source", "count", "output_dir"):
|
|
value = getattr(namespace, field, None)
|
|
if value is not None:
|
|
mapping[field] = value
|
|
for field in ("download", "resize", "dry_run"):
|
|
if getattr(namespace, field, False):
|
|
mapping[field] = True
|
|
return mapping
|
|
|
|
|
|
def _as_text(value: Any, field: str) -> str:
|
|
if not isinstance(value, str):
|
|
raise RequestFieldError(
|
|
f"{field} must be a string, got {type(value).__name__}", field=field
|
|
)
|
|
text = value.strip()
|
|
if not text:
|
|
raise RequestFieldError(f"{field} must not be empty", field=field)
|
|
return text
|
|
|
|
|
|
def _as_int(value: Any, field: str) -> int:
|
|
if isinstance(value, bool) or not isinstance(value, int):
|
|
raise RequestFieldError(
|
|
f"{field} must be an integer, got {type(value).__name__}", field=field
|
|
)
|
|
return int(value)
|
|
|
|
|
|
def _as_bool(value: Any, field: str) -> bool:
|
|
if not isinstance(value, bool):
|
|
raise RequestFieldError(
|
|
f"{field} must be a boolean, got {type(value).__name__}", field=field
|
|
)
|
|
return bool(value)
|
|
|
|
|
|
def normalize_request(raw: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""Validate ``raw`` and return the complete, typed request mapping."""
|
|
|
|
if "topic" not in raw:
|
|
raise RequestFieldError(
|
|
"topic is required",
|
|
field="topic",
|
|
details={"required_fields": ["topic"], "accepted_fields": list(REQUEST_FIELDS)},
|
|
)
|
|
topic = _as_text(raw["topic"], "topic")
|
|
|
|
size: dict[str, int | None] = {"width": None, "height": None}
|
|
for field in ("width", "height"):
|
|
if raw.get(field) is None:
|
|
continue
|
|
number = _as_int(raw[field], field)
|
|
if number <= 0:
|
|
raise RequestFieldError(f"{field} must be positive", field=field)
|
|
size[field] = number
|
|
|
|
if "format" not in raw or raw["format"] is None:
|
|
raise RequestFieldError(
|
|
"format is required",
|
|
field="format",
|
|
details={"accepted_formats": list(ACCEPTED_FORMATS)},
|
|
)
|
|
image_format = _as_text(raw["format"], "format").lower()
|
|
if image_format not in ACCEPTED_FORMATS:
|
|
raise UnsupportedFormatError(image_format)
|
|
|
|
source = DEFAULT_SOURCE
|
|
if raw.get("source") is not None:
|
|
source = _as_text(raw["source"], "source")
|
|
|
|
count = DEFAULT_LIMIT
|
|
if raw.get("count") is not None:
|
|
count = _as_int(raw["count"], "count")
|
|
if not MIN_COUNT <= count <= MAX_COUNT:
|
|
raise RequestFieldError(
|
|
f"count must be between {MIN_COUNT} and {MAX_COUNT}",
|
|
field="count",
|
|
details={"min": MIN_COUNT, "max": MAX_COUNT},
|
|
)
|
|
|
|
output_dir = DEFAULT_OUTPUT_DIR
|
|
if raw.get("output_dir") is not None:
|
|
output_dir = _as_text(raw["output_dir"], "output_dir")
|
|
|
|
return {
|
|
"topic": topic,
|
|
"width": size["width"],
|
|
"height": size["height"],
|
|
"format": image_format,
|
|
"source": source,
|
|
"count": count,
|
|
"download": _as_bool(raw.get("download", False), "download"),
|
|
"output_dir": output_dir,
|
|
"resize": _as_bool(raw.get("resize", False), "resize"),
|
|
"dry_run": _as_bool(raw.get("dry_run", False), "dry_run"),
|
|
}
|
|
|
|
|
|
def _meets_minimum_size(info: ImageInfo, request: Mapping[str, Any]) -> bool:
|
|
"""True when the *real* pixel size of an image is usable for the request.
|
|
|
|
The requested width and height are a minimum, not an exact match: any image
|
|
that is at least that large is usable because it can always be downscaled,
|
|
while a smaller image would have to be scaled up and is therefore rejected.
|
|
``--resize`` only decides whether the downloaded bytes are downscaled; it
|
|
never widens or narrows this filter.
|
|
"""
|
|
|
|
wanted_width = request.get("width")
|
|
wanted_height = request.get("height")
|
|
if wanted_width is not None and info.width < wanted_width:
|
|
return False
|
|
if wanted_height is not None and info.height < wanted_height:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _failure(candidate: SourceCandidate, fragment: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""Describe one candidate whose bytes could not be turned into an image."""
|
|
|
|
return {
|
|
"source": str(getattr(candidate, "source_id", "") or ""),
|
|
"url": str(getattr(candidate, "file_url", "") or ""),
|
|
"error": dict(fragment),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class KeptImage:
|
|
"""One candidate whose downloaded bytes really match the request."""
|
|
|
|
candidate: SourceCandidate
|
|
info: ImageInfo
|
|
payload: bytes
|
|
|
|
def to_item(self) -> dict[str, Any]:
|
|
"""Report ``source``, ``url``, ``format`` and the *actual* dimensions."""
|
|
|
|
source = str(getattr(self.candidate, "source_id", "") or "")
|
|
url = str(getattr(self.candidate, "file_url", "") or "")
|
|
item: dict[str, Any] = {
|
|
"source": source,
|
|
"url": url,
|
|
# Kept as an alias so existing consumers keep working unchanged.
|
|
"file_url": url,
|
|
"format": self.info.format,
|
|
"width": self.info.width,
|
|
"height": self.info.height,
|
|
"size_bytes": len(self.payload),
|
|
}
|
|
for key in ("title", "page_url", "mime_type"):
|
|
value = getattr(self.candidate, key, None)
|
|
if value is not None:
|
|
item[key] = value
|
|
return item
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Selection:
|
|
"""Outcome of inspecting a bounded window of downloaded candidates."""
|
|
|
|
kept: tuple[KeptImage, ...] = ()
|
|
format_rejects: int = 0
|
|
size_rejects: int = 0
|
|
failures: tuple[dict[str, Any], ...] = ()
|
|
|
|
@property
|
|
def inspected(self) -> int:
|
|
"""Number of candidates whose bytes were read successfully."""
|
|
|
|
return len(self.kept) + self.format_rejects + self.size_rejects
|
|
|
|
|
|
def _select(candidates: Sequence[SourceCandidate], request: Mapping[str, Any]) -> Selection:
|
|
"""Fetch and inspect candidates, keeping only what the real bytes justify.
|
|
|
|
Every kept item has been downloaded and parsed: its format comes from the
|
|
container signature and its width/height come from the PNG ``IHDR`` or the
|
|
ICO directory, never from what the source claimed. A candidate that serves
|
|
other-format bytes counts as a format rejection, a candidate of the wanted
|
|
format that is too small counts as a size rejection, and a candidate whose
|
|
bytes cannot be fetched or parsed is recorded as a failure instead of being
|
|
silently dropped.
|
|
"""
|
|
|
|
wanted_format = str(request["format"])
|
|
limit = int(request["count"])
|
|
kept: list[KeptImage] = []
|
|
format_rejects = 0
|
|
size_rejects = 0
|
|
failures: list[dict[str, Any]] = []
|
|
|
|
for candidate in candidates:
|
|
if len(kept) >= limit:
|
|
break
|
|
try:
|
|
payload = candidate.fetch_bytes()
|
|
except ToolkitError as error:
|
|
failures.append(_failure(candidate, error.to_dict()))
|
|
continue
|
|
except OSError as error:
|
|
failures.append(
|
|
_failure(candidate, error_fragment("fetch_failed", str(error)))
|
|
)
|
|
continue
|
|
detected = images.detect_format(payload)
|
|
if detected is None:
|
|
failures.append(
|
|
_failure(
|
|
candidate,
|
|
error_fragment(
|
|
"image_unreadable",
|
|
"downloaded payload is not a recognised image format",
|
|
details={"bytes": len(payload)},
|
|
),
|
|
)
|
|
)
|
|
continue
|
|
if detected != wanted_format:
|
|
format_rejects += 1
|
|
continue
|
|
try:
|
|
info = images.inspect_image(payload)
|
|
except ImageFormatError as error:
|
|
failures.append(_failure(candidate, error.to_dict()))
|
|
continue
|
|
if not _meets_minimum_size(info, request):
|
|
size_rejects += 1
|
|
continue
|
|
kept.append(KeptImage(candidate=candidate, info=info, payload=payload))
|
|
|
|
return Selection(
|
|
kept=tuple(kept),
|
|
format_rejects=format_rejects,
|
|
size_rejects=size_rejects,
|
|
failures=tuple(failures),
|
|
)
|
|
|
|
|
|
def _slug(candidate: SourceCandidate) -> str:
|
|
raw = str(getattr(candidate, "title", None) or getattr(candidate, "source_id", "") or "image").lower()
|
|
slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")
|
|
return (slug or "image")[:60]
|
|
|
|
|
|
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
|
ICO_HEADER_SIZE = 6
|
|
ICO_ENTRY_SIZE = 16
|
|
ICO_MAX_DIMENSION = 256
|
|
_PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
|
|
|
|
|
def _png_chunks(payload: bytes) -> list[tuple[bytes, bytes]]:
|
|
"""Split a PNG payload into ``(type, data)`` chunks."""
|
|
|
|
if not payload.startswith(PNG_SIGNATURE):
|
|
raise ImageResizeError("payload is not a PNG image")
|
|
chunks: list[tuple[bytes, bytes]] = []
|
|
offset = len(PNG_SIGNATURE)
|
|
while offset + 8 <= len(payload):
|
|
(length,) = struct.unpack(">I", payload[offset : offset + 4])
|
|
chunk_type = payload[offset + 4 : offset + 8]
|
|
data = payload[offset + 8 : offset + 8 + length]
|
|
if len(data) != length:
|
|
raise ImageResizeError("truncated PNG chunk")
|
|
chunks.append((chunk_type, data))
|
|
offset += 12 + length
|
|
if chunk_type == b"IEND":
|
|
return chunks
|
|
raise ImageResizeError("PNG image has no IEND chunk")
|
|
|
|
|
|
def _unfilter_rows(raw: bytes, width: int, height: int, bpp: int) -> list[bytearray]:
|
|
"""Undo the per-scanline PNG filters, returning one row per pixel line."""
|
|
|
|
stride = width * bpp
|
|
if len(raw) < height * (stride + 1):
|
|
raise ImageResizeError("truncated PNG image data")
|
|
rows: list[bytearray] = []
|
|
previous = bytearray(stride)
|
|
position = 0
|
|
for _ in range(height):
|
|
filter_type = raw[position]
|
|
position += 1
|
|
line = bytearray(raw[position : position + stride])
|
|
position += stride
|
|
if filter_type == 1:
|
|
for index in range(bpp, stride):
|
|
line[index] = (line[index] + line[index - bpp]) & 0xFF
|
|
elif filter_type == 2:
|
|
for index in range(stride):
|
|
line[index] = (line[index] + previous[index]) & 0xFF
|
|
elif filter_type == 3:
|
|
for index in range(stride):
|
|
left = line[index - bpp] if index >= bpp else 0
|
|
line[index] = (line[index] + ((left + previous[index]) >> 1)) & 0xFF
|
|
elif filter_type == 4:
|
|
for index in range(stride):
|
|
left = line[index - bpp] if index >= bpp else 0
|
|
up = previous[index]
|
|
up_left = previous[index - bpp] if index >= bpp else 0
|
|
estimate = left + up - up_left
|
|
to_left = abs(estimate - left)
|
|
to_up = abs(estimate - up)
|
|
to_up_left = abs(estimate - up_left)
|
|
if to_left <= to_up and to_left <= to_up_left:
|
|
predictor = left
|
|
elif to_up <= to_up_left:
|
|
predictor = up
|
|
else:
|
|
predictor = up_left
|
|
line[index] = (line[index] + predictor) & 0xFF
|
|
elif filter_type != 0:
|
|
raise ImageResizeError(f"unsupported PNG filter type {filter_type}")
|
|
previous = line
|
|
rows.append(line)
|
|
return rows
|
|
|
|
|
|
def _row_to_rgba(
|
|
row: bytearray,
|
|
width: int,
|
|
color_type: int,
|
|
depth: int,
|
|
palette: bytes,
|
|
transparency: bytes | None,
|
|
) -> bytes:
|
|
"""Convert one unfiltered scanline into 8-bit RGBA samples."""
|
|
|
|
step = depth // 8
|
|
out = bytearray()
|
|
if color_type == 3:
|
|
for x in range(width):
|
|
index = row[x]
|
|
base = index * 3
|
|
if base + 2 >= len(palette):
|
|
raise ImageResizeError("PNG palette index is out of range")
|
|
alpha = transparency[index] if transparency and index < len(transparency) else 255
|
|
out += bytes((palette[base], palette[base + 1], palette[base + 2], alpha))
|
|
return bytes(out)
|
|
for x in range(width):
|
|
offset = x * step
|
|
if color_type == 0:
|
|
value = row[offset]
|
|
out += bytes((value, value, value, 255))
|
|
elif color_type == 4:
|
|
value = row[offset]
|
|
out += bytes((value, value, value, row[offset + step]))
|
|
elif color_type == 2:
|
|
out += bytes((row[offset], row[offset + step], row[offset + 2 * step], 255))
|
|
else:
|
|
out += bytes(
|
|
(
|
|
row[offset],
|
|
row[offset + step],
|
|
row[offset + 2 * step],
|
|
row[offset + 3 * step],
|
|
)
|
|
)
|
|
return bytes(out)
|
|
|
|
|
|
def _decode_png(payload: bytes) -> tuple[int, int, list[bytes]]:
|
|
"""Decode a PNG payload into ``(width, height, rgba_rows)``."""
|
|
|
|
header: tuple[int, ...] | None = None
|
|
palette = b""
|
|
transparency: bytes | None = None
|
|
data = bytearray()
|
|
for chunk_type, chunk_data in _png_chunks(payload):
|
|
if chunk_type == b"IHDR":
|
|
if len(chunk_data) < 13:
|
|
raise ImageResizeError("truncated PNG header")
|
|
header = struct.unpack(">IIBBBBB", chunk_data[:13])
|
|
elif chunk_type == b"PLTE":
|
|
palette = chunk_data
|
|
elif chunk_type == b"tRNS":
|
|
transparency = chunk_data
|
|
elif chunk_type == b"IDAT":
|
|
data += chunk_data
|
|
if header is None:
|
|
raise ImageResizeError("PNG image has no IHDR chunk")
|
|
width, height, depth, color_type, compression, filter_method, interlace = header
|
|
if width < 1 or height < 1:
|
|
raise ImageResizeError("PNG image has no pixels")
|
|
if compression != 0 or filter_method != 0:
|
|
raise ImageResizeError("unsupported PNG compression or filter method")
|
|
if interlace != 0:
|
|
raise ImageResizeError("interlaced PNG images are not supported")
|
|
channels = _PNG_CHANNELS.get(color_type)
|
|
if channels is None:
|
|
raise ImageResizeError(f"unsupported PNG colour type {color_type}")
|
|
if depth not in (8, 16):
|
|
raise ImageResizeError(f"unsupported PNG bit depth {depth}")
|
|
if color_type == 3 and depth > 8:
|
|
raise ImageResizeError("unsupported PNG palette bit depth")
|
|
if color_type == 3 and not palette:
|
|
raise ImageResizeError("PNG palette image has no PLTE chunk")
|
|
try:
|
|
raw = zlib.decompress(bytes(data))
|
|
except zlib.error as exc:
|
|
raise ImageResizeError(f"PNG image data is corrupt: {exc}") from exc
|
|
rows = _unfilter_rows(raw, width, height, channels * (depth // 8))
|
|
return width, height, [
|
|
_row_to_rgba(row, width, color_type, depth, palette, transparency) for row in rows
|
|
]
|
|
|
|
|
|
def _encode_png(width: int, height: int, rows: Sequence[bytes]) -> bytes:
|
|
"""Encode RGBA rows as an uncompressed-filter 8-bit PNG."""
|
|
|
|
raw = bytearray()
|
|
for row in rows:
|
|
raw.append(0)
|
|
raw += row
|
|
|
|
def chunk(chunk_type: bytes, chunk_data: bytes) -> bytes:
|
|
crc = zlib.crc32(chunk_type + chunk_data) & 0xFFFFFFFF
|
|
return struct.pack(">I", len(chunk_data)) + chunk_type + chunk_data + struct.pack(">I", crc)
|
|
|
|
header = struct.pack(">IIBBBBB", width, height, 8, 6, 0, 0, 0)
|
|
return (
|
|
PNG_SIGNATURE
|
|
+ chunk(b"IHDR", header)
|
|
+ chunk(b"IDAT", zlib.compress(bytes(raw), 6))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def _ico_entries(payload: bytes) -> list[tuple[int, int, int, int]]:
|
|
"""Return ``(width, height, size, offset)`` for every ICO directory entry."""
|
|
|
|
if len(payload) < ICO_HEADER_SIZE:
|
|
raise ImageResizeError("payload is not an ICO image")
|
|
reserved, kind, count = struct.unpack("<HHH", payload[:ICO_HEADER_SIZE])
|
|
if reserved != 0 or kind != 1 or count < 1:
|
|
raise ImageResizeError("payload is not an ICO image")
|
|
entries: list[tuple[int, int, int, int]] = []
|
|
for index in range(count):
|
|
start = ICO_HEADER_SIZE + index * ICO_ENTRY_SIZE
|
|
if start + ICO_ENTRY_SIZE > len(payload):
|
|
raise ImageResizeError("truncated ICO directory")
|
|
width, height, _colours, _reserved, _planes, _bits, size, offset = struct.unpack(
|
|
"<BBBBHHII", payload[start : start + ICO_ENTRY_SIZE]
|
|
)
|
|
entries.append((width or ICO_MAX_DIMENSION, height or ICO_MAX_DIMENSION, size, offset))
|
|
return entries
|
|
|
|
|
|
def _encode_ico(png_payload: bytes, width: int, height: int) -> bytes:
|
|
"""Wrap a PNG payload in a single-entry ICO container."""
|
|
|
|
if width > ICO_MAX_DIMENSION or height > ICO_MAX_DIMENSION:
|
|
raise ImageResizeError("ICO images cannot exceed 256x256 pixels")
|
|
header = struct.pack("<HHH", 0, 1, 1)
|
|
entry = struct.pack(
|
|
"<BBBBHHII",
|
|
width % ICO_MAX_DIMENSION,
|
|
height % ICO_MAX_DIMENSION,
|
|
0,
|
|
0,
|
|
1,
|
|
32,
|
|
len(png_payload),
|
|
ICO_HEADER_SIZE + ICO_ENTRY_SIZE,
|
|
)
|
|
return header + entry + png_payload
|
|
|
|
|
|
def _decode_image(payload: bytes) -> tuple[str, int, int, list[bytes]]:
|
|
"""Decode a PNG or ICO payload into ``(container, width, height, rows)``."""
|
|
|
|
if payload.startswith(PNG_SIGNATURE):
|
|
width, height, rows = _decode_png(payload)
|
|
return "png", width, height, rows
|
|
if len(payload) >= ICO_HEADER_SIZE and payload[:4] == b"\x00\x00\x01\x00":
|
|
entries = _ico_entries(payload)
|
|
_width, _height, size, offset = max(entries, key=lambda item: item[0] * item[1])
|
|
embedded = payload[offset : offset + size]
|
|
if not embedded.startswith(PNG_SIGNATURE):
|
|
raise ImageResizeError("ICO images stored as raw bitmaps are not supported")
|
|
width, height, rows = _decode_png(embedded)
|
|
return "ico", width, height, rows
|
|
raise ImageResizeError("payload is neither a PNG nor an ICO image")
|
|
|
|
|
|
def _target_size(width: int, height: int, request: Mapping[str, Any]) -> tuple[int, int]:
|
|
"""Shrink-to-fit target inside the requested box, never enlarging."""
|
|
|
|
wanted_width = request.get("width")
|
|
wanted_height = request.get("height")
|
|
scale = 1.0
|
|
if wanted_width is not None:
|
|
scale = min(scale, wanted_width / width)
|
|
if wanted_height is not None:
|
|
scale = min(scale, wanted_height / height)
|
|
if scale >= 1.0:
|
|
return width, height
|
|
return max(1, int(round(width * scale))), max(1, int(round(height * scale)))
|
|
|
|
|
|
def _resample(rows: Sequence[bytes], width: int, height: int, target_width: int, target_height: int) -> list[bytes]:
|
|
"""Nearest-neighbour downscale of RGBA rows."""
|
|
|
|
columns = [min(width - 1, (x * width) // target_width) * 4 for x in range(target_width)]
|
|
resized: list[bytes] = []
|
|
for y in range(target_height):
|
|
source = rows[min(height - 1, (y * height) // target_height)]
|
|
line = bytearray()
|
|
for offset in columns:
|
|
line += source[offset : offset + 4]
|
|
resized.append(bytes(line))
|
|
return resized
|
|
|
|
|
|
def resize_image(payload: bytes, request: Mapping[str, Any]) -> tuple[bytes, dict[str, Any]]:
|
|
"""Downscale ``payload`` so it fits the requested size.
|
|
|
|
The requested width and height are a bounding box: the aspect ratio is
|
|
preserved, an image that already fits is left untouched and an image is
|
|
never scaled up. The returned fragment records what happened so the caller
|
|
can report the real dimensions of the stored file.
|
|
"""
|
|
|
|
container, width, height, rows = _decode_image(payload)
|
|
target_width, target_height = _target_size(width, height, request)
|
|
fragment: dict[str, Any] = {
|
|
"requested": {"width": request.get("width"), "height": request.get("height")},
|
|
"from": {"width": width, "height": height},
|
|
}
|
|
if (target_width, target_height) == (width, height):
|
|
fragment.update(
|
|
{"applied": False, "reason": "already_fits", "to": {"width": width, "height": height}}
|
|
)
|
|
return payload, fragment
|
|
png_payload = _encode_png(
|
|
target_width, target_height, _resample(rows, width, height, target_width, target_height)
|
|
)
|
|
output = (
|
|
_encode_ico(png_payload, target_width, target_height)
|
|
if container == "ico"
|
|
else png_payload
|
|
)
|
|
fragment.update({"applied": True, "to": {"width": target_width, "height": target_height}})
|
|
return output, fragment
|
|
|
|
|
|
def _download(
|
|
selected: Sequence[KeptImage], request: Mapping[str, Any]
|
|
) -> tuple[list[dict[str, Any]], int]:
|
|
"""Write the already-fetched payloads; return ``(items, failure_count)``.
|
|
|
|
The bytes verified by :func:`_select` are stored as they are (or downscaled
|
|
first when ``resize`` is enabled), so an item can never be downloaded twice
|
|
or stored in a format that was not verified. A payload that cannot be
|
|
downscaled to the requested size is reported as a failed item instead of
|
|
being silently stored at the wrong size.
|
|
"""
|
|
|
|
directory = Path(request["output_dir"]).expanduser()
|
|
try:
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
except OSError as exc:
|
|
raise SourceAccessError(
|
|
f"cannot create output directory {str(directory)!r}: {exc.strerror or exc}",
|
|
details={"output_dir": str(directory)},
|
|
) from exc
|
|
|
|
suffix = "." + str(request["format"])
|
|
items: list[dict[str, Any]] = []
|
|
failures = 0
|
|
for index, entry in enumerate(selected, start=1):
|
|
record = entry.to_item()
|
|
destination = directory / f"{index:03d}-{_slug(entry.candidate)}{suffix}"
|
|
payload = entry.payload
|
|
try:
|
|
resize_fragment = resize_image(payload, request) if request["resize"] else None
|
|
if resize_fragment is not None:
|
|
payload, resize_fragment = resize_fragment
|
|
destination.write_bytes(payload)
|
|
except (ToolkitError, OSError) as exc:
|
|
failures += 1
|
|
error = error_fragment(getattr(exc, "code", "download_failed"), str(exc))
|
|
if isinstance(exc, ToolkitError) and exc.details:
|
|
error["details"] = dict(exc.details)
|
|
record["download"] = {"ok": False, "error": error}
|
|
else:
|
|
record["download"] = {"ok": True, "path": str(destination), "bytes": len(payload)}
|
|
if resize_fragment is not None:
|
|
record["download"]["resize"] = resize_fragment
|
|
record["download"]["width"] = resize_fragment["to"]["width"]
|
|
record["download"]["height"] = resize_fragment["to"]["height"]
|
|
items.append(record)
|
|
return items, failures
|
|
|
|
|
|
def _guess_source(argv: Sequence[str]) -> str | None:
|
|
"""Best-effort source id extraction for error envelopes."""
|
|
|
|
for index, token in enumerate(argv):
|
|
if token == "--source" or token == "-s":
|
|
if index + 1 < len(argv):
|
|
return argv[index + 1]
|
|
elif token.startswith("--source="):
|
|
return token.split("=", 1)[1]
|
|
return None
|
|
|
|
|
|
def _request_from(argv: Sequence[str]) -> tuple[dict[str, Any], str, Any]:
|
|
"""Turn CLI arguments into ``(normalized_request, source_id, registry)``."""
|
|
|
|
parser = build_parser()
|
|
namespace = parser.parse_args(list(argv))
|
|
flags = _flags_to_mapping(namespace)
|
|
document = namespace.request
|
|
|
|
if document is not None:
|
|
if flags:
|
|
raise RequestError(
|
|
"a request document cannot be combined with flags: "
|
|
+ ", ".join(sorted(flags)),
|
|
details={"conflicting_flags": sorted(flags)},
|
|
accepted_fields=REQUEST_FIELDS,
|
|
)
|
|
raw = _document_to_mapping(_load_document(document))
|
|
elif flags:
|
|
raw = flags
|
|
else:
|
|
raise RequestError(
|
|
"no request given; pass a JSON request document or at least one flag",
|
|
accepted_fields=REQUEST_FIELDS,
|
|
)
|
|
|
|
request = normalize_request(raw)
|
|
registry = default_registry()
|
|
available = list(registry.available_sources())
|
|
if request["source"] not in available:
|
|
raise UnknownSourceError(request["source"], available)
|
|
return request, request["source"], registry
|
|
|
|
|
|
def _execute(argv: Sequence[str]) -> Response:
|
|
"""Run one request and return the response that must be printed.
|
|
|
|
The search path downloads a bounded window of candidates and filters that
|
|
window on the parsed bytes, so the answer distinguishes three different
|
|
situations through both the status and the exit code:
|
|
|
|
* ``ok`` -- at least one image really is the requested format and at least
|
|
the requested width and height;
|
|
* ``no_format`` -- the source had hits, but none of them really are the
|
|
requested format;
|
|
* ``no_size`` -- images of the requested format existed, but all of them
|
|
are smaller than requested;
|
|
* ``access_error`` -- the source could not be searched or served bytes that
|
|
are not readable images, which is never reported as a "no match".
|
|
"""
|
|
|
|
request, source, registry = _request_from(argv)
|
|
builder = ResponseBuilder(source=source, request=request)
|
|
|
|
if request["dry_run"]:
|
|
return builder.dry_run()
|
|
|
|
cap = candidate_cap(request["count"])
|
|
candidates = registry.search(source, request["topic"], cap)
|
|
selection = _select(candidates, request)
|
|
unreadable = list(selection.failures)
|
|
|
|
if not selection.kept:
|
|
details = {
|
|
"requested_format": request["format"],
|
|
"requested_size": {"width": request["width"], "height": request["height"]},
|
|
"minimum_size": {"width": request["width"], "height": request["height"]},
|
|
"candidate_count": len(candidates),
|
|
"candidates_inspected": selection.inspected,
|
|
"format_matches": selection.size_rejects,
|
|
"size_matches": 0,
|
|
"unreadable_candidates": len(unreadable),
|
|
}
|
|
if unreadable:
|
|
return builder.access_error(
|
|
error=error_fragment(
|
|
"candidate_unreadable",
|
|
"the source served images that could not be read",
|
|
details={**details, "failures": unreadable},
|
|
)
|
|
)
|
|
if selection.size_rejects:
|
|
return builder.build(
|
|
STATUS_NO_SIZE,
|
|
error=error_fragment(
|
|
"no_size_match",
|
|
"the source returned no images at least the requested width and height",
|
|
details=details,
|
|
),
|
|
)
|
|
return builder.build(
|
|
STATUS_NO_FORMAT,
|
|
error=error_fragment(
|
|
"no_format_match",
|
|
f"the source returned no {request['format']} images for {request['topic']!r}",
|
|
details=details,
|
|
),
|
|
)
|
|
|
|
if not request["download"]:
|
|
if unreadable:
|
|
return builder.build(
|
|
STATUS_PARTIAL_DOWNLOAD,
|
|
items=[entry.to_item() for entry in selection.kept],
|
|
error=error_fragment(
|
|
"candidate_incomplete",
|
|
f"{len(unreadable)} of {selection.inspected + len(unreadable)} "
|
|
"candidates could not be read",
|
|
details={
|
|
"failed": len(unreadable),
|
|
"total": selection.inspected + len(unreadable),
|
|
"failures": unreadable,
|
|
},
|
|
),
|
|
)
|
|
return builder.build(STATUS_OK, items=[entry.to_item() for entry in selection.kept])
|
|
|
|
items, download_failures = _download(selection.kept, request)
|
|
failed = download_failures + len(unreadable)
|
|
if download_failures == len(items):
|
|
return builder.build(
|
|
STATUS_ACCESS_ERROR,
|
|
items=items,
|
|
error=error_fragment(
|
|
"download_failed",
|
|
"every download failed",
|
|
details={"failed": download_failures, "total": len(items)},
|
|
),
|
|
)
|
|
if failed:
|
|
return builder.build(
|
|
STATUS_PARTIAL_DOWNLOAD,
|
|
items=items,
|
|
error=error_fragment(
|
|
"download_incomplete",
|
|
f"{failed} of {len(items) + len(unreadable)} images could not be handled",
|
|
details={
|
|
"failed": failed,
|
|
"total": len(items) + len(unreadable),
|
|
"failures": unreadable,
|
|
},
|
|
),
|
|
)
|
|
return builder.build(STATUS_OK, items=items)
|
|
|
|
|
|
def main(argv: Sequence[str] | None = None) -> int:
|
|
"""Entry point: emit exactly one JSON document and return the exit code."""
|
|
|
|
arguments = list(sys.argv[1:] if argv is None else argv)
|
|
try:
|
|
response = _execute(arguments)
|
|
except ToolkitError as error:
|
|
builder = ResponseBuilder(source=_guess_source(arguments))
|
|
response = builder.from_error(error)
|
|
except OSError as error:
|
|
builder = ResponseBuilder(source=_guess_source(arguments))
|
|
response = builder.build(
|
|
STATUS_ACCESS_ERROR,
|
|
error=error_fragment("access_error", str(error)),
|
|
)
|
|
response.emit()
|
|
return response.exit_code
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover - module entry point
|
|
sys.exit(main())
|