Tool Backend
f20e58e85c
Repository Validate / repository-checks (pull_request) Successful in 3s
Completed the native-process CLI for `python -m tools.example`. Added `tools/envelope.py` with the fixed status vocabulary (`ok`, `dry_run`, `invalid_request`, `no_format`, `no_size`, `access_error`, `partial_download`) mapped to exit codes 0/0/2/3/4/5/6 and the `Response`/`ResponseBuilder` JSON envelope (keys: status, source, request, items, error). Extended `tools/errors.py` with request errors (`RequestError`, `RequestDocumentError`, `RequestFieldError`, `UnsupportedFormatError`) and `ACCEPTED_FORMATS`. Rewrote `tools/example.py` as a plain executable CLI accepting either a JSON request document or flags (--topic, --width, --height, --format png|ico, --source, --count, --download, --output-dir, --resize, --dry-run), with a custom parser that turns usage errors into `invalid_request` envelopes, unknown sources reported together with the available source ids, unsupported formats rejected with the accepted values, and format/size filtering, download and partial-download handling mapped onto the shared status table. `examples/input.json` is now the canonical offline dry_run request, and `tests/test_cli.py` (new) plus a rewritten `tests/test_example.py` cover the status/exit-code table, single-JSON-document output, help text, unknown source/unsupported format rejection and the download branches.
Tests:
Facade checks run from /workspace with the prepared .venv: `python -m compileall tools tests` (tools.validate) OK; `pytest` (tools.test) 76 passed; `python -m tools.example examples/input.json` (tools.smoke) exit 0 with a single `dry_run` JSON document; `python -m tools.example --help` exits 0 and documents every parameter plus the JSON document contract. Origin target branch `origin/task/1-task` (1f929c5) is an ancestor of the work branch, so the target is integrated and the checks above ran on the integrated revision. No dependency changes were needed, so no prepare_environment update was required.
Progress:
Implemented envelope status/exit-code table and response builder, request errors, the full flag/JSON-document CLI in tools/example.py, canonical offline examples/input.json, and CLI tests; facade validate/test/smoke and --help all pass (76 tests).
Published by developer-agent through Tool Backend.
Closes #5
524 lines
17 KiB
Python
524 lines
17 KiB
Python
"""Plain native-process CLI for the image toolkit.
|
|
|
|
Usage::
|
|
|
|
python -m tools.example examples/input.json
|
|
python -m tools.example --topic "red fox" --width 640 --height 480 --format png
|
|
|
|
Exactly one machine-readable JSON document is written to stdout. The process
|
|
exit code is derived from the fixed status vocabulary in :mod:`tools.envelope`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
from .envelope import (
|
|
STATUS_ACCESS_ERROR,
|
|
STATUS_NO_FORMAT,
|
|
STATUS_NO_SIZE,
|
|
STATUS_OK,
|
|
STATUS_PARTIAL_DOWNLOAD,
|
|
Response,
|
|
ResponseBuilder,
|
|
error_fragment,
|
|
)
|
|
from .errors import (
|
|
ACCEPTED_FORMATS,
|
|
RequestDocumentError,
|
|
RequestError,
|
|
RequestFieldError,
|
|
SourceAccessError,
|
|
ToolkitError,
|
|
UnknownSourceError,
|
|
UnsupportedFormatError,
|
|
)
|
|
from .sources import DEFAULT_LIMIT, SourceCandidate, default_registry
|
|
|
|
PROG = "python -m tools.example"
|
|
|
|
#: Keys accepted in a JSON request document (``version`` is optional metadata).
|
|
REQUEST_FIELDS: tuple[str, ...] = (
|
|
"topic",
|
|
"width",
|
|
"height",
|
|
"format",
|
|
"source",
|
|
"count",
|
|
"download",
|
|
"output_dir",
|
|
"resize",
|
|
"dry_run",
|
|
)
|
|
|
|
OPTIONAL_DOCUMENT_FIELDS: tuple[str, ...] = ("version",)
|
|
|
|
DEFAULT_SOURCE = "wikimedia_commons"
|
|
DEFAULT_OUTPUT_DIR = "downloads"
|
|
MIN_COUNT = 1
|
|
MAX_COUNT = 50
|
|
|
|
#: MIME types (and filename suffixes) that count as a hit for each format.
|
|
FORMAT_MATCHERS: dict[str, tuple[str, ...]] = {
|
|
"png": ("image/png", ".png"),
|
|
"ico": ("image/x-icon", "image/vnd.microsoft.icon", "image/ico", ".ico"),
|
|
}
|
|
|
|
|
|
class _Parser(argparse.ArgumentParser):
|
|
"""Argument parser that reports bad flags as request errors."""
|
|
|
|
def error(self, message: str) -> Any: # pragma: no cover - argparse hook
|
|
raise RequestError(message, accepted_fields=REQUEST_FIELDS)
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
"""Return the CLI parser; ``--help`` documents every supported parameter."""
|
|
|
|
parser = _Parser(
|
|
prog=PROG,
|
|
description=(
|
|
"Search an image source and report the matching items as one JSON "
|
|
"document. Either pass a JSON request document or use the flags."
|
|
),
|
|
epilog=(
|
|
"Every run writes exactly one JSON document to stdout with the keys "
|
|
"status, source, request, items and error."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"request",
|
|
nargs="?",
|
|
metavar="REQUEST.json",
|
|
help=(
|
|
"path to a JSON request document holding the same fields as the "
|
|
"flags; it cannot be combined with any of the flags"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"-t",
|
|
"--topic",
|
|
metavar="TEXT",
|
|
help="search topic (required unless a request document is given)",
|
|
)
|
|
parser.add_argument(
|
|
"--width",
|
|
type=int,
|
|
metavar="PX",
|
|
help="desired image width in pixels",
|
|
)
|
|
parser.add_argument(
|
|
"--height",
|
|
type=int,
|
|
metavar="PX",
|
|
help="desired image height in pixels",
|
|
)
|
|
parser.add_argument(
|
|
"-f",
|
|
"--format",
|
|
metavar="FMT",
|
|
help="image format: " + "|".join(ACCEPTED_FORMATS),
|
|
)
|
|
parser.add_argument(
|
|
"-s",
|
|
"--source",
|
|
metavar="ID",
|
|
help=f"image source id (default: {DEFAULT_SOURCE}); unknown ids are rejected",
|
|
)
|
|
parser.add_argument(
|
|
"-c",
|
|
"--count",
|
|
type=int,
|
|
metavar="N",
|
|
help=f"how many candidates to request, {MIN_COUNT}-{MAX_COUNT} (default: {DEFAULT_LIMIT})",
|
|
)
|
|
parser.add_argument(
|
|
"--download",
|
|
action="store_true",
|
|
help="download every matching image into the output directory",
|
|
)
|
|
parser.add_argument(
|
|
"-o",
|
|
"--output-dir",
|
|
metavar="DIR",
|
|
help=f"download directory (default: ./{DEFAULT_OUTPUT_DIR})",
|
|
)
|
|
parser.add_argument(
|
|
"--resize",
|
|
action="store_true",
|
|
help=(
|
|
"accept images larger than the requested size and downscale them; "
|
|
"without this flag the size must match exactly"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
action="store_true",
|
|
help="validate the request and emit the envelope without any network access",
|
|
)
|
|
return parser
|
|
|
|
|
|
def _load_document(path: str) -> Mapping[str, Any]:
|
|
"""Read a JSON request document from ``path``."""
|
|
|
|
target = Path(path)
|
|
try:
|
|
text = target.read_text(encoding="utf-8")
|
|
except OSError as exc:
|
|
raise RequestDocumentError(
|
|
f"cannot read request document {path!r}: {exc.strerror or exc}",
|
|
details={"path": str(path)},
|
|
) from exc
|
|
try:
|
|
payload = json.loads(text)
|
|
except json.JSONDecodeError as exc:
|
|
raise RequestDocumentError(
|
|
f"request document {path!r} is not valid JSON: {exc.msg}",
|
|
details={"path": str(path), "line": exc.lineno, "column": exc.colno},
|
|
) from exc
|
|
if not isinstance(payload, Mapping):
|
|
raise RequestDocumentError(
|
|
f"request document {path!r} must contain a JSON object",
|
|
details={"path": str(path), "type": type(payload).__name__},
|
|
)
|
|
return payload
|
|
|
|
|
|
def _document_to_mapping(payload: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""Reject unknown keys and return the accepted request fields."""
|
|
|
|
accepted = REQUEST_FIELDS + OPTIONAL_DOCUMENT_FIELDS
|
|
unknown = sorted(str(key) for key in payload if key not in accepted)
|
|
if unknown:
|
|
raise RequestFieldError(
|
|
"request document contains unsupported fields: " + ", ".join(unknown),
|
|
field=unknown[0],
|
|
details={"unknown_fields": unknown, "accepted_fields": list(REQUEST_FIELDS)},
|
|
)
|
|
return {key: payload[key] for key in REQUEST_FIELDS if key in payload}
|
|
|
|
|
|
def _flags_to_mapping(namespace: argparse.Namespace) -> dict[str, Any]:
|
|
"""Collect the explicitly supplied flags into a request mapping."""
|
|
|
|
mapping: dict[str, Any] = {}
|
|
for field in ("topic", "width", "height", "format", "source", "count", "output_dir"):
|
|
value = getattr(namespace, field, None)
|
|
if value is not None:
|
|
mapping[field] = value
|
|
for field in ("download", "resize", "dry_run"):
|
|
if getattr(namespace, field, False):
|
|
mapping[field] = True
|
|
return mapping
|
|
|
|
|
|
def _as_text(value: Any, field: str) -> str:
|
|
if not isinstance(value, str):
|
|
raise RequestFieldError(
|
|
f"{field} must be a string, got {type(value).__name__}", field=field
|
|
)
|
|
text = value.strip()
|
|
if not text:
|
|
raise RequestFieldError(f"{field} must not be empty", field=field)
|
|
return text
|
|
|
|
|
|
def _as_int(value: Any, field: str) -> int:
|
|
if isinstance(value, bool) or not isinstance(value, int):
|
|
raise RequestFieldError(
|
|
f"{field} must be an integer, got {type(value).__name__}", field=field
|
|
)
|
|
return int(value)
|
|
|
|
|
|
def _as_bool(value: Any, field: str) -> bool:
|
|
if not isinstance(value, bool):
|
|
raise RequestFieldError(
|
|
f"{field} must be a boolean, got {type(value).__name__}", field=field
|
|
)
|
|
return bool(value)
|
|
|
|
|
|
def normalize_request(raw: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""Validate ``raw`` and return the complete, typed request mapping."""
|
|
|
|
if "topic" not in raw:
|
|
raise RequestFieldError(
|
|
"topic is required",
|
|
field="topic",
|
|
details={"required_fields": ["topic"], "accepted_fields": list(REQUEST_FIELDS)},
|
|
)
|
|
topic = _as_text(raw["topic"], "topic")
|
|
|
|
size: dict[str, int | None] = {"width": None, "height": None}
|
|
for field in ("width", "height"):
|
|
if raw.get(field) is None:
|
|
continue
|
|
number = _as_int(raw[field], field)
|
|
if number <= 0:
|
|
raise RequestFieldError(f"{field} must be positive", field=field)
|
|
size[field] = number
|
|
|
|
if "format" not in raw or raw["format"] is None:
|
|
raise RequestFieldError(
|
|
"format is required",
|
|
field="format",
|
|
details={"accepted_formats": list(ACCEPTED_FORMATS)},
|
|
)
|
|
image_format = _as_text(raw["format"], "format").lower()
|
|
if image_format not in ACCEPTED_FORMATS:
|
|
raise UnsupportedFormatError(image_format)
|
|
|
|
source = DEFAULT_SOURCE
|
|
if raw.get("source") is not None:
|
|
source = _as_text(raw["source"], "source")
|
|
|
|
count = DEFAULT_LIMIT
|
|
if raw.get("count") is not None:
|
|
count = _as_int(raw["count"], "count")
|
|
if not MIN_COUNT <= count <= MAX_COUNT:
|
|
raise RequestFieldError(
|
|
f"count must be between {MIN_COUNT} and {MAX_COUNT}",
|
|
field="count",
|
|
details={"min": MIN_COUNT, "max": MAX_COUNT},
|
|
)
|
|
|
|
output_dir = DEFAULT_OUTPUT_DIR
|
|
if raw.get("output_dir") is not None:
|
|
output_dir = _as_text(raw["output_dir"], "output_dir")
|
|
|
|
return {
|
|
"topic": topic,
|
|
"width": size["width"],
|
|
"height": size["height"],
|
|
"format": image_format,
|
|
"source": source,
|
|
"count": count,
|
|
"download": _as_bool(raw.get("download", False), "download"),
|
|
"output_dir": output_dir,
|
|
"resize": _as_bool(raw.get("resize", False), "resize"),
|
|
"dry_run": _as_bool(raw.get("dry_run", False), "dry_run"),
|
|
}
|
|
|
|
|
|
def _matches_format(candidate: SourceCandidate, image_format: str) -> bool:
|
|
matchers = FORMAT_MATCHERS[image_format]
|
|
mime = str(getattr(candidate, "mime_type", None) or "").split(";")[0].strip().lower()
|
|
if mime and mime in matchers:
|
|
return True
|
|
url = str(getattr(candidate, "file_url", "") or "").lower()
|
|
return any(url.endswith(suffix) for suffix in matchers if suffix.startswith("."))
|
|
|
|
|
|
def _matches_size(candidate: SourceCandidate, request: Mapping[str, Any]) -> bool:
|
|
width = request.get("width")
|
|
height = request.get("height")
|
|
if width is None and height is None:
|
|
return True
|
|
actual_width = getattr(candidate, "width", None)
|
|
actual_height = getattr(candidate, "height", None)
|
|
if isinstance(actual_width, bool) or isinstance(actual_height, bool):
|
|
return False
|
|
if not isinstance(actual_width, int) or not isinstance(actual_height, int):
|
|
return False
|
|
if request.get("resize"):
|
|
fits_width = width is None or actual_width >= width
|
|
fits_height = height is None or actual_height >= height
|
|
return fits_width and fits_height
|
|
matches_width = width is None or actual_width == width
|
|
matches_height = height is None or actual_height == height
|
|
return matches_width and matches_height
|
|
|
|
|
|
def _select(candidates: Sequence[SourceCandidate], request: Mapping[str, Any]) -> tuple[list[SourceCandidate], list[SourceCandidate]]:
|
|
"""Return ``(matching, rejected_by_size)`` split for the request."""
|
|
|
|
by_format = [c for c in candidates if _matches_format(c, request["format"])]
|
|
matching = [c for c in by_format if _matches_size(c, request)]
|
|
rejected = [c for c in by_format if not _matches_size(c, request)]
|
|
return matching, rejected
|
|
|
|
|
|
def _slug(candidate: SourceCandidate) -> str:
|
|
raw = str(getattr(candidate, "title", None) or getattr(candidate, "source_id", "") or "image").lower()
|
|
slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")
|
|
return (slug or "image")[:60]
|
|
|
|
|
|
def _download(candidates: Sequence[SourceCandidate], request: Mapping[str, Any]) -> tuple[list[dict[str, Any]], int]:
|
|
"""Download ``candidates``; return ``(items, failure_count)``."""
|
|
|
|
directory = Path(request["output_dir"]).expanduser()
|
|
try:
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
except OSError as exc:
|
|
raise SourceAccessError(
|
|
f"cannot create output directory {str(directory)!r}: {exc.strerror or exc}",
|
|
details={"output_dir": str(directory)},
|
|
) from exc
|
|
|
|
suffix = "." + str(request["format"])
|
|
items: list[dict[str, Any]] = []
|
|
failures = 0
|
|
for index, candidate in enumerate(candidates, start=1):
|
|
record = dict(candidate.to_dict())
|
|
destination = directory / f"{index:03d}-{_slug(candidate)}{suffix}"
|
|
try:
|
|
payload = candidate.fetch_bytes()
|
|
destination.write_bytes(payload)
|
|
except (ToolkitError, OSError) as exc:
|
|
failures += 1
|
|
record["download"] = {
|
|
"ok": False,
|
|
"error": error_fragment(getattr(exc, "code", "download_failed"), str(exc)),
|
|
}
|
|
else:
|
|
record["download"] = {
|
|
"ok": True,
|
|
"path": str(destination),
|
|
"bytes": len(payload),
|
|
}
|
|
items.append(record)
|
|
return items, failures
|
|
|
|
|
|
def _guess_source(argv: Sequence[str]) -> str | None:
|
|
"""Best-effort source id extraction for error envelopes."""
|
|
|
|
for index, token in enumerate(argv):
|
|
if token == "--source" or token == "-s":
|
|
if index + 1 < len(argv):
|
|
return argv[index + 1]
|
|
elif token.startswith("--source="):
|
|
return token.split("=", 1)[1]
|
|
return None
|
|
|
|
|
|
def _request_from(argv: Sequence[str]) -> tuple[dict[str, Any], str, Any]:
|
|
"""Turn CLI arguments into ``(normalized_request, source_id, registry)``."""
|
|
|
|
parser = build_parser()
|
|
namespace = parser.parse_args(list(argv))
|
|
flags = _flags_to_mapping(namespace)
|
|
document = namespace.request
|
|
|
|
if document is not None:
|
|
if flags:
|
|
raise RequestError(
|
|
"a request document cannot be combined with flags: "
|
|
+ ", ".join(sorted(flags)),
|
|
details={"conflicting_flags": sorted(flags)},
|
|
accepted_fields=REQUEST_FIELDS,
|
|
)
|
|
raw = _document_to_mapping(_load_document(document))
|
|
elif flags:
|
|
raw = flags
|
|
else:
|
|
raise RequestError(
|
|
"no request given; pass a JSON request document or at least one flag",
|
|
accepted_fields=REQUEST_FIELDS,
|
|
)
|
|
|
|
request = normalize_request(raw)
|
|
registry = default_registry()
|
|
available = list(registry.available_sources())
|
|
if request["source"] not in available:
|
|
raise UnknownSourceError(request["source"], available)
|
|
return request, request["source"], registry
|
|
|
|
|
|
def _execute(argv: Sequence[str]) -> Response:
|
|
"""Run one request and return the response that must be printed."""
|
|
|
|
request, source, registry = _request_from(argv)
|
|
builder = ResponseBuilder(source=source, request=request)
|
|
|
|
if request["dry_run"]:
|
|
return builder.dry_run()
|
|
|
|
candidates = registry.search(source, request["topic"], request["count"])
|
|
matching, rejected = _select(candidates, request)
|
|
|
|
if not matching:
|
|
details = {
|
|
"requested_format": request["format"],
|
|
"width": request["width"],
|
|
"height": request["height"],
|
|
"resize": request["resize"],
|
|
"candidate_count": len(candidates),
|
|
"format_matches": len(matching) + len(rejected),
|
|
"size_matches": 0,
|
|
}
|
|
if rejected:
|
|
return builder.build(
|
|
STATUS_NO_SIZE,
|
|
error=error_fragment(
|
|
"no_size_match",
|
|
"no image matched the requested width and height",
|
|
details=details,
|
|
),
|
|
)
|
|
return builder.build(
|
|
STATUS_NO_FORMAT,
|
|
error=error_fragment(
|
|
"no_format_match",
|
|
f"the source returned no {request['format']} images for {request['topic']!r}",
|
|
details=details,
|
|
),
|
|
)
|
|
|
|
if not request["download"]:
|
|
return builder.build(STATUS_OK, items=[c.to_dict() for c in matching])
|
|
|
|
items, failures = _download(matching, request)
|
|
if failures and failures == len(items):
|
|
return builder.build(
|
|
STATUS_ACCESS_ERROR,
|
|
items=items,
|
|
error=error_fragment(
|
|
"download_failed",
|
|
"every download failed",
|
|
details={"failed": failures, "total": len(items)},
|
|
),
|
|
)
|
|
if failures:
|
|
return builder.build(
|
|
STATUS_PARTIAL_DOWNLOAD,
|
|
items=items,
|
|
error=error_fragment(
|
|
"download_incomplete",
|
|
f"{failures} of {len(items)} downloads failed",
|
|
details={"failed": failures, "total": len(items)},
|
|
),
|
|
)
|
|
return builder.build(STATUS_OK, items=items)
|
|
|
|
|
|
def main(argv: Sequence[str] | None = None) -> int:
|
|
"""Entry point: emit exactly one JSON document and return the exit code."""
|
|
|
|
arguments = list(sys.argv[1:] if argv is None else argv)
|
|
try:
|
|
response = _execute(arguments)
|
|
except ToolkitError as error:
|
|
builder = ResponseBuilder(source=_guess_source(arguments))
|
|
response = builder.from_error(error)
|
|
except OSError as error:
|
|
builder = ResponseBuilder(source=_guess_source(arguments))
|
|
response = builder.build(
|
|
STATUS_ACCESS_ERROR,
|
|
error=error_fragment("access_error", str(error)),
|
|
)
|
|
response.emit()
|
|
return response.exit_code
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover - module entry point
|
|
sys.exit(main())
|