From 37fa9feb4aabcd873af298589871b7c17cd919cb Mon Sep 17 00:00:00 2001 From: raptorreplica Date: Mon, 12 Oct 2026 11:17:27 +1100 Subject: [PATCH] Register independently reviewed standalone terrain without approving held neighbors --- docs/hd-generation-workflow.md | 24 ++ tools/hd_job_queue.py | 11 + tools/hd_register_reviewed.py | 207 +++++++++++- tools/test_hd_register_reviewed.py | 490 +++++++++++++++++++++++++++++ 4 files changed, 729 insertions(+), 3 deletions(-) create mode 100644 tools/test_hd_register_reviewed.py diff --git a/docs/hd-generation-workflow.md b/docs/hd-generation-workflow.md index 477d5bd5..512fe339 100644 --- a/docs/hd-generation-workflow.md +++ b/docs/hd-generation-workflow.md @@ -68,3 +68,27 @@ The four-slot gate retains started requests and old requests with unknown status After interruption, recover the original yielded cell or saved result before any new submission. Report a recovered completion, or release a reservation only after confirming no request was submitted/running. Do not infer completion from age. `status` reads lease phases, ages, admission state and recent outcomes without changing reservations. Root can pause new admissions while reconciling unknown calls; existing calls may still finish and report. Eligibility currently uses the first available slot with one request per owner, without a FIFO queue. Exact sanitized service errors and UTC report times stay in ignored `data/hd/jobs/.imagegen-gate/events.jsonl`. Connection failures impose a 60–600-second cooldown and at most three failures per job in a thirty-minute retry batch. During denial or cooldown, prepare or review another job. Do not replace a failed remaster with scaled original art. + +## Registering reviewed standalone terrain + +`hd_register_reviewed.py` accepts an explicit `complete_standalone_selection` +only for a `Terrain` / `contextual_terrain_shard` job. This selection contains +complete one-frame definitions, with `frame_count = 1` and `frame = 0`; it cannot +approve part of an animation. `complete_family` must remain false. Its +`full_inventory`, `selected_assets` and `original_fallback_assets` must describe +the complete original shard and a disjoint, exhaustive partition. Fallback IDs +retain their outstanding status and cannot silently retain a previous HD binding. + +Each selected ID needs an independent material decision bound to its exact source, +raw PNG and renderer fields, plus current loader evidence at scales 1–4. The full +inventory is checked against original catalog terrain ranges, payloads, palette, +canvas and origins. A separate independent context receipt binds the combined +selection, baseline, future projection and closed changed-case proof. Registration +and queue approval reject missing, stale or unrelated evidence. Whole animation +families continue to require every original frame. + +Run the tool regression checks with +`python3 -m unittest discover -s tools -p 'test_hd_*.py'`. They include successful +registration of separately approved selections and rejection of incomplete +animations, held material, inconsistent inventory, stale artifacts and incorrect +context bindings. diff --git a/tools/hd_job_queue.py b/tools/hd_job_queue.py index 695fa41a..625c0963 100644 --- a/tools/hd_job_queue.py +++ b/tools/hd_job_queue.py @@ -325,6 +325,7 @@ def build(args): expected = {f"{asset:05x}": pictures[asset] for asset in family["assets"]} candidate_path = directory / "candidate.json" candidates = {} + proposal = {} candidate_error = None candidate_digest = None if candidate_path.is_file(): @@ -351,6 +352,16 @@ def build(args): # A corrected proposal needs fresh review; an old approval or hold must # never silently apply to newly submitted artwork. coordinator_review = None + if coordinator_review and coordinator_review.get("status") == "approved" and proposal.get("complete_standalone_selection"): + # Explicit Terrain selections must satisfy the same source, peer, + # partition and runtime guard as registration. A fallback ID never + # inherits the approval of the selected standalone definitions. + try: + from hd_register_reviewed import approved_job + approved_job((args.data / "hd").resolve(), family["id"], Path(__file__).resolve().parents[1]) + except (ValueError, KeyError, TypeError, OSError) as error: + coordinator_review = None + candidate_error = f"standalone selection approval rejected: {error}" target_rows = [] for asset in family["assets"]: source = pictures[asset] diff --git a/tools/hd_register_reviewed.py b/tools/hd_register_reviewed.py index 99c648d4..f20895a2 100644 --- a/tools/hd_register_reviewed.py +++ b/tools/hd_register_reviewed.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Register explicitly approved, source-bound complete HD families. +"""Register approved complete HD families or explicit standalone Terrain selections. Generation and a successful loader probe never approve artwork. Each job must already have a coordinator-review.json approval bound to its exact frame map, @@ -10,10 +10,185 @@ import datetime import hashlib import json import os +import re import tempfile from pathlib import Path from hd_candidate_review import binding_digest, proposal_frames +from hd_art_audit import checked_palette, validate_hd_frame +from hd_art_source import decode + + +def asset_list(value, label): + if not isinstance(value, list) or any(not isinstance(asset, str) or + re.fullmatch(r"[0-9a-f]{5}", asset) is None for asset in value): + raise ValueError(f"{label} needs canonical catalog IDs") + if len(value) != len(set(value)): + raise ValueError(f"duplicate {label}") + return set(value) + + +def standalone_selection(candidate, job, rows): + """Standalone means a complete one-frame definition, never an animation slice.""" + if candidate.get("complete_standalone_selection") is not True: + return None + if candidate.get("complete_family") is not False: + raise ValueError("standalone selection must not claim whole-family approval") + for value in [candidate, job]: + if value.get("category") != "Terrain" or value.get("kind") != "contextual_terrain_shard": + raise ValueError("standalone selection is restricted to contextual Terrain") + targets = [target["asset"] for target in job["targets"]] + inventory = asset_list(targets, "job inventory") + full = asset_list(candidate.get("full_inventory"), "full_inventory") + selected = asset_list(candidate.get("selected_assets"), "selected_assets") + fallback = asset_list(candidate.get("original_fallback_assets"), "original_fallback_assets") + if full != inventory or selected != set(rows) or not selected or selected & fallback or selected | fallback != full: + raise ValueError("standalone selected/fallback partition differs from complete job inventory") + if candidate.get("targets"): + asset_list([target["asset"] for target in candidate["targets"]], "candidate targets") + inventory_job = candidate.get("inventory_job_id", job["job_id"]) + span = re.fullmatch(r"terrain-([0-9a-f]{5})-([0-9a-f]{5})", inventory_job) + if span is None or full != {f"{asset:05x}" for asset in range(int(span[1], 16), int(span[2], 16) + 1)}: + raise ValueError("standalone full inventory differs from original Terrain shard range") + for sequence in candidate["sequences"]: + if (type(sequence["frame_count"]) is not int or sequence["frame_count"] != 1 or + len(sequence["frames"]) != 1 or type(sequence["frames"][0]["frame"]) is not int or + sequence["frames"][0]["frame"] != 0): + raise ValueError("standalone selection contains an animation or partial sequence") + return {key: sorted(value) for key, value in [ + ("full_inventory", full), ("selected_assets", selected), ("original_fallback_assets", fallback)]} + + +def receipt_json(root, receipt): + if (not isinstance(receipt, dict) or not isinstance(receipt.get("path"), str) or + re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("sha256", ""))) is None): + raise ValueError("missing hash-bound independent receipt") + relative = Path(receipt["path"]) + path = relative if relative.is_absolute() else contained(root, relative) + payload = path.read_bytes() + if hashlib.sha256(payload).hexdigest() != receipt["sha256"]: + raise ValueError("independent receipt changed") + return json.loads(payload) + + +def authentic_terrain_inventory(root, job): + """Corroborate the full shard against the real catalog, including fallbacks.""" + data = root.parent.resolve() + entries, markers = {}, {} + for line in (data / "INDEX").read_text().splitlines(): + if not line.strip(): + continue + parts = line.split() + if len(parts) != 3 or re.fullmatch(r"[0-9a-fA-F]{1,8}", parts[0]) is None: + raise ValueError("invalid original INDEX record") + identity = int(parts[0], 16) + if identity in entries or not parts[1].isdigit(): + raise ValueError("duplicate or invalid original INDEX record") + relative = Path(parts[2]) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError("unsafe original INDEX path") + path = (data / relative).resolve() + if not path.is_relative_to(data): + raise ValueError("original INDEX path escapes data") + entries[identity] = (int(parts[1]), parts[2], path) + name = relative.name + if re.fullmatch(r"(?:START|END)G[1-4]TILES", name): + if name in markers: + raise ValueError("ambiguous original Terrain marker") + markers[name] = identity + ranges = [(markers[f"STARTG{n}TILES"], markers[f"ENDG{n}TILES"]) + for n in range(1, 5) if f"STARTG{n}TILES" in markers and f"ENDG{n}TILES" in markers] + if not ranges or any(start >= end for start, end in ranges): + raise ValueError("missing or invalid original Terrain ranges") + palettes, pictures, payloads = {}, {}, {} + for target in job["targets"]: + asset = target["asset"] + identity = int(asset, 16) + if not any(start < identity < end for start, end in ranges) or identity not in entries: + raise ValueError(f"original ID is not catalog Terrain: {asset}") + size, relative, path = entries[identity] + payload = path.read_bytes() + if (relative != target["source_path"] or len(payload) != size or + hashlib.sha256(payload).hexdigest() != target["source_sha256"]): + raise ValueError(f"original inventory source differs: {asset}") + if target.get("origin") != [0, 0] or target.get("source_palette") != "PALETTE_DAT": + raise ValueError(f"original Terrain anchor/palette differs: {asset}") + palette_path = (data / "misc/PALETTE_DAT").resolve() + if not palette_path.is_relative_to(data): + raise ValueError("original palette escapes data") + palette = palette_path.read_bytes() + if hashlib.sha256(palette).hexdigest() != target["source_palette_sha256"]: + raise ValueError(f"original Terrain palette changed: {asset}") + if palette not in palettes: + palettes[palette] = checked_palette(palette) + picture = decode(path, palettes[palette]) + if list(picture.size) != target["logical_canvas"]: + raise ValueError(f"original inventory canvas differs: {asset}") + pictures[asset], payloads[asset] = picture, payload + return pictures, payloads + + +def approved_material_evidence(root, selection, review, rows, qa): + evidence = review.get("standalone_selection", {}) + if any(evidence.get(key) != value for key, value in selection.items()): + raise ValueError("review selected/fallback inventory differs") + materials = evidence.get("material_approvals", {}) + if set(materials) != set(rows): + raise ValueError("independent material map omits or adds selected IDs") + for asset, frame in rows.items(): + approval = materials[asset] + raw_sha = qa["artwork_sha256"][frame["file"]] + if (approval.get("status") != "approved" or + approval.get("frame_bindings_sha256") != binding_digest({asset: frame}) or + approval.get("source_sha256") != frame["source_sha256"] or approval.get("raw_sha256") != raw_sha): + raise ValueError(f"independent material binding differs: {asset}") + peer = receipt_json(root, approval.get("independent_review")) + reviewer = peer.get("reviewer", peer.get("owner")) + if not isinstance(reviewer, str) or not peer.get("author") or reviewer == peer["author"]: + raise ValueError(f"material review is not independent: {asset}") + records, index = peer.get("records", []), approval.get("record_index") + if type(index) is not int or not 0 <= index < len(records): + raise ValueError(f"unsafe independent decision index: {asset}") + record = records[index] + if (record.get("asset") != asset or record.get("standalone_verdict") not in ["APPROVE", "QUALIFIED_APPROVE"] or + record.get("selected_frame_binding") != frame or + record.get("source_payload_sha256") != frame["source_sha256"] or + record.get("full_untouched_raw_sha256") != raw_sha or record.get("actual_scales") != [1, 2, 3, 4]): + raise ValueError(f"independent per-ID decision does not approve exact artwork: {asset}") + + +def approved_standalone_evidence(root, selection, review, rows, qa): + approved_material_evidence(root, selection, review, rows, qa) + evidence = review["standalone_selection"] + context = evidence.get("contextual_approval", {}) + if (asset_list(context.get("selected_assets"), "context selected_assets") != set(rows) or + context.get("selected_frame_bindings_sha256") != binding_digest(rows)): + raise ValueError("combined context restriction differs from selected definitions") + peer = receipt_json(root, context.get("receipt")) + reviewer = peer.get("reviewer", peer.get("owner")) + if not isinstance(reviewer, str) or not peer.get("author") or reviewer == peer["author"]: + raise ValueError("combined contextual review is not independent") + authority = peer.get("standalone_selection_context", {}) + proposal = receipt_json(root, authority.get("candidate")) + bindings = proposal_frames(proposal) + if (asset_list(authority.get("selected_assets"), "context complete selected_assets") != set(bindings) or + authority.get("frame_bindings_sha256") != binding_digest(bindings) or not set(rows) <= set(bindings) or + any(bindings[asset] != frame for asset, frame in rows.items()) or + authority.get("technical_verdict") != "QUALIFIED_CONTEXT_INTEGRATION"): + raise ValueError("combined contextual authority differs or is incomplete") + if any(sequence["frame_count"] != 1 or len(sequence["frames"]) != 1 or sequence["frames"][0]["frame"] != 0 + for sequence in proposal["sequences"]): + raise ValueError("combined context candidate includes animation slices") + for key in ["baseline", "future_projection"]: + receipt_json(root, authority.get(key)) + closure = receipt_json(root, authority.get("changed_case_closure")) + if (closure.get("status") != "CLOSED_QUALIFIED_CONTEXT" or + closure.get("candidate_sha256") != authority["candidate"]["sha256"] or + closure.get("frame_bindings_sha256") != binding_digest(bindings) or + closure.get("baseline_sha256") != authority["baseline"]["sha256"] or + closure.get("future_projection_sha256") != authority["future_projection"]["sha256"]): + raise ValueError("changed-case closure belongs to another candidate or projection") + receipt_json(root, closure.get("changed_case_proof")) def contained(root, relative): @@ -34,7 +209,8 @@ def approved_job(root, identity, project): raise ValueError(f"job identity differs: {identity}") expected = {target["asset"]: target for target in job["targets"]} rows = proposal_frames(candidate, expected) - if not candidate.get("complete_family") or set(rows) != set(expected): + selection = standalone_selection(candidate, job, rows) + if selection is None and (not candidate.get("complete_family") or set(rows) != set(expected)): raise ValueError(f"incomplete family: {identity}") if any({frame["frame"] for frame in sequence["frames"]} != set(range(sequence["frame_count"])) for sequence in candidate["sequences"]): raise ValueError(f"complete-family claim contains a partial sequence: {identity}") @@ -52,6 +228,13 @@ def approved_job(root, identity, project): raise ValueError(f"runtime evidence differs or reports errors: {identity}") if qa.get("source_newer_than_library"): raise ValueError(f"runtime evidence used stale asset/HD code: {identity}") + if selection is not None: + candidate_sha = hashlib.sha256(candidate_path.read_bytes()).hexdigest() + if review.get("candidate_sha256") != candidate_sha or qa.get("candidate_sha256") != candidate_sha: + raise ValueError("standalone approval belongs to another candidate") + if (qa.get("source_newer_than_library") is not False or qa.get("any_game_source_newer_than_library") is not False or + not qa.get("library_sha256")): + raise ValueError("standalone runtime freshness/library identity is incomplete") library = (project / qa["library"]).resolve() if not library.is_relative_to(project / "target") or library.stat().st_mtime_ns != qa["library_modified_ns"]: raise ValueError(f"runtime library was replaced after review: {identity}") @@ -60,7 +243,12 @@ def approved_job(root, identity, project): for name in ["assets.rs", "hd.rs"]: if (project / "src" / name).stat().st_mtime_ns > qa["library_modified_ns"]: raise ValueError(f"asset/HD code changed after review: {identity}") + if selection is not None and any(path.stat().st_mtime_ns > qa["library_modified_ns"] + for path in (project / "src").rglob("*.rs")): + raise ValueError("game source changed after standalone review") reports = {frame["asset"]: frame for frame in qa["frames"]} + if selection is not None and len(reports) != len(qa["frames"]): + raise ValueError("duplicate standalone runtime ID") if set(reports) != set(rows): raise ValueError(f"runtime evidence omits family IDs: {identity}") for asset, frame in rows.items(): @@ -69,6 +257,8 @@ def approved_job(root, identity, project): raise ValueError(f"runtime source/canvas differs: {asset}") if {mask["scale"] for mask in report["hd_masks"]} != {1, 2, 3, 4}: raise ValueError(f"missing presentation scale: {asset}") + if selection is not None and (len(report["hd_masks"]) != 4 or any(type(mask["scale"]) is not int for mask in report["hd_masks"])): + raise ValueError(f"duplicate or invalid presentation scale: {asset}") source_reserved = report["source_mask"]["palette_indices_240_to_255_counts"] if not qa.get("static_card_palette_mode") and not any(source_reserved): if any(any(mask["mask"]["palette_indices_240_to_255_counts"]) for mask in report["hd_masks"]): @@ -76,6 +266,11 @@ def approved_job(root, identity, project): artwork = contained(root, frame["file"]) if hashlib.sha256(artwork.read_bytes()).hexdigest() != qa["artwork_sha256"][frame["file"]]: raise ValueError(f"raw artwork changed since review: {asset}") + if selection is not None: + pictures, payloads = authentic_terrain_inventory(root, job) + for asset, frame in rows.items(): + validate_hd_frame(root, frame, payloads[asset], pictures[asset]) + approved_standalone_evidence(root, selection, review, rows, qa) return candidate, review, rows @@ -111,6 +306,8 @@ def main(): additions = [] for identity in args.job: candidate, review, rows = approved_job(root, identity, project) + if candidate.get("complete_standalone_selection") and set(candidate["original_fallback_assets"]) & set(active): + raise ValueError("original fallback already has registered HD artwork") overlaps = set(rows) & set(active) if overlaps: if overlaps == set(rows) and all(rows[asset] == active[asset] for asset in rows): @@ -135,13 +332,17 @@ def main(): "independent_review": review["independent_review"], "frame_bindings_sha256": review["frame_bindings_sha256"], }) + if candidate.get("complete_standalone_selection"): + additions[-1]["approval_scope"] = "complete_standalone_terrain_selection" + additions[-1]["original_fallback_assets"] = candidate["original_fallback_assets"] + additions[-1]["full_inventory"] = candidate["full_inventory"] if additions and not args.dry_run: # Do not overwrite another registration completed during validation. if manifest_path.read_bytes() != original_manifest or provenance_path.read_bytes() != original_provenance: raise ValueError("active pack changed during validation; retry from the new snapshot") provenance.setdefault("reviewed_sequence_additions", []).extend(additions) provenance["registered_frame_count"] = len(active) - provenance["status"] = f"{len(active)} complete-family reviewed original picture frames registered. Whole-catalog queues remain active; HD stays optional. Render reachability is recorded separately." + provenance["status"] = f"{len(active)} reviewed original picture frames registered as complete families or explicit standalone Terrain selections. Whole-catalog queues remain active; HD stays optional. Render reachability is recorded separately." provenance["registration_updated_at_utc"] = datetime.datetime.now(datetime.timezone.utc).isoformat() # Runtime uses hd.json alone. Write its ancillary evidence first, then # replace the complete manifest in one atomic rename. diff --git a/tools/test_hd_register_reviewed.py b/tools/test_hd_register_reviewed.py new file mode 100644 index 00000000..42595818 --- /dev/null +++ b/tools/test_hd_register_reviewed.py @@ -0,0 +1,490 @@ +"""Explicit standalone Terrain selections never approve omitted or animated art.""" +import argparse +import contextlib +import copy +import hashlib +import io +import json +import os +import struct +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from PIL import Image +import hd_job_queue as queue +import hd_register_reviewed as registrar +from hd_candidate_review import binding_digest, proposal_frames + + +class StandaloneTerrainRegistrationTest(unittest.TestCase): + # Exact safe12/safe13 membership from the frozen independent source plan. + SELECTED = [ + ["10530", "10532", "10533", "10534", "10537", "10538", "10539", "1053a", "1053b", "1053c", "1053d", "1053f"], + ["10540", "10541", "10542", "10543", "10545", "10546", "10547", "10548", "10549", "1054a", "1054b", "1054d", "1054e"], + ] + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.project = Path(temporary.name) / "project" + self.data = Path(temporary.name) / "data" + self.hd = self.data / "hd" + self.hd.mkdir(parents=True) + (self.data / "misc").mkdir() + (self.project / "src").mkdir(parents=True) + for name in ["assets.rs", "hd.rs"]: + (self.project / "src" / name).write_text("synthetic source freshness fixture") + self.library = self.project / "target/fixture.rlib" + self.library.parent.mkdir() + self.library.write_bytes(b"synthetic immutable library; not a real runtime claim") + os.utime(self.library, ns=(2_000_000_000_000_000_000,) * 2) + palette = bytes(value // 4 for value in range(256) for _ in range(3)) + self.palette = self.data / "misc/PALETTE_DAT" + self.palette.write_bytes(palette) + self.entries = ["1052f 0 tiles/STARTG1TILES"] + self.jobs = [] + self.pictures = [] + all_bindings = {} + for number, selected in enumerate(self.SELECTED): + start = 0x10530 + number * 16 + identity = f"terrain-{start:05x}-{start+15:05x}" + directory = self.hd / "jobs" / identity + directory.mkdir(parents=True) + targets, sequences, records, approvals, masks, artwork = [], [], [], {}, [], {} + for offset in range(16): + asset = f"{start + offset:05x}" + relative = f"misc/source-{asset}" + payload = struct.pack("