Track deterministic HD artwork cleanup with verified lineage

tools/hd_art_derivation.py

@@ -0,0 +1,91 @@
+"""Read-only lineage checks for explicitly cleaned imagegen artwork.
+
+This records how reviewed pixels were made; it does not grant artistic approval
+or execute a recipe. Jobs without an artwork_derivation remain unchanged.
+"""
+import hashlib
+import json
+from pathlib import Path
+import re
+
+
+def checked_file(root, reference):
+    if (not isinstance(reference, dict) or set(reference) != {"path", "sha256"}
+            or not isinstance(reference["path"], str)
+            or not re.fullmatch(r"[0-9a-f]{64}", str(reference["sha256"]))):
+        raise ValueError("derivation needs an exact path/SHA-256 reference")
+    relative = Path(reference["path"])
+    if relative.is_absolute() or ".." in relative.parts or not relative.parts:
+        raise ValueError("derivation reference must be a portable relative path")
+    root = root.resolve()
+    path = (root / relative).resolve()
+    if not path.is_relative_to(root) or not path.is_file():
+        raise ValueError("derivation reference escapes its root or is not a file")
+    payload = path.read_bytes()
+    if hashlib.sha256(payload).hexdigest() != reference["sha256"]:
+        raise ValueError(f"derivation file changed: {relative}")
+    return payload
+
+
+def checked_json(root, reference):
+    value = json.loads(checked_file(root, reference))
+    if not isinstance(value, dict):
+        raise ValueError("derivation JSON must contain an object")
+    return value
+
+
+def validate_derivation(root, project, candidate, review, rows, qa):
+    reference = candidate.get("artwork_derivation")
+    if reference is None:
+        if "artwork_derivation" in candidate or "artwork_derivation" in review:
+            raise ValueError("derivation cannot be null or present only in review")
+        return None
+    if review.get("artwork_derivation") != reference:
+        raise ValueError("artistic review does not bind this derivation")
+    lineage = checked_json(root, reference)
+    if (type(lineage.get("format_version")) is not int or lineage["format_version"] != 1
+            or lineage.get("kind") != "deterministic_cleanup"
+            or not isinstance(lineage.get("method"), str) or not lineage["method"].strip()):
+        raise ValueError("unsupported or unnamed artwork derivation")
+    parents = lineage.get("parents")
+    if not isinstance(parents, list) or not parents:
+        raise ValueError("derivation needs original imagegen parents")
+    for parent in parents:
+        if not isinstance(parent, dict) or set(parent) != {"raw_png", "imagegen_receipt", "prompt"}:
+            raise ValueError("derivation parent needs raw PNG, imagegen receipt and prompt")
+        if not checked_file(root, parent["raw_png"]).startswith(b"\x89PNG\r\n\x1a\n"):
+            raise ValueError("derivation parent is not a PNG")
+        # Preserve the actual receipt verbatim. Its historical format is not
+        # rewritten to pretend that cleanup was another generator invocation.
+        checked_json(root, parent["imagegen_receipt"])
+        if not checked_file(root, parent["prompt"]).strip():
+            raise ValueError("derivation parent prompt is empty")
+    tools = lineage.get("tools")
+    if not isinstance(tools, list) or not tools:
+        raise ValueError("derivation needs a checked-in cleanup tool")
+    for tool in tools:
+        checked_file(project, tool)
+        if Path(tool["path"]).parts[0] != "tools" or Path(tool["path"]).suffix != ".py":
+            raise ValueError("derivation tool must be a project tools Python file")
+    checked_json(root, lineage.get("recipe"))
+    execution = checked_json(root, lineage.get("execution_receipt")).get("execution", {})
+    if not isinstance(execution, dict):
+        raise ValueError("derivation execution must contain an object")
+    if lineage.get("sources") != {asset: frame["source_sha256"] for asset, frame in rows.items()}:
+        raise ValueError("derivation Source inventory differs from selected artwork")
+    files = {frame["file"] for frame in rows.values()}
+    outputs = lineage.get("outputs")
+    if not isinstance(outputs, dict) or set(outputs) != files:
+        raise ValueError("derivation output inventory differs from selected artwork")
+    for path, sha in outputs.items():
+        payload = checked_file(root, {"path": path, "sha256": sha})
+        if not payload.startswith(b"\x89PNG\r\n\x1a\n") or qa.get("artwork_sha256", {}).get(path) != sha:
+            raise ValueError("derivation output differs from runtime-reviewed PNG")
+    if (execution.get("status") != "success" or execution.get("method") != lineage["method"]
+            or execution.get("recipe_sha256") != lineage["recipe"]["sha256"]
+            or execution.get("tool_sha256") not in {tool["sha256"] for tool in tools}
+            or execution.get("source_sha256") != lineage["sources"]
+            or execution.get("parent_raw_sha256") != sorted({p["raw_png"]["sha256"] for p in parents})
+            or execution.get("output_sha256") != {asset: outputs[frame["file"]] for asset, frame in rows.items()}):
+        raise ValueError("derivation execution does not bind actual recipe, tool, inputs and outputs")
+    return reference

tools/hd_job_queue.py

@@ -268,7 +268,7 @@ def contract(family):
         "submission_gate": "Prepare and inspect sources and save the prompt before acquiring. In one functions.exec orchestration, acquire one of four slots, start the reservation, submit built-in imagegen only when start.changed is true, then classify and report the actual result immediately after await before returning to the model or doing QA. See RustRaptor/docs/hd-generation-workflow.md. Started and legacy-unknown requests require explicit finish; only unstarted reservations expire. If denied or cooling down, prepare/review other jobs. Do not substitute scaling for a failed remaster.",
         "source_truth": "Keep every source orientation, silhouette, color identity, logical canvas, anchor and explicit source hash. Use each target's source_palette and source_palette_sha256; Apogee/Cygnus/Taiwan cards use their own palettes, not the gameplay palette. Inspect every frame before generating. Never invent poses or drop damaged/destroyed variants.",
         "ownership": "Only this job's target IDs may be proposed. Neighbor/context references and exact-source aliases are reference-only; never claim or overwrite another job's IDs.",
-        "raw_output": "Keep raw generated PNG bytes, prompts, source references and provenance. Do not retouch generated RGB/alpha. Atlas/square margin crops are allowed only if they preserve all meaningful visible pixels and original aspect.",
+        "raw_output": "Preserve original generated PNG bytes, prompts, source references and provenance unchanged. Separately saved deterministic pixel cleanup is allowed with an artwork_derivation lineage binding the untouched imagegen parent, checked-in tool, recipe, actual execution receipt and derived outputs. Never label cleaned pixels untouched generation. Atlas/square margin crops must preserve meaningful visible pixels and original aspect unless an explicit reviewed derivation documents the correction. See docs/hd-generation-workflow.md.",
         "full_family": "Review all frames together at logical resolution and indexed 4× with exact Rust Lanczos3 resident→presentation resampling. A good still does not complete an animation family. Record partial candidates and rejections truthfully.",
         "quality": "Check clipping, alpha halos, banking/animation centers, source bounds, pivots, tiny-projectile readability, palette fades/cycling, material continuity and intended timing. Near-black original body/backing pixels are occupied artwork, not empty background. For opaque pictures, mask overlap of 1 proves only coverage; compare source material, landmarks and internal geometry separately. Original ordinary pixels must not gain indices240..255; water/fire/individualblink source classes retain their semantics. Compare original and allscale dynamic-palette counts in runtime-qa.json. Numeric mask metrics support visual review, never replace it.",
         "terrain": "Generate each canonical tile from its own exact source; supplied 3×3 neighbors are separate continuity-review references, not scenery to copy into the target. Check all cardinal adjacency sets, repeated tile aliases and intact/destroyed state partners. Preserve coastline/topography crossings, native edge colors and exact reusable tile identity. Equal-grid atlas crops must preserve each tile's complete logical composition; reject displaced panel boundaries or foreign row strips. Review every required adjacent pair, including state changes; full opaque 32×32 terrain canvases are valid. Do not publish context-specific alternate art for one repeated ID. The validated renderer shares adjoining HD edge/corner colors only where original surface class/geometry agrees, and transitions HD/original boundaries towards the joined original layer over two logical pixels; see RustRaptor/docs/hd-terrain.md. Feathering cannot repair shifted coastlines or make a low-resolution grid acceptable; detailed interiors must remain newly authored HD art. Preserve source water indices 240..244, fire 245..249 and single-index glow/flash 250..255; ordinary terrain maps to 0..239." if terrain else None,
@@ -352,7 +352,9 @@ def build(args):
             # A corrected proposal needs fresh review; an old approval or hold must
             # never silently apply to newly submitted artwork.
             coordinator_review = None
-        if coordinator_review and coordinator_review.get("status") == "approved" and proposal.get("complete_standalone_selection"):
+        if coordinator_review and coordinator_review.get("status") == "approved" and (
+                proposal.get("complete_standalone_selection") or "artwork_derivation" in proposal
+                or "artwork_derivation" in coordinator_review):
             # Explicit Terrain selections must satisfy the same source, peer,
             # partition and runtime guard as registration. A fallback ID never
             # inherits the approval of the selected standalone definitions.
@@ -361,7 +363,7 @@ def build(args):
                 approved_job((args.data / "hd").resolve(), family["id"], Path(__file__).resolve().parents[1])
             except (ValueError, KeyError, TypeError, OSError) as error:
                 coordinator_review = None
-                candidate_error = f"standalone selection approval rejected: {error}"
+                candidate_error = f"standalone selection or derived artwork approval rejected: {error}"
         target_rows = []
         for asset in family["assets"]:
             source = pictures[asset]

tools/hd_register_reviewed.py

@@ -17,6 +17,7 @@ from pathlib import Path
 from hd_candidate_review import binding_digest, proposal_frames
 from hd_art_audit import checked_palette, validate_hd_frame
 from hd_art_source import decode
+from hd_art_derivation import validate_derivation
 from hd_frame_policy import validate_frame_policy, retained_review_version_ok, original_palette_indices
 
 
@@ -374,10 +375,15 @@ def approved_material_evidence(root, selection, review, rows, qa):
         if type(index) is not int or not 0 <= index < len(records):
             raise ValueError(f"unsafe independent decision index: {asset}")
         record = records[index]
+        derivation = review.get("artwork_derivation")
+        artwork_matches = (record.get("derived_artwork_sha256") == raw_sha and
+                           record.get("artwork_derivation_sha256") == derivation["sha256"] and
+                           "full_untouched_raw_sha256" not in record) if derivation else (
+                               record.get("full_untouched_raw_sha256") == raw_sha)
         if (record.get("asset") != asset or record.get("standalone_verdict") not in ["APPROVE", "QUALIFIED_APPROVE"] or
                 record.get("selected_frame_binding") != frame or
                 record.get("source_payload_sha256") != frame["source_sha256"] or
-                record.get("full_untouched_raw_sha256") != raw_sha or record.get("actual_scales") != [1, 2, 3, 4]):
+                not artwork_matches or record.get("actual_scales") != [1, 2, 3, 4]):
             raise ValueError(f"independent per-ID decision does not approve exact artwork: {asset}")
 
 
@@ -754,6 +760,7 @@ def approved_job(root, identity, project):
         artwork = contained(root, frame["file"])
         if hashlib.sha256(artwork.read_bytes()).hexdigest() != qa["artwork_sha256"][frame["file"]]:
             raise ValueError(f"raw artwork changed since review: {asset}")
+    validate_derivation(root, project, candidate, review, rows, qa)
     if selection is not None:
         pictures, payloads = authentic_terrain_inventory(root, job)
         validate_carried_registered_identity(root, candidate, job, rows, review)
@@ -826,6 +833,8 @@ def main():
             "independent_review": review["independent_review"],
             "frame_bindings_sha256": review["frame_bindings_sha256"],
         })
+        if candidate.get("artwork_derivation"):
+            additions[-1]["artwork_derivation"] = candidate["artwork_derivation"]
         if candidate.get("complete_standalone_selection"):
             additions[-1]["approval_scope"] = "complete_standalone_terrain_selection"
             additions[-1]["original_fallback_assets"] = candidate["original_fallback_assets"]

tools/test_hd_art_derivation.py

@@ -0,0 +1,125 @@
+"""Synthetic lineage checks; these fixtures do not claim generated art approval."""
+import copy
+import hashlib
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+from hd_art_derivation import validate_derivation
+
+
+class DerivationTest(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.project = Path(self.temp.name)
+        self.root = self.project / "data/hd"
+        self.root.mkdir(parents=True)
+        self.method = "synthetic-fixture-not-art-acceptance"
+        raw = self.put("parent.png", b"\x89PNG\r\n\x1a\nfixture-parent")
+        tool = self.put("tools/fixture.py", b"# synthetic recipe tool\n", self.project)
+        recipe = self.put("recipe.json", b'{"scope":"synthetic"}')
+        output = self.put("derived.png", b"\x89PNG\r\n\x1a\nfixture-output")
+        self.rows = {"12345": {"file": output["path"], "source_sha256": "a" * 64}}
+        self.execution = {"execution": {"status": "success", "method": self.method,
+            "recipe_sha256": recipe["sha256"], "tool_sha256": tool["sha256"],
+            "source_sha256": {"12345": "a" * 64}, "parent_raw_sha256": [raw["sha256"]],
+            "output_sha256": {"12345": output["sha256"]}}}
+        self.lineage = {"format_version": 1, "kind": "deterministic_cleanup", "method": self.method,
+            "parents": [{"raw_png": raw, "prompt": self.put("prompt.txt", b"Synthetic prompt fixture"),
+                "imagegen_receipt": self.put("parent-receipt.json", b'{"scope":"synthetic"}')}],
+            "tools": [tool], "recipe": recipe,
+            "execution_receipt": self.put_json("execution.json", self.execution),
+            "sources": {"12345": "a" * 64}, "outputs": {output["path"]: output["sha256"]}}
+        self.candidate, self.review = {}, {}
+        self.qa = {"artwork_sha256": self.lineage["outputs"].copy()}
+        self.bind()
+
+    def put(self, path, payload, root=None):
+        destination = (root or self.root) / path
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_bytes(payload)
+        return {"path": path, "sha256": hashlib.sha256(payload).hexdigest()}
+
+    def put_json(self, path, value):
+        return self.put(path, (json.dumps(value, sort_keys=True) + "\n").encode())
+
+    def bind(self):
+        ref = self.put_json("lineage.json", self.lineage)
+        self.candidate["artwork_derivation"] = ref
+        self.review["artwork_derivation"] = ref.copy()
+
+    def validate(self):
+        return validate_derivation(self.root, self.project, self.candidate, self.review, self.rows, self.qa)
+
+    def test_exact_lineage_is_read_only_and_legacy_is_optional(self):
+        before = {str(p): p.read_bytes() for p in self.project.rglob("*") if p.is_file()}
+        self.assertEqual(self.validate(), self.candidate["artwork_derivation"])
+        self.assertEqual(before, {str(p): p.read_bytes() for p in self.project.rglob("*") if p.is_file()})
+        self.assertIsNone(validate_derivation(self.root, self.project, {}, {}, {}, {}))
+
+    def test_review_cannot_use_parent_or_another_derivation(self):
+        self.review["artwork_derivation"]["sha256"] = "0" * 64
+        with self.assertRaises(ValueError): self.validate()
+        self.review.pop("artwork_derivation")
+        with self.assertRaises(ValueError): self.validate()
+
+    def test_every_input_and_output_is_hash_bound(self):
+        refs = [self.lineage["parents"][0][k] for k in ["raw_png", "prompt", "imagegen_receipt"]]
+        refs += [self.lineage[k] for k in ["recipe", "execution_receipt"]]
+        refs += [{"path": "derived.png"}, {"path": "lineage.json"}]
+        for ref in refs:
+            path = self.root / ref["path"]
+            original = path.read_bytes()
+            with self.subTest(path=path.name):
+                path.write_bytes(original + b"changed")
+                with self.assertRaises(ValueError): self.validate()
+                path.write_bytes(original)
+        tool = self.project / self.lineage["tools"][0]["path"]
+        tool.write_bytes(b"changed")
+        with self.assertRaises(ValueError): self.validate()
+
+    def test_rehashed_execution_cannot_change_recipe_tool_sources_parents_outputs_or_success(self):
+        for key in self.execution["execution"]:
+            altered = copy.deepcopy(self.execution)
+            altered["execution"][key] = "wrong"
+            with self.subTest(key=key):
+                self.lineage["execution_receipt"] = self.put_json("execution.json", altered)
+                self.bind()
+                with self.assertRaises(ValueError): self.validate()
+
+    def test_source_output_inventory_and_stale_qa_reject(self):
+        original = copy.deepcopy(self.lineage)
+        for field in ["sources", "outputs"]:
+            for value in [{}, {"extra": "0" * 64}]:
+                self.lineage = copy.deepcopy(original)
+                self.lineage[field] = value
+                self.bind()
+                with self.subTest(field=field, value=value), self.assertRaises(ValueError): self.validate()
+        self.lineage = original
+        self.bind()
+        self.qa["artwork_sha256"]["derived.png"] = original["parents"][0]["raw_png"]["sha256"]
+        with self.assertRaises(ValueError): self.validate()
+
+    def test_absolute_traversal_and_symlink_escape_reject(self):
+        outside = self.project / "outside.json"
+        outside.write_text('{"scope":"outside"}')
+        (self.root / "link.json").symlink_to(outside)
+        for path in [str(outside), "../../outside.json", "link.json"]:
+            self.lineage["recipe"] = {"path": path, "sha256": hashlib.sha256(outside.read_bytes()).hexdigest()}
+            self.bind()
+            with self.subTest(path=path), self.assertRaises(ValueError): self.validate()
+
+    def test_malformed_and_non_object_receipts_reject_as_validation_errors(self):
+        for value in [None, [], 1, {"execution": []}]:
+            self.lineage["execution_receipt"] = self.put_json("execution.json", value)
+            self.bind()
+            with self.subTest(value=value), self.assertRaises(ValueError): self.validate()
+        self.candidate["artwork_derivation"] = self.put_json("lineage.json", [])
+        self.review["artwork_derivation"] = self.candidate["artwork_derivation"]
+        with self.assertRaises(ValueError): self.validate()
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/test_hd_job_queue.py

@@ -202,6 +202,22 @@ class FamilyReviewLedgerTest(unittest.TestCase):
         self.assertEqual(ledger["family_reviews"][self.job_id]["candidate_sha256"], self.review["candidate_sha256"])
         self.assertEqual(ledger["family_reviews"][self.job_id]["frame_bindings_sha256"], self.review["frame_bindings_sha256"])
 
+    def test_derivation_removed_from_candidate_or_review_cannot_bypass_registration_guard(self):
+        for location in ["candidate", "review", "both"]:
+            candidate, review = copy.deepcopy(self.candidate), copy.deepcopy(self.review)
+            if location in ["candidate", "both"]:
+                candidate["artwork_derivation"] = {"path": "lineage.json", "sha256": "d" * 64}
+            if location in ["review", "both"]:
+                review["artwork_derivation"] = {"path": "lineage.json", "sha256": "d" * 64}
+            self.write(self.candidate_path, candidate)
+            self.write(self.review_path, review)
+            with self.subTest(location=location), patch("hd_register_reviewed.approved_job", side_effect=ValueError("bad lineage")) as guard:
+                ledger = self.build()
+                guard.assert_called_once()
+                self.assertEqual(ledger["assets"]["11000"]["status"], "candidate")
+                self.assertEqual(ledger["assets"]["11002"]["status"], "renderreachable")
+                self.assertEqual(ledger["family_reviews"], {})
+
     def test_missing_changed_receipt_and_conflicting_identities_fail_safely(self):
         legacy = self.build(legacy=True)
         normalized = queue.normalize_ledger(legacy, self.hd)

tools/test_hd_register_reviewed.py

@@ -162,6 +162,71 @@ class StandaloneTerrainRegistrationTest(unittest.TestCase):
     def approve(self, number=0):
         return registrar.approved_job(self.hd, self.jobs[number]["id"], self.project)
 
+    def install_synthetic_derivation(self):
+        row = self.jobs[0]
+        directory = row["directory"]
+        frames = proposal_frames(row["candidate"])
+        parent = directory / "synthetic-parent.png"
+        Image.new("RGBA", (8, 8), (50, 50, 50, 255)).save(parent)
+        self.write(directory / "synthetic-generation-receipt.json", {"scope": "synthetic contract only"})
+        (directory / "synthetic-prompt.txt").write_text("Synthetic prompt, no artwork acceptance")
+        self.write(directory / "synthetic-recipe.json", {"scope": "synthetic contract only"})
+        tool = self.project / "tools/synthetic-cleanup.py"
+        tool.parent.mkdir(exist_ok=True)
+        tool.write_text("# synthetic lineage fixture only\n")
+        tool_ref = {"path": "tools/synthetic-cleanup.py", "sha256": self.sha(tool.read_bytes())}
+        sources = {asset: frame["source_sha256"] for asset, frame in frames.items()}
+        outputs = {frame["file"]: row["qa"]["artwork_sha256"][frame["file"]] for frame in frames.values()}
+        execution = {"status": "success", "method": "synthetic-cleanup", "tool_sha256": tool_ref["sha256"],
+                     "recipe_sha256": self.ref(directory / "synthetic-recipe.json")["sha256"],
+                     "source_sha256": sources, "parent_raw_sha256": [self.sha(parent.read_bytes())],
+                     "output_sha256": {asset: outputs[frame["file"]] for asset, frame in frames.items()}}
+        self.write(directory / "synthetic-execution.json", {"execution": execution})
+        lineage = {"format_version": 1, "kind": "deterministic_cleanup", "method": "synthetic-cleanup",
+                   "parents": [{"raw_png": self.ref(parent),
+                       "imagegen_receipt": self.ref(directory / "synthetic-generation-receipt.json"),
+                       "prompt": self.ref(directory / "synthetic-prompt.txt")}],
+                   "tools": [tool_ref], "recipe": self.ref(directory / "synthetic-recipe.json"),
+                   "execution_receipt": self.ref(directory / "synthetic-execution.json"),
+                   "sources": sources, "outputs": outputs}
+        self.write(directory / "derivation.json", lineage)
+        reference = self.ref(directory / "derivation.json")
+        row["candidate"]["artwork_derivation"] = reference
+        row["review"]["artwork_derivation"] = reference
+        self.write(directory / "candidate.json", row["candidate"])
+        candidate_sha = self.sha((directory / "candidate.json").read_bytes())
+        row["review"]["candidate_sha256"] = row["qa"]["candidate_sha256"] = candidate_sha
+        peer_path = directory / "peer.json"
+        peer = json.loads(peer_path.read_text())
+        for record in peer["records"]:
+            if record["asset"] in frames:
+                record["derived_artwork_sha256"] = record.pop("full_untouched_raw_sha256")
+                record["artwork_derivation_sha256"] = reference["sha256"]
+        self.write(peer_path, peer)
+        for approval in row["review"]["standalone_selection"]["material_approvals"].values():
+            approval["independent_review"] = self.ref(peer_path)
+        self.write(directory / "coordinator-review.json", row["review"])
+        self.write(directory / "runtime-qa.json", row["qa"])
+
+    def test_derived_selection_passes_same_source_material_context_and_runtime_gates(self):
+        self.install_synthetic_derivation()
+        candidate, review, frames = self.approve()
+        self.assertEqual(candidate["artwork_derivation"], review["artwork_derivation"])
+        self.assertEqual(set(frames), set(self.SELECTED[0]))
+
+    def test_derived_art_cannot_be_relabelled_untouched_in_independent_review(self):
+        self.install_synthetic_derivation()
+        peer_path = self.jobs[0]["directory"] / "peer.json"
+        peer = json.loads(peer_path.read_text())
+        peer["records"][0]["full_untouched_raw_sha256"] = peer["records"][0]["derived_artwork_sha256"]
+        self.write(peer_path, peer)
+        review = self.jobs[0]["review"]
+        for approval in review["standalone_selection"]["material_approvals"].values():
+            approval["independent_review"] = self.ref(peer_path)
+        self.write(self.jobs[0]["directory"] / "coordinator-review.json", review)
+        with self.assertRaisesRegex(ValueError, "exact artwork"):
+            self.approve()
+
     def save(self, field, value, number=0):
         row = self.jobs[number]
         self.write(row["directory"] / {"candidate": "candidate.json", "job": "job.json", "review": "coordinator-review.json", "qa": "runtime-qa.json"}[field], value)