Support original sprite palettes for faithful HD terrain remasters

tools/hd_candidate_review.py

@@ -216,7 +216,9 @@ def main():
                 shutil.copyfile(source, destination)
         (isolated / "hd.json").write_text(json.dumps(manifest))
         binary = isolated / "review-bin"
-        compiler_args = ["rustc", "--edition=2021", str(project / "tools/hd_candidate_review.rs"), "--extern", f"raptor_replica={library}", "--extern", f"serde_json={serde}", "-L", f"dependency={dependency_directory}", "-L", "native=" + str(Path.home() / ".local/lib"), "-o", str(binary)]
+        review_source = project / ("tools/hd_palette_review.rs" if any(
+            frame.get("palette_constraint") is not None for frame in rows.values()) else "tools/hd_candidate_review.rs")
+        compiler_args = ["rustc", "--edition=2021", str(review_source), "--extern", f"raptor_replica={library}", "--extern", f"serde_json={serde}", "-L", f"dependency={dependency_directory}", "-L", "native=" + str(Path.home() / ".local/lib"), "-o", str(binary)]
         if static_cards:
             compiler_args.extend(["--cfg", "static_card_api"])
         subprocess.run(compiler_args, check=True)
@@ -246,6 +248,9 @@ def main():
     qa["frame_bindings_sha256"] = bindings_at_start
     qa["artwork_sha256"] = artwork_hashes
     qa["static_card_palette_mode"] = static_cards
+    if any(frame.get("palette_constraint") is not None for frame in rows.values()):
+        qa["review_helper_source"] = str(review_source.relative_to(project))
+        qa["review_helper_source_sha256"] = hashlib.sha256(review_source.read_bytes()).hexdigest()
     qa["source_newer_than_library"] = any((project / "src" / name).stat().st_mtime_ns > library_at_start.st_mtime_ns for name in ["assets.rs", "hd.rs"])
     qa["any_game_source_newer_than_library"] = any(path.stat().st_mtime_ns > library_at_start.st_mtime_ns for path in (project / "src").glob("*.rs"))
     qa_path.write_text(json.dumps(qa, indent=2) + "\n")

tools/hd_frame_policy.py

@@ -3,7 +3,7 @@ import struct
 from fractions import Fraction
 
 FRAME_KEYS = {"frame", "file", "logical_width", "logical_height", "origin",
-              "source_sha256", "crop", "atlas_grid", "preserve_reserved_coverage", "native_nearest_cues", "original_detail_cells"}
+              "source_sha256", "crop", "atlas_grid", "preserve_reserved_coverage", "native_nearest_cues", "original_detail_cells", "palette_constraint"}
 
 def validate_atlas_grid(frame):
     grid = frame.get("atlas_grid")
@@ -90,6 +90,11 @@ def validate_frame_policy(frame, source_payload=None):
     if set(frame) - FRAME_KEYS:
         raise ValueError("unknown HD frame fields")
     validate_atlas_grid(frame)
+    constraint = frame.get("palette_constraint")
+    if constraint is not None and (type(constraint) is not str or constraint != "original-painted-ordinary-v1"):
+        raise ValueError("unknown palette constraint")
+    if constraint is not None and source_payload is not None:
+        original_palette_indices(frame, source_payload)
     if type(frame.get("preserve_reserved_coverage", False)) is not bool:
         raise ValueError("preserve_reserved_coverage must be a boolean")
     cues = frame.get("native_nearest_cues", [])
@@ -141,3 +146,22 @@ def validate_frame_policy(frame, source_payload=None):
         # Runtime, audit and extended registrar approval supply actual payloads.
         if len(details) > 64 and len(details) * 4 > sum(0 <= p <= 255 for p in indices):
             raise ValueError("65..128 Original details may retain at most one quarter of painted Source")
+
+
+def original_palette_indices(frame, source_payload):
+    """Derive the exact allowed set from authentic Source, never a supplied list.
+
+    Metadata-only preparation cannot claim this validation. Rust and actual
+    audit/registration provide the checked payload and keep PNG alpha separate.
+    """
+    constraint = frame.get("palette_constraint")
+    if constraint is None:
+        return None
+    if type(constraint) is not str or constraint != "original-painted-ordinary-v1":
+        raise ValueError("unknown palette constraint")
+    sw, sh, indices = _indices(source_payload)
+    if (type(frame.get("logical_width")) is not int or type(frame.get("logical_height")) is not int
+            or (sw, sh) != (frame["logical_width"], frame["logical_height"])
+            or not indices or any(not 0 <= i < 240 for i in indices)):
+        raise ValueError("Original palette constraint requires opaque ordinary Source")
+    return sorted(set(indices))

tools/hd_palette_review.rs

@@ -0,0 +1,89 @@
+//! Diagnostic gallery linked against the already-built game library.
+//! The Python wrapper prepares an isolated pack; global hd.json is never edited.
+use raptor_replica::{assets::{Catalog, Sprite, TRANSPARENT}, hd::{HdAssets, Manifest}};
+use serde_json::{json, Value};
+use std::{env, fs, path::Path};
+
+fn ppm(path: &Path, sprite: &Sprite, palette: &[[u8; 3]; 256]) {
+    let mut bytes = format!("P6\n{} {}\n255\n", sprite.width, sprite.height).into_bytes();
+    for index in &sprite.pixels {
+        bytes.extend_from_slice(if *index == TRANSPARENT { &[20, 28, 38] } else { &palette[*index as usize] });
+    }
+    fs::write(path, bytes).unwrap();
+}
+
+fn mask(sprite: &Sprite) -> Value {
+    let points: Vec<_> = sprite.pixels.iter().enumerate()
+        .filter(|(_, pixel)| **pixel != TRANSPARENT)
+        .map(|(index, _)| (index % sprite.width, index / sprite.width)).collect();
+    if points.is_empty() { return json!({"visible_pixels": 0}); }
+    let bounds = [points.iter().map(|p| p.0).min().unwrap(), points.iter().map(|p| p.1).min().unwrap(),
+        points.iter().map(|p| p.0).max().unwrap() + 1, points.iter().map(|p| p.1).max().unwrap() + 1];
+    let centroid = [points.iter().map(|p| p.0 as f64).sum::<f64>() / points.len() as f64,
+        points.iter().map(|p| p.1 as f64).sum::<f64>() / points.len() as f64];
+    let animated_palette_counts: Vec<_> = (240..=255).map(|index| sprite.pixels.iter().filter(|pixel| **pixel == index).count()).collect();
+    json!({"visible_pixels": points.len(), "bounds": bounds, "centroid": centroid,
+        "palette_indices_240_to_255_counts": animated_palette_counts})
+}
+
+fn main() {
+    let args: Vec<_> = env::args().collect();
+    assert_eq!(args.len(), 4, "expected DATA PACK OUTPUT");
+    let output = Path::new(&args[3]); fs::create_dir_all(output).unwrap();
+    let catalog = Catalog::open(&args[1]).unwrap(); let base_palette = catalog.palette().unwrap();
+    let manifest: Manifest = serde_json::from_slice(&fs::read(Path::new(&args[2]).join("hd.json")).unwrap()).unwrap();
+    let mut hd = HdAssets::load(Path::new(&args[2]), &catalog);
+    assert!(hd.diagnostics.is_empty(), "{:?}", hd.diagnostics);
+    let mut frames = Vec::new(); let mut indexed = Vec::new();
+    for sequence in manifest.sequences { for frame in sequence.frames {
+        let id = sequence.base_asset + frame.frame;
+        let palette_name = match id { 0x1002b => "POGPAL_DAT", 0x1002d => "CYGPAL_DAT", 0x1002f => "TAIPAL_DAT", _ => "PALETTE_DAT" };
+        let palette = if palette_name == "PALETTE_DAT" { base_palette } else {
+            raptor_replica::assets::decode_palette(&catalog.named(palette_name).unwrap()).unwrap()
+        };
+        let original = catalog.sprite(id).unwrap();
+        ppm(&output.join(format!("{id:05x}-original.ppm")), &original, &palette);
+        let source_indices: Vec<u16> = original.pixels.iter().copied().filter(|&p|p<TRANSPARENT)
+            .collect::<std::collections::BTreeSet<_>>().into_iter().collect();
+        let mut constraint_scales = Vec::new();
+        let mut masks = Vec::new(); let mut logical = None; let mut indexed_scales = Vec::new();
+        for scale in 1..=4 {
+            let sprite = if palette_name == "PALETTE_DAT" {
+                hd.sprite(id, scale, &palette)
+            } else {
+                #[cfg(static_card_api)]
+                { hd.sprite_static_palette(id, scale, &palette) }
+                #[cfg(not(static_card_api))]
+                { panic!("Static-card review requires the source-specific palette API in the selected library") }
+            }.unwrap().expect("candidate rejected");
+            assert_eq!((sprite.width, sprite.height), (frame.logical_width as usize * scale, frame.logical_height as usize * scale));
+            assert!(sprite.pixels.iter().any(|p| *p != TRANSPARENT));
+            ppm(&output.join(format!("{id:05x}-hd-{scale}.ppm")), &sprite, &palette);
+            if frame.palette_constraint.is_some() {
+                constraint_scales.push(json!({"scale":scale,
+                    "visible_pixels":sprite.pixels.iter().filter(|&&p|p<TRANSPARENT).count(),
+                    "outside_original_palette":sprite.pixels.iter().filter(|&&p|p<TRANSPARENT && source_indices.binary_search(&p).is_err()).count()}));
+            }
+            indexed_scales.push(json!({"scale":scale,"width":sprite.width,"height":sprite.height,"indices":sprite.pixels})); masks.push(json!({"scale": scale, "mask": mask(&sprite)}));
+            if scale == 1 { logical = Some(sprite); }
+        }
+        indexed.push(json!({"asset":format!("{id:05x}"),"source":{"width":original.width,"height":original.height,"indices":original.pixels},"scales":indexed_scales})); let logical = logical.unwrap();
+        let intersection = original.pixels.iter().zip(&logical.pixels).filter(|(a,b)| **a != TRANSPARENT && **b != TRANSPARENT).count();
+        let union = original.pixels.iter().zip(&logical.pixels).filter(|(a,b)| **a != TRANSPARENT || **b != TRANSPARENT).count();
+        let mut report = json!({"asset": format!("{id:05x}"), "source_sha256": frame.source_sha256, "source_palette": palette_name,
+            "logical_canvas": [frame.logical_width, frame.logical_height], "source_mask": mask(&original),
+            "logical_alpha_mask_iou": intersection as f64 / union.max(1) as f64, "hd_masks": masks});
+        if let Some(mode) = frame.palette_constraint {
+            report["palette_constraint"] = json!({"policy":mode,"applied":palette_name=="PALETTE_DAT",
+                "source_indices":source_indices,
+                "source_colors":source_indices.iter().map(|&i|json!({"index":i,"RGB":palette[i as usize]})).collect::<Vec<_>>(),
+                "scales":constraint_scales});
+        }
+        frames.push(report);
+    }}
+    let report = json!({"method": "actual Rust HdAssets::load and sprite at scales 1..4; Catalog palette; no artwork edits",
+        "validated_frames": hd.coverage(), "diagnostics": hd.diagnostics, "frames": frames,
+        "limitation": "Gallery validation does not prove live scene reachability, terrain seam correctness or artistic fidelity. Opaque sprites have trivial alpha-mask overlap; compare visible source geometry manually."});
+    fs::write(output.join("runtime-qa.json"), serde_json::to_vec_pretty(&report).unwrap()).unwrap();
+    fs::write(output.join("actual-indexed-source-and-scales.json"),serde_json::to_vec_pretty(&indexed).unwrap()).unwrap(); println!("validated {} candidate frames at all four scales", frames.len());
+}

tools/hd_register_reviewed.py

@@ -17,7 +17,7 @@ from pathlib import Path
 from hd_candidate_review import binding_digest, proposal_frames
 from hd_art_audit import checked_palette, validate_hd_frame
 from hd_art_source import decode
-from hd_frame_policy import validate_frame_policy, retained_review_version_ok
+from hd_frame_policy import validate_frame_policy, retained_review_version_ok, original_palette_indices
 
 
 CARRIED_TERRAIN_V2 = "standalone-terrain-carried-identity-v2"
@@ -600,6 +600,72 @@ def validate_extended_retained_sources(project, rows):
             raise ValueError("extended Original details lack authentic catalog Source bytes")
 
 
+
+def validate_palette_constraint_approval(project, rows, review, qa):
+    """The new treatment cannot reuse an old raw-art or default-sampler approval.
+
+    Keep all existing artistic/current-QA gates. Source domains are derived here
+    from checked catalog bytes; supplied indices or RGBs never define eligibility.
+    """
+    constrained = {asset: frame for asset, frame in rows.items() if frame.get("palette_constraint") is not None}
+    if not constrained:
+        return
+    declaration = review.get("palette_constraint_review", {})
+    if (not isinstance(declaration, dict)
+            or declaration.get("policy_version") != "original-painted-ordinary-v1"
+            or declaration.get("frame_bindings_sha256") != binding_digest(rows)
+            or declaration.get("unchanged_generated_raw") is not True
+            or declaration.get("independent_allscale_and_genuine_context_review") is not True):
+        raise ValueError("explicit palette treatment and exact independent all-scale/context approval required")
+    reports = {row["asset"]: row for row in qa["frames"]}
+    for asset, frame in constrained.items():
+        authenticated = None
+        for data in (project / "data", project.parent / "data"):
+            index = data / "INDEX"
+            if not index.is_file():
+                continue
+            matches = [p for line in index.read_text().splitlines()
+                       if len(p := line.split()) == 3 and int(p[0], 16) == int(asset, 16)]
+            if len(matches) != 1:
+                continue
+            relative = Path(matches[0][2])
+            if relative.is_absolute() or ".." in relative.parts:
+                raise ValueError("palette Source path escapes catalog")
+            path = (data / relative).resolve()
+            palette_path = (data / "misc/PALETTE_DAT").resolve()
+            if not path.is_relative_to(data.resolve()) or not palette_path.is_relative_to(data.resolve()):
+                raise ValueError("palette Source path escapes catalog")
+            payload = path.read_bytes()
+            if len(payload) != int(matches[0][1]) or hashlib.sha256(payload).hexdigest() != frame["source_sha256"]:
+                continue
+            validate_frame_policy(frame, payload)
+            indices = original_palette_indices(frame, payload)
+            palette = checked_palette(palette_path.read_bytes())
+            authenticated = (indices, [{"index": i, "RGB": list(palette[i])} for i in indices])
+            break
+        if authenticated is None:
+            raise ValueError("palette treatment lacks authentic opaque ordinary catalog Source")
+        indices, colors = authenticated
+        report = reports[asset]
+        evidence = report.get("palette_constraint", {})
+        if (not isinstance(evidence, dict) or evidence.get("policy") != frame["palette_constraint"]
+                or evidence.get("applied") is not True or report.get("source_palette") != "PALETTE_DAT"
+                or not same_json(evidence.get("source_indices"), indices)
+                or not same_json(evidence.get("source_colors"), colors)):
+            raise ValueError("palette measurement is not the exact SourceClasses treatment/domain")
+        scales = evidence.get("scales", [])
+        if (not isinstance(scales, list) or any(not isinstance(x, dict) for x in scales)
+                or [x.get("scale") for x in scales] != [1, 2, 3, 4]
+                or any(type(x.get("scale")) is not int or type(x.get("visible_pixels")) is not int
+                       or type(x.get("outside_original_palette")) is not int for x in scales)):
+            raise ValueError("palette measurement needs four exact typed scale records")
+        masks = {x["scale"]: x["mask"] for x in report["hd_masks"]}
+        if any(type(masks[x["scale"]].get("visible_pixels")) is not int
+               or x["outside_original_palette"] != 0 or x["visible_pixels"] <= 0
+               or x["visible_pixels"] != masks[x["scale"]]["visible_pixels"] for x in scales):
+            raise ValueError("palette measurement violates its domain or visible sample binding")
+
+
 def approved_job(root, identity, project):
     directory = contained(root, Path("jobs") / identity)
     if directory.parent != (root / "jobs").resolve():
@@ -697,6 +763,7 @@ def approved_job(root, identity, project):
     elif retained_details and catalog_only:
         approved_catalog_only_retained_evidence(root, project, declaration, rows, qa)
     validate_extended_retained_sources(project, rows)
+    validate_palette_constraint_approval(project, rows, review, qa)
     return candidate, review, rows