Add Source-guided coherent HD terrain retexturing tools

tools/hd_terrain_cleanup.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+"""Deterministic DERIVED terrain reconstruction, never a generator-return asset.
+
+The authentic opaque ordinary Source controls geometry and low-frequency colour.
+Only a bounded monochrome fine band from the parent image survives. Material
+boundaries use the engine's ScaleNx4 shape, and interpolation never mixes material
+families. This deliberately removes coarse generated stones and colour drift.
+No asset registration, frame policy, game Source or parent image is modified.
+Requires NumPy and Pillow; run with the bundled dependency Python.
+"""
+from __future__ import annotations
+import argparse
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import stat
+import struct
+import numpy as np
+from PIL import Image
+
+METHOD = 'source-nx4-material-normalized-lowpass-plus-bounded-parent-fine-luma-v1'
+DEFAULTS = dict(output_side=512, source_sigma=1.0, residual_sigma_fine=0.65,
+                residual_sigma_coarse=2.2, residual_percentile=95.0,
+                residual_amplitude=dict(gray=2.0, olive=1.25, soil=2.5, rim=1.5))
+FAMILY_NAMES = ('gray', 'olive', 'soil', 'rim')
+
+def read_regular(path):
+    p = Path(path).absolute()
+    fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW)
+    try:
+        a = os.fstat(fd)
+        if not stat.S_ISREG(a.st_mode):
+            raise ValueError('input must be a regular file')
+        with os.fdopen(fd, 'rb', closefd=False) as f:
+            data = f.read()
+        b = os.fstat(fd)
+        if (a.st_dev, a.st_ino, a.st_size, a.st_mtime_ns) != (b.st_dev, b.st_ino, b.st_size, b.st_mtime_ns):
+            raise ValueError('input changed during read')
+    finally:
+        os.close(fd)
+    return data, dict(path=str(p), sha256=hashlib.sha256(data).hexdigest(),
+                      size=a.st_size, mtime_ns=a.st_mtime_ns, mode=a.st_mode)
+
+def decode_source(data):
+    if len(data) < 20:
+        raise ValueError('short Source PIC')
+    header = struct.unpack_from('<5i', data)
+    kind, count, offset, width, height = header
+    if (kind, count, offset) != (1, 1, 0) or not (1 <= width <= 128 and 1 <= height <= 128):
+        raise ValueError('this tool supports a single uncompressed Source PIC only')
+    if len(data) != 20 + width * height:
+        raise ValueError('Source payload size mismatch')
+    indices = np.frombuffer(data[20:], dtype=np.uint8).reshape(height, width).copy()
+    if np.any(indices >= 240):
+        raise ValueError('only fully opaque ordinary Source indices0..239 are supported')
+    return indices
+
+def palette_rgb(data):
+    if len(data) != 768:
+        raise ValueError('palette must contain256 three-channel6-bit entries')
+    a = np.minimum(np.frombuffer(data, np.uint8).reshape(256, 3).astype(np.uint16), 63)
+    return ((a << 2) | (a >> 4)).astype(np.uint8)
+
+def material_families(indices):
+    """Explicit supported Raptor palette families; no per-asset painted mask."""
+    a = np.asarray(indices)
+    out = np.full(a.shape, -1, dtype=np.int8)
+    out[a < 32] = 0
+    out[(a >= 60) & (a <= 63)] = 1
+    out[((a >= 192) & (a <= 223)) | (a == 239)] = 2
+    out[(a >= 224) & (a <= 238)] = 3
+    if np.any(out < 0):
+        raise ValueError('Source contains unsupported material family')
+    return out
+
+def scale_nx(indices, factor):
+    """Exact Scale2x/3x; Scale4x is two Scale2x passes, as render::upscale."""
+    a = np.asarray(indices)
+    if factor == 1:
+        return a.copy()
+    if factor == 4:
+        return scale_nx(scale_nx(a, 2), 2)
+    if factor not in (2, 3) or a.ndim != 2:
+        raise ValueError('ScaleNx factor must be1..4 on a plane')
+    h, w = a.shape
+    p = np.pad(a, 1, mode='edge')
+    out = np.empty((h * factor, w * factor), dtype=a.dtype)
+    for y in range(h):
+        for x in range(w):
+            nw, n, ne, west, c, east, sw, south, se = p[y:y+3, x:x+3].flatten().tolist()
+            cells = [c] * (factor * factor)
+            if n != south and west != east:
+                if factor == 2:
+                    cells = [west if west == n else c, east if n == east else c,
+                             west if west == south else c, east if south == east else c]
+                else:
+                    cells = [west if west == n else c,
+                             n if (west == n and c != ne) or (n == east and c != nw) else c,
+                             east if n == east else c,
+                             west if (west == n and c != sw) or (west == south and c != nw) else c,
+                             c,
+                             east if (n == east and c != se) or (south == east and c != ne) else c,
+                             west if west == south else c,
+                             south if (west == south and c != se) or (south == east and c != sw) else c,
+                             east if south == east else c]
+            out[y*factor:(y+1)*factor, x*factor:(x+1)*factor] = np.array(cells).reshape(factor, factor)
+    return out
+
+def gaussian(a, sigma):
+    a = np.asarray(a, dtype=np.float64)
+    if not math.isfinite(sigma) or sigma <= 0:
+        raise ValueError('sigma must be finite and positive')
+    radius = int(math.ceil(3 * sigma))
+    q = np.arange(-radius, radius + 1, dtype=np.float64)
+    k = np.exp(-0.5 * (q / sigma) ** 2)
+    k /= k.sum()
+    b = a
+    for axis in (0, 1):
+        padding = [(0, 0)] * a.ndim
+        padding[axis] = (radius, radius)
+        p = np.pad(b, padding, mode='edge')
+        b = np.zeros_like(a)
+        for i, weight in enumerate(k):
+            sl = [slice(None)] * a.ndim
+            sl[axis] = slice(i, i + a.shape[axis])
+            b += weight * p[tuple(sl)]
+    return b
+
+def resize_float(a, size):
+    a = np.asarray(a, np.float32)
+    if a.ndim == 3:
+        return np.stack([resize_float(a[..., i], size) for i in range(a.shape[2])], -1)
+    return np.asarray(Image.fromarray(a).resize(size, Image.Resampling.BILINEAR), np.float64)
+
+def validate_parameters(p, source_shape):
+    if set(p) != set(DEFAULTS):
+        raise ValueError('unknown or missing reconstruction parameter')
+    side = p['output_side']
+    if type(side) is not int or not 4 * max(source_shape) <= side <= 2048 or side % (4 * max(source_shape)):
+        raise ValueError('output must be an integer expansion of SourceNx4, at least4x')
+    for name in ('source_sigma', 'residual_sigma_fine', 'residual_sigma_coarse'):
+        if type(p[name]) not in (int, float) or not math.isfinite(p[name]) or not 0 < p[name] <= 8:
+            raise ValueError('invalid smoothing parameter')
+    if p['residual_sigma_fine'] >= p['residual_sigma_coarse']:
+        raise ValueError('fine band must be narrower than coarse band')
+    q = p['residual_percentile']
+    if type(q) not in (int, float) or not math.isfinite(q) or not 50 <= q <= 100:
+        raise ValueError('invalid residual percentile')
+    if set(p['residual_amplitude']) != set(FAMILY_NAMES):
+        raise ValueError('all material residual amplitudes required')
+    for v in p['residual_amplitude'].values():
+        if type(v) not in (int, float) or not math.isfinite(v) or not 0 <= v <= 8:
+            raise ValueError('invalid bounded residual amplitude')
+
+def reconstruct(indices, palette, generated_rgb, parameters=None):
+    p = json.loads(json.dumps(DEFAULTS if parameters is None else parameters))
+    a = np.asarray(indices)
+    if a.ndim != 2 or a.shape[0] != a.shape[1] or a.size == 0 or np.any(a >= 240) or np.any(a < 0):
+        raise ValueError('square opaque ordinary Source required')
+    validate_parameters(p, a.shape)
+    guide_indices = scale_nx(a, 4)
+    labels = material_families(guide_indices)
+    guide = np.asarray(palette)[guide_indices].astype(np.float64)
+    side = p['output_side']
+    factor = side // labels.shape[0]
+    owners = np.repeat(np.repeat(labels, factor, 0), factor, 1)
+    base = np.zeros((side, side, 3), dtype=np.float64)
+    # Normalized filtering and interpolation are performed separately per family.
+    # Thus no olive/soil/rim/gray mixture can shift a Source material boundary.
+    for family in np.unique(labels):
+        mask = (labels == family).astype(np.float64)
+        weights = resize_float(gaussian(mask, p['source_sigma']), (side, side))
+        numerator = resize_float(gaussian(guide * mask[..., None], p['source_sigma']), (side, side))
+        field = numerator / np.maximum(weights[..., None], 1e-12)
+        base[owners == family] = field[owners == family]
+    raw = np.asarray(generated_rgb)
+    if raw.ndim != 3 or raw.shape[2] != 3 or raw.dtype != np.uint8:
+        raise ValueError('generated input must be RGB uint8')
+    parent = np.asarray(Image.fromarray(raw).resize((side, side), Image.Resampling.LANCZOS), np.float64)
+    luma = parent @ np.array([0.2126, 0.7152, 0.0722])
+    band = gaussian(luma, p['residual_sigma_fine']) - gaussian(luma, p['residual_sigma_coarse'])
+    band -= band.mean()
+    divisor = max(float(np.percentile(np.abs(band), p['residual_percentile'])), 1.0)
+    fine = np.clip(band / divisor, -1, 1)
+    amplitudes = np.array([p['residual_amplitude'][n] for n in FAMILY_NAMES])
+    residual = fine * amplitudes[owners]
+    output = np.clip(np.rint(base + residual[..., None]), 0, 255).astype(np.uint8)
+    base8 = np.clip(np.rint(base), 0, 255).astype(np.uint8)
+    metrics = dict(method=METHOD, source_shape=list(a.shape), output_shape=list(output.shape),
+                   sourceNx4_indices_sha256=hashlib.sha256(guide_indices.astype('<u2').tobytes()).hexdigest(),
+                   geometry_material_labels_sha256=hashlib.sha256(owners.tobytes()).hexdigest(),
+                   material_output_pixels={FAMILY_NAMES[int(i)]: int((owners == i).sum()) for i in np.unique(owners)},
+                   generated_fine_band_std=float(band.std()), fine_normalization=divisor,
+                   maximum_retained_luma_residual=float(np.abs(residual).max()),
+                   pixels_changed_from_Source_guided_base=int(np.any(output != base8, axis=2).sum()),
+                   mean_absolute_generated_residual=float(np.abs(residual).mean()),
+                   output_RGB_sha256=hashlib.sha256(output.tobytes()).hexdigest(),
+                   output_is_derived_not_untouched=True,
+                   limits='SourceNx4 family geometry, normalized Source colour, bounded generated fine luma; no generated coarse structure or hue retained. Not a material or context approval.')
+    return output, base8, owners, metrics
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('--recipe', required=True)
+    ap.add_argument('--output', required=True)
+    args = ap.parse_args(argv)
+    recipe_path = Path(args.recipe).absolute()
+    recipe_bytes, recipe_pin = read_regular(recipe_path)
+    recipe = json.loads(recipe_bytes)
+    if recipe.get('method') != METHOD or not isinstance(recipe.get('frames'), list) or not recipe['frames']:
+        raise ValueError('wrong method or empty recipe')
+    def recipe_input(binding):
+        path = Path(binding['path'])
+        if not path.is_absolute():
+            path = recipe_path.parent / path
+        return read_regular(path)
+    palette_bytes, palette_pin = recipe_input(recipe['palette'])
+    if palette_pin['sha256'] != recipe['palette']['sha256']:
+        raise ValueError('palette identity mismatch')
+    palette = palette_rgb(palette_bytes)
+    prepared = []
+    seen = set()
+    for row in recipe['frames']:
+        asset = row['asset']
+        if not isinstance(asset, str) or len(asset) != 5 or any(c not in '0123456789abcdef' for c in asset) or asset in seen:
+            raise ValueError('invalid or duplicate asset')
+        seen.add(asset)
+        source, source_pin = recipe_input(row['Source'])
+        parent_bytes, parent_pin = recipe_input(row['parent_raw'])
+        if source_pin['sha256'] != row['Source']['sha256'] or parent_pin['sha256'] != row['parent_raw']['sha256']:
+            raise ValueError('source or parent identity mismatch')
+        indices = decode_source(source)
+        import io
+        parent = Image.open(io.BytesIO(parent_bytes))
+        if parent.convert('RGBA').getchannel('A').getextrema() != (255, 255):
+            raise ValueError('parent must be wholly opaque; no alpha repair')
+        crop = row['crop']
+        if not isinstance(crop, list) or len(crop) != 4 or any(type(v) is not int for v in crop):
+            raise ValueError('crop must be four exact integers')
+        x, y, w, h = crop
+        if min(x, y) < 0 or w <= 0 or h <= 0 or x+w > parent.width or y+h > parent.height:
+            raise ValueError('crop outside untouched parent')
+        raw = np.array(parent.convert('RGB').crop((x, y, x+w, y+h)))
+        derived, base, labels, metrics = reconstruct(indices, palette, raw, recipe['parameters'])
+        prepared.append((asset, derived, base, labels, metrics, source_pin, parent_pin, crop))
+    out = Path(args.output).absolute()
+    out.mkdir(exist_ok=False)
+    tool_bytes, tool_pin = read_regular(__file__)
+    results = []
+    for asset, derived, base, labels, metrics, source_pin, parent_pin, crop in prepared:
+        image_path = out / f'{asset}-DERIVED-source-guided-material-v1.png'
+        Image.fromarray(derived).save(image_path)
+        Image.fromarray(base).save(out / f'{asset}-SOURCE-GUIDED-BASE-diagnostic.png')
+        Image.fromarray(labels.astype(np.uint8)).save(out / f'{asset}-SOURCE-NX4-FAMILY-geometry-diagnostic.png')
+        _, output_pin = read_regular(image_path)
+        results.append(dict(asset=asset, output=output_pin, Source=source_pin, parent_raw=parent_pin,
+                            parent_crop=crop, recipe=recipe_pin, tool=tool_pin,
+                            parameters=recipe['parameters'], metrics=metrics,
+                            lineage='DERIVED scripted cleanup; generator-return parent remains untouched'))
+    for record in [recipe_pin, palette_pin] + [p for t in prepared for p in (t[5], t[6])]:
+        _, now = read_regular(record['path'])
+        if now != record:
+            raise ValueError('input changed across cleanup')
+    execution = dict(status='success', method=METHOD,
+                     recipe_sha256=recipe_pin['sha256'], tool_sha256=tool_pin['sha256'],
+                     source_sha256={r['asset']: r['Source']['sha256'] for r in results},
+                     parent_raw_sha256=sorted({r['parent_raw']['sha256'] for r in results}),
+                     output_sha256={r['asset']: r['output']['sha256'] for r in results})
+    (out / 'DERIVED-LINEAGE-AND-ACTUAL-RECONSTRUCTION-METRICS.json').write_text(
+        json.dumps(dict(status='DERIVED_OUTPUTS_NOT_MATERIAL_APPROVAL', method=METHOD,
+                        recipe=recipe_pin, tool=tool_pin, palette=palette_pin, frames=results,
+                        execution=execution), indent=2, sort_keys=True) + '\n')
+    print(json.dumps(dict(output=str(out), frames=len(results), method=METHOD), indent=2))
+
+if __name__ == '__main__':
+    main()

tools/hd_terrain_retexture.py

@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+"""Deterministic DERIVED terrain reconstruction, never a generator-return asset.
+
+The authentic opaque ordinary Source controls geometry and low-frequency colour.
+A bounded, coherent fine+midscale parent texture is retained as tone modulation. Material
+boundaries use the engine's ScaleNx4 shape, and interpolation never mixes material
+families. This flattens generated colour/lighting drift while retaining coherent material structure.
+No asset registration, frame policy, game Source or parent image is modified.
+Requires NumPy and Pillow; run with the bundled dependency Python.
+"""
+from __future__ import annotations
+import argparse
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import stat
+import struct
+import sys
+import numpy as np
+import PIL
+from PIL import Image
+
+METHOD = 'source-broad-material-guidance-coherent-parent-fine-mid-logtexture-v1'
+DEFAULTS = dict(output_side=512, source_sigma=3.0, residual_sigma_fine=0.65,
+                residual_sigma_coarse=10.0, residual_percentile=80.0,
+                texture_log_gain=dict(gray=0.38, olive=0.55, soil=0.34, rim=0.06))
+FAMILY_NAMES = ('gray', 'olive', 'soil', 'rim')
+
+def read_regular(path):
+    p = Path(path).absolute()
+    fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW)
+    try:
+        a = os.fstat(fd)
+        if not stat.S_ISREG(a.st_mode):
+            raise ValueError('input must be a regular file')
+        with os.fdopen(fd, 'rb', closefd=False) as f:
+            data = f.read()
+        b = os.fstat(fd)
+        if (a.st_dev, a.st_ino, a.st_size, a.st_mtime_ns) != (b.st_dev, b.st_ino, b.st_size, b.st_mtime_ns):
+            raise ValueError('input changed during read')
+    finally:
+        os.close(fd)
+    return data, dict(path=str(p), sha256=hashlib.sha256(data).hexdigest(),
+                      size=a.st_size, mtime_ns=a.st_mtime_ns, mode=a.st_mode)
+
+def decode_source(data):
+    if len(data) < 20:
+        raise ValueError('short Source PIC')
+    header = struct.unpack_from('<5i', data)
+    kind, count, offset, width, height = header
+    if (kind, count, offset) != (1, 1, 0) or not (1 <= width <= 128 and 1 <= height <= 128):
+        raise ValueError('this tool supports a single uncompressed Source PIC only')
+    if len(data) != 20 + width * height:
+        raise ValueError('Source payload size mismatch')
+    indices = np.frombuffer(data[20:], dtype=np.uint8).reshape(height, width).copy()
+    if np.any(indices >= 240):
+        raise ValueError('only fully opaque ordinary Source indices0..239 are supported')
+    return indices
+
+def palette_rgb(data):
+    if len(data) != 768:
+        raise ValueError('palette must contain256 three-channel6-bit entries')
+    a = np.minimum(np.frombuffer(data, np.uint8).reshape(256, 3).astype(np.uint16), 63)
+    return ((a << 2) | (a >> 4)).astype(np.uint8)
+
+def material_families(indices):
+    """Explicit supported Raptor palette families; no per-asset painted mask."""
+    a = np.asarray(indices)
+    out = np.full(a.shape, -1, dtype=np.int8)
+    out[a < 32] = 0
+    out[(a >= 60) & (a <= 63)] = 1
+    out[((a >= 192) & (a <= 223)) | (a == 239)] = 2
+    out[(a >= 224) & (a <= 238)] = 3
+    if np.any(out < 0):
+        raise ValueError('Source contains unsupported material family')
+    return out
+
+def scale_nx(indices, factor):
+    """Exact Scale2x/3x; Scale4x is two Scale2x passes, as render::upscale."""
+    a = np.asarray(indices)
+    if factor == 1:
+        return a.copy()
+    if factor == 4:
+        return scale_nx(scale_nx(a, 2), 2)
+    if factor not in (2, 3) or a.ndim != 2:
+        raise ValueError('ScaleNx factor must be1..4 on a plane')
+    h, w = a.shape
+    p = np.pad(a, 1, mode='edge')
+    out = np.empty((h * factor, w * factor), dtype=a.dtype)
+    for y in range(h):
+        for x in range(w):
+            nw, n, ne, west, c, east, sw, south, se = p[y:y+3, x:x+3].flatten().tolist()
+            cells = [c] * (factor * factor)
+            if n != south and west != east:
+                if factor == 2:
+                    cells = [west if west == n else c, east if n == east else c,
+                             west if west == south else c, east if south == east else c]
+                else:
+                    cells = [west if west == n else c,
+                             n if (west == n and c != ne) or (n == east and c != nw) else c,
+                             east if n == east else c,
+                             west if (west == n and c != sw) or (west == south and c != nw) else c,
+                             c,
+                             east if (n == east and c != se) or (south == east and c != ne) else c,
+                             west if west == south else c,
+                             south if (west == south and c != se) or (south == east and c != sw) else c,
+                             east if south == east else c]
+            out[y*factor:(y+1)*factor, x*factor:(x+1)*factor] = np.array(cells).reshape(factor, factor)
+    return out
+
+def gaussian(a, sigma):
+    a = np.asarray(a, dtype=np.float64)
+    if not math.isfinite(sigma) or sigma <= 0:
+        raise ValueError('sigma must be finite and positive')
+    radius = int(math.ceil(3 * sigma))
+    q = np.arange(-radius, radius + 1, dtype=np.float64)
+    k = np.exp(-0.5 * (q / sigma) ** 2)
+    k /= k.sum()
+    b = a
+    for axis in (0, 1):
+        padding = [(0, 0)] * a.ndim
+        padding[axis] = (radius, radius)
+        p = np.pad(b, padding, mode='edge')
+        b = np.zeros_like(a)
+        for i, weight in enumerate(k):
+            sl = [slice(None)] * a.ndim
+            sl[axis] = slice(i, i + a.shape[axis])
+            b += weight * p[tuple(sl)]
+    return b
+
+def resize_float(a, size):
+    a = np.asarray(a, np.float32)
+    if a.ndim == 3:
+        return np.stack([resize_float(a[..., i], size) for i in range(a.shape[2])], -1)
+    return np.asarray(Image.fromarray(a).resize(size, Image.Resampling.BILINEAR), np.float64)
+
+def validate_parameters(p, source_shape):
+    if set(p) != set(DEFAULTS):
+        raise ValueError('unknown or missing reconstruction parameter')
+    side = p['output_side']
+    if type(side) is not int or not 4 * max(source_shape) <= side <= 2048 or side % (4 * max(source_shape)):
+        raise ValueError('output must be an integer expansion of SourceNx4, at least4x')
+    for name in ('source_sigma', 'residual_sigma_fine', 'residual_sigma_coarse'):
+        if type(p[name]) not in (int, float) or not math.isfinite(p[name]) or not 0 < p[name] <= 16:
+            raise ValueError('invalid smoothing parameter')
+    if p['residual_sigma_fine'] >= p['residual_sigma_coarse']:
+        raise ValueError('fine band must be narrower than coarse band')
+    q = p['residual_percentile']
+    if type(q) not in (int, float) or not math.isfinite(q) or not 50 <= q <= 100:
+        raise ValueError('invalid residual percentile')
+    if set(p['texture_log_gain']) != set(FAMILY_NAMES):
+        raise ValueError('all material residual amplitudes required')
+    for v in p['texture_log_gain'].values():
+        if type(v) not in (int, float) or not math.isfinite(v) or not 0 <= v <= 0.7:
+            raise ValueError('invalid bounded log-texture gain')
+
+def reconstruct(indices, palette, generated_rgb, parameters=None):
+    p = json.loads(json.dumps(DEFAULTS if parameters is None else parameters))
+    a = np.asarray(indices)
+    if a.ndim != 2 or a.shape[0] != a.shape[1] or a.size == 0 or np.any(a >= 240) or np.any(a < 0):
+        raise ValueError('square opaque ordinary Source required')
+    validate_parameters(p, a.shape)
+    guide_indices = scale_nx(a, 4)
+    labels = material_families(guide_indices)
+    guide = np.asarray(palette)[guide_indices].astype(np.float64)
+    side = p['output_side']
+    factor = side // labels.shape[0]
+    owners = np.repeat(np.repeat(labels, factor, 0), factor, 1)
+    base = np.zeros((side, side, 3), dtype=np.float64)
+    quiet = np.ones((side, side), dtype=np.float64)
+    guide_luma = guide @ np.array([0.2126, 0.7152, 0.0722])
+    for family in np.unique(labels):
+        mask = (labels == family).astype(np.float64)
+        weights = resize_float(gaussian(mask, p['source_sigma']), (side, side))
+        numerator = resize_float(gaussian(guide * mask[..., None], p['source_sigma']), (side, side))
+        field = numerator / np.maximum(weights[..., None], 1e-12)
+        base[owners == family] = field[owners == family]
+        if family == 2:
+            second = resize_float(gaussian(guide_luma**2 * mask, p['source_sigma']), (side, side)) / np.maximum(weights, 1e-12)
+            first = field @ np.array([0.2126, 0.7152, 0.0722])
+            std = np.sqrt(np.maximum(second-first**2, 0))
+            # Calm Source soil (e.g.20400right) receives less texture strength.
+            # This map is derived from Source variance, never hand-painted.
+            quiet[owners == family] = (0.35+0.65*np.clip(std/8.0,0,1))[owners == family]
+    raw = np.asarray(generated_rgb)
+    if raw.ndim != 3 or raw.shape[2] != 3 or raw.dtype != np.uint8:
+        raise ValueError('generated input must be RGB uint8')
+    parent = np.asarray(Image.fromarray(raw).resize((side, side), Image.Resampling.LANCZOS), np.float64)
+    luma = parent @ np.array([0.2126, 0.7152, 0.0722])
+    band = gaussian(luma, p['residual_sigma_fine']) - gaussian(luma, p['residual_sigma_coarse'])
+    texture = np.zeros_like(band)
+    normalizers = {}
+    for family in np.unique(owners):
+        area = owners == family
+        centered = band-band[area].mean()
+        divisor = max(float(np.percentile(np.abs(centered[area]),p['residual_percentile'])),1.0)
+        texture[area] = np.tanh(centered[area]/divisor)
+        normalizers[FAMILY_NAMES[int(family)]] = divisor
+    strengths = np.array([p['texture_log_gain'][n]for n in FAMILY_NAMES])[owners]*quiet
+    gain = np.exp(strengths*texture)
+    output = np.clip(np.rint(base*gain[...,None]),0,255).astype(np.uint8)
+    base8 = np.clip(np.rint(base),0,255).astype(np.uint8)
+    metrics = dict(method=METHOD, source_shape=list(a.shape),output_shape=list(output.shape),
+        sourceNx4_indices_sha256=hashlib.sha256(guide_indices.astype('<u2').tobytes()).hexdigest(),
+        geometry_material_labels_sha256=hashlib.sha256(owners.tobytes()).hexdigest(),
+        material_output_pixels={FAMILY_NAMES[int(i)]:int((owners==i).sum())for i in np.unique(owners)},
+        coherent_parent_band_std=float(band.std()),per_family_normalizers=normalizers,
+        actual_tone_gain_min=float(gain.min()),actual_tone_gain_max=float(gain.max()),
+        pixels_changed_from_Source_guided_base=int(np.any(output!=base8,axis=2).sum()),
+        mean_absolute_parent_tone_modulation=float(np.abs(output.astype(float)-base8).mean()),
+        Source_derived_soil_quiet_strength_min=float(quiet[owners==2].min())if np.any(owners==2)else None,
+        Source_derived_soil_quiet_strength_max=float(quiet[owners==2].max())if np.any(owners==2)else None,
+        output_RGB_sha256=hashlib.sha256(output.tobytes()).hexdigest(),output_is_derived_not_untouched=True,
+        limits='Source masks/broad tones/rims; actual coherent generated fine+midscale luminance is hue-preserving multiplicative texture. No random noise; large generated hue/illumination drift discarded. Not material/context approval.')
+
+    return output, base8, owners, metrics
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('--recipe', required=True)
+    ap.add_argument('--output', required=True)
+    args = ap.parse_args(argv)
+    recipe_path = Path(args.recipe).absolute()
+    recipe_bytes, recipe_pin = read_regular(recipe_path)
+    recipe = json.loads(recipe_bytes)
+    if recipe.get('method') != METHOD or not isinstance(recipe.get('frames'), list) or not recipe['frames']:
+        raise ValueError('wrong method or empty recipe')
+    def recipe_input(binding):
+        path = Path(binding['path'])
+        if not path.is_absolute():
+            path = recipe_path.parent / path
+        return read_regular(path)
+    palette_bytes, palette_pin = recipe_input(recipe['palette'])
+    if palette_pin['sha256'] != recipe['palette']['sha256']:
+        raise ValueError('palette identity mismatch')
+    palette = palette_rgb(palette_bytes)
+    prepared = []
+    seen = set()
+    for row in recipe['frames']:
+        asset = row['asset']
+        if not isinstance(asset, str) or len(asset) != 5 or any(c not in '0123456789abcdef' for c in asset) or asset in seen:
+            raise ValueError('invalid or duplicate asset')
+        seen.add(asset)
+        source, source_pin = recipe_input(row['Source'])
+        parent_bytes, parent_pin = recipe_input(row['parent_raw'])
+        if source_pin['sha256'] != row['Source']['sha256'] or parent_pin['sha256'] != row['parent_raw']['sha256']:
+            raise ValueError('source or parent identity mismatch')
+        indices = decode_source(source)
+        import io
+        parent = Image.open(io.BytesIO(parent_bytes))
+        if parent.convert('RGBA').getchannel('A').getextrema() != (255, 255):
+            raise ValueError('parent must be wholly opaque; no alpha repair')
+        crop = row['crop']
+        if not isinstance(crop, list) or len(crop) != 4 or any(type(v) is not int for v in crop):
+            raise ValueError('crop must be four exact integers')
+        x, y, w, h = crop
+        if min(x, y) < 0 or w <= 0 or h <= 0 or x+w > parent.width or y+h > parent.height:
+            raise ValueError('crop outside untouched parent')
+        raw = np.array(parent.convert('RGB').crop((x, y, x+w, y+h)))
+        derived, base, labels, metrics = reconstruct(indices, palette, raw, recipe['parameters'])
+        prepared.append((asset, derived, base, labels, metrics, source_pin, parent_pin, crop))
+    out = Path(args.output).absolute()
+    out.mkdir(exist_ok=False)
+    tool_bytes, tool_pin = read_regular(__file__)
+    results = []
+    for asset, derived, base, labels, metrics, source_pin, parent_pin, crop in prepared:
+        image_path = out / f'{asset}-DERIVED-source-guided-material-v1.png'
+        Image.fromarray(derived).save(image_path)
+        Image.fromarray(base).save(out / f'{asset}-SOURCE-GUIDED-BASE-diagnostic.png')
+        Image.fromarray(labels.astype(np.uint8)).save(out / f'{asset}-SOURCE-NX4-FAMILY-geometry-diagnostic.png')
+        _, output_pin = read_regular(image_path)
+        results.append(dict(asset=asset, output=output_pin, Source=source_pin, parent_raw=parent_pin,
+                            parent_crop=crop, recipe=recipe_pin, tool=tool_pin,
+                            parameters=recipe['parameters'], metrics=metrics,
+                            lineage='DERIVED scripted cleanup; generator-return parent remains untouched'))
+    for record in [recipe_pin, palette_pin] + [p for t in prepared for p in (t[5], t[6])]:
+        _, now = read_regular(record['path'])
+        if now != record:
+            raise ValueError('input changed across cleanup')
+    runtime = dict(python=sys.version, executable=sys.executable,
+                   numpy=np.__version__, Pillow=PIL.__version__, platform=sys.platform)
+    execution = dict(status='success', method=METHOD,
+                     recipe_sha256=recipe_pin['sha256'], tool_sha256=tool_pin['sha256'],
+                     source_sha256={r['asset']: r['Source']['sha256'] for r in results},
+                     parent_raw_sha256=sorted({r['parent_raw']['sha256'] for r in results}),
+                     output_sha256={r['asset']: r['output']['sha256'] for r in results},
+                     runtime=runtime)
+    (out / 'DERIVED-LINEAGE-AND-ACTUAL-RECONSTRUCTION-METRICS.json').write_text(
+        json.dumps(dict(status='DERIVED_OUTPUTS_NOT_MATERIAL_APPROVAL', method=METHOD,
+                        recipe=recipe_pin, tool=tool_pin, palette=palette_pin, frames=results,
+                        execution=execution, runtime=runtime), indent=2, sort_keys=True) + '\n')
+    print(json.dumps(dict(output=str(out), frames=len(results), method=METHOD), indent=2))
+
+if __name__ == '__main__':
+    main()

tools/test_hd_terrain_cleanup.py

@@ -0,0 +1,119 @@
+import copy
+import json
+from pathlib import Path
+import struct
+import sys
+import tempfile
+import unittest
+import numpy as np
+from PIL import Image
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import hd_terrain_cleanup as c
+
+class TerrainCleanupTests(unittest.TestCase):
+    def setUp(self):
+        self.palette = np.zeros((256, 3), np.uint8)
+        for i in range(32): self.palette[i] = [4*i]*3
+        self.palette[60:64] = [[56,52,4],[40,40,0],[24,24,0],[12,12,0]]
+        self.palette[200:208] = [[85,40,36],[81,36,32],[73,32,28],[69,28,24],[60,24,20],[56,16,16],[40,12,8],[20,0,0]]
+        self.palette[231] = [81,60,52]
+        self.p = copy.deepcopy(c.DEFAULTS); self.p['output_side'] = 64
+
+    def test_constant_parent_has_no_retained_texture(self):
+        source = np.full((4,4),63,np.uint8)
+        out, base, labels, m = c.reconstruct(source,self.palette,np.full((64,64,3),180,np.uint8),self.p)
+        np.testing.assert_array_equal(out,base)
+        self.assertEqual(m['pixels_changed_from_Source_guided_base'],0)
+        np.testing.assert_array_equal(out,np.broadcast_to(self.palette[63],out.shape))
+
+    def test_parent_colour_and_constant_lighting_are_not_carried(self):
+        source = np.full((4,4),63,np.uint8)
+        a = np.full((64,64,3),[200,5,50],np.uint8)
+        b = np.full((64,64,3),[10,240,70],np.uint8)
+        np.testing.assert_array_equal(c.reconstruct(source,self.palette,a,self.p)[0],c.reconstruct(source,self.palette,b,self.p)[0])
+
+    def test_fine_generated_detail_is_retained_but_bounded(self):
+        yy,xx = np.indices((64,64)); raw = np.repeat((120+40*np.sin(xx*1.9)+20*np.sin(yy*2.1))[...,None],3,2).clip(0,255).astype(np.uint8)
+        out,base,labels,m = c.reconstruct(np.full((4,4),202,np.uint8),self.palette,raw,self.p)
+        self.assertGreater(m['pixels_changed_from_Source_guided_base'],1500)
+        self.assertGreater(m['mean_absolute_generated_residual'],0.5)
+        self.assertLessEqual(np.abs(out.astype(float)-base).max(),3)
+        self.assertLessEqual(m['maximum_retained_luma_residual'],2.5)
+
+    def test_exact_family_geometry_and_thin_L_are_preserved(self):
+        source = np.array([[27,27,231,63],[27,27,231,63],[27,27,231,231],[27,27,27,27]],np.uint8)
+        raw = np.random.default_rng(9).integers(0,256,(80,80,3),dtype=np.uint8)
+        out,base,labels,m = c.reconstruct(source,self.palette,raw,self.p)
+        expected = c.material_families(c.scale_nx(source,4))
+        np.testing.assert_array_equal(labels,np.repeat(np.repeat(expected,4,0),4,1))
+        self.assertTrue(np.all(out[labels==0].max(axis=1)-out[labels==0].min(axis=1)==0))
+        self.assertTrue(np.all(out[labels==1,2] <= 2))
+        self.assertTrue(np.all(out[labels==3,0] > out[labels==3,1]))
+
+    def test_smooth_source_guidance_is_not_nearest_source_copy(self):
+        source = np.array([[200,200,205,205]]*4,np.uint8)
+        out,base,_,m = c.reconstruct(source,self.palette,np.full((64,64,3),128,np.uint8),self.p)
+        nearest=np.repeat(np.repeat(self.palette[source],16,0),16,1)
+        self.assertGreater(np.any(base!=nearest,axis=2).sum(),0)
+        self.assertGreater(len(np.unique(base.reshape(-1,3),axis=0)),2)
+
+    def test_deterministic_pixels_and_geometry(self):
+        src=np.array([[27,27,27,27],[27,63,202,27],[27,231,231,27],[27,27,27,27]],np.uint8)
+        raw=np.random.default_rng(7).integers(0,256,(61,61,3),dtype=np.uint8)
+        a=c.reconstruct(src,self.palette,raw,self.p);b=c.reconstruct(src,self.palette,raw,self.p)
+        for x,y in zip(a[:3],b[:3]):np.testing.assert_array_equal(x,y)
+        self.assertEqual(a[3],b[3])
+
+    def test_scale4_is_two_scale2_and_non_nearest(self):
+        src=np.array([[1,1,2],[1,2,2],[3,3,2]],np.uint8)
+        np.testing.assert_array_equal(c.scale_nx(src,4),c.scale_nx(c.scale_nx(src,2),2))
+        self.assertFalse(np.array_equal(c.scale_nx(src,2),np.repeat(np.repeat(src,2,0),2,1)))
+
+    def test_decode_rejects_reserved_and_malformed_source(self):
+        good=struct.pack('<5i',1,1,0,2,2)+bytes([27,63,202,231])
+        self.assertEqual(c.decode_source(good).shape,(2,2))
+        for bad in [good[:-1],good[:20]+bytes([27,63,240,231]),struct.pack('<5i',2,1,0,2,2)+good[20:]]:
+            with self.assertRaises(ValueError):c.decode_source(bad)
+
+    def test_rejects_unknown_family_and_unsafe_parameters(self):
+        with self.assertRaises(ValueError):c.material_families(np.array([[100]],np.uint8))
+        for name,value in [('output_side',True),('output_side',15),('source_sigma',float('nan')),('residual_sigma_coarse',0.1)]:
+            p=copy.deepcopy(self.p);p[name]=value
+            with self.assertRaises(ValueError):c.reconstruct(np.full((4,4),63,np.uint8),self.palette,np.zeros((64,64,3),np.uint8),p)
+
+    def test_stable_reader_rejects_symlinks(self):
+        with tempfile.TemporaryDirectory()as d:
+            p=Path(d);(p/'a').write_bytes(b'fixed');(p/'link').symlink_to(p/'a')
+            with self.assertRaises(OSError):c.read_regular(p/'link')
+
+    def test_actual_cli_lineage_and_bad_crop_stops_before_mkdir(self):
+        import hashlib
+        with tempfile.TemporaryDirectory()as d:
+            p=Path(d);src=p/'source.pic';src.write_bytes(struct.pack('<5i',1,1,0,4,4)+bytes([63]*16))
+            pal=p/'palette';pal.write_bytes(bytes([0,0,0])*60+bytes([14,13,1,10,10,0,6,6,0,3,3,0])+bytes([0,0,0])*(256-64))
+            raw=p/'parent.png';yy,xx=np.indices((64,64));Image.fromarray(np.repeat((80+(xx%2)*80)[...,None],3,2).astype(np.uint8)).save(raw)
+            def binding(q):return dict(path=str(q),sha256=hashlib.sha256(q.read_bytes()).hexdigest())
+            recipe=dict(method=c.METHOD,palette=binding(pal),parameters=self.p,frames=[dict(asset='20400',Source=binding(src),parent_raw=binding(raw),crop=[0,0,64,64])])
+            rp=p/'recipe.json';rp.write_text(json.dumps(recipe));out=p/'result';c.main(['--recipe',str(rp),'--output',str(out)])
+            report=json.loads((out/'DERIVED-LINEAGE-AND-ACTUAL-RECONSTRUCTION-METRICS.json').read_text());row=report['frames'][0]
+            self.assertEqual(row['output']['sha256'],hashlib.sha256(Path(row['output']['path']).read_bytes()).hexdigest())
+            self.assertEqual(row['parent_raw']['sha256'],binding(raw)['sha256'])
+            self.assertIn('DERIVED',row['lineage'])
+            with Image.open(row['output']['path']) as saved:
+                self.assertEqual(saved.size,(64,64))
+            self.assertEqual(report['execution']['status'],'success')
+            self.assertEqual(report['execution']['source_sha256'],{'20400':binding(src)['sha256']})
+            self.assertEqual(report['execution']['output_sha256'],{'20400':row['output']['sha256']})
+            self.assertEqual(report['execution']['parent_raw_sha256'],[binding(raw)['sha256']])
+            portable=copy.deepcopy(recipe)
+            portable['palette']['path']=pal.name
+            portable['frames'][0]['Source']['path']=src.name
+            portable['frames'][0]['parent_raw']['path']=raw.name
+            portable_path=p/'portable.json';portable_path.write_text(json.dumps(portable))
+            moved=p/'relative-result';c.main(['--recipe',str(portable_path),'--output',str(moved)])
+            self.assertEqual((moved/Path(row['output']['path']).name).read_bytes(),Path(row['output']['path']).read_bytes())
+            recipe['frames'][0]['crop']=[1,0,64,64];rp.write_text(json.dumps(recipe));bad=p/'bad'
+            with self.assertRaises(ValueError):c.main(['--recipe',str(rp),'--output',str(bad)])
+            self.assertFalse(bad.exists())
+
+if __name__ == '__main__':unittest.main()

tools/test_hd_terrain_retexture.py

@@ -0,0 +1,52 @@
+import pathlib,sys,copy,unittest
+import numpy as np
+sys.path.insert(0,str(pathlib.Path(__file__).resolve().parent))
+import hd_terrain_retexture as c
+class CoherentTextureTests(unittest.TestCase):
+ def setUp(self):
+  self.palette=np.zeros((256,3),np.uint8)
+  for i in range(32):self.palette[i]=[i*4]*3
+  self.palette[60:64]=[[56,52,4],[40,40,0],[24,24,0],[12,12,0]]
+  self.palette[200:208]=[[85,40,36],[81,36,32],[73,32,28],[69,28,24],[60,24,20],[56,16,16],[40,12,8],[20,0,0]];self.palette[231]=[81,60,52]
+  self.p=copy.deepcopy(c.DEFAULTS);self.p['output_side']=64
+ def test_coherent_midscale_structure_survives_not_random_noise(self):
+  yy,xx=np.indices((64,64));lum=(120+45*np.sin(xx*2*np.pi/20)+25*np.cos(yy*2*np.pi/27));raw=np.repeat(lum[...,None],3,2).clip(0,255).astype(np.uint8)
+  out,base,_,m=c.reconstruct(np.full((4,4),27,np.uint8),self.palette,raw,self.p)
+  correlation=np.corrcoef(out[...,0].flatten(),lum.flatten())[0,1]
+  self.assertGreater(correlation,.90);self.assertGreater(out[...,0].std(),3)
+  self.assertGreater(m['mean_absolute_parent_tone_modulation'],2)
+ def test_gray_achromatic_and_olive_hue_not_shifted(self):
+  raw=np.random.default_rng(8).integers(0,256,(64,64,3),dtype=np.uint8)
+  for index in [27,63]:
+   out,_,_,m=c.reconstruct(np.full((4,4),index,np.uint8),self.palette,raw,self.p)
+   if index==27:np.testing.assert_array_equal(out[...,0],out[...,2])
+   else:self.assertTrue(np.all(out[...,2]==0));self.assertTrue(np.all(out[...,0]==out[...,1]))
+   self.assertGreaterEqual(m['actual_tone_gain_min'],np.exp(-.55));self.assertLessEqual(m['actual_tone_gain_max'],np.exp(.55))
+ def test_source_quiet_soil_receives_less_texture(self):
+  raw=np.random.default_rng(1).integers(0,256,(64,64,3),dtype=np.uint8);calm=np.full((4,4),202,np.uint8);busy=np.array([[200,207]*2,[207,200]*2]*2,np.uint8)
+  m1=c.reconstruct(calm,self.palette,raw,self.p)[3];m2=c.reconstruct(busy,self.palette,raw,self.p)[3]
+  self.assertAlmostEqual(m1['Source_derived_soil_quiet_strength_max'],.35,places=5)
+  self.assertGreater(m2['Source_derived_soil_quiet_strength_min'],.8)
+ def test_source_L_and_material_geometry_exact_with_coherent_texture(self):
+  src=np.array([[27,27,231,63],[27,27,231,202],[27,27,231,231],[27,27,27,27]],np.uint8);raw=np.random.default_rng(4).integers(0,256,(64,64,3),dtype=np.uint8)
+  a=c.reconstruct(src,self.palette,raw,self.p);b=c.reconstruct(src,self.palette,raw,self.p)
+  expected=np.repeat(np.repeat(c.material_families(c.scale_nx(src,4)),4,0),4,1)
+  np.testing.assert_array_equal(a[2],expected);np.testing.assert_array_equal(a[0],b[0]);self.assertEqual(a[3],b[3])
+ def test_constant_parent_does_not_invent_texture(self):
+  out,base,_,m=c.reconstruct(np.full((4,4),63,np.uint8),self.palette,np.full((64,64,3),170,np.uint8),self.p)
+  np.testing.assert_array_equal(out,base);self.assertEqual(m['pixels_changed_from_Source_guided_base'],0)
+ def test_invalid_gain_and_material_fail(self):
+  p=copy.deepcopy(self.p);p['texture_log_gain']['gray']=float('inf')
+  with self.assertRaises(ValueError):c.reconstruct(np.full((4,4),27,np.uint8),self.palette,np.zeros((64,64,3),np.uint8),p)
+  with self.assertRaises(ValueError):c.reconstruct(np.full((4,4),100,np.uint8),self.palette,np.zeros((64,64,3),np.uint8),self.p)
+ def test_relative_cli_receipt_binds_runtime_versions(self):
+  import tempfile,json,struct,hashlib,PIL
+  from PIL import Image
+  with tempfile.TemporaryDirectory()as d:
+   root=pathlib.Path(d);src=root/'source.pic';src.write_bytes(struct.pack('<5i',1,1,0,4,4)+bytes([63]*16));pal=root/'palette.dat';pal.write_bytes((self.palette//4).tobytes());raw=root/'parent.png';Image.fromarray(np.full((64,64,3),100,np.uint8)).save(raw)
+   def binding(p):return dict(path=p.name,sha256=hashlib.sha256(p.read_bytes()).hexdigest())
+   recipe=dict(method=c.METHOD,palette=binding(pal),parameters=self.p,frames=[dict(asset='20400',Source=binding(src),parent_raw=binding(raw),crop=[0,0,64,64])]);rp=root/'recipe.json';rp.write_text(json.dumps(recipe));out=root/'out';c.main(['--recipe',str(rp),'--output',str(out)])
+   report=json.loads((out/'DERIVED-LINEAGE-AND-ACTUAL-RECONSTRUCTION-METRICS.json').read_text());expected=dict(python=sys.version,executable=sys.executable,numpy=np.__version__,Pillow=PIL.__version__,platform=sys.platform)
+   self.assertEqual(report['runtime'],expected);self.assertEqual(report['execution']['runtime'],expected)
+   self.assertEqual(report['execution']['output_sha256']['20400'],hashlib.sha256((out/'20400-DERIVED-source-guided-material-v1.png').read_bytes()).hexdigest())
+if __name__=='__main__':unittest.main()