From f6dc5343d29697ea1083d22c8a960e085a318181 Mon Sep 17 00:00:00 2001 From: raptorreplica Date: Sun, 18 Oct 2026 10:54:33 +1100 Subject: [PATCH] Add Source-guided coherent HD terrain retexturing tools --- tools/hd_terrain_cleanup.py | 278 +++++++++++++++++++++++++++ tools/hd_terrain_retexture.py | 297 +++++++++++++++++++++++++++++ tools/test_hd_terrain_cleanup.py | 119 ++++++++++++ tools/test_hd_terrain_retexture.py | 52 +++++ 4 files changed, 746 insertions(+) create mode 100644 tools/hd_terrain_cleanup.py create mode 100644 tools/hd_terrain_retexture.py create mode 100644 tools/test_hd_terrain_cleanup.py create mode 100644 tools/test_hd_terrain_retexture.py diff --git a/tools/hd_terrain_cleanup.py b/tools/hd_terrain_cleanup.py new file mode 100644 index 00000000..c01258b3 --- /dev/null +++ b/tools/hd_terrain_cleanup.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Deterministic DERIVED terrain reconstruction, never a generator-return asset. + +The authentic opaque ordinary Source controls geometry and low-frequency colour. +Only a bounded monochrome fine band from the parent image survives. Material +boundaries use the engine's ScaleNx4 shape, and interpolation never mixes material +families. This deliberately removes coarse generated stones and colour drift. +No asset registration, frame policy, game Source or parent image is modified. +Requires NumPy and Pillow; run with the bundled dependency Python. +""" +from __future__ import annotations +import argparse +import hashlib +import json +import math +import os +from pathlib import Path +import stat +import struct +import numpy as np +from PIL import Image + +METHOD = 'source-nx4-material-normalized-lowpass-plus-bounded-parent-fine-luma-v1' +DEFAULTS = dict(output_side=512, source_sigma=1.0, residual_sigma_fine=0.65, + residual_sigma_coarse=2.2, residual_percentile=95.0, + residual_amplitude=dict(gray=2.0, olive=1.25, soil=2.5, rim=1.5)) +FAMILY_NAMES = ('gray', 'olive', 'soil', 'rim') + +def read_regular(path): + p = Path(path).absolute() + fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW) + try: + a = os.fstat(fd) + if not stat.S_ISREG(a.st_mode): + raise ValueError('input must be a regular file') + with os.fdopen(fd, 'rb', closefd=False) as f: + data = f.read() + b = os.fstat(fd) + if (a.st_dev, a.st_ino, a.st_size, a.st_mtime_ns) != (b.st_dev, b.st_ino, b.st_size, b.st_mtime_ns): + raise ValueError('input changed during read') + finally: + os.close(fd) + return data, dict(path=str(p), sha256=hashlib.sha256(data).hexdigest(), + size=a.st_size, mtime_ns=a.st_mtime_ns, mode=a.st_mode) + +def decode_source(data): + if len(data) < 20: + raise ValueError('short Source PIC') + header = struct.unpack_from('<5i', data) + kind, count, offset, width, height = header + if (kind, count, offset) != (1, 1, 0) or not (1 <= width <= 128 and 1 <= height <= 128): + raise ValueError('this tool supports a single uncompressed Source PIC only') + if len(data) != 20 + width * height: + raise ValueError('Source payload size mismatch') + indices = np.frombuffer(data[20:], dtype=np.uint8).reshape(height, width).copy() + if np.any(indices >= 240): + raise ValueError('only fully opaque ordinary Source indices0..239 are supported') + return indices + +def palette_rgb(data): + if len(data) != 768: + raise ValueError('palette must contain256 three-channel6-bit entries') + a = np.minimum(np.frombuffer(data, np.uint8).reshape(256, 3).astype(np.uint16), 63) + return ((a << 2) | (a >> 4)).astype(np.uint8) + +def material_families(indices): + """Explicit supported Raptor palette families; no per-asset painted mask.""" + a = np.asarray(indices) + out = np.full(a.shape, -1, dtype=np.int8) + out[a < 32] = 0 + out[(a >= 60) & (a <= 63)] = 1 + out[((a >= 192) & (a <= 223)) | (a == 239)] = 2 + out[(a >= 224) & (a <= 238)] = 3 + if np.any(out < 0): + raise ValueError('Source contains unsupported material family') + return out + +def scale_nx(indices, factor): + """Exact Scale2x/3x; Scale4x is two Scale2x passes, as render::upscale.""" + a = np.asarray(indices) + if factor == 1: + return a.copy() + if factor == 4: + return scale_nx(scale_nx(a, 2), 2) + if factor not in (2, 3) or a.ndim != 2: + raise ValueError('ScaleNx factor must be1..4 on a plane') + h, w = a.shape + p = np.pad(a, 1, mode='edge') + out = np.empty((h * factor, w * factor), dtype=a.dtype) + for y in range(h): + for x in range(w): + nw, n, ne, west, c, east, sw, south, se = p[y:y+3, x:x+3].flatten().tolist() + cells = [c] * (factor * factor) + if n != south and west != east: + if factor == 2: + cells = [west if west == n else c, east if n == east else c, + west if west == south else c, east if south == east else c] + else: + cells = [west if west == n else c, + n if (west == n and c != ne) or (n == east and c != nw) else c, + east if n == east else c, + west if (west == n and c != sw) or (west == south and c != nw) else c, + c, + east if (n == east and c != se) or (south == east and c != ne) else c, + west if west == south else c, + south if (west == south and c != se) or (south == east and c != sw) else c, + east if south == east else c] + out[y*factor:(y+1)*factor, x*factor:(x+1)*factor] = np.array(cells).reshape(factor, factor) + return out + +def gaussian(a, sigma): + a = np.asarray(a, dtype=np.float64) + if not math.isfinite(sigma) or sigma <= 0: + raise ValueError('sigma must be finite and positive') + radius = int(math.ceil(3 * sigma)) + q = np.arange(-radius, radius + 1, dtype=np.float64) + k = np.exp(-0.5 * (q / sigma) ** 2) + k /= k.sum() + b = a + for axis in (0, 1): + padding = [(0, 0)] * a.ndim + padding[axis] = (radius, radius) + p = np.pad(b, padding, mode='edge') + b = np.zeros_like(a) + for i, weight in enumerate(k): + sl = [slice(None)] * a.ndim + sl[axis] = slice(i, i + a.shape[axis]) + b += weight * p[tuple(sl)] + return b + +def resize_float(a, size): + a = np.asarray(a, np.float32) + if a.ndim == 3: + return np.stack([resize_float(a[..., i], size) for i in range(a.shape[2])], -1) + return np.asarray(Image.fromarray(a).resize(size, Image.Resampling.BILINEAR), np.float64) + +def validate_parameters(p, source_shape): + if set(p) != set(DEFAULTS): + raise ValueError('unknown or missing reconstruction parameter') + side = p['output_side'] + if type(side) is not int or not 4 * max(source_shape) <= side <= 2048 or side % (4 * max(source_shape)): + raise ValueError('output must be an integer expansion of SourceNx4, at least4x') + for name in ('source_sigma', 'residual_sigma_fine', 'residual_sigma_coarse'): + if type(p[name]) not in (int, float) or not math.isfinite(p[name]) or not 0 < p[name] <= 8: + raise ValueError('invalid smoothing parameter') + if p['residual_sigma_fine'] >= p['residual_sigma_coarse']: + raise ValueError('fine band must be narrower than coarse band') + q = p['residual_percentile'] + if type(q) not in (int, float) or not math.isfinite(q) or not 50 <= q <= 100: + raise ValueError('invalid residual percentile') + if set(p['residual_amplitude']) != set(FAMILY_NAMES): + raise ValueError('all material residual amplitudes required') + for v in p['residual_amplitude'].values(): + if type(v) not in (int, float) or not math.isfinite(v) or not 0 <= v <= 8: + raise ValueError('invalid bounded residual amplitude') + +def reconstruct(indices, palette, generated_rgb, parameters=None): + p = json.loads(json.dumps(DEFAULTS if parameters is None else parameters)) + a = np.asarray(indices) + if a.ndim != 2 or a.shape[0] != a.shape[1] or a.size == 0 or np.any(a >= 240) or np.any(a < 0): + raise ValueError('square opaque ordinary Source required') + validate_parameters(p, a.shape) + guide_indices = scale_nx(a, 4) + labels = material_families(guide_indices) + guide = np.asarray(palette)[guide_indices].astype(np.float64) + side = p['output_side'] + factor = side // labels.shape[0] + owners = np.repeat(np.repeat(labels, factor, 0), factor, 1) + base = np.zeros((side, side, 3), dtype=np.float64) + # Normalized filtering and interpolation are performed separately per family. + # Thus no olive/soil/rim/gray mixture can shift a Source material boundary. + for family in np.unique(labels): + mask = (labels == family).astype(np.float64) + weights = resize_float(gaussian(mask, p['source_sigma']), (side, side)) + numerator = resize_float(gaussian(guide * mask[..., None], p['source_sigma']), (side, side)) + field = numerator / np.maximum(weights[..., None], 1e-12) + base[owners == family] = field[owners == family] + raw = np.asarray(generated_rgb) + if raw.ndim != 3 or raw.shape[2] != 3 or raw.dtype != np.uint8: + raise ValueError('generated input must be RGB uint8') + parent = np.asarray(Image.fromarray(raw).resize((side, side), Image.Resampling.LANCZOS), np.float64) + luma = parent @ np.array([0.2126, 0.7152, 0.0722]) + band = gaussian(luma, p['residual_sigma_fine']) - gaussian(luma, p['residual_sigma_coarse']) + band -= band.mean() + divisor = max(float(np.percentile(np.abs(band), p['residual_percentile'])), 1.0) + fine = np.clip(band / divisor, -1, 1) + amplitudes = np.array([p['residual_amplitude'][n] for n in FAMILY_NAMES]) + residual = fine * amplitudes[owners] + output = np.clip(np.rint(base + residual[..., None]), 0, 255).astype(np.uint8) + base8 = np.clip(np.rint(base), 0, 255).astype(np.uint8) + metrics = dict(method=METHOD, source_shape=list(a.shape), output_shape=list(output.shape), + sourceNx4_indices_sha256=hashlib.sha256(guide_indices.astype(' parent.width or y+h > parent.height: + raise ValueError('crop outside untouched parent') + raw = np.array(parent.convert('RGB').crop((x, y, x+w, y+h))) + derived, base, labels, metrics = reconstruct(indices, palette, raw, recipe['parameters']) + prepared.append((asset, derived, base, labels, metrics, source_pin, parent_pin, crop)) + out = Path(args.output).absolute() + out.mkdir(exist_ok=False) + tool_bytes, tool_pin = read_regular(__file__) + results = [] + for asset, derived, base, labels, metrics, source_pin, parent_pin, crop in prepared: + image_path = out / f'{asset}-DERIVED-source-guided-material-v1.png' + Image.fromarray(derived).save(image_path) + Image.fromarray(base).save(out / f'{asset}-SOURCE-GUIDED-BASE-diagnostic.png') + Image.fromarray(labels.astype(np.uint8)).save(out / f'{asset}-SOURCE-NX4-FAMILY-geometry-diagnostic.png') + _, output_pin = read_regular(image_path) + results.append(dict(asset=asset, output=output_pin, Source=source_pin, parent_raw=parent_pin, + parent_crop=crop, recipe=recipe_pin, tool=tool_pin, + parameters=recipe['parameters'], metrics=metrics, + lineage='DERIVED scripted cleanup; generator-return parent remains untouched')) + for record in [recipe_pin, palette_pin] + [p for t in prepared for p in (t[5], t[6])]: + _, now = read_regular(record['path']) + if now != record: + raise ValueError('input changed across cleanup') + execution = dict(status='success', method=METHOD, + recipe_sha256=recipe_pin['sha256'], tool_sha256=tool_pin['sha256'], + source_sha256={r['asset']: r['Source']['sha256'] for r in results}, + parent_raw_sha256=sorted({r['parent_raw']['sha256'] for r in results}), + output_sha256={r['asset']: r['output']['sha256'] for r in results}) + (out / 'DERIVED-LINEAGE-AND-ACTUAL-RECONSTRUCTION-METRICS.json').write_text( + json.dumps(dict(status='DERIVED_OUTPUTS_NOT_MATERIAL_APPROVAL', method=METHOD, + recipe=recipe_pin, tool=tool_pin, palette=palette_pin, frames=results, + execution=execution), indent=2, sort_keys=True) + '\n') + print(json.dumps(dict(output=str(out), frames=len(results), method=METHOD), indent=2)) + +if __name__ == '__main__': + main() diff --git a/tools/hd_terrain_retexture.py b/tools/hd_terrain_retexture.py new file mode 100644 index 00000000..3acf699e --- /dev/null +++ b/tools/hd_terrain_retexture.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""Deterministic DERIVED terrain reconstruction, never a generator-return asset. + +The authentic opaque ordinary Source controls geometry and low-frequency colour. +A bounded, coherent fine+midscale parent texture is retained as tone modulation. Material +boundaries use the engine's ScaleNx4 shape, and interpolation never mixes material +families. This flattens generated colour/lighting drift while retaining coherent material structure. +No asset registration, frame policy, game Source or parent image is modified. +Requires NumPy and Pillow; run with the bundled dependency Python. +""" +from __future__ import annotations +import argparse +import hashlib +import json +import math +import os +from pathlib import Path +import stat +import struct +import sys +import numpy as np +import PIL +from PIL import Image + +METHOD = 'source-broad-material-guidance-coherent-parent-fine-mid-logtexture-v1' +DEFAULTS = dict(output_side=512, source_sigma=3.0, residual_sigma_fine=0.65, + residual_sigma_coarse=10.0, residual_percentile=80.0, + texture_log_gain=dict(gray=0.38, olive=0.55, soil=0.34, rim=0.06)) +FAMILY_NAMES = ('gray', 'olive', 'soil', 'rim') + +def read_regular(path): + p = Path(path).absolute() + fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW) + try: + a = os.fstat(fd) + if not stat.S_ISREG(a.st_mode): + raise ValueError('input must be a regular file') + with os.fdopen(fd, 'rb', closefd=False) as f: + data = f.read() + b = os.fstat(fd) + if (a.st_dev, a.st_ino, a.st_size, a.st_mtime_ns) != (b.st_dev, b.st_ino, b.st_size, b.st_mtime_ns): + raise ValueError('input changed during read') + finally: + os.close(fd) + return data, dict(path=str(p), sha256=hashlib.sha256(data).hexdigest(), + size=a.st_size, mtime_ns=a.st_mtime_ns, mode=a.st_mode) + +def decode_source(data): + if len(data) < 20: + raise ValueError('short Source PIC') + header = struct.unpack_from('<5i', data) + kind, count, offset, width, height = header + if (kind, count, offset) != (1, 1, 0) or not (1 <= width <= 128 and 1 <= height <= 128): + raise ValueError('this tool supports a single uncompressed Source PIC only') + if len(data) != 20 + width * height: + raise ValueError('Source payload size mismatch') + indices = np.frombuffer(data[20:], dtype=np.uint8).reshape(height, width).copy() + if np.any(indices >= 240): + raise ValueError('only fully opaque ordinary Source indices0..239 are supported') + return indices + +def palette_rgb(data): + if len(data) != 768: + raise ValueError('palette must contain256 three-channel6-bit entries') + a = np.minimum(np.frombuffer(data, np.uint8).reshape(256, 3).astype(np.uint16), 63) + return ((a << 2) | (a >> 4)).astype(np.uint8) + +def material_families(indices): + """Explicit supported Raptor palette families; no per-asset painted mask.""" + a = np.asarray(indices) + out = np.full(a.shape, -1, dtype=np.int8) + out[a < 32] = 0 + out[(a >= 60) & (a <= 63)] = 1 + out[((a >= 192) & (a <= 223)) | (a == 239)] = 2 + out[(a >= 224) & (a <= 238)] = 3 + if np.any(out < 0): + raise ValueError('Source contains unsupported material family') + return out + +def scale_nx(indices, factor): + """Exact Scale2x/3x; Scale4x is two Scale2x passes, as render::upscale.""" + a = np.asarray(indices) + if factor == 1: + return a.copy() + if factor == 4: + return scale_nx(scale_nx(a, 2), 2) + if factor not in (2, 3) or a.ndim != 2: + raise ValueError('ScaleNx factor must be1..4 on a plane') + h, w = a.shape + p = np.pad(a, 1, mode='edge') + out = np.empty((h * factor, w * factor), dtype=a.dtype) + for y in range(h): + for x in range(w): + nw, n, ne, west, c, east, sw, south, se = p[y:y+3, x:x+3].flatten().tolist() + cells = [c] * (factor * factor) + if n != south and west != east: + if factor == 2: + cells = [west if west == n else c, east if n == east else c, + west if west == south else c, east if south == east else c] + else: + cells = [west if west == n else c, + n if (west == n and c != ne) or (n == east and c != nw) else c, + east if n == east else c, + west if (west == n and c != sw) or (west == south and c != nw) else c, + c, + east if (n == east and c != se) or (south == east and c != ne) else c, + west if west == south else c, + south if (west == south and c != se) or (south == east and c != sw) else c, + east if south == east else c] + out[y*factor:(y+1)*factor, x*factor:(x+1)*factor] = np.array(cells).reshape(factor, factor) + return out + +def gaussian(a, sigma): + a = np.asarray(a, dtype=np.float64) + if not math.isfinite(sigma) or sigma <= 0: + raise ValueError('sigma must be finite and positive') + radius = int(math.ceil(3 * sigma)) + q = np.arange(-radius, radius + 1, dtype=np.float64) + k = np.exp(-0.5 * (q / sigma) ** 2) + k /= k.sum() + b = a + for axis in (0, 1): + padding = [(0, 0)] * a.ndim + padding[axis] = (radius, radius) + p = np.pad(b, padding, mode='edge') + b = np.zeros_like(a) + for i, weight in enumerate(k): + sl = [slice(None)] * a.ndim + sl[axis] = slice(i, i + a.shape[axis]) + b += weight * p[tuple(sl)] + return b + +def resize_float(a, size): + a = np.asarray(a, np.float32) + if a.ndim == 3: + return np.stack([resize_float(a[..., i], size) for i in range(a.shape[2])], -1) + return np.asarray(Image.fromarray(a).resize(size, Image.Resampling.BILINEAR), np.float64) + +def validate_parameters(p, source_shape): + if set(p) != set(DEFAULTS): + raise ValueError('unknown or missing reconstruction parameter') + side = p['output_side'] + if type(side) is not int or not 4 * max(source_shape) <= side <= 2048 or side % (4 * max(source_shape)): + raise ValueError('output must be an integer expansion of SourceNx4, at least4x') + for name in ('source_sigma', 'residual_sigma_fine', 'residual_sigma_coarse'): + if type(p[name]) not in (int, float) or not math.isfinite(p[name]) or not 0 < p[name] <= 16: + raise ValueError('invalid smoothing parameter') + if p['residual_sigma_fine'] >= p['residual_sigma_coarse']: + raise ValueError('fine band must be narrower than coarse band') + q = p['residual_percentile'] + if type(q) not in (int, float) or not math.isfinite(q) or not 50 <= q <= 100: + raise ValueError('invalid residual percentile') + if set(p['texture_log_gain']) != set(FAMILY_NAMES): + raise ValueError('all material residual amplitudes required') + for v in p['texture_log_gain'].values(): + if type(v) not in (int, float) or not math.isfinite(v) or not 0 <= v <= 0.7: + raise ValueError('invalid bounded log-texture gain') + +def reconstruct(indices, palette, generated_rgb, parameters=None): + p = json.loads(json.dumps(DEFAULTS if parameters is None else parameters)) + a = np.asarray(indices) + if a.ndim != 2 or a.shape[0] != a.shape[1] or a.size == 0 or np.any(a >= 240) or np.any(a < 0): + raise ValueError('square opaque ordinary Source required') + validate_parameters(p, a.shape) + guide_indices = scale_nx(a, 4) + labels = material_families(guide_indices) + guide = np.asarray(palette)[guide_indices].astype(np.float64) + side = p['output_side'] + factor = side // labels.shape[0] + owners = np.repeat(np.repeat(labels, factor, 0), factor, 1) + base = np.zeros((side, side, 3), dtype=np.float64) + quiet = np.ones((side, side), dtype=np.float64) + guide_luma = guide @ np.array([0.2126, 0.7152, 0.0722]) + for family in np.unique(labels): + mask = (labels == family).astype(np.float64) + weights = resize_float(gaussian(mask, p['source_sigma']), (side, side)) + numerator = resize_float(gaussian(guide * mask[..., None], p['source_sigma']), (side, side)) + field = numerator / np.maximum(weights[..., None], 1e-12) + base[owners == family] = field[owners == family] + if family == 2: + second = resize_float(gaussian(guide_luma**2 * mask, p['source_sigma']), (side, side)) / np.maximum(weights, 1e-12) + first = field @ np.array([0.2126, 0.7152, 0.0722]) + std = np.sqrt(np.maximum(second-first**2, 0)) + # Calm Source soil (e.g.20400right) receives less texture strength. + # This map is derived from Source variance, never hand-painted. + quiet[owners == family] = (0.35+0.65*np.clip(std/8.0,0,1))[owners == family] + raw = np.asarray(generated_rgb) + if raw.ndim != 3 or raw.shape[2] != 3 or raw.dtype != np.uint8: + raise ValueError('generated input must be RGB uint8') + parent = np.asarray(Image.fromarray(raw).resize((side, side), Image.Resampling.LANCZOS), np.float64) + luma = parent @ np.array([0.2126, 0.7152, 0.0722]) + band = gaussian(luma, p['residual_sigma_fine']) - gaussian(luma, p['residual_sigma_coarse']) + texture = np.zeros_like(band) + normalizers = {} + for family in np.unique(owners): + area = owners == family + centered = band-band[area].mean() + divisor = max(float(np.percentile(np.abs(centered[area]),p['residual_percentile'])),1.0) + texture[area] = np.tanh(centered[area]/divisor) + normalizers[FAMILY_NAMES[int(family)]] = divisor + strengths = np.array([p['texture_log_gain'][n]for n in FAMILY_NAMES])[owners]*quiet + gain = np.exp(strengths*texture) + output = np.clip(np.rint(base*gain[...,None]),0,255).astype(np.uint8) + base8 = np.clip(np.rint(base),0,255).astype(np.uint8) + metrics = dict(method=METHOD, source_shape=list(a.shape),output_shape=list(output.shape), + sourceNx4_indices_sha256=hashlib.sha256(guide_indices.astype(' parent.width or y+h > parent.height: + raise ValueError('crop outside untouched parent') + raw = np.array(parent.convert('RGB').crop((x, y, x+w, y+h))) + derived, base, labels, metrics = reconstruct(indices, palette, raw, recipe['parameters']) + prepared.append((asset, derived, base, labels, metrics, source_pin, parent_pin, crop)) + out = Path(args.output).absolute() + out.mkdir(exist_ok=False) + tool_bytes, tool_pin = read_regular(__file__) + results = [] + for asset, derived, base, labels, metrics, source_pin, parent_pin, crop in prepared: + image_path = out / f'{asset}-DERIVED-source-guided-material-v1.png' + Image.fromarray(derived).save(image_path) + Image.fromarray(base).save(out / f'{asset}-SOURCE-GUIDED-BASE-diagnostic.png') + Image.fromarray(labels.astype(np.uint8)).save(out / f'{asset}-SOURCE-NX4-FAMILY-geometry-diagnostic.png') + _, output_pin = read_regular(image_path) + results.append(dict(asset=asset, output=output_pin, Source=source_pin, parent_raw=parent_pin, + parent_crop=crop, recipe=recipe_pin, tool=tool_pin, + parameters=recipe['parameters'], metrics=metrics, + lineage='DERIVED scripted cleanup; generator-return parent remains untouched')) + for record in [recipe_pin, palette_pin] + [p for t in prepared for p in (t[5], t[6])]: + _, now = read_regular(record['path']) + if now != record: + raise ValueError('input changed across cleanup') + runtime = dict(python=sys.version, executable=sys.executable, + numpy=np.__version__, Pillow=PIL.__version__, platform=sys.platform) + execution = dict(status='success', method=METHOD, + recipe_sha256=recipe_pin['sha256'], tool_sha256=tool_pin['sha256'], + source_sha256={r['asset']: r['Source']['sha256'] for r in results}, + parent_raw_sha256=sorted({r['parent_raw']['sha256'] for r in results}), + output_sha256={r['asset']: r['output']['sha256'] for r in results}, + runtime=runtime) + (out / 'DERIVED-LINEAGE-AND-ACTUAL-RECONSTRUCTION-METRICS.json').write_text( + json.dumps(dict(status='DERIVED_OUTPUTS_NOT_MATERIAL_APPROVAL', method=METHOD, + recipe=recipe_pin, tool=tool_pin, palette=palette_pin, frames=results, + execution=execution, runtime=runtime), indent=2, sort_keys=True) + '\n') + print(json.dumps(dict(output=str(out), frames=len(results), method=METHOD), indent=2)) + +if __name__ == '__main__': + main() diff --git a/tools/test_hd_terrain_cleanup.py b/tools/test_hd_terrain_cleanup.py new file mode 100644 index 00000000..b9efc7c3 --- /dev/null +++ b/tools/test_hd_terrain_cleanup.py @@ -0,0 +1,119 @@ +import copy +import json +from pathlib import Path +import struct +import sys +import tempfile +import unittest +import numpy as np +from PIL import Image +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import hd_terrain_cleanup as c + +class TerrainCleanupTests(unittest.TestCase): + def setUp(self): + self.palette = np.zeros((256, 3), np.uint8) + for i in range(32): self.palette[i] = [4*i]*3 + self.palette[60:64] = [[56,52,4],[40,40,0],[24,24,0],[12,12,0]] + self.palette[200:208] = [[85,40,36],[81,36,32],[73,32,28],[69,28,24],[60,24,20],[56,16,16],[40,12,8],[20,0,0]] + self.palette[231] = [81,60,52] + self.p = copy.deepcopy(c.DEFAULTS); self.p['output_side'] = 64 + + def test_constant_parent_has_no_retained_texture(self): + source = np.full((4,4),63,np.uint8) + out, base, labels, m = c.reconstruct(source,self.palette,np.full((64,64,3),180,np.uint8),self.p) + np.testing.assert_array_equal(out,base) + self.assertEqual(m['pixels_changed_from_Source_guided_base'],0) + np.testing.assert_array_equal(out,np.broadcast_to(self.palette[63],out.shape)) + + def test_parent_colour_and_constant_lighting_are_not_carried(self): + source = np.full((4,4),63,np.uint8) + a = np.full((64,64,3),[200,5,50],np.uint8) + b = np.full((64,64,3),[10,240,70],np.uint8) + np.testing.assert_array_equal(c.reconstruct(source,self.palette,a,self.p)[0],c.reconstruct(source,self.palette,b,self.p)[0]) + + def test_fine_generated_detail_is_retained_but_bounded(self): + yy,xx = np.indices((64,64)); raw = np.repeat((120+40*np.sin(xx*1.9)+20*np.sin(yy*2.1))[...,None],3,2).clip(0,255).astype(np.uint8) + out,base,labels,m = c.reconstruct(np.full((4,4),202,np.uint8),self.palette,raw,self.p) + self.assertGreater(m['pixels_changed_from_Source_guided_base'],1500) + self.assertGreater(m['mean_absolute_generated_residual'],0.5) + self.assertLessEqual(np.abs(out.astype(float)-base).max(),3) + self.assertLessEqual(m['maximum_retained_luma_residual'],2.5) + + def test_exact_family_geometry_and_thin_L_are_preserved(self): + source = np.array([[27,27,231,63],[27,27,231,63],[27,27,231,231],[27,27,27,27]],np.uint8) + raw = np.random.default_rng(9).integers(0,256,(80,80,3),dtype=np.uint8) + out,base,labels,m = c.reconstruct(source,self.palette,raw,self.p) + expected = c.material_families(c.scale_nx(source,4)) + np.testing.assert_array_equal(labels,np.repeat(np.repeat(expected,4,0),4,1)) + self.assertTrue(np.all(out[labels==0].max(axis=1)-out[labels==0].min(axis=1)==0)) + self.assertTrue(np.all(out[labels==1,2] <= 2)) + self.assertTrue(np.all(out[labels==3,0] > out[labels==3,1])) + + def test_smooth_source_guidance_is_not_nearest_source_copy(self): + source = np.array([[200,200,205,205]]*4,np.uint8) + out,base,_,m = c.reconstruct(source,self.palette,np.full((64,64,3),128,np.uint8),self.p) + nearest=np.repeat(np.repeat(self.palette[source],16,0),16,1) + self.assertGreater(np.any(base!=nearest,axis=2).sum(),0) + self.assertGreater(len(np.unique(base.reshape(-1,3),axis=0)),2) + + def test_deterministic_pixels_and_geometry(self): + src=np.array([[27,27,27,27],[27,63,202,27],[27,231,231,27],[27,27,27,27]],np.uint8) + raw=np.random.default_rng(7).integers(0,256,(61,61,3),dtype=np.uint8) + a=c.reconstruct(src,self.palette,raw,self.p);b=c.reconstruct(src,self.palette,raw,self.p) + for x,y in zip(a[:3],b[:3]):np.testing.assert_array_equal(x,y) + self.assertEqual(a[3],b[3]) + + def test_scale4_is_two_scale2_and_non_nearest(self): + src=np.array([[1,1,2],[1,2,2],[3,3,2]],np.uint8) + np.testing.assert_array_equal(c.scale_nx(src,4),c.scale_nx(c.scale_nx(src,2),2)) + self.assertFalse(np.array_equal(c.scale_nx(src,2),np.repeat(np.repeat(src,2,0),2,1))) + + def test_decode_rejects_reserved_and_malformed_source(self): + good=struct.pack('<5i',1,1,0,2,2)+bytes([27,63,202,231]) + self.assertEqual(c.decode_source(good).shape,(2,2)) + for bad in [good[:-1],good[:20]+bytes([27,63,240,231]),struct.pack('<5i',2,1,0,2,2)+good[20:]]: + with self.assertRaises(ValueError):c.decode_source(bad) + + def test_rejects_unknown_family_and_unsafe_parameters(self): + with self.assertRaises(ValueError):c.material_families(np.array([[100]],np.uint8)) + for name,value in [('output_side',True),('output_side',15),('source_sigma',float('nan')),('residual_sigma_coarse',0.1)]: + p=copy.deepcopy(self.p);p[name]=value + with self.assertRaises(ValueError):c.reconstruct(np.full((4,4),63,np.uint8),self.palette,np.zeros((64,64,3),np.uint8),p) + + def test_stable_reader_rejects_symlinks(self): + with tempfile.TemporaryDirectory()as d: + p=Path(d);(p/'a').write_bytes(b'fixed');(p/'link').symlink_to(p/'a') + with self.assertRaises(OSError):c.read_regular(p/'link') + + def test_actual_cli_lineage_and_bad_crop_stops_before_mkdir(self): + import hashlib + with tempfile.TemporaryDirectory()as d: + p=Path(d);src=p/'source.pic';src.write_bytes(struct.pack('<5i',1,1,0,4,4)+bytes([63]*16)) + pal=p/'palette';pal.write_bytes(bytes([0,0,0])*60+bytes([14,13,1,10,10,0,6,6,0,3,3,0])+bytes([0,0,0])*(256-64)) + raw=p/'parent.png';yy,xx=np.indices((64,64));Image.fromarray(np.repeat((80+(xx%2)*80)[...,None],3,2).astype(np.uint8)).save(raw) + def binding(q):return dict(path=str(q),sha256=hashlib.sha256(q.read_bytes()).hexdigest()) + recipe=dict(method=c.METHOD,palette=binding(pal),parameters=self.p,frames=[dict(asset='20400',Source=binding(src),parent_raw=binding(raw),crop=[0,0,64,64])]) + rp=p/'recipe.json';rp.write_text(json.dumps(recipe));out=p/'result';c.main(['--recipe',str(rp),'--output',str(out)]) + report=json.loads((out/'DERIVED-LINEAGE-AND-ACTUAL-RECONSTRUCTION-METRICS.json').read_text());row=report['frames'][0] + self.assertEqual(row['output']['sha256'],hashlib.sha256(Path(row['output']['path']).read_bytes()).hexdigest()) + self.assertEqual(row['parent_raw']['sha256'],binding(raw)['sha256']) + self.assertIn('DERIVED',row['lineage']) + with Image.open(row['output']['path']) as saved: + self.assertEqual(saved.size,(64,64)) + self.assertEqual(report['execution']['status'],'success') + self.assertEqual(report['execution']['source_sha256'],{'20400':binding(src)['sha256']}) + self.assertEqual(report['execution']['output_sha256'],{'20400':row['output']['sha256']}) + self.assertEqual(report['execution']['parent_raw_sha256'],[binding(raw)['sha256']]) + portable=copy.deepcopy(recipe) + portable['palette']['path']=pal.name + portable['frames'][0]['Source']['path']=src.name + portable['frames'][0]['parent_raw']['path']=raw.name + portable_path=p/'portable.json';portable_path.write_text(json.dumps(portable)) + moved=p/'relative-result';c.main(['--recipe',str(portable_path),'--output',str(moved)]) + self.assertEqual((moved/Path(row['output']['path']).name).read_bytes(),Path(row['output']['path']).read_bytes()) + recipe['frames'][0]['crop']=[1,0,64,64];rp.write_text(json.dumps(recipe));bad=p/'bad' + with self.assertRaises(ValueError):c.main(['--recipe',str(rp),'--output',str(bad)]) + self.assertFalse(bad.exists()) + +if __name__ == '__main__':unittest.main() diff --git a/tools/test_hd_terrain_retexture.py b/tools/test_hd_terrain_retexture.py new file mode 100644 index 00000000..4f2ff006 --- /dev/null +++ b/tools/test_hd_terrain_retexture.py @@ -0,0 +1,52 @@ +import pathlib,sys,copy,unittest +import numpy as np +sys.path.insert(0,str(pathlib.Path(__file__).resolve().parent)) +import hd_terrain_retexture as c +class CoherentTextureTests(unittest.TestCase): + def setUp(self): + self.palette=np.zeros((256,3),np.uint8) + for i in range(32):self.palette[i]=[i*4]*3 + self.palette[60:64]=[[56,52,4],[40,40,0],[24,24,0],[12,12,0]] + self.palette[200:208]=[[85,40,36],[81,36,32],[73,32,28],[69,28,24],[60,24,20],[56,16,16],[40,12,8],[20,0,0]];self.palette[231]=[81,60,52] + self.p=copy.deepcopy(c.DEFAULTS);self.p['output_side']=64 + def test_coherent_midscale_structure_survives_not_random_noise(self): + yy,xx=np.indices((64,64));lum=(120+45*np.sin(xx*2*np.pi/20)+25*np.cos(yy*2*np.pi/27));raw=np.repeat(lum[...,None],3,2).clip(0,255).astype(np.uint8) + out,base,_,m=c.reconstruct(np.full((4,4),27,np.uint8),self.palette,raw,self.p) + correlation=np.corrcoef(out[...,0].flatten(),lum.flatten())[0,1] + self.assertGreater(correlation,.90);self.assertGreater(out[...,0].std(),3) + self.assertGreater(m['mean_absolute_parent_tone_modulation'],2) + def test_gray_achromatic_and_olive_hue_not_shifted(self): + raw=np.random.default_rng(8).integers(0,256,(64,64,3),dtype=np.uint8) + for index in [27,63]: + out,_,_,m=c.reconstruct(np.full((4,4),index,np.uint8),self.palette,raw,self.p) + if index==27:np.testing.assert_array_equal(out[...,0],out[...,2]) + else:self.assertTrue(np.all(out[...,2]==0));self.assertTrue(np.all(out[...,0]==out[...,1])) + self.assertGreaterEqual(m['actual_tone_gain_min'],np.exp(-.55));self.assertLessEqual(m['actual_tone_gain_max'],np.exp(.55)) + def test_source_quiet_soil_receives_less_texture(self): + raw=np.random.default_rng(1).integers(0,256,(64,64,3),dtype=np.uint8);calm=np.full((4,4),202,np.uint8);busy=np.array([[200,207]*2,[207,200]*2]*2,np.uint8) + m1=c.reconstruct(calm,self.palette,raw,self.p)[3];m2=c.reconstruct(busy,self.palette,raw,self.p)[3] + self.assertAlmostEqual(m1['Source_derived_soil_quiet_strength_max'],.35,places=5) + self.assertGreater(m2['Source_derived_soil_quiet_strength_min'],.8) + def test_source_L_and_material_geometry_exact_with_coherent_texture(self): + src=np.array([[27,27,231,63],[27,27,231,202],[27,27,231,231],[27,27,27,27]],np.uint8);raw=np.random.default_rng(4).integers(0,256,(64,64,3),dtype=np.uint8) + a=c.reconstruct(src,self.palette,raw,self.p);b=c.reconstruct(src,self.palette,raw,self.p) + expected=np.repeat(np.repeat(c.material_families(c.scale_nx(src,4)),4,0),4,1) + np.testing.assert_array_equal(a[2],expected);np.testing.assert_array_equal(a[0],b[0]);self.assertEqual(a[3],b[3]) + def test_constant_parent_does_not_invent_texture(self): + out,base,_,m=c.reconstruct(np.full((4,4),63,np.uint8),self.palette,np.full((64,64,3),170,np.uint8),self.p) + np.testing.assert_array_equal(out,base);self.assertEqual(m['pixels_changed_from_Source_guided_base'],0) + def test_invalid_gain_and_material_fail(self): + p=copy.deepcopy(self.p);p['texture_log_gain']['gray']=float('inf') + with self.assertRaises(ValueError):c.reconstruct(np.full((4,4),27,np.uint8),self.palette,np.zeros((64,64,3),np.uint8),p) + with self.assertRaises(ValueError):c.reconstruct(np.full((4,4),100,np.uint8),self.palette,np.zeros((64,64,3),np.uint8),self.p) + def test_relative_cli_receipt_binds_runtime_versions(self): + import tempfile,json,struct,hashlib,PIL + from PIL import Image + with tempfile.TemporaryDirectory()as d: + root=pathlib.Path(d);src=root/'source.pic';src.write_bytes(struct.pack('<5i',1,1,0,4,4)+bytes([63]*16));pal=root/'palette.dat';pal.write_bytes((self.palette//4).tobytes());raw=root/'parent.png';Image.fromarray(np.full((64,64,3),100,np.uint8)).save(raw) + def binding(p):return dict(path=p.name,sha256=hashlib.sha256(p.read_bytes()).hexdigest()) + recipe=dict(method=c.METHOD,palette=binding(pal),parameters=self.p,frames=[dict(asset='20400',Source=binding(src),parent_raw=binding(raw),crop=[0,0,64,64])]);rp=root/'recipe.json';rp.write_text(json.dumps(recipe));out=root/'out';c.main(['--recipe',str(rp),'--output',str(out)]) + report=json.loads((out/'DERIVED-LINEAGE-AND-ACTUAL-RECONSTRUCTION-METRICS.json').read_text());expected=dict(python=sys.version,executable=sys.executable,numpy=np.__version__,Pillow=PIL.__version__,platform=sys.platform) + self.assertEqual(report['runtime'],expected);self.assertEqual(report['execution']['runtime'],expected) + self.assertEqual(report['execution']['output_sha256']['20400'],hashlib.sha256((out/'20400-DERIVED-source-guided-material-v1.png').read_bytes()).hexdigest()) +if __name__=='__main__':unittest.main()