Remaster fifteen original pictures with reproducible artwork cleanup

Add the final boss picture, three coherent terrain remasters and complete DSHP20G3/WORMRITE catalogue families. Preserve generated parents, portable recipes, derivation receipts and separate artistic/context reviews.

Validate 5500 HD pictures with zero rejects and all 45 mods. Review three unchanged 1080p SDL controls, preserve existing runtime/data bindings, and update Linux/Windows shared-data metadata and progress records. Cleanup tools pass 101 tests plus 12 TARGT input-safety tests; TARGT artwork remains pending.
This commit is contained in:
raptorreplica committed 2026-10-18 11:35:24 +11:00
1 parent f6dc5343d2
commit 00304e25c1
109 files changed
+22122 -81

No files matched your search

+276
View File
@@ -0,0 +1,276 @@
"""Explicit DERIVED RGB-preserving Source-geometry cleanup for mechanical families.
Unlike a Source-color/high-pass reconstruction, this keeps complete generated RGB
panels. It registers a declared parent viewport, uses actual Original Scale4x
coverage, fills only uncovered boundary samples from nearby generated RGB, and
shares complete RGB on Source-identical stationary pixels. Functional Source
palette cells are explicit Original replacements. Inputs are never overwritten.
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import sys
from fractions import Fraction
from pathlib import Path
import numpy as np
from PIL import Image, __version__ as PILLOW_VERSION
import hd_frame_policy
from hd_frame_policy import _indices
METHOD = "derived-generated-RGB-SourceScale4-geometry-coherent-dynamic-regions-v2"
def digest(data):
return hashlib.sha256(data).hexdigest()
def checked(binding, base):
if not isinstance(binding, dict) or set(binding) != {"path", "sha256"}:
raise ValueError("input binding needs exactly path and sha256")
p = Path(binding["path"])
if not p.is_absolute():
p = base / p
data = p.read_bytes()
if digest(data) != binding["sha256"]:
raise ValueError(f"input hash mismatch: {p}")
return p.resolve(), data
def scale2(source):
"""Original render::upscale Scale2x; label256 remains transparent."""
h, w = source.shape
b = np.concatenate([source[:1], source[:-1]], axis=0)
d = np.concatenate([source[:, :1], source[:, :-1]], axis=1)
f = np.concatenate([source[:, 1:], source[:, -1:]], axis=1)
g = np.concatenate([source[1:], source[-1:]], axis=0)
condition = (b != g) & (d != f)
out = np.empty((h * 2, w * 2), dtype=np.uint16)
out[0::2, 0::2] = np.where(condition & (d == b), d, source)
out[0::2, 1::2] = np.where(condition & (b == f), f, source)
out[1::2, 0::2] = np.where(condition & (d == g), d, source)
out[1::2, 1::2] = np.where(condition & (g == f), f, source)
return out
def fill_generated_rgb(rgb, trusted, mask, source4, max_distance):
"""Deterministic nearest generated RGB, preferring matching Source labels.
No Source RGB fallback or synthesized texture is used. A frame without a
nearby generated donor is rejected. Decisions never cascade through fills.
"""
donors = trusted & mask
missing = mask & ~trusted
result = rgb.copy()
distances = []
for y, x in np.argwhere(missing):
y0, y1 = max(0, y - max_distance), min(mask.shape[0], y + max_distance + 1)
x0, x1 = max(0, x - max_distance), min(mask.shape[1], x + max_distance + 1)
dy, dx = np.nonzero(donors[y0:y1, x0:x1])
if not len(dy):
raise ValueError(f"no generated RGB donor within bound at ({x},{y})")
yy, xx = dy + y0, dx + x0
distance = (yy-y)**2 + (xx-x)**2
allowed = distance <= max_distance**2
if not np.any(allowed):
raise ValueError(f"no generated RGB donor within distance at ({x},{y})")
yy, xx, distance = yy[allowed], xx[allowed], distance[allowed]
# Exact Source label preference prevents crossing an adjacent material
# when an equally nearby native material donor actually exists.
same = source4[yy, xx] == source4[y, x]
if np.any(same):
yy, xx, distance = yy[same], xx[same], distance[same]
choice = np.lexsort((xx, yy, distance))[0]
result[y, x] = rgb[yy[choice], xx[choice]]
distances.append(int(distance[choice]))
return result, dict(filled_generated_RGB_samples=int(missing.sum()),
maximum_donor_squared_distance=max(distances, default=0),
no_Source_RGB_or_noise_fallback=True)
def dynamic_regions(sources):
"""Source-observed changing parts, with a fixed one-native-cell halo.
Keep whole parent RGB within each 8-connected changing component's bounding
box. Otherwise isolated unchanged Source labels inside a changing cockpit
can become a visible mosaic of stationary and phase-specific RGB.
"""
changed = np.any(sources != sources[0], axis=0)
h, w = changed.shape
seen = set()
rectangles = []
for y, x in np.argwhere(changed):
start = (int(y), int(x))
if start in seen:
continue
todo, component = [start], []
seen.add(start)
while todo:
yy, xx = todo.pop()
component.append((yy, xx))
for ny in range(max(0, yy-1), min(h, yy+2)):
for nx in range(max(0, xx-1), min(w, xx+2)):
if changed[ny, nx] and (ny, nx) not in seen:
seen.add((ny, nx))
todo.append((ny, nx))
rectangles.append([max(0, min(x for y,x in component)-1),
max(0, min(y for y,x in component)-1),
min(w, max(x for y,x in component)+2),
min(h, max(y for y,x in component)+2)])
return sorted(set(tuple(r) for r in rectangles))
def render_family(recipe, base):
if not isinstance(recipe, dict) or set(recipe) - {
"format_version", "label", "palette", "output_scale", "max_fill_distance_pixels",
"shared_backing_anchor", "frames"
}:
raise ValueError("unknown recipe fields")
if type(recipe.get("format_version")) is not int or recipe["format_version"] != 1:
raise ValueError("unsupported recipe version")
if type(recipe.get("output_scale")) is not int or recipe["output_scale"] != 4:
raise ValueError("output_scale must be exactly4 for actual SourceScale4 coverage")
bound = recipe.get("max_fill_distance_pixels", 12)
if type(bound) is not int or not 1 <= bound <= 24:
raise ValueError("donor distance must be integer1..24 output pixels")
frames = recipe.get("frames")
if not isinstance(frames, list) or not 1 <= len(frames) <= 16:
raise ValueError("need1..16 complete-family frames")
anchor = recipe.get("shared_backing_anchor", 0)
if type(anchor) is not int or not 0 <= anchor < len(frames):
raise ValueError("invalid stationary backing anchor")
palette_path, palette_bytes = checked(recipe["palette"], base)
if len(palette_bytes) != 768 or max(palette_bytes) > 63:
raise ValueError("need exact768-byte six-bit Original palette")
p6 = np.frombuffer(palette_bytes, np.uint8).reshape(256,3).astype(np.uint16)
palette = ((p6 << 2) | (p6 >> 4)).astype(np.uint8)
loaded, inputs, assets = [], {str(palette_path): digest(palette_bytes)}, set()
shape = None
for f in frames:
if not isinstance(f, dict) or set(f) != {
"asset", "source", "raw_png", "viewport", "sampling_offset_native", "origin"
}:
raise ValueError("frame needs exact declared source/raw/viewport/offset/origin fields")
asset = f["asset"]
if not isinstance(asset, str) or not asset or any(c not in "0123456789abcdef" for c in asset) or asset in assets:
raise ValueError("need unique lowercase hex asset IDs")
assets.add(asset)
sp, sb = checked(f["source"], base)
w, h, cells = _indices(sb)
if shape is None:
shape = (h,w)
if shape != (h,w) or w*h*16 > 4_194_304:
raise ValueError("inconsistent or excessive complete-family canvas")
if not isinstance(f["origin"], list) or len(f["origin"]) != 2 or any(type(x) is not int for x in f["origin"]):
raise ValueError("need literal two-integer native origin")
source = np.asarray(cells,np.uint16).reshape(h,w)
source4 = scale2(scale2(source))
mask = source4 < 256
rp, rb = checked(f["raw_png"], base)
if len(rb) > 32*1024*1024:
raise ValueError("parent PNG exceeds32MiB")
if not isinstance(f["viewport"], list) or len(f["viewport"]) != 4:
raise ValueError("need four exact rational parent viewport bounds")
l,t,r,b = [Fraction(x) for x in f["viewport"]]
if not isinstance(f["sampling_offset_native"], list) or len(f["sampling_offset_native"]) != 2:
raise ValueError("need exact rational two-coordinate registration offset")
ox,oy = [Fraction(x) for x in f["sampling_offset_native"]]
if abs(ox)>2 or abs(oy)>2:
raise ValueError("registration exceeds two native cells")
actual = [l+ox*(r-l)/w,t+oy*(b-t)/h,r+ox*(r-l)/w,b+oy*(b-t)/h]
with Image.open(io.BytesIO(rb)) as image:
if image.format != "PNG" or image.width*image.height > 4_194_304:
raise ValueError("parent must be bounded PNG")
if not 0<=l<r<=image.width or not 0<=t<b<=image.height:
raise ValueError("declared parent viewport exceeds image")
pixels = np.asarray(image.convert("RGBA").transform(
(w*4,h*4),Image.Transform.EXTENT,tuple(map(float,actual)),Image.Resampling.BICUBIC)).copy()
trusted = pixels[:,:,3]>=128
rgb, filling = fill_generated_rgb(pixels[:,:,:3],trusted,mask,source4,bound)
loaded.append(dict(frame=f,source=source,source4=source4,mask=mask,rgb=rgb,
sampled=pixels,trusted=trusted,filling=filling,
actual_sampling_bounds=[str(x)for x in actual]))
inputs[str(sp)] = digest(sb)
inputs[str(rp)] = digest(rb)
# Whole RGB, not just high-frequency noise, is shared on truly identical
# SourceScale4 labels. Changed Source labels keep each phase's parent RGB.
s4 = np.stack([x['source4']for x in loaded])
common = np.all(s4==s4[anchor],axis=0) & (s4[anchor]<256)
regions = dynamic_regions(np.stack([x['source'] for x in loaded]))
for l,t,r,b in regions:
common[t*4:b*4,l*4:r*4] = False
stationary_rgb = loaded[anchor]['rgb'].copy()
outputs, reports = [], []
for item in loaded:
f, source, mask = item['frame'],item['source'],item['mask']
h,w = source.shape
rgb = item['rgb'].copy()
shared_changed = int(np.any(rgb[common]!=stationary_rgb[common],axis=1).sum())
rgb[common] = stationary_rgb[common]
functional_samples = 0
for y,x in np.argwhere((source>=240)&(source<256)):
rgb[y*4:(y+1)*4,x*4:(x+1)*4] = palette[source[y,x]]
functional_samples += int(mask[y*4:(y+1)*4,x*4:(x+1)*4].sum())
rgba = np.zeros((h*4,w*4,4),np.uint8)
rgba[:,:,:3][mask] = rgb[mask]
rgba[:,:,3][mask] = 255
source_nn = np.repeat(np.repeat(source,4,axis=0),4,axis=1)
valid_nn = source_nn<256
disagreement = np.zeros(mask.shape,bool)
disagreement[mask&valid_nn] = np.any(rgb[mask&valid_nn]!=palette[source_nn[mask&valid_nn]],axis=1)
if int(disagreement.sum()) < int(mask.sum())//4:
raise ValueError("insufficient complete generated RGB variation; refusing SourceNN HD relabel")
outputs.append(rgba)
reports.append(dict(asset=f['asset'],logical_width=w,logical_height=h,origin=f['origin'],
Source_painted_cells=int((source<256).sum()),SourceScale4_opaque_samples=int(mask.sum()),
derived_alpha_exact_SourceScale4=True,actual_sampling_bounds=item['actual_sampling_bounds'],
removed_generated_outside_SourceScale4=int((item['trusted']&~mask).sum()),
original_sampled_RGB_preserved_before_stationary_share=int((item['trusted']&mask).sum()),
stationary_full_RGB_replacements=shared_changed,
functional_Original_RGB_samples=functional_samples,
generated_RGB_different_from_SourceNN_samples=int(disagreement.sum()),
**item['filling']))
for rgba in outputs:
assert np.array_equal(rgba[:,:,:3][common],outputs[anchor][:,:,:3][common])
return outputs, dict(method=METHOD,output_class='DERIVED_SCRIPTED_RGBA_NOT_UNTOUCHED_GENERATION',
input_hashes=inputs,output_scale=4,shared_SourceScale4_labels=int(common.sum()),stationary_anchor_asset=frames[anchor]['asset'],
Source_derived_dynamic_component_halo_native=1,dynamic_regions_native=[list(r) for r in regions],
RGB_policy='Preserve full generated RGB, except declared nearest generated-RGB fills, Source-identical full-RGB backing share, and literal functional Source palette cells; no noise/highpass reconstruction.',
source_mask='Actual Original Scale4x = Scale2x twice; complete native canvas and origin retained.',
frames=reports,
decoder_module=dict(path=str(Path(hd_frame_policy.__file__).resolve()),
sha256=digest(Path(hd_frame_policy.__file__).read_bytes())),
python_version=sys.version,numpy_version=np.__version__,pillow_version=PILLOW_VERSION)
def run_recipe(recipe_path, output):
recipe_path, output = Path(recipe_path),Path(output)
if output.exists():
raise ValueError('output directory already exists; no inputs or old outputs overwritten')
rb = recipe_path.read_bytes();recipe = json.loads(rb)
arrays,report = render_family(recipe,recipe_path.resolve().parent)
payloads=[]
for f,array in zip(recipe['frames'],arrays):
b=io.BytesIO();Image.fromarray(array).save(b,format='PNG',compress_level=9)
payloads.append((f['asset'],b.getvalue(),array))
output.mkdir(parents=True)
for asset,data,array in payloads:
(output/(asset+'-derived.png')).write_bytes(data)
execution=dict(status='success',method=METHOD,recipe_sha256=digest(rb),tool_sha256=digest(Path(__file__).read_bytes()),
source_sha256={f['asset']:f['source']['sha256']for f in recipe['frames']},
parent_raw_sha256=sorted({f['raw_png']['sha256']for f in recipe['frames']}),
output_sha256={asset:digest(data)for asset,data,_ in payloads})
report['execution']=execution
report['outputs']=[dict(asset=asset,file=asset+'-derived.png',sha256=digest(data),RGBA_sha256=digest(array.tobytes()),bytes=len(data))for asset,data,array in payloads]
(output/'normalized-execution.json').write_text(json.dumps(execution,indent=2,sort_keys=True)+'\n')
(output/'derived-provenance.json').write_text(json.dumps(report,indent=2,sort_keys=True)+'\n')
return report
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('recipe',type=Path);p.add_argument('output_new_directory',type=Path)
a=p.parse_args();r=run_recipe(a.recipe,a.output_new_directory);print(json.dumps(r['execution'],sort_keys=True))
if __name__=='__main__':
main()
+265
View File
@@ -0,0 +1,265 @@
"""Source-guided DERIVED hatch reconstruction with generated continuous RGB.
Source supplies alpha, cover/core geometry and broad material/exposure statistics.
Generated RGB supplies continuous mechanism form, warning/contact detail and real
soil mid/fine structure. No random texture or per-cell Source RGB copying occurs.
Every input is SHA-bound; every output is new and explicitly derived.
"""
from pathlib import Path
from collections import deque
import argparse, hashlib, io, json, platform, re, sys
import numpy as np
import PIL
from PIL import Image, ImageFilter
from hd_frame_policy import _indices
METHOD = 'derived-continuous-generated-hatch-Source-broad-tone-phase-rebuild-v1'
def sha(body): return hashlib.sha256(body).hexdigest()
def checked(ref, base):
if type(ref) is not dict or set(ref) != {'path', 'sha256'}:
raise ValueError('input requires exactly path/sha256')
p = Path(ref['path']); p = p if p.is_absolute() else base/p
if p.is_symlink() or not p.is_file(): raise ValueError('regular nonsymlink input required')
b = p.read_bytes()
if sha(b) != ref['sha256']: raise ValueError('input SHA mismatch')
return b
def box(mask):
y,x = np.nonzero(mask)
if not len(x): raise ValueError('empty required region')
return [int(x.min()), int(y.min()), int(x.max()+1), int(y.max()+1)]
def main_painted_bounds(rgba):
"""Largest alpha>=128 four-connected component; disconnected fringe excluded."""
mask = rgba[:,:,3] >= 128
seen = np.zeros(mask.shape, bool); h,w = mask.shape; best=[]
for sy,sx in zip(*np.nonzero(mask)):
if seen[sy,sx]: continue
todo=deque([(int(sy),int(sx))]); seen[sy,sx]=True; comp=[]
while todo:
y,x=todo.popleft(); comp.append((y,x))
for yy,xx in ((y-1,x),(y,x-1),(y,x+1),(y+1,x)):
if 0<=yy<h and 0<=xx<w and mask[yy,xx] and not seen[yy,xx]:
seen[yy,xx]=True; todo.append((yy,xx))
if len(comp)>len(best): best=comp
if len(best)<16: raise ValueError('no substantial painted component')
ys,xs=np.array(best).T
return [int(xs.min()),int(ys.min()),int(xs.max()+1),int(ys.max()+1)]
def resize_rgb(a, size):
return np.asarray(Image.fromarray(np.clip(a,0,255).astype('uint8'), 'RGB').resize(size, Image.Resampling.LANCZOS)).astype(float)
def blur(a, radius):
return np.asarray(Image.fromarray(np.clip(a,0,255).astype('uint8'), 'RGB').filter(ImageFilter.GaussianBlur(radius))).astype(float)
def broad_soil(generated, source_rgb, source_mask, scale):
"""Source mean/broad modulation + correlated generated low/mid/fine detail.
Radius4 logical cells explicitly removes Source pixel-grid colour changes.
The generated residuals are deterministic RGB texture, never random noise.
"""
if not source_mask.any(): raise ValueError('no authentic soil palette samples')
mean=source_rgb[source_mask].mean(0)
filled=np.broadcast_to(mean, source_rgb.shape).copy(); filled[source_mask]=source_rgb[source_mask]
source_low=resize_rgb(blur(filled,4), (generated.shape[1],generated.shape[0]))
gen_small=blur(generated, max(1,scale*.35)); gen_large=blur(generated, scale*3)
low=gen_large-gen_large.mean((0,1)); mid=gen_small-gen_large; fine=generated-gen_small
return np.clip(mean + .30*(source_low-mean) + .18*low + .38*mid + .48*fine,0,255)
def warning_rows(source):
return np.flatnonzero(((source>=75)&(source<=95)).sum(1) >= max(4,source.shape[1]//6))
def core_donor_mask(image, core_bounds, scale):
"""Only the center neighbourhood can donate core RGB, never warm rim soil."""
warm=(image[:,:,0]>image[:,:,1]*1.1)&(image[:,:,1]>image[:,:,2]*1.8)&(image[:,:,0]>35)
b=core_bounds; region=np.zeros(warm.shape,bool)
region[max(0,(b[1]-3)*scale):min(image.shape[0],(b[3]+3)*scale),
max(0,(b[0]-3)*scale):min(image.shape[1],(b[2]+3)*scale)]=True
return warm&region
def map_parent(rgba, source, scale):
b=main_painted_bounds(rgba); sb=box(source<256)
out=np.zeros((source.shape[0]*scale,source.shape[1]*scale,3),float)
out[sb[1]*scale:sb[3]*scale,sb[0]*scale:sb[2]*scale]=resize_rgb(rgba[b[1]:b[3],b[0]:b[2],:3],((sb[2]-sb[0])*scale,(sb[3]-sb[1])*scale))
return out,b,sb
def validate_recipe(recipe):
keys={'format_version','method','output_scale','palette','raw_png','layout','frames','soil_donor_frame','closed_donor_frame','module_donor_frame','warning_donor_frame'}
if type(recipe) is not dict or set(recipe)!=keys or type(recipe['format_version']) is not int or recipe['format_version']!=1 or recipe['method']!=METHOD:
raise ValueError('unknown recipe format/method')
s=recipe['output_scale']
if type(s) is not int or not 4<=s<=16: raise ValueError('scale must be integer4..16')
layout=recipe['layout']
if type(layout) is not dict or set(layout)!={'columns','rows'} or any(type(v) is not int or v<=0 for v in layout.values()):
raise ValueError('invalid layout')
frames=recipe['frames']
if not isinstance(frames,list) or not 2<=len(frames)<=32 or len(frames)!=layout['columns']*layout['rows']:
raise ValueError('complete exact slot inventory required')
assets=[];source_paths=[]
for f in frames:
if type(f) is not dict or set(f)!={'asset','source','origin'}: raise ValueError('frame fields invalid')
asset=f['asset']
if type(asset) is not str or not re.fullmatch(r'(?:0|[1-9a-f][0-9a-f]{0,7})',asset) or int(asset,16)>0xffffffff:
raise ValueError('asset must be a canonical lowercase u32 hex ID, never a path')
assets.append(int(asset,16))
origin=f['origin']
if type(origin) is not list or len(origin)!=2 or any(type(v) is not int or v!=0 for v in origin):
raise ValueError('origin must contain two literal integer zeros')
source=f['source']
if type(source) is not dict or set(source)!={'path','sha256'} or type(source['path']) is not str or not source['path']:
raise ValueError('Source needs literal path/sha256')
source_paths.append(Path(source['path']))
if len(set(assets))!=len(assets): raise ValueError('duplicate asset')
if assets!=list(range(assets[0],assets[0]+len(assets))): raise ValueError('asset IDs must follow complete contiguous numeric Source order')
base=str(source_paths[0])
if source_paths!=[Path(base+('.'+str(i) if i else '')) for i in range(len(frames))]:
raise ValueError('Source picture paths must follow base,base.1,... family order')
for name in ('soil_donor_frame','closed_donor_frame','module_donor_frame','warning_donor_frame'):
if type(recipe[name]) is not int or not 0<=recipe[name]<len(frames): raise ValueError('bad donor frame')
def render(recipe, base):
validate_recipe(recipe); scale=recipe['output_scale']
pb=checked(recipe['palette'],base)
if len(pb)!=768: raise ValueError('exact VGA palette required')
p=np.minimum(np.frombuffer(pb,dtype='uint8').astype('uint16'),63)
palette=((p<<2)|(p>>4)).astype('uint8').reshape(256,3)
raw=Image.open(io.BytesIO(checked(recipe['raw_png'],base)))
if raw.format!='PNG' or raw.width*raw.height>8_000_000: raise ValueError('bounded PNG required')
rgba=np.asarray(raw.convert('RGBA')); cols=recipe['layout']['columns']; rows=recipe['layout']['rows']
sources=[]; mapped=[]; transforms=[]
for i,f in enumerate(recipe['frames']):
w,h,c=_indices(checked(f['source'],base)); src=np.asarray(c,dtype='uint16').reshape(h,w)
if (w,h)!=(32,32): raise ValueError('this method requires authentic32x32 hatch canvases')
if np.any((src<256)&~(((src>=194)&(src<=239))|(src<32)|((src>=75)&(src<=95)))):
raise ValueError('unsupported Source materials')
c0=i%cols; r0=i//cols; x0=c0*raw.width//cols; x1=(c0+1)*raw.width//cols; y0=r0*raw.height//rows; y1=(r0+1)*raw.height//rows
m,b,sb=map_parent(rgba[y0:y1,x0:x1],src,scale)
sources.append(src); mapped.append(m)
transforms.append(dict(asset=f['asset'],declared_raw_slot=[x0,y0,x1,y1],own_painted_bounds=[b[0]+x0,b[1]+y0,b[2]+x0,b[3]+y0],Source_painted_bounds=sb,
nonuniform_scale_xy=[(sb[2]-sb[0])*scale/(b[2]-b[0]),(sb[3]-sb[1])*scale/(b[3]-b[1])]))
source=np.stack(sources); h,w=source.shape[1:]
if not np.all(source[0]<256): raise ValueError('first frame must be a full opaque soil tile')
static=np.all(source[1:]==source[1],axis=0)&(source[1]<256)
# Broad phase statistics below exclude the moving support teeth and border.
yy,xx=np.indices((h,w))
module=mapped[recipe['module_donor_frame']].copy()
# Remove the donor's luminous center before inserting each authentic phase's
# own core. The surrounding generated dark well supplies the replacement;
# otherwise the donor core would leak around a smaller/dimmer Source core.
donor_source=source[recipe['module_donor_frame']]
donor_core=(donor_source>=75)&(donor_source<=95)
for row in warning_rows(donor_source): donor_core[row]=False
if not donor_core.any(): raise ValueError('module donor needs authentic core')
donor_warm=core_donor_mask(module,box(donor_core),scale)
if donor_warm.any():
db=box(donor_warm); near=np.zeros(donor_warm.shape,bool)
near[max(0,db[1]-scale):db[3]+scale,max(0,db[0]-scale):db[2]+scale]=True
dark=near&(module.max(2)<30)&~donor_warm
if not dark.any(): raise ValueError('no generated dark-well donor')
module[donor_warm]=np.median(module[dark],axis=0)
reference_roi=(xx>=7)&(xx<26)&(yy>=6)&(yy<28)&(source[-1]<32)
big_ref=np.repeat(np.repeat(reference_roi,scale,0),scale,1)
donor_p90=max(1,float(np.percentile(module[big_ref].mean(1),90)))
soil_donor=mapped[recipe['soil_donor_frame']]
soil0=broad_soil(soil_donor,palette[np.minimum(source[0],255)],(source[0]>=194)&(source[0]<=239),scale)
closed=mapped[recipe['closed_donor_frame']]
quiet=broad_soil(closed,palette[np.minimum(source[1],255)],(source[1]>=194)&(source[1]<=239),scale)
# Shared rim keeps the generated continuous edge texture but matches one
# family-wide authentic brown mean; its pixels never vary across phases.
border_source=palette[source[1][static]].mean(0)
border_pixels=closed[np.repeat(np.repeat(static,scale,0),scale,1)]
border_ratio=np.clip(border_source/np.maximum(1,border_pixels.mean(0)),.25,1.5)
common_border=closed*border_ratio
# Reuse one complete generated warning/contact assembly. Moving the entire
# strip avoids Source-pixel brick reconstruction and phase-to-phase drift.
warning_image=mapped[recipe['warning_donor_frame']]
rr,gg,bb=warning_image[:,:,0],warning_image[:,:,1],warning_image[:,:,2]
warm=(rr>gg*1.1)&(gg>bb*1.8)&(gg>45)&(rr>80)
found=np.flatnonzero(warm.sum(1)>scale*8)
if not len(found): raise ValueError('generated warning assembly missing')
wy0,wy1=int(found.min()),int(found.max()+1)
warning_strip=resize_rgb(warning_image[wy0:wy1,4*scale:27*scale],(23*scale,scale))
contact_strip=resize_rgb(warning_image[wy1:min(warning_image.shape[0],wy1+3*scale),4*scale:27*scale],(23*scale,2*scale))
outputs=[]; metrics=[]
for i,src in enumerate(sources):
if i==0:
rgb=soil0.copy(); gain=0.; warn=None; core_bounds=None
else:
warnrows=warning_rows(src); warn=int(warnrows[0]) if len(warnrows) else None
start=max(6,warn+4) if warn is not None else 6
roi=(xx>=7)&(xx<26)&(yy>=start)&(yy<28)&(src<32)
values=palette[src[roi]].mean(1)
target_p90=float(np.percentile(values,90)) if len(values) else 4.
gain=float(np.clip(target_p90/donor_p90,0,1.6))
rgb=np.clip(module*gain,0,255)
end=warn if warn is not None else (28 if i in (1,2) else 5)
if end>5:
# A single generated soil cover translates upward, exposing the
# shared fixed mechanism rather than a new render in each slot.
offset=(28-end)*scale
rgb[5*scale:end*scale,4*scale:27*scale]=quiet[5*scale+offset:28*scale,4*scale:27*scale]
if i>=2:
topmean=palette[src[5,4:27]].mean(0); soilmean=palette[src[(src>=194)&(src<=239)]].mean(0)
ratio=np.clip(topmean/np.maximum(1,soilmean),.35,1.3)
rgb[5*scale:6*scale,4*scale:27*scale]*=ratio
if warn is not None:
strip=warning_strip.copy()
actual_values=palette[src[warn][(src[warn]>=75)&(src[warn]<=95)]]
target=actual_values.mean(0)
donor=strip[(strip[:,:,0]>strip[:,:,1]*1.1)&(strip[:,:,1]>strip[:,:,2]*1.8)]
if not len(donor): raise ValueError('empty warning color')
ratio=np.clip(target/np.maximum(1,donor.mean(0)),.1,2)
strip=np.clip(strip*ratio,0,255)
rgb[warn*scale:(warn+1)*scale,4*scale:27*scale]=strip
rgb[(warn+1)*scale:(warn+3)*scale,4*scale:27*scale]=contact_strip
core=((src>=75)&(src<=95))
if warn is not None: core[warn]=False
core_bounds=None
if core.any():
cb=box(core); core_bounds=cb
own=mapped[i]; wr=core_donor_mask(own,cb,scale)
wr[:(warn+3)*scale if warn is not None else 7*scale]=False
if not wr.any(): raise ValueError('generated own-phase core missing')
ob=box(wr); patch=resize_rgb(own[ob[1]:ob[3],ob[0]:ob[2]],((cb[2]-cb[0])*scale,(cb[3]-cb[1])*scale))
target=palette[src[core]].mean(0); patch*=target/np.maximum(1,patch.mean((0,1)))
rgb[cb[1]*scale:cb[3]*scale,cb[0]*scale:cb[2]*scale]=np.clip(patch,0,255)
bigstatic=np.repeat(np.repeat(static,scale,0),scale,1)
rgb[bigstatic]=common_border[bigstatic]
alpha=np.repeat(np.repeat(src<256,scale,0),scale,1)
out=np.zeros((h*scale,w*scale,4),dtype='uint8');out[alpha,:3]=np.rint(np.clip(rgb[alpha],0,255)).astype('uint8');out[alpha,3]=255
outputs.append(out)
metrics.append(dict(asset=recipe['frames'][i]['asset'],Source_alpha_exact=True,warning_Source_y=warn,body_broad_exposure_gain=gain,Source_core_bounds=core_bounds,
meaningful_generated_content='continuous module RGB, complete generated warning/contact assembly, own-phase core texture, correlated generated soil mid/fine details',
RGB_not_untouched='interpolation, broad material color correction and phase gain are intentional derived edits'))
shared=np.repeat(np.repeat(static,scale,0),scale,1)
assert all(np.array_equal(o[shared],outputs[1][shared]) for o in outputs[2:])
return outputs,dict(method=METHOD,transforms=transforms,frames=metrics,static_Source_cells=int(static.sum()),
reference_shape=[w,h],output_scale=scale,Source_RGB_nearest_or_random_noise=False,
broad_Source_radius_logical_cells=4,shared_module_donor_frame=recipe['module_donor_frame'])
def run(recipe_path,output):
recipe_path=Path(recipe_path);output=Path(output)
if output.exists(): raise ValueError('output exists; no overwrite')
body=recipe_path.read_bytes(); recipe=json.loads(body); arrays,result=render(recipe,recipe_path.parent)
payloads={}
for f,a in zip(recipe['frames'],arrays):
stream=io.BytesIO();Image.fromarray(a,'RGBA').save(stream,format='PNG',compress_level=9);payloads[f['asset']]=stream.getvalue()
output.mkdir(parents=True)
for asset,b in payloads.items(): (output/(asset+'-derived.png')).write_bytes(b)
result['execution']=dict(status='success',method=METHOD,recipe_sha256=sha(body),tool_sha256=sha(Path(__file__).read_bytes()),
source_sha256={f['asset']:f['source']['sha256'] for f in recipe['frames']},
parent_raw_sha256=sorted({recipe['raw_png']['sha256']}),output_sha256={k:sha(v) for k,v in payloads.items()})
decoder=Path(sys.modules[_indices.__module__].__file__).resolve()
result['implementation_bindings']=dict(
decoder=dict(path=str(decoder),sha256=sha(decoder.read_bytes())),
runtime=dict(python=platform.python_version(),numpy=np.__version__,pillow=PIL.__version__,executable=sys.executable),
pixel_math_parent_tool_sha256='2ba29bea099d30105f61541504ba882712bda42be2d1c363e668226035bc23d2',
successor_scope='input validation and receipt metadata only; rendering math unchanged')
(output/'execution-and-derived-provenance.json').write_text(json.dumps(result,indent=2)+'\n')
return result
if __name__=='__main__':
parser=argparse.ArgumentParser();parser.add_argument('recipe',type=Path);parser.add_argument('new_output',type=Path)
args=parser.parse_args();print(json.dumps(run(args.recipe,args.new_output)))
+274
View File
@@ -0,0 +1,274 @@
"""Derived family cleanup preserving continuous generated RGB, not Source cells.
Source controls canvas/alpha, material-region statistics and the moving envelope.
Generated RGB supplies all body form and shading. Only explicit functional blue
cue cells are copied from Source RGB. Inputs and prior outputs are never changed.
"""
from __future__ import annotations
import argparse
from collections import deque
from fractions import Fraction
import hashlib
import io
import json
from pathlib import Path
import platform
import numpy as np
from PIL import Image, ImageFilter, __version__ as PILLOW_VERSION
import hd_frame_policy
METHOD = "derived-generated-worm-rgb-broad-material-source-alpha-shared-backing-v1"
def sha(data):
return hashlib.sha256(data).hexdigest()
def checked(binding, base):
if type(binding) is not dict or set(binding) != {"path", "sha256"}:
raise ValueError("input needs exactly path/sha256")
path = Path(binding["path"])
if not path.is_absolute():
path = base / path
data = path.read_bytes()
if sha(data) != binding["sha256"]:
raise ValueError(f"input hash mismatch: {path}")
return path.resolve(), data
def up(mask, scale):
return np.repeat(np.repeat(mask, scale, axis=0), scale, axis=1)
def nearest_generated_fill(rgb, trusted, occupied):
"""Deterministic nearest Manhattan donor, with raster/N,W,E,S tie order."""
if not trusted.any():
raise ValueError("no visible generated RGB donors")
owner_y, owner_x = np.indices(trusted.shape)
known = trusted.copy()
queue = deque((int(y), int(x)) for y, x in np.argwhere(trusted))
height, width = trusted.shape
while queue:
y, x = queue.popleft()
for ny, nx in ((y-1, x), (y, x-1), (y, x+1), (y+1, x)):
if 0 <= ny < height and 0 <= nx < width and not known[ny, nx]:
known[ny, nx] = True
owner_y[ny, nx], owner_x[ny, nx] = owner_y[y, x], owner_x[y, x]
queue.append((ny, nx))
missing = occupied & ~trusted
filled = rgb.copy()
filled[missing] = rgb[owner_y[missing], owner_x[missing]]
return filled, int(missing.sum())
def inner_weight(mask, feather):
"""Blend inside a genuine moving envelope; outside is exactly stationary."""
current = Image.fromarray((mask * 255).astype(np.uint8))
weight = np.zeros(mask.shape, dtype=np.float64)
for _ in range(feather):
weight += np.asarray(current) > 0
current = current.filter(ImageFilter.MinFilter(3))
return weight / feather
def region_gain(target, generated, label):
if len(target) == 0 or len(generated) == 0:
raise ValueError(f"empty {label} material sample")
target_mean, raw_mean = target.mean(axis=0), generated.mean(axis=0)
gain = np.clip(target_mean / np.maximum(raw_mean, 1), 0.25, 2.0)
return gain, target_mean, raw_mean
def render_family(recipe, base):
if (type(recipe) is not dict or set(recipe) !=
{"format_version", "label", "palette", "output_scale", "feather_logical_pixels", "frames"}
or type(recipe["format_version"]) is not int or recipe["format_version"] != 1):
raise ValueError("unknown recipe fields/version")
scale = recipe["output_scale"]
feather = recipe["feather_logical_pixels"]
if type(scale) is not int or not 4 <= scale <= 16:
raise ValueError("output_scale needs integer4..16")
if type(feather) is not int or not 1 <= feather <= 2:
raise ValueError("feather needs integer1..2 logical pixels")
frames = recipe["frames"]
if type(frames) is not list or not 2 <= len(frames) <= 64:
raise ValueError("needs complete family2..64")
palette_path, palette_data = checked(recipe["palette"], base)
if len(palette_data) != 768:
raise ValueError("Original palette needs768 bytes")
palette = np.minimum(np.frombuffer(palette_data, dtype=np.uint8), 63)
palette = ((palette << 2) | (palette >> 4)).reshape(256, 3)
loaded, ids, shape = [], set(), None
inputs = {str(palette_path): sha(palette_data)}
for frame in frames:
if type(frame) is not dict or set(frame) != {"asset", "source", "parent_raw", "logical_canvas", "origin", "atlas_grid", "raw_painted_bounds"}:
raise ValueError("unknown frame fields")
asset = frame["asset"]
if type(asset) is not str or not asset or asset in ids or any(c not in "0123456789abcdef" for c in asset):
raise ValueError("unique lowercase hexadecimal IDs required")
ids.add(asset)
source_path, source_data = checked(frame["source"], base)
width, height, cells = hd_frame_policy._indices(source_data)
if shape is not None and shape != (height, width):
raise ValueError("family logical canvas differs")
shape = (height, width)
if (type(frame["logical_canvas"]) is not list or type(frame["origin"]) is not list
or any(type(v) is not int for v in frame["logical_canvas"]+frame["origin"])
or frame["logical_canvas"] != [width, height] or frame["origin"] != [0, 0]):
raise ValueError("Source canvas/origin must remain exact")
if width * height * scale * scale > 4_194_304:
raise ValueError("output exceeds runtime pixel ceiling")
source = np.asarray(cells, dtype=np.uint16).reshape(height, width)
if not (source < 256).any():
raise ValueError("empty Source")
raw_path, raw_data = checked(frame["parent_raw"], base)
if len(raw_data) > 32 * 1024 * 1024:
raise ValueError("raw exceeds32MiB")
with Image.open(io.BytesIO(raw_data)) as opened:
if opened.format != "PNG" or opened.width * opened.height > 4_194_304:
raise ValueError("parent needs bounded PNG")
raw = opened.convert("RGBA")
if raw.getchannel("A").getextrema()[0] != 0:
raise ValueError("parent must have actual transparent alpha")
grid = frame["atlas_grid"]
if (type(grid) is not dict or set(grid) != {"columns", "rows", "column", "row"}
or any(type(v) is not int for v in grid.values())):
raise ValueError("exact integer atlas_grid required")
cols, rows, col, row = (grid[k] for k in ("columns", "rows", "column", "row"))
if not (1 <= cols <= 64 and 1 <= rows <= 64 and 0 <= col < cols and 0 <= row < rows):
raise ValueError("invalid atlas grid")
if Fraction(raw.width, cols * width) != Fraction(raw.height, rows * height):
raise ValueError("grid aspect differs from logical canvas")
bounds = frame["raw_painted_bounds"]
if type(bounds) is not list or len(bounds) != 4 or any(type(v) is not int for v in bounds):
raise ValueError("raw painted bounds need integer4")
l, t, r, b = bounds
if not (Fraction(col*raw.width, cols) <= l < r <= Fraction((col+1)*raw.width, cols)
and Fraction(row*raw.height, rows) <= t < b <= Fraction((row+1)*raw.height, rows)):
raise ValueError("painted bounds exceed predeclared cell")
# Alignment is the actual whole body's threshold bounds within the
# predeclared atlas cell, never a caller-chosen crop that hides artwork.
cell_l, cell_t = col*raw.width//cols, row*raw.height//rows
cell_r = ((col+1)*raw.width+cols-1)//cols
cell_b = ((row+1)*raw.height+rows-1)//rows
cell_alpha = np.asarray(raw.getchannel("A"))[cell_t:cell_b, cell_l:cell_r]
raw_y, raw_x = np.nonzero(cell_alpha >= 128)
actual_bounds = ([int(raw_x.min())+cell_l, int(raw_y.min())+cell_t,
int(raw_x.max()+1)+cell_l, int(raw_y.max()+1)+cell_t] if len(raw_x) else [])
if bounds != actual_bounds:
raise ValueError("declared painted bounds differ from actual whole-cell alpha>=128")
ys, xs = np.nonzero(source < 256)
sl, st, sr, sb = int(xs.min()), int(ys.min()), int(xs.max()+1), int(ys.max()+1)
ax, ay = (r-l)/((sr-sl)*scale), (b-t)/((sb-st)*scale)
affine = [ax, 0, l-ax*sl*scale, 0, ay, t-ay*st*scale]
registered = np.array(raw.transform((width*scale, height*scale), Image.Transform.AFFINE,
affine, Image.Resampling.BICUBIC))
occupied = up(source < 256, scale)
trusted = (registered[:, :, 3] >= 128) & occupied
if trusted.sum() / occupied.sum() < 0.7:
raise ValueError("most Source support lacks generated RGB; refusing relabel")
if not np.any(registered[trusted, :3] != registered[trusted, :3][0]):
raise ValueError("parent contains no generated spatial RGB variation")
rgb, gaps = nearest_generated_fill(registered[:, :, :3], trusted, occupied)
loaded.append(dict(frame=frame, source=source, rgb=rgb.astype(np.float64), affine=affine,
gaps=gaps, trusted=int(trusted.sum()), source_bounds=[sl, st, sr, sb]))
inputs[str(source_path)] = sha(source_data)
inputs[str(raw_path)] = sha(raw_data)
sources = np.stack([item["source"] for item in loaded])
common = np.all(sources == sources[0], axis=0) & (sources[0] < 256)
changing = np.any(sources != sources[0], axis=0)
common_pixels = up(common, scale)
moving_weight = inner_weight(up(changing, scale), feather*scale)[:, :, None]
if not common.any() or not changing.any():
raise ValueError("family needs both stationary and genuinely changing Source regions")
shared = np.median(np.stack([item["rgb"] for item in loaded]), axis=0)
source0_rgb = palette[np.minimum(sources[0], 255)]
blue0 = (source0_rgb[:, :, 2].astype(int) > source0_rgb[:, :, 0].astype(int)+4) & (source0_rgb[:, :, 2].astype(int) > source0_rgb[:, :, 1].astype(int)+4)
backing_sample = common & ~blue0
backing_gain, backing_target, backing_raw = region_gain(source0_rgb[backing_sample], shared[up(backing_sample, scale)], "backing")
shared = np.clip(shared * backing_gain, 0, 255)
outputs, reports = [], []
for item in loaded:
source, rgb = item["source"], item["rgb"]
source_rgb = palette[np.minimum(source, 255)]
values = source_rgb.astype(int)
warm = (source < 256) & changing & (values[:, :, 0] >= 32) & (values[:, :, 0] > values[:, :, 1]*1.15) & (values[:, :, 0] > values[:, :, 2]*1.15)
gain, target_mean, raw_mean = region_gain(source_rgb[warm], rgb[up(warm, scale)], "moving warm")
moving = np.clip(rgb * gain, 0, 255)
result = np.rint(shared*(1-moving_weight) + moving*moving_weight).astype(np.uint8)
blue = (source < 256) & (values[:, :, 2] > values[:, :, 0]+4) & (values[:, :, 2] > values[:, :, 1]+4)
cue_pixels = up(blue, scale)
exact_cues = up(source_rgb, scale)
result[cue_pixels] = exact_cues[cue_pixels]
occupied = up(source < 256, scale)
rgba = np.zeros((*occupied.shape, 4), dtype=np.uint8)
rgba[:, :, :3][occupied] = result[occupied]
rgba[:, :, 3][occupied] = 255
internal = 0
for y, x in np.argwhere(source < 256):
cell = result[y*scale:(y+1)*scale, x*scale:(x+1)*scale]
internal += int(np.any(cell != cell[0, 0], axis=2).sum())
if internal == 0:
raise ValueError("no generated spatial detail remains")
outputs.append(rgba)
reports.append(dict(asset=item["frame"]["asset"], logical_canvas=[shape[1], shape[0]], origin=[0, 0],
Source_painted_cells=int((source < 256).sum()), registered_generated_pixels=item["trusted"],
nearest_generated_donor_gap_pixels=item["gaps"], fullcanvas_output_to_raw_affine=item["affine"],
Source_painted_bounds=item["source_bounds"], warm_Source_sample_cells=int(warm.sum()),
warm_material_RGB_gain=gain.tolist(), warm_Source_sample_mean=target_mean.tolist(),
warm_generated_sample_mean_before_gain=raw_mean.tolist(), functional_blue_Source_RGB_cells=int(blue.sum()),
internally_nonconstant_generated_RGB_pixels=internal))
for rgba in outputs[1:]:
if not np.array_equal(rgba[common_pixels], outputs[0][common_pixels]):
raise AssertionError("stationary Source backing differs between frames")
report = dict(method=METHOD, output_class="DERIVED_GENERATED_RGB_GEOMETRY_AND_BROAD_MATERIAL_CLEANUP",
input_hashes=inputs, output_scale=scale, Source_stationary_cells=int(common.sum()),
Source_changing_union_cells=int(changing.sum()), shared_backing_RGB_gain=backing_gain.tolist(),
backing_Source_sample_mean=backing_target.tolist(), backing_generated_sample_mean=backing_raw.tolist(),
per_Source_cell_RGB_reconstruction=False, generated_RGB_controls_continuous_form=True,
frames=reports, decoder_module=dict(path=str(Path(hd_frame_policy.__file__).resolve()), sha256=sha(Path(hd_frame_policy.__file__).read_bytes())),
python_version=platform.python_version(), python_implementation=platform.python_implementation(),
numpy_version=np.__version__, pillow_version=PILLOW_VERSION)
return outputs, report
def run_recipe(recipe_path, output):
recipe_path, output = Path(recipe_path), Path(output)
if output.exists():
raise ValueError("output exists; never overwrite prior artwork")
data = recipe_path.read_bytes()
recipe = json.loads(data)
arrays, report = render_family(recipe, recipe_path.resolve().parent)
payloads = []
for pixels, frame in zip(arrays, report["frames"]):
buffer = io.BytesIO()
Image.fromarray(pixels).save(buffer, format="PNG", compress_level=9)
png = buffer.getvalue()
name = frame["asset"]+"-derived.png"
frame["output"] = dict(file=name, sha256=sha(png), bytes=len(png), RGBA_sha256=sha(pixels.tobytes()))
payloads.append((name, png))
report["execution"] = dict(status="success", method=METHOD, recipe_sha256=sha(data), tool_sha256=sha(Path(__file__).read_bytes()),
source_sha256={f["asset"]: f["source"]["sha256"] for f in recipe["frames"]},
parent_raw_sha256=sorted({f["parent_raw"]["sha256"] for f in recipe["frames"]}),
output_sha256={f["asset"]: f["output"]["sha256"] for f in report["frames"]})
output.mkdir(parents=True)
for name, png in payloads:
(output/name).write_bytes(png)
(output/"derived-provenance.json").write_text(json.dumps(report, indent=2, sort_keys=True)+"\n")
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("recipe", type=Path)
parser.add_argument("output_new_directory", type=Path)
args = parser.parse_args()
report = run_recipe(args.recipe, args.output_new_directory)
print(json.dumps(dict(execution=report["execution"], method=METHOD)))
if __name__ == "__main__":
main()
+78
View File
@@ -0,0 +1,78 @@
import hashlib
import importlib.util
import json
import pathlib
import struct
import sys
import tempfile
import unittest
import numpy as np
from PIL import Image, __version__ as PILLOW_VERSION
_spec=importlib.util.spec_from_file_location('dshp20_cleanup',pathlib.Path(__file__).with_name('hd_dshp20_cleanup.py'))
c=importlib.util.module_from_spec(_spec);_spec.loader.exec_module(c)
class RGBGeometryTests(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.root=pathlib.Path(self.tmp.name)
self.palette=self.root/'palette';self.palette.write_bytes(bytes([i%64 for i in range(768)]))
def tearDown(self):self.tmp.cleanup()
def binding(self,p):return {'path':str(p),'sha256':hashlib.sha256(p.read_bytes()).hexdigest()}
def source(self,values,name):
h,w=values.shape;data=bytearray(struct.pack('<5i',0,1,31,w,h))
for y,row in enumerate(values):
x=0
while x<w:
if row[x]==256:x+=1;continue
stop=x
while stop<w and row[stop]<256:stop+=1
data.extend(struct.pack('<4i',x,y,y*w+x,stop-x));data.extend(bytes(row[x:stop].tolist()));x=stop
data.extend(struct.pack('<4i',0,0,-1,0));p=self.root/name;p.write_bytes(data);return p
def frame(self,values,rgb,asset,alpha=None):
p=self.root/(asset+'.png');h,w=values.shape
arr=np.zeros((h*4,w*4,4),np.uint8);arr[:,:,:3]=rgb;arr[:,:,3]=255 if alpha is None else alpha
Image.fromarray(arr).save(p)
return {'asset':asset,'source':self.binding(self.source(values,asset+'.pic')),'raw_png':self.binding(p),'viewport':[0,0,w*4,h*4],'sampling_offset_native':[0,0],'origin':[0,0]}
def recipe(self,frames):return {'format_version':1,'palette':self.binding(self.palette),'output_scale':4,'max_fill_distance_pixels':12,'shared_backing_anchor':0,'frames':frames}
def pattern(self,h,w,offset=0):
y,x=np.indices((h*4,w*4));return np.stack([(x*7+80+offset)%230,(y*11+90+offset)%230,((x//4+y//4)*23+50+offset)%230],axis=-1).astype(np.uint8)
def test_broad_generated_panels_and_RGB_are_preserved_not_highpass_noise(self):
s=np.full((4,6),10);rgb=self.pattern(4,6)
out,report=c.render_family(self.recipe([self.frame(s,rgb,'a')]),self.root)
self.assertTrue(np.array_equal(out[0][:,:,:3],rgb));self.assertEqual(report['frames'][0]['filled_generated_RGB_samples'],0)
def test_source_scale4_mask_removes_overflow_and_holes_use_generated_RGB(self):
s=np.full((5,5),10);s[0,0]=256;s[4,4]=256;rgb=self.pattern(5,5);alpha=np.full((20,20),255,np.uint8);alpha[8,8]=0
out,report=c.render_family(self.recipe([self.frame(s,rgb,'a',alpha)]),self.root);mask=c.scale2(c.scale2(s))<256
self.assertTrue(np.array_equal(out[0][:,:,3]>0,mask));self.assertTrue(np.array_equal(out[0][2,10,:3],rgb[2,10]));self.assertEqual(report['frames'][0]['filled_generated_RGB_samples'],1);self.assertGreater(report['frames'][0]['removed_generated_outside_SourceScale4'],0)
self.assertTrue(any(np.array_equal(out[0][8,8,:3],v)for v in rgb.reshape(-1,3)))
def test_stationary_RGB_shared_but_changed_Source_phase_is_not_frozen(self):
a=np.full((6,6),10);b=a.copy();b[3,3]=11
out,report=c.render_family(self.recipe([self.frame(a,self.pattern(6,6),'a'),self.frame(b,self.pattern(6,6,33),'b')]),self.root)
self.assertTrue(np.array_equal(out[0][0,0],out[1][0,0]));self.assertFalse(np.array_equal(out[0][13,13],out[1][13,13]));self.assertGreater(report['frames'][1]['stationary_full_RGB_replacements'],0)
# Unchanged labels inside a coherent moving part keep its whole RGB.
self.assertFalse(np.array_equal(out[0][9,9],out[1][9,9]))
def test_functional_Source_palette_RGB_replaces_wrong_parent_marker(self):
s=np.full((3,4),10);s[1,1]=252;s[1,2]=253;out,_=c.render_family(self.recipe([self.frame(s,self.pattern(3,4),'a')]),self.root)
p=np.frombuffer(self.palette.read_bytes(),np.uint8).reshape(256,3);p=(p<<2)|(p>>4)
self.assertTrue(np.all(out[0][4:8,4:8,:3]==p[252]));self.assertTrue(np.all(out[0][4:8,8:12,:3]==p[253]))
def test_scale2_has_actual_kernel_corner_not_nearest_replication(self):
s=np.array([[1,2],[2,2]],np.uint16)
expected=np.array([[1,1,2,2],[1,2,2,2],[2,2,2,2],[2,2,2,2]],np.uint16)
self.assertTrue(np.array_equal(c.scale2(s),expected))
def test_normalized_execution_hashes_and_input_immutability(self):
s=np.full((3,4),10);recipe=self.recipe([self.frame(s,self.pattern(3,4),'a')]);rp=self.root/'recipe.json';rp.write_text(json.dumps(recipe));before={p:p.read_bytes()for p in self.root.iterdir()if p.is_file()};report=c.run_recipe(rp,self.root/'out');e=report['execution']
self.assertEqual(set(e),{'status','method','recipe_sha256','tool_sha256','source_sha256','parent_raw_sha256','output_sha256'});self.assertEqual(e['status'],'success');self.assertEqual(e['output_sha256']['a'],hashlib.sha256((self.root/'out/a-derived.png').read_bytes()).hexdigest());self.assertEqual(e['parent_raw_sha256'],sorted(set(e['parent_raw_sha256'])))
self.assertEqual(report['decoder_module'],{'path':str(pathlib.Path(c.hd_frame_policy.__file__).resolve()),'sha256':hashlib.sha256(pathlib.Path(c.hd_frame_policy.__file__).read_bytes()).hexdigest()})
self.assertEqual(report['python_version'],sys.version);self.assertEqual(report['numpy_version'],np.__version__);self.assertEqual(report['pillow_version'],PILLOW_VERSION)
for p,data in before.items():self.assertEqual(p.read_bytes(),data)
with self.assertRaisesRegex(ValueError,'already exists'):c.run_recipe(rp,self.root/'out')
def test_missing_generated_donor_refuses_SourceNN_or_noise_fallback(self):
s=np.full((3,4),10);f=self.frame(s,self.pattern(3,4),'a',np.zeros((12,16),np.uint8))
with self.assertRaisesRegex(ValueError,'no generated RGB donor'):c.render_family(self.recipe([f]),self.root)
def test_bad_hash_and_bool_scale_rejected_before_output(self):
f=self.frame(np.full((3,4),10),self.pattern(3,4),'a');r=self.recipe([f]);r['output_scale']=True
with self.assertRaises(ValueError):c.render_family(r,self.root)
r['output_scale']=4;r['frames'][0]['source']['sha256']='0'*64
with self.assertRaisesRegex(ValueError,'hash mismatch'):c.render_family(r,self.root)
if __name__=='__main__':unittest.main(verbosity=2)
+151
View File
@@ -0,0 +1,151 @@
"""Portable synthetic behavioural tests; no engine, image service or old corpus."""
from pathlib import Path
import hashlib, json, tempfile, unittest, struct
import numpy as np
from PIL import Image
import hd_targt_rebuild as t
class RebuildTests(unittest.TestCase):
def test_main_component_excludes_disconnected_fringe(self):
a=np.zeros((20,30,4),dtype='uint8');a[4:18,7:26,3]=255;a[0,0,3]=255
self.assertEqual(t.main_painted_bounds(a),[7,4,26,18])
def test_nonuniform_mapping_keeps_source_canvas_alpha_region(self):
a=np.zeros((12,30,4),dtype='uint8');a[2:10,4:28,:3]=[90,70,50];a[2:10,4:28,3]=255
s=np.full((32,32),256,dtype='uint16');s[4:29,3:28]=195
image,b,sb=t.map_parent(a,s,4)
self.assertEqual(b,[4,2,28,10]);self.assertEqual(sb,[3,4,28,29])
self.assertEqual(image.shape,(128,128,3));self.assertTrue(np.all(image[16:116,12:112]==[90,70,50]))
self.assertEqual(int(image[:16].sum()),0)
def test_generated_correlated_texture_survives_quiet_soil(self):
y,x=np.indices((128,128));v=80+20*np.sin(x*.35)+10*np.cos(y*.8)
g=np.stack([v,v*.65,v*.5],2);src=np.full((32,32,3),[100,60,45],dtype='uint8')
out=t.broad_soil(g,src,np.ones((32,32),bool),4)
self.assertGreater(float(out[:,:,0].std()),1)
self.assertLess(float(out[:,:,0].std()),float(g[:,:,0].std()))
self.assertGreater(float(np.corrcoef((g-g.mean((0,1)))[:,:,0].ravel(),(out-out.mean((0,1)))[:,:,0].ravel())[0,1]),.8)
def test_source_pixel_checker_not_copied_into_broad_material(self):
y,x=np.indices((32,32));src=np.repeat((((x+y)%2)*140+40)[:,:,None],3,2).astype('uint8')
g=np.full((128,128,3),90.,dtype=float);out=t.broad_soil(g,src,np.ones((32,32),bool),4)
self.assertLess(float(out[:,:,0].std()),1)
def test_core_sampling_rejects_distant_warm_brown_rim(self):
a=np.zeros((128,128,3),float);a[16:120,12:16]=[110,55,12]
a[68:80,60:68]=[100,35,3]
mask=t.core_donor_mask(a,[15,17,17,20],4)
self.assertEqual(t.box(mask),[60,68,68,80])
def fixture(self,root):
def ref(p):return dict(path=str(p),sha256=hashlib.sha256(p.read_bytes()).hexdigest())
pal=np.zeros((256,3),dtype='uint8');pal[:32]=np.arange(31,-1,-1)[:,None]*2
pal[195]=[26,15,11];pal[81]=[59,44,7];pal[95]=[16,4,0]
pp=root/'palette';pp.write_bytes(pal.tobytes())
sources=[];frames=[]
for i in range(4):
s=np.full((32,32),256,dtype='uint16')
if i==0:s[:]=195
else:
s[4:29,3:28]=195
if i>=2:s[5:28,4:27]=31;s[12:25,10:22]=14;s[17:20,15:17]=95
if i==2:s[5:16,4:27]=195;s[16,4:27:2]=81;s[17:19,4:7]=10
p=root/('TEST_PIC'+('.'+str(i) if i else ''))
payload=struct.pack('<iiiii',0,0,0,32,32)
for y in range(32):
xs=np.flatnonzero(s[y]<256)
if len(xs):
l,r=int(xs[0]),int(xs[-1]+1);payload+=struct.pack('<iiii',l,y,0,r-l)+s[y,l:r].astype('uint8').tobytes()
payload+=struct.pack('<iiii',0,0,-1,0);p.write_bytes(payload)
sources.append(s);frames.append(dict(asset=str(i),source=ref(p),origin=[0,0]))
raw=np.zeros((80,4*80,4),dtype='uint8')
for i in range(4):
x=i*80;a=raw[:,x:x+80];y0,y1,x0,x1=(5,75,4,76) if i==0 else (10,70,10,70)
a[y0:y1,x0:x1,:3]=[100,62,45];a[y0:y1,x0:x1,3]=255
if i>=2:
a[12:68,12:68,:3]=[8,8,8];a[32:56,28:56,:3]=[110,110,110]
a[37:49,35:47,:3]=[8,8,8];a[39:47,38:44,:3]=[90,30,3]
if i==2:
a[12:40,12:68,:3]=[100,62,45];a[40:43,12:68,:3]=[235,170,25];a[43:49,12:22,:3]=[100,100,100]
rp=root/'raw.png';Image.fromarray(raw).save(rp)
recipe=dict(format_version=1,method=t.METHOD,output_scale=4,palette=ref(pp),raw_png=ref(rp),layout=dict(columns=4,rows=1),
frames=frames,soil_donor_frame=0,closed_donor_frame=1,module_donor_frame=3,warning_donor_frame=2)
return recipe,sources
def test_full_assembly_exact_alpha_warning_position_static_and_normalized_execution(self):
with tempfile.TemporaryDirectory() as td:
p=Path(td);recipe,sources=self.fixture(p);outputs,result=t.render(recipe,p)
for o,s in zip(outputs,sources):self.assertTrue(np.array_equal(o[:,:,3],np.repeat(np.repeat(s<256,4,0),4,1)*255))
self.assertEqual(result['frames'][2]['warning_Source_y'],16)
strip=outputs[2][16*4:17*4,4*4:27*4,:3]
self.assertGreater(float(strip[:,:,0].mean()),float(strip[:,:,2].mean())*2)
static=np.all(np.stack(sources)[1:]==sources[1],axis=0)&(sources[1]<256);big=np.repeat(np.repeat(static,4,0),4,1)
self.assertTrue(np.array_equal(outputs[1][big],outputs[3][big]))
rp=p/'recipe.json';rp.write_text(json.dumps(recipe));result=t.run(rp,p/'new')
self.assertEqual(set(result['execution']),{'status','method','recipe_sha256','tool_sha256','source_sha256','parent_raw_sha256','output_sha256'})
self.assertEqual(result['execution']['parent_raw_sha256'],[recipe['raw_png']['sha256']])
self.assertEqual(len(result['execution']['output_sha256']),4)
bindings=result['implementation_bindings']
decoder=Path(bindings['decoder']['path'])
self.assertEqual(bindings['decoder']['sha256'],hashlib.sha256(decoder.read_bytes()).hexdigest())
self.assertEqual(set(bindings['runtime']),{'python','numpy','pillow','executable'})
self.assertTrue(all(type(v) is str and v for v in bindings['runtime'].values()))
with self.assertRaisesRegex(ValueError,'no overwrite'):t.run(rp,p/'new')
def test_bad_hash_fails_before_creating_output(self):
with tempfile.TemporaryDirectory() as td:
p=Path(td);recipe,_=self.fixture(p);recipe['frames'][2]['source']['sha256']='0'*64
rp=p/'recipe.json';rp.write_text(json.dumps(recipe))
with self.assertRaisesRegex(ValueError,'SHA mismatch'):t.run(rp,p/'absent')
self.assertFalse((p/'absent').exists())
def test_duplicate_partition_origin_and_bool_scale_rejected(self):
with tempfile.TemporaryDirectory() as td:
recipe,_=self.fixture(Path(td))
for mutation in ('duplicate','partition','origin','boolscale'):
r=json.loads(json.dumps(recipe))
if mutation=='duplicate':r['frames'][1]['asset']=r['frames'][0]['asset']
if mutation=='partition':r['frames'].pop()
if mutation=='origin':r['frames'][0]['origin']=[1,0]
if mutation=='boolscale':r['output_scale']=True
with self.assertRaises(ValueError):t.validate_recipe(r)
def test_path_escape_and_absolute_asset_fail_before_output_or_sentinel_write(self):
with tempfile.TemporaryDirectory() as td:
p=Path(td);recipe,_=self.fixture(p)
sentinels=[p/'sentinel-derived.png',p/'absolute-sentinel-derived.png']
for sentinel in sentinels:sentinel.write_bytes(b'KEEP-EXISTING-ART-EXACT')
for i,bad in enumerate(['../sentinel',str(p/'absolute-sentinel'),'0/../sentinel','../0','',None,False,'0x10152','ABC','0001','100000000']):
r=json.loads(json.dumps(recipe));r['frames'][0]['asset']=bad
rp=p/('bad'+str(i)+'.json');rp.write_text(json.dumps(r));out=p/('absent'+str(i))
with self.assertRaisesRegex(ValueError,'canonical lowercase'):t.run(rp,out)
self.assertFalse(out.exists())
for sentinel in sentinels:self.assertEqual(sentinel.read_bytes(),b'KEEP-EXISTING-ART-EXACT')
def test_boolean_version_and_origin_are_not_integer_identity(self):
with tempfile.TemporaryDirectory() as td:
recipe,_=self.fixture(Path(td))
for mutate in ('version','origin'):
r=json.loads(json.dumps(recipe))
if mutate=='version':r['format_version']=True
else:r['frames'][0]['origin']=[False,0]
with self.assertRaises(ValueError):t.validate_recipe(r)
def test_complete_numeric_asset_and_Source_picture_order(self):
with tempfile.TemporaryDirectory() as td:
recipe,_=self.fixture(Path(td))
for mutate in ('gap','reversed','source_swapped'):
r=json.loads(json.dumps(recipe))
if mutate=='gap':r['frames'][-1]['asset']='5'
if mutate=='reversed':r['frames'][0]['asset'],r['frames'][1]['asset']=r['frames'][1]['asset'],r['frames'][0]['asset']
if mutate=='source_swapped':r['frames'][1]['source'],r['frames'][2]['source']=r['frames'][2]['source'],r['frames'][1]['source']
with self.assertRaisesRegex(ValueError,'order'):t.validate_recipe(r)
def test_repeated_synthetic_render_is_byte_deterministic(self):
with tempfile.TemporaryDirectory() as td:
p=Path(td);recipe,_=self.fixture(p)
a,ma=t.render(recipe,p);b,mb=t.render(recipe,p)
self.assertEqual(ma,mb)
for x,y in zip(a,b):self.assertEqual(x.tobytes(),y.tobytes())
if __name__=='__main__':unittest.main()
+140
View File
@@ -0,0 +1,140 @@
"""Focused synthetic alignment/form/animation/provenance checks, no game data."""
import copy
import hashlib
import json
from pathlib import Path
import struct
import tempfile
import unittest
import numpy as np
from PIL import Image
import hd_worm_cleanup as cleanup
class WormCleanup(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.sources = []
for phase in range(2):
cells = np.full((8, 8), 256, dtype=np.uint16)
cells[1:7, 1:7] = 10
cells[2:6, 2:6] = 20+phase
cells[3, 6] = 30
cells[6, 2] = 30
self.sources.append(cells)
payload = bytearray(struct.pack('<5i', 0, 0, 0, 8, 8))
for y, x in np.argwhere(cells < 256):
payload += struct.pack('<4i', int(x), int(y), int(y*8+x), 1)+bytes([cells[y, x]])
payload += struct.pack('<4i', 0, 0, -1, 0)
(self.root/f'Source{phase}').write_bytes(payload)
palette = bytearray(768)
for index, rgb in [(10, [12, 12, 12]), (20, [25, 15, 12]), (21, [36, 24, 18]), (30, [0, 0, 14])]:
palette[index*3:index*3+3] = bytes(rgb)
(self.root/'palette').write_bytes(palette)
self.raw = np.zeros((64, 128, 4), dtype=np.uint8)
for phase in range(2):
for y in range(8, 56):
for x in range(8, 56):
warm = 16 <= x < 48 and 16 <= y < 48
rgb = [70+x*2, 30+y, 25+x//2] if warm else [25+x//2+phase*8, 25+y//2+phase*8, 25+(x+y)//4+phase*8]
self.raw[y, x+phase*64] = [*rgb, 255]
self.save_raw()
bind = lambda p: {'path':p,'sha256':cleanup.sha((self.root/p).read_bytes())}
self.recipe = dict(format_version=1, label='synthetic only', palette=bind('palette'), output_scale=8,
feather_logical_pixels=1, frames=[dict(asset=f'3003{phase}', source=bind(f'Source{phase}'),
parent_raw=bind('raw.png'), logical_canvas=[8,8], origin=[0,0],
atlas_grid=dict(columns=2,rows=1,column=phase,row=0),
raw_painted_bounds=[8+phase*64,8,56+phase*64,56]) for phase in range(2)])
def save_raw(self):
Image.fromarray(self.raw).save(self.root/'raw.png')
def rebind_raw(self, recipe):
for frame in recipe['frames']:
frame['parent_raw']['sha256'] = cleanup.sha((self.root/'raw.png').read_bytes())
def test_source_alpha_cues_and_continuous_generated_form_survive(self):
outputs, report = cleanup.render_family(self.recipe, self.root)
self.assertFalse(report['per_Source_cell_RGB_reconstruction'])
for source, output, frame in zip(self.sources, outputs, report['frames']):
occupied = cleanup.up(source<256,8)
self.assertTrue(np.array_equal(output[:,:,3]==255,occupied))
self.assertTrue(np.all(output[~occupied]==0))
self.assertTrue(np.all(output[24:32,48:56,:3]==[0,0,56]))
# Source warm cells are one flat color, but generated gradients must
# remain different across cells and within each cell after cleanup.
self.assertGreater(np.ptp(output[24:40,24:40,0]),15)
self.assertNotEqual(float(output[24:32,24:32,0].mean()),float(output[24:32,32:40,0].mean()))
self.assertEqual(frame['functional_blue_Source_RGB_cells'],2)
def test_stationary_generated_backing_shared_and_early_phase_dimmer(self):
outputs, report = cleanup.render_family(self.recipe, self.root)
common = cleanup.up(self.sources[0]==self.sources[1],8)
self.assertTrue(np.array_equal(outputs[0][common],outputs[1][common]))
self.assertLess(outputs[0][24:40,24:40,:3].mean(),outputs[1][24:40,24:40,:3].mean())
self.assertEqual(report['Source_changing_union_cells'],16)
def test_generated_gap_fill_is_local_and_never_source_rgb(self):
rgb=np.array([[[10,20,30],[90,80,70]],[[60,50,40],[1,2,3]]],dtype=np.uint8)
trusted=np.array([[True,False],[False,True]])
filled,count=cleanup.nearest_generated_fill(rgb,trusted,np.ones((2,2),dtype=bool))
self.assertEqual(count,2)
self.assertTrue(np.array_equal(filled[trusted],rgb[trusted]))
self.assertTrue(np.array_equal(filled[0,1],rgb[0,0]))
self.assertTrue(np.array_equal(filled[1,0],rgb[0,0]))
def test_blend_never_modifies_stationary_region(self):
mask=np.zeros((12,12),dtype=bool);mask[2:10,2:10]=True
weight=cleanup.inner_weight(mask,4)
self.assertTrue(np.all(weight[~mask]==0))
self.assertEqual(weight[2,2],0.25)
self.assertEqual(weight[5,5],1)
def test_invalid_identity_bounds_grid_origin_fail_before_output(self):
mutations=[lambda r:r['frames'][0]['source'].update(sha256='0'*64),
lambda r:r['frames'][0].update(origin=[False,0]),
lambda r:r['frames'][0].update(origin=[1,0]),
lambda r:r['frames'][0].update(raw_painted_bounds=[9,8,56,56]),
lambda r:r['frames'][0]['atlas_grid'].update(column=2),
lambda r:r['frames'][0]['atlas_grid'].update(rows=2),
lambda r:r.update(output_scale=True)]
for mutate in mutations:
with self.subTest(mutate=mutate):
recipe=copy.deepcopy(self.recipe);mutate(recipe)
with self.assertRaises(ValueError):cleanup.render_family(recipe,self.root)
def test_opaque_and_detail_free_parents_rejected(self):
self.raw[:,:,3]=255;self.save_raw();r=copy.deepcopy(self.recipe);self.rebind_raw(r)
with self.assertRaisesRegex(ValueError,'transparent alpha'):cleanup.render_family(r,self.root)
self.raw[:]=0
self.raw[8:56,8:56]=[80,50,40,255];self.raw[8:56,72:120]=[80,50,40,255]
self.save_raw();self.rebind_raw(r)
with self.assertRaisesRegex(ValueError,'no generated spatial RGB'):cleanup.render_family(r,self.root)
def test_repeatability_and_full_execution_decoder_version_bindings(self):
recipe=self.root/'recipe.json';recipe.write_text(json.dumps(self.recipe,sort_keys=True)+'\n')
original={p.name:p.read_bytes() for p in self.root.iterdir() if p.is_file()}
reports=[cleanup.run_recipe(recipe,self.root/f'derived{i}') for i in range(2)]
self.assertEqual(reports[0],reports[1])
report=reports[0];execution=report['execution']
self.assertEqual(execution['status'],'success')
self.assertEqual(execution['method'],cleanup.METHOD)
self.assertEqual(execution['recipe_sha256'],hashlib.sha256(recipe.read_bytes()).hexdigest())
self.assertEqual(execution['tool_sha256'],cleanup.sha(Path(cleanup.__file__).read_bytes()))
self.assertEqual(report['decoder_module']['sha256'],cleanup.sha(Path(cleanup.hd_frame_policy.__file__).read_bytes()))
for field in ['python_version','numpy_version','pillow_version']:self.assertTrue(report[field])
self.assertEqual(execution['source_sha256'],{f['asset']:f['source']['sha256'] for f in self.recipe['frames']})
self.assertEqual(len(execution['parent_raw_sha256']),1)
for frame in self.recipe['frames']:
name=frame['asset']+'-derived.png';p=self.root/'derived0'/name
self.assertEqual(p.read_bytes(),(self.root/'derived1'/name).read_bytes())
self.assertEqual(execution['output_sha256'][frame['asset']],cleanup.sha(p.read_bytes()))
for name,data in original.items():self.assertEqual((self.root/name).read_bytes(),data)
with self.assertRaisesRegex(ValueError,'output exists'):cleanup.run_recipe(recipe,self.root/'derived0')
if __name__=='__main__':unittest.main()