Remaster eighteen original terrain pictures
Add the complete organic forest family and two Source-guided terrain tiles with portable reproduction tools and review evidence. Preserve existing game assets and restore fifteen referenced archival metadata files. Validation: 157 maintained HD-tool tests; installed HD validation 5539/5539 with zero rejects; 45 mods; three 1080p SDL controls unchanged. SourceF binaries and gameplay are unchanged. Native Windows validation for the preceding 5521-picture commit passed.
This commit is contained in:
raptorreplica committed
1 parent
e0d114e657
commit
f3b30df0f8
136 files changed
+35104
-60
No files matched your search
@@ -0,0 +1,197 @@
|
||||
"""PRIVATE new recipe: explicit Source palette groups + coherent generated detail.
|
||||
|
||||
Never imports a moving dependency or patches legacy material_families. The exact
|
||||
reviewed 0e4d source is loaded from checked bytes, and only its audited reader,
|
||||
decoder, palette, ScaleNx and filtering primitives are reused. Old recipes keep
|
||||
their own entrypoint and behaviour. No engine/registration/World operations.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
import argparse, ctypes, errno, hashlib, io, json, math, os, re, shutil, stat, sys, tempfile, types
|
||||
import numpy as np
|
||||
import PIL
|
||||
from PIL import Image
|
||||
|
||||
METHOD='source-declared-material-groups-coherent-parent-fine-mid-logtexture-v1'
|
||||
DEPENDENCY_SHA256='0e4d7d5a4f7a7ddf6e396e757691156dfc74d231d9925744c342078532b89a98'
|
||||
PARAMETERS={'output_side','source_sigma','residual_sigma_fine','residual_sigma_coarse','residual_percentile'}
|
||||
|
||||
def digest(b): return hashlib.sha256(b).hexdigest()
|
||||
|
||||
def binding(value):
|
||||
if type(value) is not dict or set(value)!={'path','sha256'} or type(value['path']) is not str or not value['path'] or type(value['sha256']) is not str or not re.fullmatch('[0-9a-f]{64}',value['sha256']):
|
||||
raise ValueError('input needs exactly nonempty path and lowercase SHA256')
|
||||
return value
|
||||
|
||||
def load_dependency(ref,base):
|
||||
binding(ref)
|
||||
if ref['sha256']!=DEPENDENCY_SHA256: raise ValueError('only exact reviewed0e4d dependency is allowed')
|
||||
p=Path(ref['path']);p=p if p.is_absolute() else base/p
|
||||
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
try:
|
||||
a=os.fstat(fd)
|
||||
if not stat.S_ISREG(a.st_mode) or a.st_size>100_000: raise ValueError('bounded regular dependency required')
|
||||
with os.fdopen(fd,'rb',closefd=False) as stream:body=stream.read()
|
||||
b=os.fstat(fd)
|
||||
if (a.st_dev,a.st_ino,a.st_size,a.st_mtime_ns)!=(b.st_dev,b.st_ino,b.st_size,b.st_mtime_ns):raise ValueError('dependency changed during read')
|
||||
finally:os.close(fd)
|
||||
if digest(body)!=DEPENDENCY_SHA256:raise ValueError('dependency hash mismatch')
|
||||
module=types.ModuleType('private_exact_0e4d_terrain_primitives');module.__file__=str(p.absolute())
|
||||
exec(compile(body,str(p),'exec'),module.__dict__)
|
||||
_,pin=module.read_regular(p)
|
||||
if pin['sha256']!=DEPENDENCY_SHA256:raise ValueError('dependency changed after load')
|
||||
return module,pin
|
||||
|
||||
def groups_validated(groups,used=None):
|
||||
if type(groups) is not list or not 1<=len(groups)<=32:raise ValueError('material_groups must contain1..32 explicit groups')
|
||||
names=set();members=set()
|
||||
for group in groups:
|
||||
if type(group) is not dict or set(group)!={'name','indices','texture_log_gain','quiet_from_source_variance'}:raise ValueError('exact group fields required')
|
||||
name=group['name']
|
||||
if type(name) is not str or not re.fullmatch('[a-z][a-z0-9_]{0,31}',name) or name in names:raise ValueError('unique slug material names required')
|
||||
names.add(name);indices=group['indices']
|
||||
if type(indices) is not list or not indices or any(type(i) is not int or not 0<=i<240 for i in indices):raise ValueError('group indices must be literal ordinary0..239 integers')
|
||||
if indices!=sorted(set(indices)):raise ValueError('group index sets must be sorted unique')
|
||||
if members.intersection(indices):raise ValueError('material index sets overlap')
|
||||
members.update(indices);gain=group['texture_log_gain']
|
||||
if type(gain) not in (int,float) or not math.isfinite(gain) or not 0<=gain<=.7:raise ValueError('bounded finite texture gain required')
|
||||
if type(group['quiet_from_source_variance']) is not bool:raise ValueError('quiet control must be boolean')
|
||||
if used is not None and not set(map(int,used))<=members:raise ValueError('used Source indices not completely covered by declared groups')
|
||||
return groups
|
||||
|
||||
def validate_parameters(p,shape):
|
||||
if type(p) is not dict or set(p)!=PARAMETERS:raise ValueError('exact reconstruction parameters required')
|
||||
side=p['output_side']
|
||||
if type(side) is not int or not 4*max(shape)<=side<=1024 or side%(4*max(shape)):raise ValueError('output side must be bounded integer expansion of SourceNx4')
|
||||
for key in ('source_sigma','residual_sigma_fine','residual_sigma_coarse'):
|
||||
v=p[key]
|
||||
if type(v) not in (int,float) or not math.isfinite(v) or not 0<v<=16:raise ValueError('invalid positive finite sigma')
|
||||
if p['residual_sigma_fine']>=p['residual_sigma_coarse']:raise ValueError('fine band must be narrower than coarse')
|
||||
v=p['residual_percentile']
|
||||
if type(v) not in (int,float) or not math.isfinite(v) or not 50<=v<=100:raise ValueError('invalid percentile')
|
||||
|
||||
def validate_recipe(r):
|
||||
if type(r) is not dict or set(r)!={'format_version','method','dependency','palette','material_groups','parameters','frames'} or type(r['format_version']) is not int or r['format_version']!=1 or r['method']!=METHOD:raise ValueError('new versioned recipe required')
|
||||
binding(r['dependency']);binding(r['palette']);groups_validated(r['material_groups'])
|
||||
frames=r['frames']
|
||||
if type(frames) is not list or not 1<=len(frames)<=64:raise ValueError('bounded nonempty frame inventory required')
|
||||
seen=[]
|
||||
for f in frames:
|
||||
if type(f) is not dict or set(f)!={'asset','Source','parent_raw','crop','origin'}:raise ValueError('exact frame fields required')
|
||||
asset=f['asset']
|
||||
if type(asset) is not str or not re.fullmatch('(0|[1-9a-f][0-9a-f]{0,7})',asset) or int(asset,16)>0xffffffff:raise ValueError('canonical lowercase u32 hex asset required, never a path')
|
||||
seen.append(int(asset,16));binding(f['Source']);binding(f['parent_raw'])
|
||||
if type(f['origin']) is not list or len(f['origin'])!=2 or any(type(v) is not int or v!=0 for v in f['origin']):raise ValueError('literal zero origin required')
|
||||
crop=f['crop']
|
||||
if type(crop) is not list or len(crop)!=4 or any(type(v) is not int for v in crop) or min(crop[:2])<0 or min(crop[2:])<=0:raise ValueError('exact positive integer crop required')
|
||||
if len(set(seen))!=len(seen) or seen!=sorted(seen):raise ValueError('unique numeric asset order required')
|
||||
validate_parameters(r['parameters'],(1,1))
|
||||
if len(frames)*r['parameters']['output_side']**2>16_777_216:raise ValueError('prepared output pixel budget exceeded')
|
||||
|
||||
def reconstruct(indices,palette,generated_rgb,parameters,groups,kernel):
|
||||
a=np.asarray(indices);palette=np.asarray(palette);raw=np.asarray(generated_rgb)
|
||||
if a.ndim!=2 or a.shape[0]!=a.shape[1] or not a.size or a.dtype.kind not in 'iu' or np.any(a<0) or np.any(a>=240):raise ValueError('square opaque ordinary integer Source required')
|
||||
if palette.shape!=(256,3) or palette.dtype!=np.uint8:raise ValueError('exact RGB uint8 palette required')
|
||||
if raw.ndim!=3 or raw.shape[2]!=3 or raw.dtype!=np.uint8 or not raw.size:raise ValueError('generated RGB uint8 required')
|
||||
validate_parameters(parameters,a.shape);groups_validated(groups,np.unique(a));p=parameters
|
||||
guide_indices=kernel.scale_nx(a,4)
|
||||
lookup=np.full(240,-1,np.int16)
|
||||
for i,g in enumerate(groups):lookup[g['indices']]=i
|
||||
labels=lookup[guide_indices];guide=palette[guide_indices].astype(float);side=p['output_side'];factor=side//labels.shape[0]
|
||||
owners=np.repeat(np.repeat(labels,factor,0),factor,1);base=np.zeros((side,side,3),float);quiet=np.ones((side,side),float)
|
||||
guide_luma=guide@np.array([.2126,.7152,.0722])
|
||||
for i in np.unique(labels):
|
||||
mask=(labels==i).astype(float)
|
||||
weights=kernel.resize_float(kernel.gaussian(mask,p['source_sigma']),(side,side))
|
||||
numerator=kernel.resize_float(kernel.gaussian(guide*mask[...,None],p['source_sigma']),(side,side))
|
||||
field=numerator/np.maximum(weights[...,None],1e-12);area=owners==i;base[area]=field[area]
|
||||
if groups[int(i)]['quiet_from_source_variance']:
|
||||
second=kernel.resize_float(kernel.gaussian(guide_luma**2*mask,p['source_sigma']),(side,side))/np.maximum(weights,1e-12)
|
||||
first=field@np.array([.2126,.7152,.0722]);std=np.sqrt(np.maximum(second-first**2,0))
|
||||
quiet[area]=(.35+.65*np.clip(std/8.,0,1))[area]
|
||||
parent=np.asarray(Image.fromarray(raw).resize((side,side),Image.Resampling.LANCZOS),float)
|
||||
luma=parent@np.array([.2126,.7152,.0722]);band=kernel.gaussian(luma,p['residual_sigma_fine'])-kernel.gaussian(luma,p['residual_sigma_coarse'])
|
||||
if float(band.std())<.25:raise ValueError('parent lacks coherent fine/mid generated detail; Source-only fallback forbidden')
|
||||
texture=np.zeros_like(band);normalizers={}
|
||||
for i in np.unique(owners):
|
||||
area=owners==i;centered=band-band[area].mean();divisor=max(float(np.percentile(np.abs(centered[area]),p['residual_percentile'])),1.)
|
||||
texture[area]=np.tanh(centered[area]/divisor);normalizers[groups[int(i)]['name']]=divisor
|
||||
strengths=np.array([g['texture_log_gain'] for g in groups])[owners]*quiet;gain=np.exp(strengths*texture)
|
||||
output=np.clip(np.rint(base*gain[...,None]),0,255).astype(np.uint8);base8=np.clip(np.rint(base),0,255).astype(np.uint8)
|
||||
changed=int(np.any(output!=base8,axis=2).sum())
|
||||
if changed<max(1,math.ceil(side*side*.01)):raise ValueError('generated modulation too small; no Source-only output')
|
||||
metrics=dict(method=METHOD,source_shape=list(a.shape),sourceNx4_indices_sha256=digest(guide_indices.astype('<u2').tobytes()),
|
||||
material_labels_sha256=digest(owners.astype('<i2').tobytes()),group_output_pixels={groups[int(i)]['name']:int((owners==i).sum()) for i in np.unique(owners)},
|
||||
used_Source_indices_by_group={g['name']:sorted(set(g['indices'])&set(map(int,np.unique(a)))) for g in groups},coherent_parent_band_std=float(band.std()),
|
||||
per_group_normalizers=normalizers,actual_tone_gain_min=float(gain.min()),actual_tone_gain_max=float(gain.max()),pixels_changed_from_Source_guided_base=changed,
|
||||
mean_absolute_parent_tone_modulation=float(np.abs(output.astype(float)-base8).mean()),output_RGB_sha256=digest(output.tobytes()),
|
||||
limits='Explicit Source palette groups/SourceNx geometry and broad masked RGB; coherent actual generated fine+mid luminance retained multiplicatively. No random noise/SourceNN fallback, material/context approval or reserved semantics.')
|
||||
return output,base8,owners,metrics
|
||||
|
||||
def publish_no_replace(stage,target):
|
||||
"""Atomic fresh-directory publication; unsupported platforms fail closed."""
|
||||
if os.name=='nt':os.rename(stage,target);return
|
||||
if not sys.platform.startswith('linux'):raise OSError(errno.ENOTSUP,'atomic no-replace publication unsupported')
|
||||
libc=ctypes.CDLL(None,use_errno=True)
|
||||
if not hasattr(libc,'renameat2'):raise OSError(errno.ENOTSUP,'renameat2 unavailable')
|
||||
fn=libc.renameat2;fn.argtypes=[ctypes.c_int,ctypes.c_char_p,ctypes.c_int,ctypes.c_char_p,ctypes.c_uint];fn.restype=ctypes.c_int
|
||||
if fn(-100,os.fsencode(stage),-100,os.fsencode(target),1)!=0:
|
||||
code=ctypes.get_errno();raise OSError(code,os.strerror(code),str(target))
|
||||
|
||||
def run(recipe_path,output):
|
||||
recipe_path=Path(recipe_path).absolute();output=Path(output).absolute()
|
||||
if output.exists() or output.is_symlink():raise ValueError('output exists; never overwrite')
|
||||
if not output.parent.is_dir():raise ValueError('output parent must already exist')
|
||||
# JSON is read with no-follow before loading the exact allowlisted primitive.
|
||||
fd=os.open(recipe_path,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):raise ValueError('regular recipe required')
|
||||
with os.fdopen(fd,'rb',closefd=False) as stream:body=stream.read(1_000_001)
|
||||
finally:os.close(fd)
|
||||
if len(body)>1_000_000:raise ValueError('recipe too large')
|
||||
r=json.loads(body);validate_recipe(r);kernel,dependency_pin=load_dependency(r['dependency'],recipe_path.parent)
|
||||
_,recipe_pin=kernel.read_regular(recipe_path)
|
||||
if recipe_pin['sha256']!=digest(body):raise ValueError('recipe changed during load')
|
||||
_,tool_pin=kernel.read_regular(__file__)
|
||||
inputs=[recipe_pin,dependency_pin,tool_pin]
|
||||
def checked(ref):
|
||||
binding(ref);p=Path(ref['path']);p=p if p.is_absolute() else recipe_path.parent/p
|
||||
b,pin=kernel.read_regular(p)
|
||||
if pin['sha256']!=ref['sha256']:raise ValueError('input SHA mismatch')
|
||||
inputs.append(pin);return b,pin
|
||||
pb,palette_pin=checked(r['palette']);palette=kernel.palette_rgb(pb);prepared=[]
|
||||
for f in r['frames']:
|
||||
sb,source_pin=checked(f['Source']);indices=kernel.decode_source(sb);parent_bytes,parent_pin=checked(f['parent_raw'])
|
||||
if len(parent_bytes)>32*1024*1024:raise ValueError('parent byte ceiling exceeded')
|
||||
im=Image.open(io.BytesIO(parent_bytes))
|
||||
if im.format!='PNG' or im.width*im.height>8_000_000:raise ValueError('bounded PNG parent required')
|
||||
if im.convert('RGBA').getchannel('A').getextrema()!=(255,255):raise ValueError('opaque parent required; alpha repair forbidden')
|
||||
x,y,w,h=f['crop']
|
||||
if x+w>im.width or y+h>im.height:raise ValueError('crop outside parent')
|
||||
raw=np.asarray(im.convert('RGB').crop((x,y,x+w,y+h)));derived,_,_,metrics=reconstruct(indices,palette,raw,r['parameters'],r['material_groups'],kernel)
|
||||
stream=io.BytesIO();Image.fromarray(derived).save(stream,format='PNG');prepared.append((f,stream.getvalue(),metrics,source_pin,parent_pin))
|
||||
output_hashes={f['asset']:digest(png) for f,png,_,_,_ in prepared}
|
||||
execution=dict(status='success',method=METHOD,recipe_sha256=recipe_pin['sha256'],tool_sha256=tool_pin['sha256'],
|
||||
source_sha256={f['asset']:s['sha256'] for f,_,_,s,_ in prepared},parent_raw_sha256=sorted({p['sha256'] for _,_,_,_,p in prepared}),output_sha256=output_hashes)
|
||||
receipt=dict(status='DERIVED_OUTPUTS_NOT_MATERIAL_APPROVAL',execution=execution,recipe=recipe_pin,tool=tool_pin,dependency=dependency_pin,
|
||||
decoder=dict(name='decode_source',module_sha256=dependency_pin['sha256']),runtime=dict(python=sys.version,executable=sys.executable,numpy=np.__version__,Pillow=PIL.__version__,platform=sys.platform),
|
||||
material_groups=r['material_groups'],material_groups_sha256=digest(json.dumps(r['material_groups'],sort_keys=True,separators=(',',':')).encode()),
|
||||
frames=[dict(asset=f['asset'],Source=s,parent_raw=p,crop=f['crop'],metrics=m) for f,_,m,s,p in prepared])
|
||||
stage=Path(tempfile.mkdtemp(prefix='.'+output.name+'.stage-',dir=output.parent))
|
||||
try:
|
||||
for f,png,_,_,_ in prepared:(stage/(f['asset']+'-DERIVED-declared-material-v1.png')).write_bytes(png)
|
||||
(stage/'normalized-execution-object.json').write_text(json.dumps(dict(execution=execution),indent=2)+'\n')
|
||||
(stage/'DERIVED-LINEAGE-AND-MATERIAL-GROUP-METRICS.json').write_text(json.dumps(receipt,indent=2,sort_keys=True)+'\n')
|
||||
for old in inputs:
|
||||
_,now=kernel.read_regular(old['path'])
|
||||
if now!=old:raise ValueError('input changed across reconstruction')
|
||||
publish_no_replace(stage,output)
|
||||
except BaseException:
|
||||
shutil.rmtree(stage,ignore_errors=True);raise
|
||||
return receipt
|
||||
|
||||
def main(argv=None):
|
||||
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--recipe',required=True);parser.add_argument('--output',required=True);args=parser.parse_args(argv)
|
||||
result=run(args.recipe,args.output);print(json.dumps(dict(status='success',output=str(Path(args.output).absolute()),frames=len(result['frames']),method=METHOD)))
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,258 @@
|
||||
"""PRIVATE new recipe: explicit Source palette groups + coherent generated detail.
|
||||
|
||||
Never imports a moving dependency or patches legacy material_families. The exact
|
||||
reviewed 0e4d source is loaded from checked bytes, and only its audited reader,
|
||||
decoder, palette, ScaleNx and filtering primitives are reused. Old recipes keep
|
||||
their own entrypoint and behaviour. No engine/registration/World operations.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
import argparse, ctypes, errno, hashlib, io, json, math, os, re, shutil, stat, sys, tempfile, types
|
||||
import numpy as np
|
||||
import PIL
|
||||
from PIL import Image
|
||||
|
||||
METHOD='source-declared-material-regions-coherent-parent-fine-mid-logtexture-v2'
|
||||
DEPENDENCY_SHA256='0e4d7d5a4f7a7ddf6e396e757691156dfc74d231d9925744c342078532b89a98'
|
||||
PARAMETERS={'output_side','source_sigma','residual_sigma_fine','residual_sigma_coarse','residual_percentile'}
|
||||
|
||||
def digest(b): return hashlib.sha256(b).hexdigest()
|
||||
|
||||
def binding(value):
|
||||
if type(value) is not dict or set(value)!={'path','sha256'} or type(value['path']) is not str or not value['path'] or type(value['sha256']) is not str or not re.fullmatch('[0-9a-f]{64}',value['sha256']):
|
||||
raise ValueError('input needs exactly nonempty path and lowercase SHA256')
|
||||
return value
|
||||
|
||||
def load_dependency(ref,base):
|
||||
binding(ref)
|
||||
if ref['sha256']!=DEPENDENCY_SHA256: raise ValueError('only exact reviewed0e4d dependency is allowed')
|
||||
p=Path(ref['path']);p=p if p.is_absolute() else base/p
|
||||
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
try:
|
||||
a=os.fstat(fd)
|
||||
if not stat.S_ISREG(a.st_mode) or a.st_size>100_000: raise ValueError('bounded regular dependency required')
|
||||
with os.fdopen(fd,'rb',closefd=False) as stream:body=stream.read()
|
||||
b=os.fstat(fd)
|
||||
if (a.st_dev,a.st_ino,a.st_size,a.st_mtime_ns)!=(b.st_dev,b.st_ino,b.st_size,b.st_mtime_ns):raise ValueError('dependency changed during read')
|
||||
finally:os.close(fd)
|
||||
if digest(body)!=DEPENDENCY_SHA256:raise ValueError('dependency hash mismatch')
|
||||
module=types.ModuleType('private_exact_0e4d_terrain_primitives');module.__file__=str(p.absolute())
|
||||
exec(compile(body,str(p),'exec'),module.__dict__)
|
||||
_,pin=module.read_regular(p)
|
||||
if pin['sha256']!=DEPENDENCY_SHA256:raise ValueError('dependency changed after load')
|
||||
return module,pin
|
||||
|
||||
def groups_validated(groups,used=None):
|
||||
if type(groups) is not list or not 1<=len(groups)<=32:raise ValueError('material_groups must contain1..32 explicit groups')
|
||||
names=set();members=set()
|
||||
for group in groups:
|
||||
if type(group) is not dict or set(group)!={'name','indices','texture_log_gain','quiet_from_source_variance'}:raise ValueError('exact group fields required')
|
||||
name=group['name']
|
||||
if type(name) is not str or not re.fullmatch('[a-z][a-z0-9_]{0,31}',name) or name in names:raise ValueError('unique slug material names required')
|
||||
names.add(name);indices=group['indices']
|
||||
if type(indices) is not list or not indices or any(type(i) is not int or not 0<=i<240 for i in indices):raise ValueError('group indices must be literal ordinary0..239 integers')
|
||||
if indices!=sorted(set(indices)):raise ValueError('group index sets must be sorted unique')
|
||||
if members.intersection(indices):raise ValueError('material index sets overlap')
|
||||
members.update(indices);gain=group['texture_log_gain']
|
||||
if type(gain) not in (int,float) or not math.isfinite(gain) or not 0<=gain<=.7:raise ValueError('bounded finite texture gain required')
|
||||
if type(group['quiet_from_source_variance']) is not bool:raise ValueError('quiet control must be boolean')
|
||||
if used is not None and not set(map(int,used))<=members:raise ValueError('used Source indices not completely covered by declared groups')
|
||||
return groups
|
||||
|
||||
def validate_parameters(p,shape):
|
||||
if type(p) is not dict or set(p)!=PARAMETERS:raise ValueError('exact reconstruction parameters required')
|
||||
side=p['output_side']
|
||||
if type(side) is not int or not 4*max(shape)<=side<=1024 or side%(4*max(shape)):raise ValueError('output side must be bounded integer expansion of SourceNx4')
|
||||
for key in ('source_sigma','residual_sigma_fine','residual_sigma_coarse'):
|
||||
v=p[key]
|
||||
if type(v) not in (int,float) or not math.isfinite(v) or not 0<v<=16:raise ValueError('invalid positive finite sigma')
|
||||
if p['residual_sigma_fine']>=p['residual_sigma_coarse']:raise ValueError('fine band must be narrower than coarse')
|
||||
v=p['residual_percentile']
|
||||
if type(v) not in (int,float) or not math.isfinite(v) or not 50<=v<=100:raise ValueError('invalid percentile')
|
||||
|
||||
def validate_recipe(r):
|
||||
if type(r) is not dict or set(r)!={'format_version','method','dependency','palette','material_groups','parameters','frames'} or type(r['format_version']) is not int or r['format_version']!=2 or r['method']!=METHOD:raise ValueError('new versioned recipe required')
|
||||
binding(r['dependency']);binding(r['palette']);groups_validated(r['material_groups'])
|
||||
frames=r['frames']
|
||||
if type(frames) is not list or not 1<=len(frames)<=64:raise ValueError('bounded nonempty frame inventory required')
|
||||
seen=[]
|
||||
for f in frames:
|
||||
required={'asset','Source','parent_raw','crop','origin'}
|
||||
if type(f) is not dict or not required<=set(f) or set(f)-required-{'material_parent_regions'}:raise ValueError('exact frame fields required')
|
||||
asset=f['asset']
|
||||
if type(asset) is not str or not re.fullmatch('(0|[1-9a-f][0-9a-f]{0,7})',asset) or int(asset,16)>0xffffffff:raise ValueError('canonical lowercase u32 hex asset required, never a path')
|
||||
seen.append(int(asset,16));binding(f['Source']);binding(f['parent_raw'])
|
||||
if type(f['origin']) is not list or len(f['origin'])!=2 or any(type(v) is not int or v!=0 for v in f['origin']):raise ValueError('literal zero origin required')
|
||||
crop=f['crop']
|
||||
if type(crop) is not list or len(crop)!=4 or any(type(v) is not int for v in crop) or min(crop[:2])<0 or min(crop[2:])<=0:raise ValueError('exact positive integer crop required')
|
||||
validate_regions(f.get('material_parent_regions',{}),crop,r['material_groups'])
|
||||
if len(set(seen))!=len(seen) or seen!=sorted(seen):raise ValueError('unique numeric asset order required')
|
||||
validate_parameters(r['parameters'],(1,1))
|
||||
if len(frames)*r['parameters']['output_side']**2>16_777_216:raise ValueError('prepared output pixel budget exceeded')
|
||||
|
||||
def validate_donor(donor,crop):
|
||||
if type(donor) is not list or len(donor)!=4 or any(type(v) is not int for v in donor) or min(donor[2:])<=0:
|
||||
raise ValueError('donor must be four literal integers with positive dimensions')
|
||||
x,y,w,h=donor;fx,fy,fw,fh=crop
|
||||
if not (fx<x and fy<y and x+w<fx+fw and y+h<fy+fh):
|
||||
raise ValueError('donor must lie strictly inside its authenticated frame crop')
|
||||
|
||||
def validate_regions(regions,crop,groups):
|
||||
if type(regions) is not dict:raise ValueError('material_parent_regions must be an object')
|
||||
names={g['name'] for g in groups}
|
||||
if any(type(name) is not str or name not in names for name in regions):raise ValueError('region key must name a declared group')
|
||||
for donor in regions.values():validate_donor(donor,crop)
|
||||
|
||||
def reflect_axis(coordinates,length):
|
||||
if type(length) is not int or length<=0:raise ValueError('positive reflection length required')
|
||||
a=np.asarray(coordinates)
|
||||
if a.dtype.kind not in 'iu':raise ValueError('integer reflection coordinates required')
|
||||
if length==1:return np.zeros(a.shape,np.int64)
|
||||
phase=np.mod(a,2*(length-1))
|
||||
return np.where(phase<length,phase,2*(length-1)-phase).astype(np.int64)
|
||||
|
||||
def reflect_parent_region(parent,crop,donor):
|
||||
"""Exact raw-density reflection, anchored at the donor's original offset.
|
||||
|
||||
No resize precedes the full-frame field. In the donor's original rectangle
|
||||
the field is byte-identical to raw RGB; outside it, reflect without repeating
|
||||
edge samples. Final output resizing happens only in reconstruct().
|
||||
"""
|
||||
parent=np.asarray(parent)
|
||||
if parent.ndim!=3 or parent.shape[2]!=3 or parent.dtype!=np.uint8:raise ValueError('raw parent RGB uint8 required')
|
||||
if type(crop) is not list or len(crop)!=4 or any(type(v) is not int for v in crop):raise ValueError('integer frame crop required')
|
||||
fx,fy,fw,fh=crop
|
||||
if min(fx,fy)<0 or min(fw,fh)<=0 or fx+fw>parent.shape[1] or fy+fh>parent.shape[0]:raise ValueError('frame crop outside parent')
|
||||
validate_donor(donor,crop);x,y,w,h=donor
|
||||
xs=x+reflect_axis(np.arange(fw,dtype=np.int64)+fx-x,w)
|
||||
ys=y+reflect_axis(np.arange(fh,dtype=np.int64)+fy-y,h)
|
||||
field=parent[ys[:,None],xs[None,:],:].copy()
|
||||
original=parent[y:y+h,x:x+w]
|
||||
if not np.array_equal(field[y-fy:y-fy+h,x-fx:x-fx+w],original):raise AssertionError('native donor samples moved')
|
||||
return field,dict(parent_crop=crop,raw_donor=donor,original_donor_offset=[x-fx,y-fy],native_reflected_field_dimensions=[fw,fh],
|
||||
donor_RGB_sha256=digest(original.tobytes()),reflected_native_field_RGB_sha256=digest(field.tobytes()),
|
||||
donor_original_position_samples_exact=True,reflection='original-offset reflect101, no edge repetition; length1 repeats sole sample',
|
||||
resampling_before_native_frame_field=False)
|
||||
|
||||
def reconstruct(indices,palette,generated_rgb,parameters,groups,kernel,material_parent_fields=None):
|
||||
a=np.asarray(indices);palette=np.asarray(palette);raw=np.asarray(generated_rgb)
|
||||
if a.ndim!=2 or a.shape[0]!=a.shape[1] or not a.size or a.dtype.kind not in 'iu' or np.any(a<0) or np.any(a>=240):raise ValueError('square opaque ordinary integer Source required')
|
||||
if palette.shape!=(256,3) or palette.dtype!=np.uint8:raise ValueError('exact RGB uint8 palette required')
|
||||
if raw.ndim!=3 or raw.shape[2]!=3 or raw.dtype!=np.uint8 or not raw.size:raise ValueError('generated RGB uint8 required')
|
||||
validate_parameters(parameters,a.shape);groups_validated(groups,np.unique(a));p=parameters
|
||||
fields={} if material_parent_fields is None else material_parent_fields
|
||||
if type(fields) is not dict or set(fields)-{g['name'] for g in groups}:raise ValueError('texture fields require declared group keys')
|
||||
for field in fields.values():
|
||||
if not isinstance(field,np.ndarray) or field.dtype!=np.uint8 or field.shape!=raw.shape:raise ValueError('native material field must match the full raw frame crop shape/density')
|
||||
guide_indices=kernel.scale_nx(a,4)
|
||||
lookup=np.full(240,-1,np.int16)
|
||||
for i,g in enumerate(groups):lookup[g['indices']]=i
|
||||
labels=lookup[guide_indices];guide=palette[guide_indices].astype(float);side=p['output_side'];factor=side//labels.shape[0]
|
||||
owners=np.repeat(np.repeat(labels,factor,0),factor,1);base=np.zeros((side,side,3),float);quiet=np.ones((side,side),float)
|
||||
guide_luma=guide@np.array([.2126,.7152,.0722])
|
||||
for i in np.unique(labels):
|
||||
mask=(labels==i).astype(float)
|
||||
weights=kernel.resize_float(kernel.gaussian(mask,p['source_sigma']),(side,side))
|
||||
numerator=kernel.resize_float(kernel.gaussian(guide*mask[...,None],p['source_sigma']),(side,side))
|
||||
field=numerator/np.maximum(weights[...,None],1e-12);area=owners==i;base[area]=field[area]
|
||||
if groups[int(i)]['quiet_from_source_variance']:
|
||||
second=kernel.resize_float(kernel.gaussian(guide_luma**2*mask,p['source_sigma']),(side,side))/np.maximum(weights,1e-12)
|
||||
first=field@np.array([.2126,.7152,.0722]);std=np.sqrt(np.maximum(second-first**2,0))
|
||||
quiet[area]=(.35+.65*np.clip(std/8.,0,1))[area]
|
||||
parent=np.asarray(Image.fromarray(raw).resize((side,side),Image.Resampling.LANCZOS),float)
|
||||
luma=parent@np.array([.2126,.7152,.0722]);band=kernel.gaussian(luma,p['residual_sigma_fine'])-kernel.gaussian(luma,p['residual_sigma_coarse'])
|
||||
if not fields and float(band.std())<.25:raise ValueError('parent lacks coherent fine/mid generated detail; Source-only fallback forbidden')
|
||||
texture=np.zeros_like(band);normalizers={};band_stds={}
|
||||
for i in np.unique(owners):
|
||||
name=groups[int(i)]['name'];group_band=band
|
||||
if name in fields:
|
||||
material_parent=np.asarray(Image.fromarray(fields[name]).resize((side,side),Image.Resampling.LANCZOS),float)
|
||||
material_luma=material_parent@np.array([.2126,.7152,.0722])
|
||||
group_band=kernel.gaussian(material_luma,p['residual_sigma_fine'])-kernel.gaussian(material_luma,p['residual_sigma_coarse'])
|
||||
band_stds[name]=float(group_band.std())
|
||||
if band_stds[name]<.25:raise ValueError('parent lacks coherent fine/mid generated detail for group '+name)
|
||||
area=owners==i;centered=group_band-group_band[area].mean();divisor=max(float(np.percentile(np.abs(centered[area]),p['residual_percentile'])),1.)
|
||||
texture[area]=np.tanh(centered[area]/divisor);normalizers[groups[int(i)]['name']]=divisor
|
||||
strengths=np.array([g['texture_log_gain'] for g in groups])[owners]*quiet;gain=np.exp(strengths*texture)
|
||||
output=np.clip(np.rint(base*gain[...,None]),0,255).astype(np.uint8);base8=np.clip(np.rint(base),0,255).astype(np.uint8)
|
||||
changed=int(np.any(output!=base8,axis=2).sum())
|
||||
if changed<max(1,math.ceil(side*side*.01)):raise ValueError('generated modulation too small; no Source-only output')
|
||||
metrics=dict(method=METHOD,source_shape=list(a.shape),sourceNx4_indices_sha256=digest(guide_indices.astype('<u2').tobytes()),
|
||||
material_labels_sha256=digest(owners.astype('<i2').tobytes()),group_output_pixels={groups[int(i)]['name']:int((owners==i).sum()) for i in np.unique(owners)},
|
||||
used_Source_indices_by_group={g['name']:sorted(set(g['indices'])&set(map(int,np.unique(a)))) for g in groups},coherent_parent_band_std=float(band.std()),
|
||||
per_group_normalizers=normalizers,per_group_parent_band_std=band_stds,material_parent_field_names=sorted(fields),actual_tone_gain_min=float(gain.min()),actual_tone_gain_max=float(gain.max()),pixels_changed_from_Source_guided_base=changed,
|
||||
mean_absolute_parent_tone_modulation=float(np.abs(output.astype(float)-base8).mean()),output_RGB_sha256=digest(output.tobytes()),
|
||||
limits='Explicit Source palette groups/SourceNx geometry and broad masked RGB; coherent actual generated fine+mid luminance retained multiplicatively. No random noise/SourceNN fallback, material/context approval or reserved semantics.')
|
||||
return output,base8,owners,metrics
|
||||
|
||||
def publish_no_replace(stage,target):
|
||||
"""Atomic fresh-directory publication; unsupported platforms fail closed."""
|
||||
if os.name=='nt':os.rename(stage,target);return
|
||||
if not sys.platform.startswith('linux'):raise OSError(errno.ENOTSUP,'atomic no-replace publication unsupported')
|
||||
libc=ctypes.CDLL(None,use_errno=True)
|
||||
if not hasattr(libc,'renameat2'):raise OSError(errno.ENOTSUP,'renameat2 unavailable')
|
||||
fn=libc.renameat2;fn.argtypes=[ctypes.c_int,ctypes.c_char_p,ctypes.c_int,ctypes.c_char_p,ctypes.c_uint];fn.restype=ctypes.c_int
|
||||
if fn(-100,os.fsencode(stage),-100,os.fsencode(target),1)!=0:
|
||||
code=ctypes.get_errno();raise OSError(code,os.strerror(code),str(target))
|
||||
|
||||
def run(recipe_path,output):
|
||||
recipe_path=Path(recipe_path).absolute();output=Path(output).absolute()
|
||||
if output.exists() or output.is_symlink():raise ValueError('output exists; never overwrite')
|
||||
if not output.parent.is_dir():raise ValueError('output parent must already exist')
|
||||
# JSON is read with no-follow before loading the exact allowlisted primitive.
|
||||
fd=os.open(recipe_path,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):raise ValueError('regular recipe required')
|
||||
with os.fdopen(fd,'rb',closefd=False) as stream:body=stream.read(1_000_001)
|
||||
finally:os.close(fd)
|
||||
if len(body)>1_000_000:raise ValueError('recipe too large')
|
||||
r=json.loads(body);validate_recipe(r);kernel,dependency_pin=load_dependency(r['dependency'],recipe_path.parent)
|
||||
_,recipe_pin=kernel.read_regular(recipe_path)
|
||||
if recipe_pin['sha256']!=digest(body):raise ValueError('recipe changed during load')
|
||||
_,tool_pin=kernel.read_regular(__file__)
|
||||
inputs=[recipe_pin,dependency_pin,tool_pin]
|
||||
def checked(ref):
|
||||
binding(ref);p=Path(ref['path']);p=p if p.is_absolute() else recipe_path.parent/p
|
||||
b,pin=kernel.read_regular(p)
|
||||
if pin['sha256']!=ref['sha256']:raise ValueError('input SHA mismatch')
|
||||
inputs.append(pin);return b,pin
|
||||
pb,palette_pin=checked(r['palette']);palette=kernel.palette_rgb(pb);prepared=[]
|
||||
for f in r['frames']:
|
||||
sb,source_pin=checked(f['Source']);indices=kernel.decode_source(sb);parent_bytes,parent_pin=checked(f['parent_raw'])
|
||||
if len(parent_bytes)>32*1024*1024:raise ValueError('parent byte ceiling exceeded')
|
||||
im=Image.open(io.BytesIO(parent_bytes))
|
||||
if im.format!='PNG' or im.width*im.height>8_000_000:raise ValueError('bounded PNG parent required')
|
||||
if im.convert('RGBA').getchannel('A').getextrema()!=(255,255):raise ValueError('opaque parent required; alpha repair forbidden')
|
||||
x,y,w,h=f['crop']
|
||||
if x+w>im.width or y+h>im.height:raise ValueError('crop outside parent')
|
||||
parent_rgb=np.asarray(im.convert('RGB'));raw=parent_rgb[y:y+h,x:x+w].copy();fields={};region_proofs={}
|
||||
for name,donor in f.get('material_parent_regions',{}).items():
|
||||
fields[name],region_proofs[name]=reflect_parent_region(parent_rgb,f['crop'],donor)
|
||||
derived,_,_,metrics=reconstruct(indices,palette,raw,r['parameters'],r['material_groups'],kernel,fields)
|
||||
metrics['material_parent_region_proofs']=region_proofs
|
||||
stream=io.BytesIO();Image.fromarray(derived).save(stream,format='PNG');prepared.append((f,stream.getvalue(),metrics,source_pin,parent_pin))
|
||||
output_hashes={f['asset']:digest(png) for f,png,_,_,_ in prepared}
|
||||
execution=dict(status='success',method=METHOD,recipe_sha256=recipe_pin['sha256'],tool_sha256=tool_pin['sha256'],
|
||||
source_sha256={f['asset']:s['sha256'] for f,_,_,s,_ in prepared},parent_raw_sha256=sorted({p['sha256'] for _,_,_,_,p in prepared}),output_sha256=output_hashes)
|
||||
receipt=dict(status='DERIVED_OUTPUTS_NOT_MATERIAL_APPROVAL',execution=execution,recipe=recipe_pin,tool=tool_pin,dependency=dependency_pin,
|
||||
decoder=dict(name='decode_source',module_sha256=dependency_pin['sha256']),runtime=dict(python=sys.version,executable=sys.executable,numpy=np.__version__,Pillow=PIL.__version__,platform=sys.platform),
|
||||
material_groups=r['material_groups'],material_groups_sha256=digest(json.dumps(r['material_groups'],sort_keys=True,separators=(',',':')).encode()),
|
||||
frames=[dict(asset=f['asset'],Source=s,parent_raw=p,crop=f['crop'],metrics=m) for f,_,m,s,p in prepared])
|
||||
stage=Path(tempfile.mkdtemp(prefix='.'+output.name+'.stage-',dir=output.parent))
|
||||
try:
|
||||
for f,png,_,_,_ in prepared:(stage/(f['asset']+'-DERIVED-declared-material-v1.png')).write_bytes(png)
|
||||
(stage/'normalized-execution-object.json').write_text(json.dumps(dict(execution=execution),indent=2)+'\n')
|
||||
(stage/'DERIVED-LINEAGE-AND-MATERIAL-GROUP-METRICS.json').write_text(json.dumps(receipt,indent=2,sort_keys=True)+'\n')
|
||||
for old in inputs:
|
||||
_,now=kernel.read_regular(old['path'])
|
||||
if now!=old:raise ValueError('input changed across reconstruction')
|
||||
publish_no_replace(stage,output)
|
||||
except BaseException:
|
||||
shutil.rmtree(stage,ignore_errors=True);raise
|
||||
return receipt
|
||||
|
||||
def main(argv=None):
|
||||
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--recipe',required=True);parser.add_argument('--output',required=True);args=parser.parse_args(argv)
|
||||
result=run(args.recipe,args.output);print(json.dumps(dict(status='success',output=str(Path(args.output).absolute()),frames=len(result['frames']),method=METHOD)))
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,237 @@
|
||||
from pathlib import Path
|
||||
import copy,hashlib,json,struct,tempfile,unittest,importlib.util
|
||||
from unittest import mock
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import hd_terrain_group_retexture as c
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
DEP={'path':str(ROOT/'tools/hd_terrain_retexture.py'),'sha256':c.DEPENDENCY_SHA256}
|
||||
|
||||
class DeclaredGroupsTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):cls.kernel,cls.dep_pin=c.load_dependency(DEP,Path('.'))
|
||||
def setUp(self):
|
||||
self.groups=[dict(name='gray',indices=[30,31],texture_log_gain=.2,quiet_from_source_variance=False),dict(name='olive',indices=[62],texture_log_gain=.35,quiet_from_source_variance=True),dict(name='forest',indices=list(range(96,112)),texture_log_gain=.48,quiet_from_source_variance=True),dict(name='brown',indices=[237,238,239],texture_log_gain=.18,quiet_from_source_variance=True)]
|
||||
self.params=dict(output_side=64,source_sigma=3.,residual_sigma_fine=.65,residual_sigma_coarse=10.,residual_percentile=80.)
|
||||
self.pal=np.zeros((256,3),np.uint8)
|
||||
self.pal[:32]=np.arange(31,-1,-1)[:,None]*8
|
||||
self.pal[62]=[24,24,0]
|
||||
for i in range(96,112):self.pal[i]=[max(0,100-(i-96)*7),max(12,140-(i-96)*8),0]
|
||||
self.pal[111]=[0,12,0];self.pal[237:240]=[[48,32,28],[32,16,12],[16,8,4]]
|
||||
yy,xx=np.indices((64,64));self.wave=120+40*np.sin(xx*2*np.pi/20)+24*np.cos(yy*2*np.pi/27)
|
||||
self.parent=np.stack([self.wave*.9,self.wave*.3,self.wave*1.2],2).clip(0,255).astype(np.uint8)
|
||||
def render(self,src,groups=None):return c.reconstruct(src,self.pal,self.parent,self.params,self.groups if groups is None else groups,self.kernel)
|
||||
def fixture(self,p,frames=1):
|
||||
palette=p/'palette.dat';palette.write_bytes((self.pal//4).tobytes());parent=p/'parent.png';Image.fromarray(self.parent).save(parent)
|
||||
ref=lambda q:dict(path=q.name,sha256=hashlib.sha256(q.read_bytes()).hexdigest())
|
||||
source=p/'source.pic';source.write_bytes(struct.pack('<5i',1,1,0,4,4)+bytes([111,111,31,31,103,103,62,62,237,237,111,111,31,31,111,111]))
|
||||
recipe=dict(format_version=2,method=c.METHOD,dependency=DEP,palette=ref(palette),material_groups=self.groups,parameters=self.params,
|
||||
frames=[dict(asset=format(0x2061d+i,'x'),Source=ref(source),parent_raw=ref(parent),crop=[0,0,64,64],origin=[0,0]) for i in range(frames)])
|
||||
rp=p/'recipe.json';rp.write_text(json.dumps(recipe));return recipe,rp,source
|
||||
def test_explicit_forest_support_and_legacy_globals_unchanged(self):
|
||||
before=copy.deepcopy(self.kernel.DEFAULTS);names=self.kernel.FAMILY_NAMES
|
||||
with self.assertRaisesRegex(ValueError,'unsupported material'):self.kernel.material_families(np.full((4,4),100,np.uint8))
|
||||
out,base,owners,m=self.render(np.full((4,4),100,np.uint8))
|
||||
self.assertGreater(m['pixels_changed_from_Source_guided_base'],40);self.assertTrue(np.all(owners==2))
|
||||
self.assertEqual(self.kernel.DEFAULTS,before);self.assertEqual(self.kernel.FAMILY_NAMES,names)
|
||||
def test_disjoint_complete_used_coverage_and_reserved_rejection(self):
|
||||
for mutate in ('overlap','missing','reserved','duplicate','bool','unsorted'):
|
||||
g=copy.deepcopy(self.groups)
|
||||
if mutate=='overlap':g[-1]['indices']=[31,237,238,239]
|
||||
if mutate=='missing':g[2]['indices'].remove(111)
|
||||
if mutate=='reserved':g[-1]['indices'].append(240)
|
||||
if mutate=='duplicate':g[-1]['indices']=[237,237,238,239]
|
||||
if mutate=='bool':g[0]['indices']=[False,30,31]
|
||||
if mutate=='unsorted':g[0]['indices']=[31,30]
|
||||
with self.assertRaises(ValueError):c.groups_validated(g,[111])
|
||||
def test_names_gains_and_flags_are_strict(self):
|
||||
for name,value in [('name','../green'),('texture_log_gain',True),('texture_log_gain',float('nan')),('texture_log_gain',.71),('quiet_from_source_variance',1)]:
|
||||
g=copy.deepcopy(self.groups);g[2][name]=value
|
||||
with self.assertRaises(ValueError):c.groups_validated(g)
|
||||
g=copy.deepcopy(self.groups);g[-1]['name']=g[0]['name']
|
||||
with self.assertRaises(ValueError):c.groups_validated(g)
|
||||
def test_SourceNx_material_owner_geometry_and_determinism(self):
|
||||
src=np.array([[31,31,103,103],[31,62,62,103],[237,237,103,111],[239,239,111,111]],np.uint8)
|
||||
a=self.render(src);b=self.render(src);lookup=np.full(240,-1,np.int16)
|
||||
for i,g in enumerate(self.groups):lookup[g['indices']]=i
|
||||
expected=np.repeat(np.repeat(lookup[self.kernel.scale_nx(src,4)],4,0),4,1)
|
||||
np.testing.assert_array_equal(a[2],expected);np.testing.assert_array_equal(a[0],b[0]);self.assertEqual(a[3],b[3])
|
||||
def test_coherent_generated_fine_mid_structure_is_retained(self):
|
||||
out,base,_,m=self.render(np.full((4,4),100,np.uint8))
|
||||
self.assertGreater(float(np.corrcoef(out[:,:,1].ravel(),self.wave.ravel())[0,1]),.9)
|
||||
self.assertGreater(float(out[:,:,1].std()),1);self.assertGreater(m['mean_absolute_parent_tone_modulation'],.3)
|
||||
def test_Source_color_family_zero_channels_and_gray_preserved(self):
|
||||
for idx in [30,62,111]:
|
||||
out,base,_,m=self.render(np.full((4,4),idx,np.uint8))
|
||||
if idx==30:np.testing.assert_array_equal(out[:,:,0],out[:,:,2])
|
||||
if idx==62:np.testing.assert_array_equal(out[:,:,0],out[:,:,1]);self.assertTrue(np.all(out[:,:,2]==0))
|
||||
if idx==111:self.assertTrue(np.all(out[:,:,[0,2]]==0));self.assertGreater(out[:,:,1].max(),out[:,:,1].min())
|
||||
self.assertGreaterEqual(m['actual_tone_gain_min'],np.exp(-.7));self.assertLessEqual(m['actual_tone_gain_max'],np.exp(.7))
|
||||
def test_flat_parent_and_zero_gain_do_not_silently_return_Source(self):
|
||||
with self.assertRaisesRegex(ValueError,'lacks coherent'):c.reconstruct(np.full((4,4),111,np.uint8),self.pal,np.full((64,64,3),90,np.uint8),self.params,self.groups,self.kernel)
|
||||
g=copy.deepcopy(self.groups)
|
||||
for z in g:z['texture_log_gain']=0
|
||||
with self.assertRaisesRegex(ValueError,'modulation too small'):self.render(np.full((4,4),111,np.uint8),g)
|
||||
def test_old_four_family_declaration_reproduces_old_math_exactly(self):
|
||||
groups=[dict(name='gray',indices=list(range(32)),texture_log_gain=.38,quiet_from_source_variance=False),dict(name='olive',indices=list(range(60,64)),texture_log_gain=.55,quiet_from_source_variance=False),dict(name='soil',indices=list(range(192,224))+[239],texture_log_gain=.34,quiet_from_source_variance=True),dict(name='rim',indices=list(range(224,239)),texture_log_gain=.06,quiet_from_source_variance=False)]
|
||||
palette=self.pal.copy();palette[202]=[60,24,20];palette[231]=[81,60,52];palette[63]=[12,12,0]
|
||||
src=np.array([[30,30,231,63],[30,30,231,202],[30,30,231,231],[30,30,30,30]],np.uint8)
|
||||
p=dict(self.params,texture_log_gain=dict(gray=.38,olive=.55,soil=.34,rim=.06))
|
||||
a=self.kernel.reconstruct(src,palette,self.parent,p);b=c.reconstruct(src,palette,self.parent,self.params,groups,self.kernel)
|
||||
for i in range(3):np.testing.assert_array_equal(a[i],b[i])
|
||||
def test_Source_reserved_wrong_format_and_float_planes_fail(self):
|
||||
for src in [np.full((4,4),240,np.uint8),np.full((4,4),111.,float),np.ones((3,4),np.uint8)]:
|
||||
with self.assertRaises(ValueError):self.render(src)
|
||||
with self.assertRaises(ValueError):self.kernel.decode_source(struct.pack('<5i',0,1,0,4,4)+bytes([111]*16))
|
||||
def test_relative_atomic_success_normalized_receipt_and_all_bindings(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);r,rp,src=self.fixture(p,2);out=p/'new';result=c.run(rp,out)
|
||||
execution=result['execution'];self.assertEqual(set(execution),{'status','method','recipe_sha256','tool_sha256','source_sha256','parent_raw_sha256','output_sha256'})
|
||||
self.assertEqual(result['dependency']['sha256'],c.DEPENDENCY_SHA256);self.assertEqual(result['decoder']['module_sha256'],c.DEPENDENCY_SHA256)
|
||||
self.assertTrue(result['runtime']['numpy']);self.assertTrue(result['runtime']['Pillow']);self.assertTrue(result['runtime']['python'])
|
||||
for asset,want in execution['output_sha256'].items():self.assertEqual(hashlib.sha256((out/(asset+'-DERIVED-declared-material-v1.png')).read_bytes()).hexdigest(),want)
|
||||
self.assertEqual(json.loads((out/'normalized-execution-object.json').read_text()),dict(execution=execution))
|
||||
before={x.name:x.read_bytes() for x in out.iterdir()}
|
||||
with self.assertRaisesRegex(ValueError,'never overwrite'):c.run(rp,out)
|
||||
self.assertEqual(before,{x.name:x.read_bytes() for x in out.iterdir()});self.assertFalse(list(p.glob('.new.stage-*')))
|
||||
def test_typed_asset_path_escape_and_source_sentinels_preserved(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);r,rp,src=self.fixture(p);sentinel=p/'sentinel-DERIVED-declared-material-v1.png';sentinel.write_bytes(b'EXISTING')
|
||||
for i,bad in enumerate(['../sentinel',str(p/'sentinel'),'0x2061d','2061D','02061d',False,17,'',None]):
|
||||
z=copy.deepcopy(r);z['frames'][0]['asset']=bad;rp.write_text(json.dumps(z));out=p/('absent'+str(i))
|
||||
with self.assertRaisesRegex(ValueError,'canonical'):c.run(rp,out)
|
||||
self.assertFalse(out.exists());self.assertEqual(sentinel.read_bytes(),b'EXISTING')
|
||||
def test_boolean_version_origin_crop_and_duplicate_ids_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r,_,_=self.fixture(Path(d),2)
|
||||
for mutate in ('version','origin','crop','duplicate','order'):
|
||||
z=copy.deepcopy(r)
|
||||
if mutate=='version':z['format_version']=True
|
||||
if mutate=='origin':z['frames'][0]['origin']=[False,0]
|
||||
if mutate=='crop':z['frames'][0]['crop'][0]=False
|
||||
if mutate=='duplicate':z['frames'][1]['asset']=z['frames'][0]['asset']
|
||||
if mutate=='order':z['frames'].reverse()
|
||||
with self.assertRaises(ValueError):c.validate_recipe(z)
|
||||
def test_dependency_forgery_and_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);fake=p/'fake.py';fake.write_text('raise RuntimeError("MUST NEVER EXECUTE")')
|
||||
with self.assertRaisesRegex(ValueError,'hash mismatch'):c.load_dependency(dict(path=str(fake),sha256=c.DEPENDENCY_SHA256),p)
|
||||
with self.assertRaisesRegex(ValueError,'only exact'):c.load_dependency(dict(path=str(fake),sha256='0'*64),p)
|
||||
link=p/'link.py';link.symlink_to(DEP['path'])
|
||||
with self.assertRaises(OSError):c.load_dependency(dict(path=str(link),sha256=c.DEPENDENCY_SHA256),p)
|
||||
def test_bad_raw_hash_transparency_crop_and_Source_symlink_leave_no_outputs(self):
|
||||
for mutation in ('hash','alpha','crop','symlink'):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);r,rp,src=self.fixture(p);out=p/'absent'
|
||||
if mutation=='hash':r['frames'][0]['parent_raw']['sha256']='0'*64
|
||||
if mutation=='alpha':
|
||||
im=np.concatenate([self.parent,np.full((64,64,1),254,np.uint8)],2);Image.fromarray(im).save(p/'parent.png');r['frames'][0]['parent_raw']['sha256']=hashlib.sha256((p/'parent.png').read_bytes()).hexdigest()
|
||||
if mutation=='crop':r['frames'][0]['crop']=[1,0,64,64]
|
||||
if mutation=='symlink':
|
||||
link=p/'link.pic';link.symlink_to(src);r['frames'][0]['Source']['path']=link.name
|
||||
rp.write_text(json.dumps(r))
|
||||
with self.assertRaises((ValueError,OSError)):c.run(rp,out)
|
||||
self.assertFalse(out.exists());self.assertFalse(list(p.glob('.absent.stage-*')))
|
||||
def test_atomic_publish_failure_rolls_back_stage_only(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);r,rp,src=self.fixture(p);raw_before=(p/'parent.png').read_bytes();source_before=src.read_bytes()
|
||||
with mock.patch.object(c,'publish_no_replace',side_effect=OSError('synthetic atomic failure')):
|
||||
with self.assertRaisesRegex(OSError,'synthetic atomic'):c.run(rp,p/'absent')
|
||||
self.assertFalse((p/'absent').exists());self.assertFalse(list(p.glob('.absent.stage-*')));self.assertEqual(src.read_bytes(),source_before);self.assertEqual((p/'parent.png').read_bytes(),raw_before)
|
||||
def test_racing_existing_directory_is_never_replaced(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);stage=p/'stage';stage.mkdir();(stage/'new').write_bytes(b'NEW');target=p/'target';target.mkdir();ino=target.stat().st_ino
|
||||
with self.assertRaises(OSError):c.publish_no_replace(stage,target)
|
||||
self.assertEqual(target.stat().st_ino,ino);self.assertFalse(list(target.iterdir()));self.assertTrue((stage/'new').exists())
|
||||
def test_input_changed_during_prepare_prevents_publication(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);r,rp,src=self.fixture(p);real=c.reconstruct
|
||||
def mutate(*args,**kwargs):
|
||||
result=real(*args,**kwargs);src.write_bytes(src.read_bytes()+b'CHANGED');return result
|
||||
with mock.patch.object(c,'reconstruct',side_effect=mutate):
|
||||
with self.assertRaisesRegex(ValueError,'changed across'):c.run(rp,p/'absent')
|
||||
self.assertFalse((p/'absent').exists());self.assertFalse(list(p.glob('.absent.stage-*')))
|
||||
|
||||
def test_reflection_integer_geometry_no_edge_repeat(self):
|
||||
np.testing.assert_array_equal(c.reflect_axis(np.arange(-2,7),3),[2,1,0,1,2,1,0,1,2])
|
||||
np.testing.assert_array_equal(c.reflect_axis(np.arange(-3,4),1),np.zeros(7,int))
|
||||
with self.assertRaises(ValueError):c.reflect_axis(np.array([.5]),3)
|
||||
|
||||
def test_original_donor_location_and_all_RGB_samples_match_before_resize(self):
|
||||
y,x=np.indices((20,22));parent=np.stack([x*7,y*9,(x+y)*5],2).astype(np.uint8)
|
||||
crop=[2,3,14,13];donor=[6,7,5,4];field,proof=c.reflect_parent_region(parent,crop,donor)
|
||||
self.assertEqual(field.shape,(13,14,3));np.testing.assert_array_equal(field[4:8,4:9],parent[7:11,6:11])
|
||||
self.assertEqual(proof['donor_RGB_sha256'],hashlib.sha256(parent[7:11,6:11].tobytes()).hexdigest())
|
||||
self.assertFalse(proof['resampling_before_native_frame_field'])
|
||||
xs=6+c.reflect_axis(np.arange(14)+2-6,5);ys=7+c.reflect_axis(np.arange(13)+3-7,4)
|
||||
np.testing.assert_array_equal(field,parent[ys[:,None],xs[None,:]])
|
||||
|
||||
def test_reflection_keeps_original_pixel_density_not_stretched_donor(self):
|
||||
y,x=np.indices((80,80));parent=np.stack([x,x+y,y],2).astype(np.uint8)
|
||||
field,_=c.reflect_parent_region(parent,[1,1,64,64],[20,20,16,16])
|
||||
# Thirty sample period, unit steps; a donor enlarged to64 first cannot satisfy.
|
||||
np.testing.assert_array_equal(field[20,0:30],field[20,30:60])
|
||||
self.assertTrue(set(np.abs(np.diff(field[20,:,0].astype(int))))<={1})
|
||||
|
||||
def test_no_region_and_empty_region_arrays_exact_frozen_v1(self):
|
||||
path=ROOT/'tools/fixtures/hd_terrain_group_retexture_v1.py'
|
||||
self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(),'425db50b811edec71568bc82e7aca7cbfacc330e5dd78f276951062bee18c8d7')
|
||||
spec=importlib.util.spec_from_file_location('frozen_groups_v1',path);v1=importlib.util.module_from_spec(spec);spec.loader.exec_module(v1)
|
||||
src=np.array([[31,31,103,103],[31,62,62,103],[237,237,103,111],[239,239,111,111]],np.uint8)
|
||||
old=v1.reconstruct(src,self.pal,self.parent,self.params,self.groups,self.kernel)
|
||||
for fields in (None,{}):
|
||||
new=c.reconstruct(src,self.pal,self.parent,self.params,self.groups,self.kernel,fields)
|
||||
for i in range(3):np.testing.assert_array_equal(new[i],old[i])
|
||||
|
||||
def test_only_declared_Source_owner_receives_its_changed_region_band(self):
|
||||
src=np.full((4,4),100,np.uint8);src[:,2:]=31
|
||||
y,x=np.indices((64,64));one=np.repeat((110+30*np.sin(x/3.))[...,None],3,2).astype(np.uint8)
|
||||
two=np.repeat((110+30*np.cos(y/4.))[...,None],3,2).astype(np.uint8)
|
||||
a=c.reconstruct(src,self.pal,self.parent,self.params,self.groups,self.kernel,{'forest':one})
|
||||
b=c.reconstruct(src,self.pal,self.parent,self.params,self.groups,self.kernel,{'forest':two})
|
||||
changed=np.any(a[0]!=b[0],2);self.assertGreater(int(changed.sum()),20)
|
||||
self.assertTrue(np.all(a[2][changed]==2));np.testing.assert_array_equal(a[0][a[2]==0],b[0][b[2]==0])
|
||||
|
||||
def test_fullcrop_wrong_edge_changes_cannot_enter_complete_regional_fields(self):
|
||||
y,x=np.indices((64,64));wave=8*np.sin(x/2.)+6*np.cos(y/3.)
|
||||
level=np.where(x+y<63,35,145)+wave;parent=np.repeat(level[...,None],3,2).clip(0,255).astype(np.uint8)
|
||||
crop=[0,0,64,64];regions={'dark':[3,3,16,16],'light':[45,45,16,16]}
|
||||
altered=parent.copy();keep=np.zeros((64,64),bool)
|
||||
for xx,yy,w,h in regions.values():keep[yy:yy+h,xx:xx+w]=True
|
||||
altered[~keep]=np.repeat(((x*5+y*3)%240)[...,None],3,2)[~keep]
|
||||
groups=[dict(name='dark',indices=list(range(26,32)),texture_log_gain=.38,quiet_from_source_variance=False),dict(name='light',indices=list(range(26)),texture_log_gain=.38,quiet_from_source_variance=False)]
|
||||
src=np.array([[28,28,28,21],[28,28,21,21],[28,28,21,21],[28,21,21,21]],np.uint8)
|
||||
fields_a={k:c.reflect_parent_region(parent,crop,r)[0] for k,r in regions.items()};fields_b={k:c.reflect_parent_region(altered,crop,r)[0] for k,r in regions.items()}
|
||||
for name in fields_a:np.testing.assert_array_equal(fields_a[name],fields_b[name])
|
||||
a=c.reconstruct(src,self.pal,parent,self.params,groups,self.kernel,fields_a);b=c.reconstruct(src,self.pal,altered,self.params,groups,self.kernel,fields_b)
|
||||
np.testing.assert_array_equal(a[0],b[0]);np.testing.assert_array_equal(a[2],b[2])
|
||||
self.assertLessEqual(a[3]['actual_tone_gain_max'],np.exp(.38));self.assertGreaterEqual(a[3]['actual_tone_gain_min'],np.exp(-.38))
|
||||
|
||||
def test_region_keys_rectangles_and_strict_interior_are_validated(self):
|
||||
crop=[10,20,60,60]
|
||||
for regions in [None,[],{'unknown':[12,22,5,5]},{'forest':[10,22,5,5]},{'forest':[12,20,5,5]},{'forest':[12,22,58,5]},{'forest':[12,22,5,58]},{'forest':[12,22,0,5]},{'forest':[False,22,5,5]},{'forest':[12.,22,5,5]},{'forest':{'rect':[12,22,5,5]}}]:
|
||||
with self.assertRaises(ValueError):c.validate_regions(regions,crop,self.groups)
|
||||
c.validate_regions({},crop,self.groups);c.validate_regions({'forest':[12,22,5,5]},crop,self.groups)
|
||||
|
||||
def test_material_field_dtype_shape_and_unknown_key_reject(self):
|
||||
src=np.full((4,4),100,np.uint8)
|
||||
for fields in [{'unknown':self.parent},{'forest':self.parent.astype(float)},{'forest':self.parent[:16,:16]},[]]:
|
||||
with self.assertRaises(ValueError):c.reconstruct(src,self.pal,self.parent,self.params,self.groups,self.kernel,fields)
|
||||
|
||||
def test_regional_CLI_receipt_binds_actual_raw_donor_and_reflected_field(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);r,rp,src=self.fixture(p);r['frames'][0]['material_parent_regions']={'forest':[4,4,20,20]};rp.write_text(json.dumps(r))
|
||||
receipt=c.run(rp,p/'new');proof=receipt['frames'][0]['metrics']['material_parent_region_proofs']['forest']
|
||||
raw=np.array(Image.open(p/'parent.png').convert('RGB'));field,expected=c.reflect_parent_region(raw,[0,0,64,64],[4,4,20,20])
|
||||
self.assertEqual(proof,expected);self.assertEqual(proof['donor_RGB_sha256'],hashlib.sha256(raw[4:24,4:24].tobytes()).hexdigest())
|
||||
self.assertEqual(proof['reflected_native_field_RGB_sha256'],hashlib.sha256(field.tobytes()).hexdigest())
|
||||
self.assertEqual(set(receipt['execution']),{'status','method','recipe_sha256','tool_sha256','source_sha256','parent_raw_sha256','output_sha256'})
|
||||
|
||||
def test_invalid_region_never_creates_output_or_changes_parent(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=Path(d);r,rp,src=self.fixture(p);raw=(p/'parent.png').read_bytes();s=src.read_bytes();r['frames'][0]['material_parent_regions']={'forest':[62,62,5,5]};rp.write_text(json.dumps(r))
|
||||
with self.assertRaisesRegex(ValueError,'strictly inside'):c.run(rp,p/'absent')
|
||||
self.assertFalse((p/'absent').exists());self.assertEqual((p/'parent.png').read_bytes(),raw);self.assertEqual(src.read_bytes(),s)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user