Add verified multipart Linux and Windows GitHub releases
This commit is contained in:
1 parent
443d77852b
commit
d1f5b7e5a2
8 files changed
+1152
-1
No files matched your search
@@ -0,0 +1,368 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export a frozen, unchanged playable package as independently extractable ZIPs.
|
||||
|
||||
No builds, data staging, runtime-manifest rewriting, Git changes or uploads.
|
||||
Each platform core plus ALL shared data ZIPs extracts into one RustRaptor folder.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
from assemble_runtime import checked_file, digest
|
||||
|
||||
MAX_ASSET_BYTES = 2 * 1024**3 # GitHub requires each release asset UNDER 2 GiB.
|
||||
DEFAULT_VOLUME_BYTES = 1_500_000_000
|
||||
ROOT_DISK_FLOOR = 1_800_000_000
|
||||
SKIPPED = {"target", "dist", ".reference", ".local", "derived", "__pycache__", ".git"}
|
||||
|
||||
|
||||
def read_json(path):
|
||||
return json.loads(Path(path).read_bytes())
|
||||
|
||||
|
||||
def write_new_json(path, value):
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("x", encoding="utf-8") as stream:
|
||||
json.dump(value, stream, indent=2)
|
||||
stream.write("\n")
|
||||
|
||||
|
||||
def git_commit(package):
|
||||
return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=package, text=True).strip()
|
||||
|
||||
|
||||
def require_clean_workspace(package):
|
||||
if subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=all"], cwd=package):
|
||||
raise ValueError("release exports require a clean, fully committed workspace")
|
||||
|
||||
|
||||
def snapshot_identity(package):
|
||||
return {"commit": git_commit(package),
|
||||
"runtime_manifest_sha256": digest(package / "runtime-manifest.json"),
|
||||
"windows_manifest_sha256": digest(package / "runtime-windows-manifest.json")}
|
||||
|
||||
|
||||
def check_snapshot(package, plan):
|
||||
require_clean_workspace(package)
|
||||
if snapshot_identity(package) != plan["snapshot"]:
|
||||
raise ValueError("source commit or runtime manifests changed after release planning")
|
||||
|
||||
|
||||
def entry(package, name, expected=None):
|
||||
path = checked_file(package, name)
|
||||
safe_member("RustRaptor/" + name)
|
||||
return {"name": name, "bytes": path.stat().st_size,
|
||||
"sha256": expected if expected is not None else digest(path)}
|
||||
|
||||
|
||||
def safe_member(name):
|
||||
"""All exports are portable ordinary files below the one installation root."""
|
||||
if not isinstance(name, str) or not name.startswith("RustRaptor/") or "\\" in name:
|
||||
raise ValueError("unsafe release member path")
|
||||
parts = name.split("/")
|
||||
reserved = {"CON", "PRN", "AUX", "NUL"} | {f"{prefix}{i}" for prefix in ("COM", "LPT") for i in range(1, 10)}
|
||||
if any(not part or part in (".", "..") or part.endswith((".", " "))
|
||||
or any(ord(c) < 32 or c in '<>:"|?*' for c in part)
|
||||
or part.split(".")[0].upper() in reserved for part in parts):
|
||||
raise ValueError("unsafe or nonportable release member path")
|
||||
return name
|
||||
|
||||
|
||||
def selected_runtime_files(package):
|
||||
"""Explicit finite no-follow inventory; skipped private namespaces stay private."""
|
||||
files = []
|
||||
visited = 0
|
||||
|
||||
def excluded(relative):
|
||||
if relative.parts[0] == "data":
|
||||
return False # The complete physical runtime tree is mandatory.
|
||||
return (bool(SKIPPED.intersection(relative.parts))
|
||||
or any(part.startswith(".runtime-data-") for part in relative.parts)
|
||||
or (len(relative.parts) >= 4 and relative.parts[0] == "mods"
|
||||
and relative.parts[2:4] == ("art", "source")))
|
||||
|
||||
def walk(directory, depth):
|
||||
nonlocal visited
|
||||
if depth > 128:
|
||||
raise ValueError("release source tree exceeds the 128-level inventory bound")
|
||||
with os.scandir(directory) as stream:
|
||||
children = sorted(stream, key=lambda child: child.name)
|
||||
for child in children:
|
||||
path = Path(child.path)
|
||||
relative = path.relative_to(package)
|
||||
if excluded(relative):
|
||||
continue
|
||||
visited += 1
|
||||
if visited > 1_000_000:
|
||||
raise ValueError("release source tree exceeds the one-million-entry inventory bound")
|
||||
metadata = child.stat(follow_symlinks=False)
|
||||
if stat.S_ISLNK(metadata.st_mode):
|
||||
raise ValueError(f"symlink in selected release tree: {relative}")
|
||||
if stat.S_ISDIR(metadata.st_mode):
|
||||
walk(path, depth + 1)
|
||||
elif stat.S_ISREG(metadata.st_mode):
|
||||
files.append(path)
|
||||
else:
|
||||
raise ValueError(f"non-regular member in selected release tree: {relative}")
|
||||
walk(package, 0)
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def partition(rows, budget):
|
||||
if type(budget) is not int or not 0 < budget <= DEFAULT_VOLUME_BYTES:
|
||||
raise ValueError("ZIP volume budget must be positive and at most 1,500,000,000 bytes")
|
||||
groups, current, size = [], [], 0
|
||||
for row in rows:
|
||||
if row["bytes"] > budget:
|
||||
raise ValueError(f"one runtime file exceeds ZIP volume budget: {row['name']}")
|
||||
if current and size + row["bytes"] > budget:
|
||||
groups.append(current)
|
||||
current, size = [], 0
|
||||
current.append(row)
|
||||
size += row["bytes"]
|
||||
if current:
|
||||
groups.append(current)
|
||||
return groups
|
||||
|
||||
|
||||
def build_plan(package, expected_commit, budget=DEFAULT_VOLUME_BYTES):
|
||||
package = Path(package).resolve(strict=True)
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", expected_commit):
|
||||
raise ValueError("an exact final tested Git commit is required")
|
||||
require_clean_workspace(package)
|
||||
snapshot = snapshot_identity(package)
|
||||
if snapshot["commit"] != expected_commit:
|
||||
raise ValueError("package HEAD is not the requested tested checkpoint")
|
||||
selected = selected_runtime_files(package)
|
||||
manifest = read_json(package / "runtime-manifest.json")
|
||||
windows = read_json(package / "runtime-windows-manifest.json")
|
||||
hd = read_json(package / "data/hd/hd.json")
|
||||
hd_count = sum(len(sequence["frames"]) for sequence in hd["sequences"])
|
||||
if hd_count != manifest["approved_hd_frames"] or windows["approved_hd_frames"] != hd_count:
|
||||
raise ValueError("shared data and platform HD counts differ")
|
||||
if digest(package / "bin/raptor-rust") != manifest["binary_sha256"]:
|
||||
raise ValueError("included Linux executable differs from its runtime manifest")
|
||||
for name, expected in windows["files"].items():
|
||||
if digest(checked_file(package, name)) != expected:
|
||||
raise ValueError(f"included Windows payload differs: {name}")
|
||||
rows = [entry(package, name, expected) for name, expected in sorted(manifest["files"].items())]
|
||||
if any(not row["name"].startswith("data/") for row in rows):
|
||||
raise ValueError("shared runtime manifest must contain data/ paths only")
|
||||
if len(rows) != manifest["data_files"] or sum(row["bytes"] for row in rows) != manifest["data_bytes"]:
|
||||
raise ValueError("shared runtime file count/bytes differ from its manifest")
|
||||
for sequence in hd["sequences"]:
|
||||
for frame in sequence["frames"]:
|
||||
checked_file(package / "data/hd", frame["file"])
|
||||
if "data/hd/" + frame["file"] not in manifest["files"]:
|
||||
raise ValueError("active HD artwork is absent from the shared runtime manifest")
|
||||
physical_data = {path.relative_to(package).as_posix() for path in selected
|
||||
if path.relative_to(package).parts[0] == "data"}
|
||||
if physical_data != set(manifest["files"]):
|
||||
raise ValueError("runtime manifest does not cover the complete physical data tree")
|
||||
groups = partition(rows, budget)
|
||||
assets = [{"name": f"RustRaptor-data-{index:02d}-of-{len(groups):02d}.zip",
|
||||
"kind": "shared-data", "files": group,
|
||||
"uncompressed_bytes": sum(row["bytes"] for row in group)}
|
||||
for index, group in enumerate(groups, 1)]
|
||||
core = [entry(package, path.relative_to(package).as_posix()) for path in selected
|
||||
if path.relative_to(package).parts[0] != "data"]
|
||||
for platform in ("linux", "windows"):
|
||||
if platform == "linux":
|
||||
excluded = {"bin/raptor-rust.exe", "bin/SDL2.dll", "bin/SDL2-WINDOWS-LICENSE.txt",
|
||||
"run.cmd", "runtime-windows-manifest.json"}
|
||||
files = [row for row in core if row["name"] not in excluded]
|
||||
else:
|
||||
files = [row for row in core if row["name"] not in {"bin/raptor-rust", "run.sh"}
|
||||
and not row["name"].startswith("lib/")]
|
||||
if sum(row["bytes"] for row in files) > DEFAULT_VOLUME_BYTES:
|
||||
raise ValueError("platform core exceeds the safe single-asset budget")
|
||||
assets.append({"name": f"RustRaptor-{platform}-x86_64.zip", "kind": platform,
|
||||
"files": files, "uncompressed_bytes": sum(row["bytes"] for row in files)})
|
||||
plan = {"format_version": 1, "package": str(package), "snapshot": snapshot,
|
||||
"catalog_entries": manifest["catalog_entries"], "approved_hd_frames": hd_count,
|
||||
"bundled_mods": len(list((package / "mods").glob("*/mod.toml"))),
|
||||
"data_files": len(rows), "data_bytes": manifest["data_bytes"],
|
||||
"volume_uncompressed_budget": budget, "assets": assets,
|
||||
"install": "Extract your platform ZIP and every shared-data ZIP into the same directory; run RustRaptor/run.sh or RustRaptor/run.cmd.",
|
||||
"payload_scope": "Complete unchanged runtime-manifest data plus explicitly inventoried mods, source, docs and platform libraries. No proof/lineage files from the manifest are silently removed."}
|
||||
for platform in ("linux", "windows"):
|
||||
installation = [row["name"].casefold() for asset in assets
|
||||
if asset["kind"] in ("shared-data", platform) for row in asset["files"]]
|
||||
if len(installation) != len(set(installation)):
|
||||
raise ValueError("case-insensitive duplicate installation path across assets")
|
||||
for asset in assets:
|
||||
seen = set()
|
||||
for row in asset["files"]:
|
||||
key = row["name"].casefold()
|
||||
if key in seen:
|
||||
raise ValueError("case-insensitive duplicate release path")
|
||||
seen.add(key)
|
||||
check_snapshot(package, plan)
|
||||
return plan
|
||||
|
||||
|
||||
def source_identity(metadata):
|
||||
return metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns
|
||||
|
||||
|
||||
@contextmanager
|
||||
def opened_source(package, name, expected_size):
|
||||
relative = Path(name)
|
||||
if relative.is_absolute() or not relative.parts or any(part in (".", "..") for part in relative.parts):
|
||||
raise ValueError("unsafe release member path")
|
||||
flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW | os.O_NONBLOCK
|
||||
directory = os.open("/", flags | os.O_DIRECTORY)
|
||||
source = None
|
||||
try:
|
||||
for part in package.parts[1:] + relative.parts[:-1]:
|
||||
following = os.open(part, flags | os.O_DIRECTORY, dir_fd=directory)
|
||||
os.close(directory)
|
||||
directory = following
|
||||
source = os.open(relative.name, flags, dir_fd=directory)
|
||||
before = os.fstat(source)
|
||||
if not stat.S_ISREG(before.st_mode) or before.st_size != expected_size:
|
||||
raise ValueError("release source is not the planned regular file")
|
||||
with os.fdopen(source, "rb", closefd=False) as stream:
|
||||
yield stream, before
|
||||
if source_identity(before) != source_identity(os.fstat(source)):
|
||||
raise ValueError("release source changed while being read")
|
||||
finally:
|
||||
if source is not None:
|
||||
os.close(source)
|
||||
os.close(directory)
|
||||
|
||||
|
||||
def write_asset(plan_path, asset_name, output):
|
||||
plan_path, output = Path(plan_path), Path(output)
|
||||
plan = read_json(plan_path)
|
||||
package = Path(plan["package"]).resolve(strict=True)
|
||||
check_snapshot(package, plan)
|
||||
matches = [asset for asset in plan["assets"] if asset["name"] == asset_name]
|
||||
if len(matches) != 1 or Path(asset_name).name != asset_name:
|
||||
raise ValueError("choose one exact planned release asset")
|
||||
asset = matches[0]
|
||||
if output.name != asset_name:
|
||||
raise ValueError("output filename must equal the planned asset name")
|
||||
if output.exists():
|
||||
raise ValueError("archive output already exists; never overwrite exports")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
if shutil.disk_usage(package).free < ROOT_DISK_FLOOR:
|
||||
raise ValueError("workspace free space is below the 1.8GB floor")
|
||||
# Deflate worst-case and ZIP overhead fit this generous reserve. /tmp may
|
||||
# be tmpfs: only one volume is held and no complete runtime tree is copied.
|
||||
if shutil.disk_usage(output.parent).free < asset["uncompressed_bytes"] + 128 * 1024**2:
|
||||
raise ValueError("insufficient output space for one archive and reserve")
|
||||
if (output.parent.stat().st_dev == package.stat().st_dev
|
||||
and shutil.disk_usage(package).free < ROOT_DISK_FLOOR + asset["uncompressed_bytes"] + 128 * 1024**2):
|
||||
raise ValueError("same-filesystem output would cross the workspace 1.8GB floor")
|
||||
measured = {}
|
||||
with zipfile.ZipFile(output, "x", zipfile.ZIP_DEFLATED, compresslevel=6, allowZip64=True) as archive:
|
||||
for row in asset["files"]:
|
||||
h = hashlib.sha256()
|
||||
with opened_source(package, row["name"], row["bytes"]) as (original, before):
|
||||
info = zipfile.ZipInfo("RustRaptor/" + row["name"], time.localtime(before.st_mtime)[:6])
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.create_system = 3
|
||||
info.external_attr = (before.st_mode & 0xFFFF) << 16
|
||||
with archive.open(info, "w", force_zip64=True) as member:
|
||||
remaining = row["bytes"]
|
||||
while remaining:
|
||||
block = original.read(min(128 * 1024, remaining))
|
||||
if not block:
|
||||
raise ValueError("release source ended before its planned size")
|
||||
remaining -= len(block)
|
||||
h.update(block)
|
||||
member.write(block)
|
||||
if original.read(1):
|
||||
raise ValueError("release source grew beyond its planned size")
|
||||
if h.hexdigest() != row["sha256"]:
|
||||
raise ValueError(f"release source changed or differs from plan: {row['name']}")
|
||||
measured[row["name"]] = h.hexdigest()
|
||||
if output.stat().st_size >= MAX_ASSET_BYTES:
|
||||
raise ValueError("release asset reaches GitHub's 2GiB limit")
|
||||
with zipfile.ZipFile(output) as archive:
|
||||
if archive.testzip() is not None:
|
||||
raise ValueError("release ZIP failed CRC validation")
|
||||
if archive.namelist() != ["RustRaptor/" + row["name"] for row in asset["files"]]:
|
||||
raise ValueError("release ZIP membership differs from plan")
|
||||
check_snapshot(package, plan)
|
||||
return {"status": "ZIP_CRC_AND_COMPLETE_PAYLOAD_SHA256_PASS", "asset": asset_name,
|
||||
"kind": asset["kind"], "bytes": output.stat().st_size, "sha256": digest(output),
|
||||
"plan_sha256": digest(plan_path), "checkpoint": plan["snapshot"],
|
||||
"payload_files": len(measured), "payload_sha256": measured,
|
||||
"root_disk_floor_after_bytes": shutil.disk_usage(package).free,
|
||||
"builds_or_source_manifest_mutations": 0}
|
||||
|
||||
|
||||
def checksums(plan_path, receipts, output, manifest_output=None, tag=None):
|
||||
plan_path, receipts, output = Path(plan_path), Path(receipts), Path(output)
|
||||
plan = read_json(plan_path)
|
||||
if manifest_output is not None and (not isinstance(tag, str) or not re.fullmatch(r"v[0-9A-Za-z._-]+", tag)):
|
||||
raise ValueError("a safe exact release tag is required")
|
||||
lines, published = [], []
|
||||
for asset in plan["assets"]:
|
||||
receipt = read_json(receipts / (asset["name"] + ".json"))
|
||||
if (receipt.get("status") != "ZIP_CRC_AND_COMPLETE_PAYLOAD_SHA256_PASS"
|
||||
or receipt.get("asset") != asset["name"]
|
||||
or receipt.get("kind") != asset["kind"]
|
||||
or receipt.get("checkpoint") != plan["snapshot"]
|
||||
or receipt.get("payload_files") != len(asset["files"])
|
||||
or receipt.get("plan_sha256") != digest(plan_path)
|
||||
or receipt.get("payload_sha256") != {row["name"]: row["sha256"] for row in asset["files"]}
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", receipt.get("sha256", ""))
|
||||
or not 0 < receipt.get("bytes", 0) < MAX_ASSET_BYTES):
|
||||
raise ValueError("incomplete or mismatched release asset receipt")
|
||||
lines.append(f"{receipt['sha256']} {asset['name']}\n")
|
||||
published.append({"name": asset["name"], "kind": asset["kind"], "bytes": receipt["bytes"],
|
||||
"sha256": receipt["sha256"], "files": asset["files"]})
|
||||
with output.open("x", encoding="ascii") as stream:
|
||||
stream.writelines(lines)
|
||||
if manifest_output is not None:
|
||||
write_new_json(manifest_output, {"format_version": 1, "tag": tag, "checkpoint": plan["snapshot"],
|
||||
"approved_hd_frames": plan["approved_hd_frames"], "data_files": plan["data_files"],
|
||||
"data_bytes": plan["data_bytes"], "bundled_mods": plan["bundled_mods"], "assets": published})
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
planning = commands.add_parser("plan")
|
||||
planning.add_argument("--package", type=Path, required=True)
|
||||
planning.add_argument("--expected-commit", required=True)
|
||||
planning.add_argument("--output", type=Path, required=True)
|
||||
exporting = commands.add_parser("archive")
|
||||
exporting.add_argument("--plan", type=Path, required=True)
|
||||
exporting.add_argument("--asset", required=True)
|
||||
exporting.add_argument("--output", type=Path, required=True)
|
||||
exporting.add_argument("--receipt", type=Path, required=True)
|
||||
summing = commands.add_parser("checksums")
|
||||
summing.add_argument("--plan", type=Path, required=True)
|
||||
summing.add_argument("--receipts", type=Path, required=True)
|
||||
summing.add_argument("--output", type=Path, required=True)
|
||||
summing.add_argument("--manifest-output", type=Path, required=True)
|
||||
summing.add_argument("--tag", required=True)
|
||||
args = parser.parse_args()
|
||||
if args.command == "plan":
|
||||
value = build_plan(args.package, args.expected_commit)
|
||||
write_new_json(args.output, value)
|
||||
print(json.dumps({key: value[key] for key in ("snapshot", "approved_hd_frames", "bundled_mods", "data_files", "data_bytes")}))
|
||||
elif args.command == "archive":
|
||||
value = write_asset(args.plan, args.asset, args.output)
|
||||
write_new_json(args.receipt, value)
|
||||
print(json.dumps({key: value[key] for key in ("status", "asset", "bytes", "sha256", "payload_files")}))
|
||||
else:
|
||||
checksums(args.plan, args.receipts, args.output, args.manifest_output, args.tag)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Small real-ZIP release fixtures; no production archives or game build."""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import zipfile
|
||||
|
||||
import export_release as release
|
||||
|
||||
COMMIT = "a" * 40
|
||||
|
||||
|
||||
class ReleaseExportTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.package = self.root / "package"
|
||||
self.package.mkdir()
|
||||
fixtures = {"data/INDEX": b"10000 4 misc/pic\n", "data/misc/pic": b"base",
|
||||
"data/hd/jobs/reviewed/derived/picture.png": b"accepted artwork",
|
||||
"data/hd/jobs/reviewed/proofs/actual-proof.json": b"{}",
|
||||
"bin/raptor-rust": b"Linux exe", "bin/raptor-rust.exe": b"Windows exe",
|
||||
"bin/SDL2.dll": b"Windows SDL", "lib/libSDL2-2.0.so.0": b"Linux SDL",
|
||||
"run.sh": b"#!/bin/sh\n", "run.cmd": b"@echo off\n",
|
||||
"mods/example/mod.toml": b'id = "example"\n',
|
||||
"mods/example/art/sd.png": b"SD", "mods/example/art/hd.png": b"HD",
|
||||
"mods/example/sounds/fire.wav": b"sound", "mods/example/music/track.wav": b"music",
|
||||
"mods/example/art/source/draft.png": b"private source art",
|
||||
"src/main.rs": b"fn main() {}", "docs/modding.md": b"API docs",
|
||||
"target/cache": b"build cache", ".reference/private": b"private review"}
|
||||
for name, value in fixtures.items():
|
||||
self.put(name, value)
|
||||
(self.package / "run.sh").chmod(0o755)
|
||||
self.put_json("data/hd/hd.json", {"sequences": [{"frames": [{"file": "jobs/reviewed/derived/picture.png"}]}]})
|
||||
data = {p.relative_to(self.package).as_posix(): release.digest(p)
|
||||
for p in (self.package / "data").rglob("*") if p.is_file()}
|
||||
self.put_json("runtime-manifest.json", {"files": data, "approved_hd_frames": 1,
|
||||
"binary_sha256": release.digest(self.package / "bin/raptor-rust"),
|
||||
"data_files": len(data), "data_bytes": sum((self.package / n).stat().st_size for n in data),
|
||||
"catalog_entries": 1})
|
||||
self.put_json("runtime-windows-manifest.json", {"files": {
|
||||
n: release.digest(self.package / n) for n in ["bin/raptor-rust.exe", "bin/SDL2.dll", "run.cmd"]},
|
||||
"approved_hd_frames": 1})
|
||||
patch.object(release, "require_clean_workspace").start()
|
||||
self.mock_commit = patch.object(release, "git_commit", return_value=COMMIT).start()
|
||||
self.addCleanup(patch.stopall)
|
||||
|
||||
def put(self, name, value):
|
||||
path = self.package / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(value)
|
||||
|
||||
def put_json(self, name, value):
|
||||
self.put(name, (json.dumps(value) + "\n").encode())
|
||||
|
||||
def plan(self):
|
||||
value = release.build_plan(self.package, COMMIT)
|
||||
path = self.root / "plan.json"
|
||||
release.write_new_json(path, value)
|
||||
return path, value
|
||||
|
||||
def test_complete_data_mods_audio_and_platform_split(self):
|
||||
_, plan = self.plan()
|
||||
data, linux, windows = plan["assets"]
|
||||
self.assertIn("data/hd/jobs/reviewed/derived/picture.png", {r["name"] for r in data["files"]})
|
||||
self.assertIn("data/hd/jobs/reviewed/proofs/actual-proof.json", {r["name"] for r in data["files"]})
|
||||
for asset in (linux, windows):
|
||||
names = {r["name"] for r in asset["files"]}
|
||||
self.assertTrue({"mods/example/mod.toml", "mods/example/art/sd.png", "mods/example/art/hd.png",
|
||||
"mods/example/sounds/fire.wav", "mods/example/music/track.wav", "src/main.rs"} <= names)
|
||||
self.assertFalse({"target/cache", ".reference/private", "mods/example/art/source/draft.png"} & names)
|
||||
self.assertIn("bin/raptor-rust", {r["name"] for r in linux["files"]})
|
||||
self.assertNotIn("bin/raptor-rust.exe", {r["name"] for r in linux["files"]})
|
||||
self.assertNotIn("lib/libSDL2-2.0.so.0", {r["name"] for r in windows["files"]})
|
||||
self.assertEqual(plan["bundled_mods"], 1)
|
||||
|
||||
def test_partition_covers_every_file_once_under_budget(self):
|
||||
rows = [{"name": str(i), "bytes": i} for i in range(1, 7)]
|
||||
groups = release.partition(rows, 7)
|
||||
self.assertEqual([r for group in groups for r in group], rows)
|
||||
self.assertTrue(all(sum(r["bytes"] for r in group) <= 7 for group in groups))
|
||||
with self.assertRaises(ValueError):
|
||||
release.partition(rows, 5)
|
||||
|
||||
def test_real_zip_relocation_complete_hashes_and_linux_permissions(self):
|
||||
path, plan = self.plan()
|
||||
relocated = self.root / "Portable Ω folder"
|
||||
for asset in plan["assets"]:
|
||||
archive = self.root / asset["name"]
|
||||
receipt = release.write_asset(path, asset["name"], archive)
|
||||
self.assertEqual(receipt["status"], "ZIP_CRC_AND_COMPLETE_PAYLOAD_SHA256_PASS")
|
||||
with zipfile.ZipFile(archive) as packed:
|
||||
self.assertIsNone(packed.testzip())
|
||||
packed.extractall(relocated)
|
||||
if asset["kind"] == "linux":
|
||||
self.assertEqual((packed.getinfo("RustRaptor/run.sh").external_attr >> 16) & 0o777, 0o755)
|
||||
for row in asset["files"]:
|
||||
self.assertEqual(release.digest(relocated / "RustRaptor" / row["name"]), row["sha256"])
|
||||
|
||||
def test_modified_same_size_data_is_rejected(self):
|
||||
path, plan = self.plan()
|
||||
self.put("data/misc/pic", b"evil")
|
||||
with self.assertRaisesRegex(ValueError, "differs from plan"):
|
||||
release.write_asset(path, plan["assets"][0]["name"], self.root / plan["assets"][0]["name"])
|
||||
|
||||
def test_manifest_or_commit_change_rejected(self):
|
||||
path, plan = self.plan()
|
||||
self.mock_commit.return_value = "b" * 40
|
||||
with self.assertRaisesRegex(ValueError, "changed after"):
|
||||
release.write_asset(path, plan["assets"][0]["name"], self.root / plan["assets"][0]["name"])
|
||||
|
||||
def test_no_overwrite_or_unplanned_output_name(self):
|
||||
path, plan = self.plan()
|
||||
asset = plan["assets"][0]["name"]
|
||||
with self.assertRaisesRegex(ValueError, "filename"):
|
||||
release.write_asset(path, asset, self.root / "wrong.zip")
|
||||
output = self.root / asset
|
||||
output.write_bytes(b"existing export")
|
||||
with self.assertRaisesRegex(ValueError, "already exists"):
|
||||
release.write_asset(path, asset, output)
|
||||
self.assertEqual(output.read_bytes(), b"existing export")
|
||||
|
||||
def test_symlink_and_fifo_sources_are_rejected_without_blocking(self):
|
||||
outside = self.root / "outside"
|
||||
outside.write_bytes(b"base")
|
||||
source = self.package / "data/misc/pic"
|
||||
source.unlink()
|
||||
source.symlink_to(outside)
|
||||
with self.assertRaises(OSError):
|
||||
with release.opened_source(self.package, "data/misc/pic", 4):
|
||||
self.fail("symlink source was accepted")
|
||||
source.unlink()
|
||||
os.mkfifo(source)
|
||||
with self.assertRaisesRegex(ValueError, "regular file"):
|
||||
with release.opened_source(self.package, "data/misc/pic", 0):
|
||||
self.fail("FIFO source was accepted")
|
||||
|
||||
def test_open_descriptor_growth_is_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError, "changed while"):
|
||||
with release.opened_source(self.package, "data/misc/pic", 4) as (stream, _):
|
||||
with (self.package / "data/misc/pic").open("ab") as growing:
|
||||
growing.write(b"extra")
|
||||
self.assertEqual(stream.read(5), b"basee")
|
||||
|
||||
def test_checksums_require_complete_matching_actual_receipts(self):
|
||||
path, plan = self.plan()
|
||||
receipts = self.root / "receipts"
|
||||
receipts.mkdir()
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
release.checksums(path, receipts, self.root / "SHA256SUMS")
|
||||
for asset in plan["assets"]:
|
||||
result = release.write_asset(path, asset["name"], self.root / asset["name"])
|
||||
release.write_new_json(receipts / (asset["name"] + ".json"), result)
|
||||
release.checksums(path, receipts, self.root / "SHA256SUMS")
|
||||
self.assertEqual(len((self.root / "SHA256SUMS").read_text().splitlines()), len(plan["assets"]))
|
||||
record = receipts / (plan["assets"][0]["name"] + ".json")
|
||||
value = release.read_json(record)
|
||||
value["payload_sha256"] = {}
|
||||
record.write_text(json.dumps(value))
|
||||
with self.assertRaisesRegex(ValueError, "mismatched"):
|
||||
release.checksums(path, receipts, self.root / "bad-sums")
|
||||
|
||||
def test_active_hd_missing_from_manifest_rejected(self):
|
||||
self.put_json("data/hd/hd.json", {"sequences": [{"frames": [{"file": "missing.png"}]}]})
|
||||
manifest = release.read_json(self.package / "runtime-manifest.json")
|
||||
manifest["files"]["data/hd/hd.json"] = release.digest(self.package / "data/hd/hd.json")
|
||||
manifest["data_bytes"] = sum((self.package / name).stat().st_size for name in manifest["files"])
|
||||
self.put_json("runtime-manifest.json", manifest)
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
release.build_plan(self.package, COMMIT)
|
||||
|
||||
def test_complete_physical_data_must_match_manifest(self):
|
||||
self.put("data/forgotten.wav", b"required sound")
|
||||
with self.assertRaisesRegex(ValueError, "complete physical"):
|
||||
self.plan()
|
||||
|
||||
def test_nonportable_paths_rejected(self):
|
||||
for name in ("../outside", "RustRaptor/a/../b", "RustRaptor/C:evil", "RustRaptor/a\\b", "RustRaptor/CON.txt", "RustRaptor/a."):
|
||||
with self.subTest(name=name), self.assertRaises(ValueError):
|
||||
release.safe_member(name)
|
||||
|
||||
def test_release_manifest_binds_all_payload_and_snapshot(self):
|
||||
path, plan = self.plan()
|
||||
receipts = self.root / "receipts"
|
||||
receipts.mkdir()
|
||||
for asset in plan["assets"]:
|
||||
result = release.write_asset(path, asset["name"], self.root / asset["name"])
|
||||
release.write_new_json(receipts / (asset["name"] + ".json"), result)
|
||||
manifest = self.root / "release-manifest.json"
|
||||
release.checksums(path, receipts, self.root / "SHA256SUMS", manifest, "v0.1.0-preview.test")
|
||||
actual = release.read_json(manifest)
|
||||
self.assertEqual(actual["checkpoint"], plan["snapshot"])
|
||||
self.assertEqual(actual["tag"], "v0.1.0-preview.test")
|
||||
self.assertEqual([a["files"] for a in actual["assets"]], [a["files"] for a in plan["assets"]])
|
||||
|
||||
def test_data_directory_symlink_is_rejected(self):
|
||||
outside = self.root / "outside-data"
|
||||
outside.mkdir()
|
||||
(outside / "required.png").write_bytes(b"required")
|
||||
(self.package / "data/linked").symlink_to(outside, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symlink"):
|
||||
self.plan()
|
||||
|
||||
def test_mod_directory_symlink_is_rejected(self):
|
||||
outside = self.root / "outside-mod"
|
||||
outside.mkdir()
|
||||
(outside / "mod.toml").write_text('id="linked"')
|
||||
(self.package / "mods/linked").symlink_to(outside, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symlink"):
|
||||
self.plan()
|
||||
|
||||
def test_unmanifested_data_fifo_is_rejected_without_blocking(self):
|
||||
os.mkfifo(self.package / "data/fifo")
|
||||
with self.assertRaisesRegex(ValueError, "non-regular"):
|
||||
self.plan()
|
||||
|
||||
def test_skipped_private_symlinks_and_fifo_do_not_enter_release(self):
|
||||
(self.package / "target/link").symlink_to(self.root, target_is_directory=True)
|
||||
os.mkfifo(self.package / ".reference/fifo")
|
||||
_, plan = self.plan()
|
||||
self.assertFalse(any("target/" in row["name"] or ".reference/" in row["name"]
|
||||
for asset in plan["assets"] for row in asset["files"]))
|
||||
|
||||
def test_selected_core_symlink_and_fifo_are_rejected(self):
|
||||
for kind in ("symlink", "fifo"):
|
||||
with self.subTest(kind=kind):
|
||||
path = self.package / "mods/example/bad"
|
||||
if kind == "symlink":
|
||||
path.symlink_to(self.package / "src/main.rs")
|
||||
else:
|
||||
os.mkfifo(path)
|
||||
try:
|
||||
with self.assertRaisesRegex(ValueError, "symlink|non-regular"):
|
||||
self.plan()
|
||||
finally:
|
||||
path.unlink()
|
||||
|
||||
def test_checksum_receipt_identity_and_invalid_tag_rejected_before_output(self):
|
||||
path, plan = self.plan()
|
||||
receipts = self.root / "receipts"
|
||||
receipts.mkdir()
|
||||
for asset in plan["assets"]:
|
||||
result = release.write_asset(path, asset["name"], self.root / asset["name"])
|
||||
release.write_new_json(receipts / (asset["name"] + ".json"), result)
|
||||
record = receipts / (plan["assets"][0]["name"] + ".json")
|
||||
original = release.read_json(record)
|
||||
for field, bad in (("checkpoint", {}), ("kind", "wrong"), ("payload_files", -1)):
|
||||
value = dict(original)
|
||||
value[field] = bad
|
||||
record.write_text(json.dumps(value))
|
||||
output = self.root / (field + "-SHA256SUMS")
|
||||
with self.assertRaisesRegex(ValueError, "mismatched"):
|
||||
release.checksums(path, receipts, output)
|
||||
self.assertFalse(output.exists())
|
||||
record.write_text(json.dumps(original))
|
||||
output = self.root / "invalid-tag-SHA256SUMS"
|
||||
with self.assertRaisesRegex(ValueError, "release tag"):
|
||||
release.checksums(path, receipts, output, self.root / "invalid-manifest.json", "../bad")
|
||||
self.assertFalse(output.exists())
|
||||
self.assertFalse((self.root / "invalid-manifest.json").exists())
|
||||
|
||||
|
||||
class LinuxInstallerTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
script = Path(__file__).parents[1] / "release/Install-Linux.sh"
|
||||
source = script.read_text().split("# BEGIN INSTALLER PYTHON\n", 1)[1].split("# END INSTALLER PYTHON", 1)[0]
|
||||
cls.api = {"__name__": "installer_test"}
|
||||
exec(compile(source, str(script), "exec"), cls.api)
|
||||
|
||||
def test_real_long_path_payload_hash_and_executable_mode(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
archive = root / "one.zip"
|
||||
name = "data/" + ("long-component/" * 25) + "picture.png"
|
||||
body = b"complete accepted payload"
|
||||
with zipfile.ZipFile(archive, "w") as packed:
|
||||
info = zipfile.ZipInfo("RustRaptor/" + name)
|
||||
info.external_attr = 0o100755 << 16
|
||||
packed.writestr(info, body)
|
||||
row = {"name": name, "bytes": len(body), "sha256": release.hashlib.sha256(body).hexdigest()}
|
||||
self.api["extract"](archive, root, [row])
|
||||
actual = root / "RustRaptor" / name
|
||||
self.assertEqual(actual.read_bytes(), body)
|
||||
self.assertEqual(actual.stat().st_mode & 0o777, 0o755)
|
||||
with self.assertRaises(FileExistsError):
|
||||
self.api["extract"](archive, root, [row])
|
||||
|
||||
def test_unexpected_member_symlink_and_bad_hash_rejected(self):
|
||||
for case in ("unexpected", "symlink", "hash", "traversal"):
|
||||
with self.subTest(case=case), tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
archive = root / "one.zip"
|
||||
name = "RustRaptor/file" if case != "traversal" else "RustRaptor/../escape"
|
||||
with zipfile.ZipFile(archive, "w") as packed:
|
||||
info = zipfile.ZipInfo(name)
|
||||
info.external_attr = (0o120777 if case == "symlink" else 0o100644) << 16
|
||||
packed.writestr(info, b"body")
|
||||
row = {"name": "other" if case == "unexpected" else name[len("RustRaptor/"):],
|
||||
"bytes": 4, "sha256": "0" * 64 if case == "hash" else release.hashlib.sha256(b"body").hexdigest()}
|
||||
with self.assertRaises(ValueError):
|
||||
self.api["extract"](archive, root, [row])
|
||||
|
||||
def test_existing_symlink_parent_is_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
outside = root / "outside"
|
||||
outside.mkdir()
|
||||
(root / "RustRaptor").symlink_to(outside, target_is_directory=True)
|
||||
archive = root / "one.zip"
|
||||
with zipfile.ZipFile(archive, "w") as packed:
|
||||
packed.writestr("RustRaptor/file", b"body")
|
||||
row = {"name": "file", "bytes": 4, "sha256": release.hashlib.sha256(b"body").hexdigest()}
|
||||
with self.assertRaisesRegex(ValueError, "parent"):
|
||||
self.api["extract"](archive, root, [row])
|
||||
self.assertFalse((outside / "file").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user