Add verified multipart Linux and Windows GitHub releases

tools/export_release.py

@@ -0,0 +1,368 @@
+#!/usr/bin/env python3
+"""Export a frozen, unchanged playable package as independently extractable ZIPs.
+
+No builds, data staging, runtime-manifest rewriting, Git changes or uploads.
+Each platform core plus ALL shared data ZIPs extracts into one RustRaptor folder.
+"""
+import argparse
+from contextlib import contextmanager
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import stat
+import subprocess
+import time
+import zipfile
+
+from assemble_runtime import checked_file, digest
+
+MAX_ASSET_BYTES = 2 * 1024**3  # GitHub requires each release asset UNDER 2 GiB.
+DEFAULT_VOLUME_BYTES = 1_500_000_000
+ROOT_DISK_FLOOR = 1_800_000_000
+SKIPPED = {"target", "dist", ".reference", ".local", "derived", "__pycache__", ".git"}
+
+
+def read_json(path):
+    return json.loads(Path(path).read_bytes())
+
+
+def write_new_json(path, value):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("x", encoding="utf-8") as stream:
+        json.dump(value, stream, indent=2)
+        stream.write("\n")
+
+
+def git_commit(package):
+    return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=package, text=True).strip()
+
+
+def require_clean_workspace(package):
+    if subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=all"], cwd=package):
+        raise ValueError("release exports require a clean, fully committed workspace")
+
+
+def snapshot_identity(package):
+    return {"commit": git_commit(package),
+            "runtime_manifest_sha256": digest(package / "runtime-manifest.json"),
+            "windows_manifest_sha256": digest(package / "runtime-windows-manifest.json")}
+
+
+def check_snapshot(package, plan):
+    require_clean_workspace(package)
+    if snapshot_identity(package) != plan["snapshot"]:
+        raise ValueError("source commit or runtime manifests changed after release planning")
+
+
+def entry(package, name, expected=None):
+    path = checked_file(package, name)
+    safe_member("RustRaptor/" + name)
+    return {"name": name, "bytes": path.stat().st_size,
+            "sha256": expected if expected is not None else digest(path)}
+
+
+def safe_member(name):
+    """All exports are portable ordinary files below the one installation root."""
+    if not isinstance(name, str) or not name.startswith("RustRaptor/") or "\\" in name:
+        raise ValueError("unsafe release member path")
+    parts = name.split("/")
+    reserved = {"CON", "PRN", "AUX", "NUL"} | {f"{prefix}{i}" for prefix in ("COM", "LPT") for i in range(1, 10)}
+    if any(not part or part in (".", "..") or part.endswith((".", " "))
+           or any(ord(c) < 32 or c in '<>:"|?*' for c in part)
+           or part.split(".")[0].upper() in reserved for part in parts):
+        raise ValueError("unsafe or nonportable release member path")
+    return name
+
+
+def selected_runtime_files(package):
+    """Explicit finite no-follow inventory; skipped private namespaces stay private."""
+    files = []
+    visited = 0
+
+    def excluded(relative):
+        if relative.parts[0] == "data":
+            return False  # The complete physical runtime tree is mandatory.
+        return (bool(SKIPPED.intersection(relative.parts))
+                or any(part.startswith(".runtime-data-") for part in relative.parts)
+                or (len(relative.parts) >= 4 and relative.parts[0] == "mods"
+                    and relative.parts[2:4] == ("art", "source")))
+
+    def walk(directory, depth):
+        nonlocal visited
+        if depth > 128:
+            raise ValueError("release source tree exceeds the 128-level inventory bound")
+        with os.scandir(directory) as stream:
+            children = sorted(stream, key=lambda child: child.name)
+        for child in children:
+            path = Path(child.path)
+            relative = path.relative_to(package)
+            if excluded(relative):
+                continue
+            visited += 1
+            if visited > 1_000_000:
+                raise ValueError("release source tree exceeds the one-million-entry inventory bound")
+            metadata = child.stat(follow_symlinks=False)
+            if stat.S_ISLNK(metadata.st_mode):
+                raise ValueError(f"symlink in selected release tree: {relative}")
+            if stat.S_ISDIR(metadata.st_mode):
+                walk(path, depth + 1)
+            elif stat.S_ISREG(metadata.st_mode):
+                files.append(path)
+            else:
+                raise ValueError(f"non-regular member in selected release tree: {relative}")
+    walk(package, 0)
+    return sorted(files)
+
+
+def partition(rows, budget):
+    if type(budget) is not int or not 0 < budget <= DEFAULT_VOLUME_BYTES:
+        raise ValueError("ZIP volume budget must be positive and at most 1,500,000,000 bytes")
+    groups, current, size = [], [], 0
+    for row in rows:
+        if row["bytes"] > budget:
+            raise ValueError(f"one runtime file exceeds ZIP volume budget: {row['name']}")
+        if current and size + row["bytes"] > budget:
+            groups.append(current)
+            current, size = [], 0
+        current.append(row)
+        size += row["bytes"]
+    if current:
+        groups.append(current)
+    return groups
+
+
+def build_plan(package, expected_commit, budget=DEFAULT_VOLUME_BYTES):
+    package = Path(package).resolve(strict=True)
+    if not re.fullmatch(r"[0-9a-f]{40}", expected_commit):
+        raise ValueError("an exact final tested Git commit is required")
+    require_clean_workspace(package)
+    snapshot = snapshot_identity(package)
+    if snapshot["commit"] != expected_commit:
+        raise ValueError("package HEAD is not the requested tested checkpoint")
+    selected = selected_runtime_files(package)
+    manifest = read_json(package / "runtime-manifest.json")
+    windows = read_json(package / "runtime-windows-manifest.json")
+    hd = read_json(package / "data/hd/hd.json")
+    hd_count = sum(len(sequence["frames"]) for sequence in hd["sequences"])
+    if hd_count != manifest["approved_hd_frames"] or windows["approved_hd_frames"] != hd_count:
+        raise ValueError("shared data and platform HD counts differ")
+    if digest(package / "bin/raptor-rust") != manifest["binary_sha256"]:
+        raise ValueError("included Linux executable differs from its runtime manifest")
+    for name, expected in windows["files"].items():
+        if digest(checked_file(package, name)) != expected:
+            raise ValueError(f"included Windows payload differs: {name}")
+    rows = [entry(package, name, expected) for name, expected in sorted(manifest["files"].items())]
+    if any(not row["name"].startswith("data/") for row in rows):
+        raise ValueError("shared runtime manifest must contain data/ paths only")
+    if len(rows) != manifest["data_files"] or sum(row["bytes"] for row in rows) != manifest["data_bytes"]:
+        raise ValueError("shared runtime file count/bytes differ from its manifest")
+    for sequence in hd["sequences"]:
+        for frame in sequence["frames"]:
+            checked_file(package / "data/hd", frame["file"])
+            if "data/hd/" + frame["file"] not in manifest["files"]:
+                raise ValueError("active HD artwork is absent from the shared runtime manifest")
+    physical_data = {path.relative_to(package).as_posix() for path in selected
+                     if path.relative_to(package).parts[0] == "data"}
+    if physical_data != set(manifest["files"]):
+        raise ValueError("runtime manifest does not cover the complete physical data tree")
+    groups = partition(rows, budget)
+    assets = [{"name": f"RustRaptor-data-{index:02d}-of-{len(groups):02d}.zip",
+               "kind": "shared-data", "files": group,
+               "uncompressed_bytes": sum(row["bytes"] for row in group)}
+              for index, group in enumerate(groups, 1)]
+    core = [entry(package, path.relative_to(package).as_posix()) for path in selected
+            if path.relative_to(package).parts[0] != "data"]
+    for platform in ("linux", "windows"):
+        if platform == "linux":
+            excluded = {"bin/raptor-rust.exe", "bin/SDL2.dll", "bin/SDL2-WINDOWS-LICENSE.txt",
+                        "run.cmd", "runtime-windows-manifest.json"}
+            files = [row for row in core if row["name"] not in excluded]
+        else:
+            files = [row for row in core if row["name"] not in {"bin/raptor-rust", "run.sh"}
+                     and not row["name"].startswith("lib/")]
+        if sum(row["bytes"] for row in files) > DEFAULT_VOLUME_BYTES:
+            raise ValueError("platform core exceeds the safe single-asset budget")
+        assets.append({"name": f"RustRaptor-{platform}-x86_64.zip", "kind": platform,
+                       "files": files, "uncompressed_bytes": sum(row["bytes"] for row in files)})
+    plan = {"format_version": 1, "package": str(package), "snapshot": snapshot,
+            "catalog_entries": manifest["catalog_entries"], "approved_hd_frames": hd_count,
+            "bundled_mods": len(list((package / "mods").glob("*/mod.toml"))),
+            "data_files": len(rows), "data_bytes": manifest["data_bytes"],
+            "volume_uncompressed_budget": budget, "assets": assets,
+            "install": "Extract your platform ZIP and every shared-data ZIP into the same directory; run RustRaptor/run.sh or RustRaptor/run.cmd.",
+            "payload_scope": "Complete unchanged runtime-manifest data plus explicitly inventoried mods, source, docs and platform libraries. No proof/lineage files from the manifest are silently removed."}
+    for platform in ("linux", "windows"):
+        installation = [row["name"].casefold() for asset in assets
+                        if asset["kind"] in ("shared-data", platform) for row in asset["files"]]
+        if len(installation) != len(set(installation)):
+            raise ValueError("case-insensitive duplicate installation path across assets")
+    for asset in assets:
+        seen = set()
+        for row in asset["files"]:
+            key = row["name"].casefold()
+            if key in seen:
+                raise ValueError("case-insensitive duplicate release path")
+            seen.add(key)
+    check_snapshot(package, plan)
+    return plan
+
+
+def source_identity(metadata):
+    return metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns
+
+
+@contextmanager
+def opened_source(package, name, expected_size):
+    relative = Path(name)
+    if relative.is_absolute() or not relative.parts or any(part in (".", "..") for part in relative.parts):
+        raise ValueError("unsafe release member path")
+    flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW | os.O_NONBLOCK
+    directory = os.open("/", flags | os.O_DIRECTORY)
+    source = None
+    try:
+        for part in package.parts[1:] + relative.parts[:-1]:
+            following = os.open(part, flags | os.O_DIRECTORY, dir_fd=directory)
+            os.close(directory)
+            directory = following
+        source = os.open(relative.name, flags, dir_fd=directory)
+        before = os.fstat(source)
+        if not stat.S_ISREG(before.st_mode) or before.st_size != expected_size:
+            raise ValueError("release source is not the planned regular file")
+        with os.fdopen(source, "rb", closefd=False) as stream:
+            yield stream, before
+        if source_identity(before) != source_identity(os.fstat(source)):
+            raise ValueError("release source changed while being read")
+    finally:
+        if source is not None:
+            os.close(source)
+        os.close(directory)
+
+
+def write_asset(plan_path, asset_name, output):
+    plan_path, output = Path(plan_path), Path(output)
+    plan = read_json(plan_path)
+    package = Path(plan["package"]).resolve(strict=True)
+    check_snapshot(package, plan)
+    matches = [asset for asset in plan["assets"] if asset["name"] == asset_name]
+    if len(matches) != 1 or Path(asset_name).name != asset_name:
+        raise ValueError("choose one exact planned release asset")
+    asset = matches[0]
+    if output.name != asset_name:
+        raise ValueError("output filename must equal the planned asset name")
+    if output.exists():
+        raise ValueError("archive output already exists; never overwrite exports")
+    output.parent.mkdir(parents=True, exist_ok=True)
+    if shutil.disk_usage(package).free < ROOT_DISK_FLOOR:
+        raise ValueError("workspace free space is below the 1.8GB floor")
+    # Deflate worst-case and ZIP overhead fit this generous reserve. /tmp may
+    # be tmpfs: only one volume is held and no complete runtime tree is copied.
+    if shutil.disk_usage(output.parent).free < asset["uncompressed_bytes"] + 128 * 1024**2:
+        raise ValueError("insufficient output space for one archive and reserve")
+    if (output.parent.stat().st_dev == package.stat().st_dev
+            and shutil.disk_usage(package).free < ROOT_DISK_FLOOR + asset["uncompressed_bytes"] + 128 * 1024**2):
+        raise ValueError("same-filesystem output would cross the workspace 1.8GB floor")
+    measured = {}
+    with zipfile.ZipFile(output, "x", zipfile.ZIP_DEFLATED, compresslevel=6, allowZip64=True) as archive:
+        for row in asset["files"]:
+            h = hashlib.sha256()
+            with opened_source(package, row["name"], row["bytes"]) as (original, before):
+                info = zipfile.ZipInfo("RustRaptor/" + row["name"], time.localtime(before.st_mtime)[:6])
+                info.compress_type = zipfile.ZIP_DEFLATED
+                info.create_system = 3
+                info.external_attr = (before.st_mode & 0xFFFF) << 16
+                with archive.open(info, "w", force_zip64=True) as member:
+                    remaining = row["bytes"]
+                    while remaining:
+                        block = original.read(min(128 * 1024, remaining))
+                        if not block:
+                            raise ValueError("release source ended before its planned size")
+                        remaining -= len(block)
+                        h.update(block)
+                        member.write(block)
+                    if original.read(1):
+                        raise ValueError("release source grew beyond its planned size")
+            if h.hexdigest() != row["sha256"]:
+                raise ValueError(f"release source changed or differs from plan: {row['name']}")
+            measured[row["name"]] = h.hexdigest()
+    if output.stat().st_size >= MAX_ASSET_BYTES:
+        raise ValueError("release asset reaches GitHub's 2GiB limit")
+    with zipfile.ZipFile(output) as archive:
+        if archive.testzip() is not None:
+            raise ValueError("release ZIP failed CRC validation")
+        if archive.namelist() != ["RustRaptor/" + row["name"] for row in asset["files"]]:
+            raise ValueError("release ZIP membership differs from plan")
+    check_snapshot(package, plan)
+    return {"status": "ZIP_CRC_AND_COMPLETE_PAYLOAD_SHA256_PASS", "asset": asset_name,
+            "kind": asset["kind"], "bytes": output.stat().st_size, "sha256": digest(output),
+            "plan_sha256": digest(plan_path), "checkpoint": plan["snapshot"],
+            "payload_files": len(measured), "payload_sha256": measured,
+            "root_disk_floor_after_bytes": shutil.disk_usage(package).free,
+            "builds_or_source_manifest_mutations": 0}
+
+
+def checksums(plan_path, receipts, output, manifest_output=None, tag=None):
+    plan_path, receipts, output = Path(plan_path), Path(receipts), Path(output)
+    plan = read_json(plan_path)
+    if manifest_output is not None and (not isinstance(tag, str) or not re.fullmatch(r"v[0-9A-Za-z._-]+", tag)):
+        raise ValueError("a safe exact release tag is required")
+    lines, published = [], []
+    for asset in plan["assets"]:
+        receipt = read_json(receipts / (asset["name"] + ".json"))
+        if (receipt.get("status") != "ZIP_CRC_AND_COMPLETE_PAYLOAD_SHA256_PASS"
+                or receipt.get("asset") != asset["name"]
+                or receipt.get("kind") != asset["kind"]
+                or receipt.get("checkpoint") != plan["snapshot"]
+                or receipt.get("payload_files") != len(asset["files"])
+                or receipt.get("plan_sha256") != digest(plan_path)
+                or receipt.get("payload_sha256") != {row["name"]: row["sha256"] for row in asset["files"]}
+                or not re.fullmatch(r"[0-9a-f]{64}", receipt.get("sha256", ""))
+                or not 0 < receipt.get("bytes", 0) < MAX_ASSET_BYTES):
+            raise ValueError("incomplete or mismatched release asset receipt")
+        lines.append(f"{receipt['sha256']}  {asset['name']}\n")
+        published.append({"name": asset["name"], "kind": asset["kind"], "bytes": receipt["bytes"],
+                          "sha256": receipt["sha256"], "files": asset["files"]})
+    with output.open("x", encoding="ascii") as stream:
+        stream.writelines(lines)
+    if manifest_output is not None:
+        write_new_json(manifest_output, {"format_version": 1, "tag": tag, "checkpoint": plan["snapshot"],
+            "approved_hd_frames": plan["approved_hd_frames"], "data_files": plan["data_files"],
+            "data_bytes": plan["data_bytes"], "bundled_mods": plan["bundled_mods"], "assets": published})
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    commands = parser.add_subparsers(dest="command", required=True)
+    planning = commands.add_parser("plan")
+    planning.add_argument("--package", type=Path, required=True)
+    planning.add_argument("--expected-commit", required=True)
+    planning.add_argument("--output", type=Path, required=True)
+    exporting = commands.add_parser("archive")
+    exporting.add_argument("--plan", type=Path, required=True)
+    exporting.add_argument("--asset", required=True)
+    exporting.add_argument("--output", type=Path, required=True)
+    exporting.add_argument("--receipt", type=Path, required=True)
+    summing = commands.add_parser("checksums")
+    summing.add_argument("--plan", type=Path, required=True)
+    summing.add_argument("--receipts", type=Path, required=True)
+    summing.add_argument("--output", type=Path, required=True)
+    summing.add_argument("--manifest-output", type=Path, required=True)
+    summing.add_argument("--tag", required=True)
+    args = parser.parse_args()
+    if args.command == "plan":
+        value = build_plan(args.package, args.expected_commit)
+        write_new_json(args.output, value)
+        print(json.dumps({key: value[key] for key in ("snapshot", "approved_hd_frames", "bundled_mods", "data_files", "data_bytes")}))
+    elif args.command == "archive":
+        value = write_asset(args.plan, args.asset, args.output)
+        write_new_json(args.receipt, value)
+        print(json.dumps({key: value[key] for key in ("status", "asset", "bytes", "sha256", "payload_files")}))
+    else:
+        checksums(args.plan, args.receipts, args.output, args.manifest_output, args.tag)
+
+
+if __name__ == "__main__":
+    main()