Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 23 additions & 6 deletions scripts/publish_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import os
import re
import subprocess
import shutil
import sys
import tempfile
import zipfile
Expand All @@ -39,6 +40,14 @@
DEFAULT_REPO = "interscript/interscript-ml"


def canonical_filename(meta: dict) -> str:
"""Published filenames derive from the zip's own metadata (id +
precision), never from the local staging path — a /tmp name like
ld-int4.zip must not leak into the index entry or asset name: the
runtime resolver matches volume files by this pattern."""
return f"{meta['id']}-{meta['precision']}.zip"


def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess:
return subprocess.run(cmd, check=True, **kwargs)

Expand Down Expand Up @@ -166,18 +175,26 @@ def main() -> None:
raise SystemExit(f"metadata id {meta['id']!r} != requested {args.model_id!r}")
key = args.key or args.model_id

whole_sha = sha256_file(args.zip)
size = args.zip.stat().st_size
canonical = canonical_filename(meta)
zip_path = args.zip
if zip_path.name != canonical:
staged = Path(tempfile.mkdtemp(prefix="publish-")) / canonical
shutil.copyfile(zip_path, staged)
print(f"staging {zip_path.name} -> {canonical}")
zip_path = staged

whole_sha = sha256_file(zip_path)
size = zip_path.stat().st_size

assets = [args.zip]
assets = [zip_path]
if size > SPLIT_THRESHOLD:
print(f"{size:,} bytes > {SPLIT_THRESHOLD:,}; splitting for the GitHub asset cap")
parts = split_zip(args.zip, 1_500_000_000)
parts = split_zip(zip_path, 1_500_000_000)
assets = [part for part, _, _ in parts]

tag = args.model_id
with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as fh:
fh.write(release_notes(key, meta, args.zip.name, size, whole_sha, assets))
fh.write(release_notes(key, meta, canonical, size, whole_sha, assets))
notes_path = fh.name

existing = subprocess.run(
Expand Down Expand Up @@ -225,7 +242,7 @@ def main() -> None:
# models/<family>/<id>.metadata.yaml, e.g. models/heb-diac/heb-diac-1.0...
family = key.rsplit("-", 1)[0]
upsert_models_yaml(wt_models, key,
entry_block(key, meta, args.zip.name, whole_sha,
entry_block(key, meta, canonical, whole_sha,
size, assets, args.repo, tag))
model_dir = worktree / "models" / family
model_dir.mkdir(parents=True, exist_ok=True)
Expand Down
29 changes: 29 additions & 0 deletions tests/test_publish_model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
"""Canonical artifact naming: the index entry and release asset name
come from the zip's metadata, never the local staging path."""

from __future__ import annotations

import sys
from pathlib import Path

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))

pytest.importorskip("yaml")

from publish_model import canonical_filename # noqa: E402


def test_filename_from_metadata() -> None:
meta = {"id": "ara-diac-layerdrop-1.0", "precision": "int4"}
assert canonical_filename(meta) == "ara-diac-layerdrop-1.0-int4.zip"


def test_staging_path_cannot_leak() -> None:
# a /tmp staging name like ld-int4.zip must never become the
# published filename — the runtime resolver matches volume files
# by the canonical pattern
meta = {"id": "ara-diac-layerdrop-1.0", "precision": "fp32"}
assert canonical_filename(meta) != "ld-fp32.zip"
Loading