noonghunnaandClaude Opus 4.7
3ae74bfdcf
Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
659 lines
29 KiB
Bash
Executable File
659 lines
29 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
# test-pullgate-download.sh — v0.8.0 [E] STEP E2 (club-3090 #141 / #147).
|
||
#
|
||
# Contract test for CONTRACT-3: the HF download stage. The test IS the spec;
|
||
# the code is fixed to it. NO live network / GPU / Docker — a recorded-
|
||
# fixture fetcher replays snapshot + HF HEAD x-linked-etag responses and a
|
||
# real tmp staging tree is exercised so atomic-move + .incomplete cleanup
|
||
# are asserted on disk.
|
||
#
|
||
# Coverage:
|
||
# * download_set == the CONTRACT-3 reconciled union (weights + index +
|
||
# REQUIRED_METADATA incl. vocab.json/merges.txt AND *.jinja).
|
||
# * fetched-set == sized-set: allow_patterns passed to snapshot_download
|
||
# IS literally download_set() (nothing outside the set is fetched);
|
||
# deriver.sized_download_gb sizes exactly that set; gates.c2a_disk
|
||
# consumes the SHARED download_set() (no drift).
|
||
# * SHA pass: every *.safetensors verified vs the HF API
|
||
# siblings[].lfs.sha256 (the deriver-already-fetched _hf_api payload) ==
|
||
# sha256(file) -> ok, atomic move into the CONTRACT-2 host --model dir,
|
||
# .incomplete gone. (E2-fix-2: the verify SOURCE moved off the
|
||
# Xet-redirect-fragile HEAD x-linked-etag onto the API lfs.sha256.)
|
||
# * E2-fix-2 regression: a Xet-backed repo whose redirect-following HEAD
|
||
# returns NO x-linked-etag (head_etag -> None) but whose API publishes
|
||
# siblings[].lfs.sha256 matching the file -> ok=True/sha_verified=True
|
||
# (this exact on-rig scenario, Qwen/Qwen2.5-0.5B-Instruct, FALSE-failed
|
||
# "no-etag" before the fix).
|
||
# * no-etag -> HARD fail (DownloadResult.failure=="no-etag") when a
|
||
# *.safetensors has NO retrievable API lfs.sha256 (genuinely
|
||
# unverifiable), .incomplete tree deleted, NOT skipped (the deliberate
|
||
# opposite of setup.sh:437) — CONTRACT-3 hard-fail-if-unverifiable
|
||
# preserved; only the hash source moved (token kept "no-etag").
|
||
# * sha-mismatch -> failure + .incomplete cleanup, no final dir.
|
||
# * gated-401 mid-fetch -> failure="gated-401" + cleanup, no partials.
|
||
# * E1 dtype header-probe now LIVE: a quantized repo lacking torch_dtype
|
||
# resolves --dtype bfloat16 via the standardized probe-fetcher wiring
|
||
# (was fail-closed in E1).
|
||
# * DEFAULT real fetcher shells out to the `hf` CLI (NOT a
|
||
# huggingface_hub lib-import): resolves hf->huggingface-cli, builds
|
||
# argv with one --include per download_set entry (set == download_set,
|
||
# no extra), --local-dir == staging; subprocess.run MOCKED. The on-rig
|
||
# E5 ModuleNotFoundError regression fix.
|
||
# * NEITHER hf NOR huggingface-cli -> STRUCTURED DownloadResult(failure==
|
||
# "hf-cli-missing") with the canonical actionable message — NOT a
|
||
# raised/swallowed ModuleNotFoundError.
|
||
|
||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
cd "$ROOT_DIR"
|
||
export PYTHONPATH="$ROOT_DIR${PYTHONPATH:+:$PYTHONPATH}"
|
||
|
||
python3 - "$ROOT_DIR" <<'PY'
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import struct
|
||
import sys
|
||
import tempfile
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
root = Path(sys.argv[1])
|
||
sys.path.insert(0, str(root))
|
||
|
||
from scripts.lib.profiles import deriver as D # noqa: E402
|
||
from scripts.lib.profiles import downloader as DL # noqa: E402
|
||
from scripts.lib import generate_compose as gc # noqa: E402
|
||
from scripts.lib.profiles.einput import EInput # noqa: E402
|
||
|
||
failures: list[str] = []
|
||
|
||
|
||
def check(cond: bool, msg: str) -> None:
|
||
if cond:
|
||
print(f"PASS: {msg}")
|
||
else:
|
||
print(f"FAIL: {msg}", file=sys.stderr)
|
||
failures.append(msg)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# A sharded quantized repo (awq) with the full reconciled metadata spread:
|
||
# weights + index + config/generation_config/tokenizer*/special_tokens_map
|
||
# + vocab.json + merges.txt + a *.jinja chat template + an adapter (must
|
||
# be EXCLUDED) + an unrelated README (must NOT be fetched).
|
||
# ---------------------------------------------------------------------------
|
||
API = {
|
||
"siblings": [
|
||
{"rfilename": "config.json", "size": 900},
|
||
{"rfilename": "generation_config.json", "size": 120},
|
||
{"rfilename": "tokenizer.json", "size": 1_800_000},
|
||
{"rfilename": "tokenizer_config.json", "size": 4_000},
|
||
{"rfilename": "tokenizer.model", "size": 500_000},
|
||
{"rfilename": "special_tokens_map.json", "size": 300},
|
||
{"rfilename": "vocab.json", "size": 800_000},
|
||
{"rfilename": "merges.txt", "size": 400_000},
|
||
{"rfilename": "chat_template.jinja", "size": 2_000},
|
||
{"rfilename": "model.safetensors.index.json", "size": 50_000},
|
||
{"rfilename": "model-00001-of-00002.safetensors", "size": 6_000_000_000},
|
||
{"rfilename": "model-00002-of-00002.safetensors", "size": 5_000_000_000},
|
||
{"rfilename": "adapter_model.safetensors", "size": 90_000_000},
|
||
{"rfilename": "README.md", "size": 12_000},
|
||
]
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 1. download_set == the CONTRACT-3 reconciled union.
|
||
# ---------------------------------------------------------------------------
|
||
ds = D.download_set(API)
|
||
expected = [
|
||
"model-00001-of-00002.safetensors",
|
||
"model-00002-of-00002.safetensors",
|
||
"model.safetensors.index.json",
|
||
"config.json",
|
||
"generation_config.json",
|
||
"tokenizer.json",
|
||
"tokenizer_config.json",
|
||
"tokenizer.model",
|
||
"special_tokens_map.json",
|
||
"vocab.json",
|
||
"merges.txt",
|
||
"chat_template.jinja",
|
||
]
|
||
check(ds == expected,
|
||
f"download_set == reconciled union (weights+index+REQUIRED_METADATA"
|
||
f"+jinja, adapter+README excluded) got={ds}")
|
||
check("adapter_model.safetensors" not in ds,
|
||
"download_set excludes the LoRA/adapter safetensors")
|
||
check("README.md" not in ds, "download_set excludes unrelated repo files")
|
||
check("chat_template.jinja" in ds and "vocab.json" in ds
|
||
and "merges.txt" in ds,
|
||
"download_set reconciles BOTH v2's *.jinja AND legacy vocab/merges")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 2. sized_download_gb sizes EXACTLY download_set; gates.c2a_disk consumes
|
||
# the SHARED set (no drift between [C2a] / E2 / E3).
|
||
# ---------------------------------------------------------------------------
|
||
sized = D.sized_download_gb(API)
|
||
manual = round(
|
||
sum(
|
||
s["size"]
|
||
for s in API["siblings"]
|
||
if s["rfilename"] in set(ds)
|
||
) / (1024 ** 3),
|
||
4,
|
||
)
|
||
check(abs(sized - manual) < 1e-6,
|
||
f"sized_download_gb sums exactly download_set ({sized} == {manual})")
|
||
# adapter NOT counted (would add 90 MB if drift existed)
|
||
no_adapter = round(
|
||
sum(s["size"] for s in API["siblings"]
|
||
if s["rfilename"] in set(ds)) / (1024 ** 3), 4)
|
||
check(abs(sized - no_adapter) < 1e-6,
|
||
"sized footprint excludes the adapter (download_set is authority)")
|
||
|
||
from scripts.lib.profiles import gates as G # noqa: E402
|
||
|
||
der_for_c2a = SimpleNamespace(profile={"_hf_api": API})
|
||
|
||
|
||
class FakeStat:
|
||
def __init__(self, free_gb: float):
|
||
self.f_frsize = 4096
|
||
self.f_bavail = int(free_gb * (1024 ** 3) / 4096)
|
||
|
||
|
||
c2a = G.c2a_disk(der_for_c2a, statvfs=lambda p: FakeStat(500.0))
|
||
check(abs(c2a.required_gb - round(sized * 1.2, 4)) < 1e-3,
|
||
f"gates.c2a_disk sizes the SHARED download_set() "
|
||
f"(required={c2a.required_gb} == sized*1.2={round(sized*1.2,4)})")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixture fetcher: replays snapshot (writes EXACTLY allow_patterns) + HEAD
|
||
# x-linked-etag. Records the allow_patterns it was asked to fetch so we can
|
||
# assert fetched-set == sized-set (nothing extra).
|
||
# ---------------------------------------------------------------------------
|
||
class FixtureFetcher:
|
||
def __init__(self, *, etags=None, gated_on_head=None,
|
||
gated_on_snapshot=False, disk_on_snapshot=False,
|
||
bad_bytes=None):
|
||
# etags: {filename: etag_or_None}; gated_on_head: filename set
|
||
self.etags = etags or {}
|
||
self.gated_on_head = set(gated_on_head or ())
|
||
self.gated_on_snapshot = gated_on_snapshot
|
||
self.disk_on_snapshot = disk_on_snapshot
|
||
self.bad_bytes = bad_bytes or {} # filename -> bytes (sha won't match)
|
||
self.allow_seen = None
|
||
self.head_calls: list[str] = []
|
||
|
||
def snapshot(self, repo_id, local_dir, allow_patterns):
|
||
self.allow_seen = list(allow_patterns)
|
||
if self.gated_on_snapshot:
|
||
raise DL.GatedError("401")
|
||
if self.disk_on_snapshot:
|
||
raise DL.DiskError("ENOSPC")
|
||
written = []
|
||
rootp = Path(local_dir)
|
||
for name in allow_patterns:
|
||
p = rootp / name
|
||
p.parent.mkdir(parents=True, exist_ok=True)
|
||
if name in self.bad_bytes:
|
||
p.write_bytes(self.bad_bytes[name])
|
||
else:
|
||
# deterministic content so the etag fixture == sha256(file)
|
||
p.write_bytes(f"CONTENT::{name}".encode())
|
||
written.append(name)
|
||
return written
|
||
|
||
def head_etag(self, repo_id, filename):
|
||
self.head_calls.append(filename)
|
||
if filename in self.gated_on_head:
|
||
return "__gated-401__"
|
||
return self.etags.get(filename, None)
|
||
|
||
|
||
def content_sha(name: str) -> str:
|
||
return hashlib.sha256(f"CONTENT::{name}".encode()).hexdigest()
|
||
|
||
|
||
import copy # noqa: E402
|
||
|
||
|
||
def api_with_lfs(sha_for=None, drop_lfs=()):
|
||
"""A deep copy of API where every *.safetensors sibling carries an
|
||
`lfs.sha256` == sha256("CONTENT::<name>") (the deterministic content the
|
||
FixtureFetcher writes) — i.e. the canonical per-file LFS SHA256 the
|
||
deriver stashes in `_hf_api`. `sha_for` overrides specific filenames'
|
||
lfs.sha256; `drop_lfs` removes lfs entirely for a filename (a genuinely
|
||
unverifiable *.safetensors -> CONTRACT-3 hard `no-etag`)."""
|
||
sha_for = sha_for or {}
|
||
a = copy.deepcopy(API)
|
||
for s in a["siblings"]:
|
||
n = s["rfilename"]
|
||
if not n.endswith(".safetensors"):
|
||
continue
|
||
if n in drop_lfs:
|
||
s.pop("lfs", None)
|
||
continue
|
||
sha = sha_for.get(n, content_sha(n))
|
||
s["lfs"] = {"sha256": sha, "size": s.get("size", 0)}
|
||
return a
|
||
|
||
|
||
def mk_einput(slug, hf_home, api=None):
|
||
der = SimpleNamespace(
|
||
slug=slug,
|
||
profile={
|
||
"_hf_api": api if api is not None else api_with_lfs(),
|
||
"selected_weight_files": [
|
||
"model-00001-of-00002.safetensors",
|
||
"model-00002-of-00002.safetensors",
|
||
],
|
||
"weight_format": "awq",
|
||
"torch_dtype": None,
|
||
},
|
||
)
|
||
return EInput(
|
||
slug=slug, terminal="proceed", is_override_accepted=False,
|
||
der=der, runtime={}, selected_files=[], hf_home=Path(hf_home),
|
||
c2a=None, hardware_sm=8.6, visible_gpu_count=1,
|
||
per_gpu_vram_mib=[24576], selected_gpu_indices=[0],
|
||
selected_gpu_vram_mib=[24576], topology_summary="(RTX 3090, 24576)",
|
||
club3090_commit="deadbeef", diagnostics={},
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3. SHA pass -> ok, atomic move, .incomplete gone, fetched-set == sized-set.
|
||
# ---------------------------------------------------------------------------
|
||
with tempfile.TemporaryDirectory() as td:
|
||
ei = mk_einput("Org/Awq-Model", td)
|
||
good_etags = {
|
||
n: content_sha(n) for n in ds if n.endswith(".safetensors")
|
||
}
|
||
f = FixtureFetcher(etags=good_etags)
|
||
r = DL.download_model(ei, fetcher=f)
|
||
check(r.ok and r.failure is None,
|
||
f"SHA-pass: DownloadResult ok (got ok={r.ok} fail={r.failure})")
|
||
check(r.sha_verified is True, "SHA-pass: sha_verified True")
|
||
check(sorted(r.files) == sorted(ds),
|
||
f"SHA-pass: fetched-set == download_set (got {sorted(r.files)})")
|
||
# fetched-set == sized-set: allow_patterns IS literally download_set
|
||
check(f.allow_seen == ds,
|
||
f"allow_patterns passed to snapshot IS literally download_set "
|
||
f"(got {f.allow_seen})")
|
||
final = DL.pull_dir(Path(td), "Org/Awq-Model")
|
||
check(final.is_dir(), f"SHA-pass: final --model dir created at {final}")
|
||
check(not (final / ".incomplete").exists(),
|
||
"SHA-pass: .incomplete tree removed after atomic move")
|
||
check(all((final / n).exists() for n in ds),
|
||
"SHA-pass: every download_set file present in the final dir")
|
||
check(r.local_dir == str(final),
|
||
"SHA-pass: DownloadResult.local_dir is the host --model dir")
|
||
check(r.bytes > 0, "SHA-pass: bytes accounted")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3b. E2-fix-2 REGRESSION (the exact on-rig E5 scenario):
|
||
# a Xet-backed repo. The redirect-following HEAD into cas-bridge.
|
||
# xethub.hf.co carries a CAS-blob ETag but NO x-linked-etag, so
|
||
# head_etag() -> None for every *.safetensors. BEFORE the fix this
|
||
# false-failed failure=="no-etag" even though the canonical LFS SHA256
|
||
# IS published in the HF model API siblings[].lfs.sha256 (which the
|
||
# deriver already fetched into _hf_api). AFTER the fix: verification
|
||
# uses the API lfs.sha256 -> ok=True, sha_verified=True. (On-rig:
|
||
# `pull.sh Qwen/Qwen2.5-0.5B-Instruct` reached [E], hf-download ran,
|
||
# then false `no-etag`.)
|
||
# ---------------------------------------------------------------------------
|
||
with tempfile.TemporaryDirectory() as td:
|
||
# API publishes the canonical lfs.sha256 (== sha256 of the fixture
|
||
# content) for every shard — exactly what /api/models?blobs=true does.
|
||
ei = mk_einput("Qwen/Qwen2.5-0.5B-Instruct", td)
|
||
# Xet redirect: HEAD sees NO x-linked-etag for ANY *.safetensors.
|
||
xet_no_etag = {
|
||
n: None for n in ds if n.endswith(".safetensors")
|
||
}
|
||
f = FixtureFetcher(etags=xet_no_etag)
|
||
r = DL.download_model(ei, fetcher=f)
|
||
check(r.ok is True and r.failure is None,
|
||
f"E2-fix-2: Xet-redirect HEAD has NO x-linked-etag yet verify "
|
||
f"PASSES off the API lfs.sha256 (got ok={r.ok} fail={r.failure})"
|
||
f" — this exact case FALSE-failed 'no-etag' before the fix")
|
||
check(r.sha_verified is True,
|
||
"E2-fix-2: sha_verified True via API lfs.sha256 (redirect-immune)")
|
||
# the HEAD seam WAS still consulted (gated-401 probe) — proves we did
|
||
# NOT just delete the HEAD call, only stopped trusting its hash value.
|
||
check(set(f.head_calls) == {
|
||
n for n in ds if n.endswith(".safetensors")},
|
||
"E2-fix-2: HEAD still consulted as the gated-401 probe seam "
|
||
f"(head_calls={sorted(f.head_calls)})")
|
||
final = DL.pull_dir(Path(td), "Qwen/Qwen2.5-0.5B-Instruct")
|
||
check(final.is_dir() and not (final / ".incomplete").exists(),
|
||
"E2-fix-2: atomic move + .incomplete cleanup unchanged")
|
||
check(all((final / n).exists() for n in ds),
|
||
"E2-fix-2: every download_set file promoted to the final dir")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3c. E2-fix-2: SHA verification is sourced from the API lfs.sha256, NOT
|
||
# the HEAD value — a WRONG/STALE HEAD x-linked-etag must NOT cause a
|
||
# false sha-mismatch when the file matches the API lfs.sha256.
|
||
# ---------------------------------------------------------------------------
|
||
with tempfile.TemporaryDirectory() as td:
|
||
ei = mk_einput("Org/StaleHeadEtag", td)
|
||
bogus = {n: "deadbeef" * 8 for n in ds if n.endswith(".safetensors")}
|
||
f = FixtureFetcher(etags=bogus) # HEAD lies; API lfs.sha256 is truth
|
||
r = DL.download_model(ei, fetcher=f)
|
||
check(r.ok is True and r.failure is None and r.sha_verified is True,
|
||
f"E2-fix-2: a wrong HEAD x-linked-etag is IGNORED; verify uses "
|
||
f"API lfs.sha256 (got ok={r.ok} fail={r.failure})")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 4. no-etag -> HARD fail (NOT skip), .incomplete deleted.
|
||
# CONTRACT-3 hard-fail-if-unverifiable PRESERVED through E2-fix-2: the
|
||
# trusted hash now comes from the API siblings[].lfs.sha256, so the
|
||
# genuinely-unverifiable case is "a *.safetensors with NO lfs.sha256 in
|
||
# the API payload" (NOT a blank HEAD x-linked-etag — that is now the
|
||
# EXPECTED Xet-redirect state, see regression case 3b). HEAD etags are
|
||
# fully present here to PROVE the hard fail comes from the missing API
|
||
# hash, not the HEAD seam. Failure token UNCHANGED ("no-etag").
|
||
# ---------------------------------------------------------------------------
|
||
with tempfile.TemporaryDirectory() as td:
|
||
# shard 2 has NO lfs.sha256 in the API -> genuinely unverifiable.
|
||
api_no_lfs = api_with_lfs(drop_lfs={"model-00002-of-00002.safetensors"})
|
||
ei = mk_einput("Org/NoEtag", td, api=api_no_lfs)
|
||
# every HEAD x-linked-etag present & correct: proves the hard fail is
|
||
# from the MISSING API lfs.sha256, not the (now-ignored) HEAD value.
|
||
etags = {n: content_sha(n) for n in ds if n.endswith(".safetensors")}
|
||
f = FixtureFetcher(etags=etags)
|
||
r = DL.download_model(ei, fetcher=f)
|
||
check(r.ok is False and r.failure == "no-etag",
|
||
f"no-etag: HARD fail failure=='no-etag' when a *.safetensors has "
|
||
f"NO API lfs.sha256 (got ok={r.ok} fail={r.failure}) — NOT "
|
||
f"skipped like setup.sh:437; token kept across the source move")
|
||
final = DL.pull_dir(Path(td), "Org/NoEtag")
|
||
check(not (final / ".incomplete").exists(),
|
||
"no-etag: .incomplete tree deleted (no corrupt residue)")
|
||
check(not any(p.is_file() for p in final.rglob("*")) if final.exists()
|
||
else True,
|
||
"no-etag: no partial/corrupt files left behind")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 5. sha-mismatch -> fail + cleanup.
|
||
# ---------------------------------------------------------------------------
|
||
with tempfile.TemporaryDirectory() as td:
|
||
# API lfs.sha256 (mk_einput default) says one thing; written bytes are
|
||
# different -> mismatch. HEAD etags present & correct to prove the
|
||
# mismatch is judged against the API hash, not the (ignored) HEAD value.
|
||
ei = mk_einput("Org/ShaBad", td)
|
||
etags = {n: content_sha(n) for n in ds if n.endswith(".safetensors")}
|
||
f = FixtureFetcher(
|
||
etags=etags,
|
||
bad_bytes={"model-00001-of-00002.safetensors": b"CORRUPT"},
|
||
)
|
||
r = DL.download_model(ei, fetcher=f)
|
||
check(r.ok is False and r.failure == "sha-mismatch",
|
||
f"sha-mismatch: fail failure=='sha-mismatch' (got {r.failure})")
|
||
final = DL.pull_dir(Path(td), "Org/ShaBad")
|
||
check(not (final / ".incomplete").exists(),
|
||
"sha-mismatch: .incomplete deleted")
|
||
check(not final.exists() or not any(
|
||
p.name.endswith(".safetensors") for p in final.rglob("*")),
|
||
"sha-mismatch: no staged weights promoted to final")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 6. gated-401 (on snapshot AND on HEAD) -> fail + cleanup, no partials.
|
||
# ---------------------------------------------------------------------------
|
||
with tempfile.TemporaryDirectory() as td:
|
||
ei = mk_einput("Org/Gated", td)
|
||
f = FixtureFetcher(gated_on_snapshot=True)
|
||
r = DL.download_model(ei, fetcher=f)
|
||
check(r.ok is False and r.failure == "gated-401",
|
||
f"gated-401 (snapshot): failure=='gated-401' (got {r.failure})")
|
||
final = DL.pull_dir(Path(td), "Org/Gated")
|
||
check(not (final / ".incomplete").exists(),
|
||
"gated-401: .incomplete deleted, no partials")
|
||
|
||
with tempfile.TemporaryDirectory() as td:
|
||
ei = mk_einput("Org/GatedHead", td)
|
||
etags = {n: content_sha(n) for n in ds if n.endswith(".safetensors")}
|
||
f = FixtureFetcher(
|
||
etags=etags,
|
||
gated_on_head={"model-00001-of-00002.safetensors"},
|
||
)
|
||
r = DL.download_model(ei, fetcher=f)
|
||
check(r.ok is False and r.failure == "gated-401",
|
||
f"gated-401 (HEAD mid-verify): failure=='gated-401' "
|
||
f"(got {r.failure})")
|
||
final = DL.pull_dir(Path(td), "Org/GatedHead")
|
||
check(not (final / ".incomplete").exists(),
|
||
"gated-401 (HEAD): .incomplete deleted")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 7. disk failure -> failure="disk" + cleanup.
|
||
# ---------------------------------------------------------------------------
|
||
with tempfile.TemporaryDirectory() as td:
|
||
ei = mk_einput("Org/Disk", td)
|
||
f = FixtureFetcher(disk_on_snapshot=True)
|
||
r = DL.download_model(ei, fetcher=f)
|
||
check(r.ok is False and r.failure == "disk",
|
||
f"disk: failure=='disk' (got {r.failure})")
|
||
final = DL.pull_dir(Path(td), "Org/Disk")
|
||
check(not (final / ".incomplete").exists(),
|
||
"disk: .incomplete deleted")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 8. E1 dtype header-probe NOW LIVE via standardized probe-fetcher wiring.
|
||
# A quantized (awq) repo with NO torch_dtype: in E1 step (2) was dead
|
||
# (diagnostics["fetcher"] never set) -> fail-closed missing-torch-dtype.
|
||
# E2's downloader.set_probe_fetcher() standardizes the wiring; with a
|
||
# fixture fetcher exposing a bf16 safetensors header, _resolve_compute_
|
||
# dtype now resolves --dtype bfloat16 (NOT fail-closed).
|
||
# ---------------------------------------------------------------------------
|
||
def st_header(obj):
|
||
blob = json.dumps(obj).encode("utf-8")
|
||
return struct.pack("<Q", len(blob)) + blob
|
||
|
||
|
||
class HeaderProbeFetcher:
|
||
"""Replays the deriver's bounded header probe: range-GET 0-7 then the
|
||
header bytes. Exposes a bf16 storage dtype."""
|
||
def __init__(self, slug, weight):
|
||
self._url = f"{D._HF_RESOLVE}/{slug}/resolve/main/{weight}"
|
||
self._blob = st_header({"__metadata__": {},
|
||
"w": {"dtype": "BF16"}})
|
||
|
||
def get(self, url, headers=None, range_=None):
|
||
if url != self._url or range_ is None:
|
||
return D.FetchResponse(status=404, body=b"")
|
||
lo, hi = range_
|
||
return D.FetchResponse(status=206, body=self._blob[lo:hi + 1])
|
||
|
||
|
||
SLUG = "Org/Awq-NoTorchDtype"
|
||
WEIGHT = "model-00001-of-00002.safetensors"
|
||
der = SimpleNamespace(
|
||
slug=SLUG,
|
||
profile={
|
||
"weight_format": "awq",
|
||
"torch_dtype": None, # NO config torch_dtype
|
||
"selected_weight_files": [WEIGHT, "model-00002-of-00002.safetensors"],
|
||
},
|
||
)
|
||
ei = EInput(
|
||
slug=SLUG, terminal="proceed", is_override_accepted=False, der=der,
|
||
runtime={}, selected_files=[], hf_home=Path("/data/hf"), c2a=None,
|
||
hardware_sm=8.6, visible_gpu_count=1, per_gpu_vram_mib=[24576],
|
||
selected_gpu_indices=[0], selected_gpu_vram_mib=[24576],
|
||
topology_summary="(RTX 3090, 24576)", club3090_commit="deadbeef",
|
||
diagnostics={},
|
||
)
|
||
|
||
# Before wiring: E1 step (2) is dead -> fail-closed missing-torch-dtype.
|
||
dt0, rej0 = gc._resolve_compute_dtype(ei)
|
||
check(dt0 == "" and rej0 == "missing-torch-dtype",
|
||
f"E1 baseline: no probe fetcher -> fail-closed missing-torch-dtype "
|
||
f"(got dt={dt0!r} rej={rej0!r})")
|
||
|
||
# E2 standardizer wires diagnostics['fetcher']; probe is now LIVE.
|
||
DL.set_probe_fetcher(ei, HeaderProbeFetcher(SLUG, WEIGHT))
|
||
check(ei.diagnostics.get("fetcher") is not None,
|
||
"set_probe_fetcher populated einput.diagnostics['fetcher']")
|
||
dt1, rej1 = gc._resolve_compute_dtype(ei)
|
||
check(dt1 == "bfloat16" and rej1 is None,
|
||
f"E1 dtype probe NOW LIVE: awg-no-torch_dtype resolves --dtype "
|
||
f"bfloat16 via the standardized probe-fetcher (was fail-closed in "
|
||
f"E1); got dt={dt1!r} rej={rej1!r}")
|
||
|
||
# default_probe_fetcher() yields the canonical real bounded fetcher.
|
||
check(isinstance(D.default_probe_fetcher(), D.HttpFetcher),
|
||
"deriver.default_probe_fetcher() == canonical HttpFetcher")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 9. DEFAULT REAL FETCHER (no injected fixture) shells out to the `hf` CLI:
|
||
# resolves `hf` -> `huggingface-cli`, builds argv with EXACTLY one
|
||
# `--include` per download_set entry (set == download_set, no extra
|
||
# patterns), --local-dir == staging. subprocess.run is MOCKED (no live
|
||
# network/subprocess); the mock materializes the include files so the
|
||
# rest of download_model (SHA verify / atomic stage) runs unchanged.
|
||
# This is the on-rig E5 regression fix: the prior default did
|
||
# `from huggingface_hub import snapshot_download` -> ModuleNotFoundError
|
||
# under the bare python3 pull.sh runs.
|
||
# ---------------------------------------------------------------------------
|
||
import subprocess as _subprocess # noqa: E402
|
||
|
||
_orig_which = DL.shutil.which
|
||
_orig_run = DL.subprocess.run
|
||
|
||
captured = {}
|
||
|
||
|
||
def fake_which_hf(name):
|
||
# `hf` present, `huggingface-cli` also present (hf must WIN).
|
||
if name == "hf":
|
||
return "/usr/bin/hf"
|
||
if name == "huggingface-cli":
|
||
return "/usr/bin/huggingface-cli"
|
||
return None
|
||
|
||
|
||
def fake_run_ok(argv, **kw):
|
||
captured["argv"] = list(argv)
|
||
captured["env"] = kw.get("env")
|
||
# Materialize EXACTLY the --include set into --local-dir so the
|
||
# downstream SHA/stage logic (unchanged) operates on real files.
|
||
local_dir = None
|
||
includes = []
|
||
i = 0
|
||
while i < len(argv):
|
||
if argv[i] == "--local-dir":
|
||
local_dir = argv[i + 1]
|
||
i += 2
|
||
continue
|
||
if argv[i] == "--include":
|
||
includes.append(argv[i + 1])
|
||
i += 2
|
||
continue
|
||
i += 1
|
||
captured["includes"] = includes
|
||
captured["local_dir"] = local_dir
|
||
rootp = Path(local_dir)
|
||
for name in includes:
|
||
p = rootp / name
|
||
p.parent.mkdir(parents=True, exist_ok=True)
|
||
p.write_bytes(f"CONTENT::{name}".encode())
|
||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||
|
||
|
||
with tempfile.TemporaryDirectory() as td:
|
||
ei = mk_einput("Org/RealCli", td)
|
||
DL.shutil.which = fake_which_hf
|
||
DL.subprocess.run = fake_run_ok
|
||
try:
|
||
# NO fetcher= -> the DEFAULT real HubFetcher (CLI subprocess) runs.
|
||
# head_etag still hits urllib (network) — short-circuit it so the
|
||
# SHA stage uses our deterministic content fixture.
|
||
good_etags = {
|
||
n: content_sha(n) for n in ds if n.endswith(".safetensors")
|
||
}
|
||
_orig_head = DL.HubFetcher.head_etag
|
||
DL.HubFetcher.head_etag = (
|
||
lambda self, repo_id, filename: good_etags.get(filename)
|
||
)
|
||
try:
|
||
r = DL.download_model(ei)
|
||
finally:
|
||
DL.HubFetcher.head_etag = _orig_head
|
||
finally:
|
||
DL.shutil.which = _orig_which
|
||
DL.subprocess.run = _orig_run
|
||
|
||
argv = captured.get("argv", [])
|
||
check(argv[:1] == ["/usr/bin/hf"],
|
||
f"real fetcher resolves `hf` (wins over huggingface-cli); "
|
||
f"argv[0]={argv[:1]}")
|
||
check(argv[1:4] == ["download", "Org/RealCli", "--local-dir"],
|
||
f"real fetcher argv == `hf download <slug> --local-dir ...` "
|
||
f"(got {argv[1:4]})")
|
||
staging = str(DL.incomplete_dir(Path(td), "Org/RealCli"))
|
||
check(captured.get("local_dir") == staging,
|
||
f"--local-dir == the .incomplete staging dir (got "
|
||
f"{captured.get('local_dir')!r} want {staging!r})")
|
||
inc = captured.get("includes", [])
|
||
check(inc == ds,
|
||
f"ONE --include per download_set entry, in order, set == "
|
||
f"download_set, NOTHING extra (got {inc})")
|
||
check(set(inc) == set(ds) and len(inc) == len(ds),
|
||
"real fetcher --include set is EXACTLY download_set (Codex-r7 "
|
||
"High-1 exact-set guarantee holds via CLI transport)")
|
||
check(r.ok and r.failure is None and r.sha_verified is True,
|
||
f"real CLI fetcher: SHA verify / atomic stage logic UNCHANGED "
|
||
f"(ok={r.ok} fail={r.failure} sha={r.sha_verified})")
|
||
final = DL.pull_dir(Path(td), "Org/RealCli")
|
||
check(final.is_dir() and not (final / ".incomplete").exists(),
|
||
"real CLI fetcher: atomic move + .incomplete cleanup unchanged")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 10. NEITHER `hf` NOR `huggingface-cli` resolvable -> STRUCTURED
|
||
# DownloadResult(ok=False, failure="hf-cli-missing") with the canonical
|
||
# actionable message — NOT a raised/swallowed ModuleNotFoundError (the
|
||
# exact on-rig E5 failure mode this fix closes).
|
||
# ---------------------------------------------------------------------------
|
||
with tempfile.TemporaryDirectory() as td:
|
||
ei = mk_einput("Org/NoCli", td)
|
||
DL.shutil.which = lambda name: None # nothing on PATH
|
||
try:
|
||
raised = None
|
||
try:
|
||
r = DL.download_model(ei)
|
||
except BaseException as exc: # MUST NOT happen — structured only
|
||
raised = exc
|
||
r = None
|
||
finally:
|
||
DL.shutil.which = _orig_which
|
||
|
||
check(raised is None,
|
||
f"missing-CLI is NOT a raised exception (the E5 "
|
||
f"ModuleNotFoundError mode is closed); raised={raised!r}")
|
||
check(r is not None and r.ok is False
|
||
and r.failure == "hf-cli-missing",
|
||
f"missing-CLI -> structured DownloadResult(failure="
|
||
f"'hf-cli-missing') (got ok={getattr(r,'ok',None)} "
|
||
f"fail={getattr(r,'failure',None)})")
|
||
check(r is not None and "hf download" not in (r.detail or "")
|
||
and "huggingface-cli" in (r.detail or "")
|
||
and "uv tool install" in (r.detail or ""),
|
||
f"missing-CLI carries the canonical actionable detail "
|
||
f"(setup.sh:418-421 install hint); detail={getattr(r,'detail',None)!r}")
|
||
final = DL.pull_dir(Path(td), "Org/NoCli")
|
||
check(not (final / ".incomplete").exists(),
|
||
"missing-CLI: .incomplete tree deleted (no residue)")
|
||
|
||
if failures:
|
||
print(f"\n{len(failures)} assertion(s) failed.", file=sys.stderr)
|
||
sys.exit(1)
|
||
print("\nAll E2 download-stage (CONTRACT-3) assertions passed.")
|
||
PY
|
||
|
||
echo "test-pullgate-download.sh OK"
|