diff --git a/src/gpu/distill_specs.yaml b/src/gpu/distill_specs.yaml index c50b107..50dc4ca 100644 --- a/src/gpu/distill_specs.yaml +++ b/src/gpu/distill_specs.yaml @@ -527,3 +527,25 @@ heb-diac-small-s46: train: hebrew-v4/train.jsonl val: hebrew-v4/val.jsonl out: rababa_hebrew_distill_small/run-002-s46 +heb-diac-small-s46-layerdrop: + # Item 04 (TODO.publish-client): cross-lingual depth-cut test — the + # Arabic width/depth finding replicated on Hebrew. Single variable + # vs run-002-s46: encoder 12->6 via verbatim layer copy (layer_drop), + # same teacher, corpus, and epochs. Gate: the Arabic depth premium + # (1.21pp at 6ep) predicts a comparable lite-vs-full gap here. + teacher: rababa_hebrew/run-s46-phonikud-plus/run-002-gold-ft/best + teacher_volume: rababa + student_init: google/byt5-small + layer_drop: 'true' + student_config: + d_model: 1472 + d_kv: 64 + d_ff: 3584 + num_heads: 6 + enc_layers: 6 + dec_layers: 4 + feed_forward_proj: gated-gelu + train: hebrew-v4/train.jsonl + val: hebrew-v4/val.jsonl + out: rababa_hebrew_distill_small/run-003-s46-layerdrop + note: Hebrew lite rung; cross-lingual depth-cut replication diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index 7f70aa6..8d9eba0 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -311,6 +311,10 @@ def distill(spec_id: str, epochs: int = 3, alpha: float = 0.5, temperature: floa cfg = T5Config(**spec["student_config"]) student = T5ForConditionalGeneration(cfg).to(device) + if spec.get("layer_drop"): + # depth-cut students keep the pretrained init: verbatim + # layer copy, not the random init the tiny tier uses + _maybe_stitch(spec_id, spec, student) else: student = AutoModelForSeq2SeqLM.from_pretrained(spec["student_init"]).to(device) student.train()