From fe1470888a617fcb4de05aef95cd740ed84b8264 Mon Sep 17 00:00:00 2001
From: Ronald Tse <ronald.tse@gmail.com>
Date: Fri, 4 Sep 2026 13:48:32 +0200
Subject: [PATCH] =?UTF-8?q?feat(specs):=20heb-diac-small-s46-layerdrop=20?=
 =?UTF-8?q?=E2=80=94=20cross-lingual=20depth-cut=20replication?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Item 04: the Arabic width/depth finding replicated on Hebrew — encoder
12->6 verbatim layer copy from pretrained ByT5-small, single variable
vs run-002-s46 (same s46 teacher, hebrew-v4 corpus, 3 epochs). Wires
_maybe_stitch into the logit-KD custom-config branch so depth-cut
students keep the pretrained init there too (previously random-init,
which the capacity law rejects).
---
 src/gpu/distill_specs.yaml | 22 ++++++++++++++++++++++
 src/gpu/modal_distill.py   |  4 ++++
 2 files changed, 26 insertions(+)

diff --git a/src/gpu/distill_specs.yaml b/src/gpu/distill_specs.yaml
index c50b107..50dc4ca 100644
--- a/src/gpu/distill_specs.yaml
+++ b/src/gpu/distill_specs.yaml
@@ -527,3 +527,25 @@ heb-diac-small-s46:
   train: hebrew-v4/train.jsonl
   val: hebrew-v4/val.jsonl
   out: rababa_hebrew_distill_small/run-002-s46
+heb-diac-small-s46-layerdrop:
+  # Item 04 (TODO.publish-client): cross-lingual depth-cut test — the
+  # Arabic width/depth finding replicated on Hebrew. Single variable
+  # vs run-002-s46: encoder 12->6 via verbatim layer copy (layer_drop),
+  # same teacher, corpus, and epochs. Gate: the Arabic depth premium
+  # (1.21pp at 6ep) predicts a comparable lite-vs-full gap here.
+  teacher: rababa_hebrew/run-s46-phonikud-plus/run-002-gold-ft/best
+  teacher_volume: rababa
+  student_init: google/byt5-small
+  layer_drop: 'true'
+  student_config:
+    d_model: 1472
+    d_kv: 64
+    d_ff: 3584
+    num_heads: 6
+    enc_layers: 6
+    dec_layers: 4
+    feed_forward_proj: gated-gelu
+  train: hebrew-v4/train.jsonl
+  val: hebrew-v4/val.jsonl
+  out: rababa_hebrew_distill_small/run-003-s46-layerdrop
+  note: Hebrew lite rung; cross-lingual depth-cut replication
diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py
index 7f70aa6..8d9eba0 100644
--- a/src/gpu/modal_distill.py
+++ b/src/gpu/modal_distill.py
@@ -311,6 +311,10 @@ def distill(spec_id: str, epochs: int = 3, alpha: float = 0.5, temperature: floa
 
         cfg = T5Config(**spec["student_config"])
         student = T5ForConditionalGeneration(cfg).to(device)
+        if spec.get("layer_drop"):
+            # depth-cut students keep the pretrained init: verbatim
+            # layer copy, not the random init the tiny tier uses
+            _maybe_stitch(spec_id, spec, student)
     else:
         student = AutoModelForSeq2SeqLM.from_pretrained(spec["student_init"]).to(device)
     student.train()