From 095244db45de8258f1ba2e90b02634a4dccf791b Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Tue, 1 Sep 2026 15:54:48 +0800 Subject: [PATCH] feat: support GLM-5 Next expert defusion --- README.md | 2 +- defuser/model_registry.py | 9 +++++++++ defuser/modeling/moe_experts_interface.py | 4 ++++ pyproject.toml | 2 +- tests/test_candidate_coverage.py | 19 +++++++++++++++++++ tests/test_meta_model_defusion.py | 15 +++++++++++++++ 6 files changed, 49 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 45b301c..360eb58 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Defuser currently supports the following `transformers>=5.3.0` `model_type` valu | Pattern | Supported model types | Defused op performed ⚙️ | | --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --- | | Standard routed expert tensors 🧱 | `deepseek_v2`, `dots1`, `ernie4_5_moe`, `ernie4_5_vl_moe`, `exaone_moe`, `flex_olmo`, `glm4_moe_lite`, `glm4v_moe`, `hunyuan_v1_moe`, `jamba`, `laguna`, `lfm2_moe`, `minimax`, `minimax_m2`, `olmoe`, `qwen3_vl_moe`, `solar_open`, `solar_open2` | Splits fused expert tensors or registered expert buffers into numbered expert `nn.Linear` modules with per-expert `gate_proj`, `up_proj`, and `down_proj`. | -| Mixed sparse and shared experts | `deepseek_v3`, `deepseek_v32`, `deepseek_v4`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text`, `qwen4_exp`, `qwen4_exp_text` | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. | +| Mixed sparse and shared experts | `deepseek_v3`, `deepseek_v32`, `deepseek_v4`, `glm5_next`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text`, `qwen4_exp`, `qwen4_exp_text` | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. | | Transposed or packed expert tensors | `gpt_oss`, `phimoe` | Splits transposed fused expert `gate_up_proj` tensors into per-expert `gate_proj` + `up_proj`, preserves expert bias when present, and converts expert tensors into numbered expert `nn.Linear` modules. | | Flattened expert layout | `dbrx` | Rebuilds the flattened DBRX expert FFN weights into numbered expert `gate_proj`, `up_proj`, and `down_proj` `nn.Linear` modules. | | Batched expert-input execution | `llama4` | Runtime expert tensor defusion plus preservation of the llama4 batched expert-input execution contract. | diff --git a/defuser/model_registry.py b/defuser/model_registry.py index 39a620d..6dc674e 100644 --- a/defuser/model_registry.py +++ b/defuser/model_registry.py @@ -199,6 +199,15 @@ class PATCH(str, Enum): "glm_image": { "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, }, + "glm5_next": { + "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, + PATCH.EXPERTS_DEFUSE: [ + { + "module_class": "transformers.models.glm5_next.modeling_glm5_next.Glm5NextTextExperts", + "forward_impl": "linear_loop", + } + ], + }, "glm_moe_dsa": { "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, }, diff --git a/defuser/modeling/moe_experts_interface.py b/defuser/modeling/moe_experts_interface.py index ad5b35a..730db89 100644 --- a/defuser/modeling/moe_experts_interface.py +++ b/defuser/modeling/moe_experts_interface.py @@ -282,6 +282,10 @@ def _matching_experts_defuse_spec(module: nn.Module, specs: list[dict]) -> dict def _install_instance_forward(module: nn.Module, implementation: str) -> None: """Attach a generic forward implementation directly to one experts module.""" + if implementation == LINEAR_LOOP_IMPL: + module.forward = MethodType(linear_loop_experts_forward, module) + return + if implementation == BATCHED_INPUT_IMPL: module.forward = MethodType(batched_input_experts_forward, module) return diff --git a/pyproject.toml b/pyproject.toml index 516c4c1..e65d0fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "Defuser" -version = "0.0.26" +version = "0.0.27" description = "Model defuser helper for HF Transformers." readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_candidate_coverage.py b/tests/test_candidate_coverage.py index 716bb12..24fee1b 100644 --- a/tests/test_candidate_coverage.py +++ b/tests/test_candidate_coverage.py @@ -177,6 +177,24 @@ def _standard_hidden(case: dict) -> torch.Tensor: "route_indices": [[0], [1], [2], [3], [0]], "expert_attrs": ("gate_proj", "up_proj", "down_proj"), }, + { + "model_type": "glm5_next", + "module_path": "transformers.models.glm5_next.modeling_glm5_next", + "class_name": "Glm5NextTextExperts", + "config_module": "transformers.models.glm5_next.configuration_glm5_next", + "config_name": "Glm5NextTextConfig", + "config_updates": { + "hidden_size": 64, + "moe_intermediate_size": 32, + "n_routed_experts": 4, + "hidden_act": "silu", + "swiglu_limit": 1.0, + "_experts_implementation": "grouped_mm", + }, + "input_dim": 64, + "route_indices": [[0], [1], [2], [3], [0]], + "expert_attrs": ("gate_proj", "up_proj", "down_proj"), + }, { "model_type": "dots1", "module_path": "transformers.models.dots1.modeling_dots1", @@ -576,6 +594,7 @@ def _standard_hidden(case: dict) -> torch.Tensor: "glm4_moe_lite", "glm4v", "glm4v_moe", + "glm5_next", "glm_image", "glm_moe_dsa", "glm_ocr", diff --git a/tests/test_meta_model_defusion.py b/tests/test_meta_model_defusion.py index 3dd96cf..cab8a45 100644 --- a/tests/test_meta_model_defusion.py +++ b/tests/test_meta_model_defusion.py @@ -166,6 +166,8 @@ def _build_model_config(case: dict): } elif model_type == "glm4_moe": config.first_k_dense_replace = -1 + elif model_type == "glm5_next": + config.text_config.mlp_layer_types = ["sparse"] * config.text_config.num_hidden_layers elif model_type == "glm_moe_dsa": config.mlp_layer_types = ["sparse"] * config.num_hidden_layers elif model_type == "granitemoehybrid": @@ -500,6 +502,19 @@ def _validate_defused_module(case: dict, module) -> None: "target_class_paths": ("transformers.models.glm_image.modeling_glm_image.GlmImageTextMLP",), "validator": "dense_split", }, + { + "model_type": "glm5_next", + "mode": "convert", + "model_module": "transformers.models.glm5_next.modeling_glm5_next", + "model_class": "Glm5NextForConditionalGeneration", + "config_module": "transformers.models.glm5_next.configuration_glm5_next", + "config_class": "Glm5NextConfig", + "target_class_paths": ( + "transformers.models.glm5_next.modeling_glm5_next.Glm5NextTextExperts", + ), + "validator": "experts", + "min_targets": 2, + }, { "model_type": "glm_moe_dsa", "mode": "convert",