Add MoE LoRA converter (Unsloth fused format -> vLLM per-expert format)

Converts an Unsloth-trained, MoE-expert-targeting LoRA adapter into
vLLM's expected per-expert format so it can be served live through
vLLM's FusedMoE LoRA support, instead of requiring a full merge-and-
reload per adapter switch.

Includes:
- moe_lora_convert_vllm.py -- the converter
- moe_lora_convert_validate.py -- three-way validation (round-trip
  bit-exactness, per-expert delta match against Unsloth's real merge
  function, key coverage)
- README.md -- usage plus the full findings report: the tensor-layout
  investigation (including a wrong turn worth recording), all five
  vLLM-serving validation gates, and the GB10 infrastructure lessons
  from developing this

Developed and validated against Qwen3-30B-A3B (48 layers, 128 experts,
r=16 LoRA).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015dwQp8JCqhM2MRFHYwMyxp

moe_lora_convert_vllm.py

@@ -0,0 +1,133 @@
+"""Convert an Unsloth fused-per-layer MoE LoRA adapter (target_parameters:
+mlp.experts.gate_up_proj / mlp.experts.down_proj -- one tensor per layer
+covering all 128 experts) into vLLM's expected per-expert format
+(experts.{e}.{gate_proj,up_proj,down_proj}), per MOE_LORA_LAYOUT_PLAN.md §3.
+
+Layout used (corrected 2026-09-09, see MOE_LORA_LAYOUT_REPORT.md §4):
+  lora_A [E*r, in]:  expert e -> contiguous rows [e*r:(e+1)*r]
+  lora_B [out, E*r]: expert e -> contiguous columns [e*r:(e+1)*r]   <- NOT strided
+  gate_up_proj's 2*inter-wide output is concatenated: gate=[0:inter], up=[inter:2*inter]
+  orientation [E, out, in], no transpose needed relative to the raw parameter
+
+Usage:
+  python3 moe_lora_convert_vllm.py <src_adapter_dir> <dst_adapter_dir>
+"""
+import json
+import re
+import sys
+from pathlib import Path
+
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+EXPERT_PREFIX_RE = re.compile(r"^(base_model\.model\.model\.layers\.(\d+)\.mlp\.experts\.)(?:base_layer\.)?lora_A\.weight$")
+
+
+def find_layer_expert_keys(all_keys, layer_prefix):
+    """Return {A_key: B_key} for the two expert-LoRA (gate_up, down) pairs in one
+    layer. Which key is which target_parameter is NOT resolved here -- PEFT's
+    nesting order between the two target_parameters isn't guaranteed, so the
+    caller disambiguates gate_up vs down by tensor shape instead."""
+    a_keys = [k for k in all_keys if k.startswith(layer_prefix) and k.endswith("lora_A.weight")]
+    if len(a_keys) != 2:
+        raise ValueError(f"expected exactly 2 expert lora_A keys under {layer_prefix}, found {len(a_keys)}: {a_keys}")
+    return {a_key: a_key.replace("lora_A.weight", "lora_B.weight") for a_key in a_keys}
+
+
+def convert(src_dir: str, dst_dir: str, E: int = 128, r: int = 16):
+    src_dir = Path(src_dir)
+    dst_dir = Path(dst_dir)
+    dst_dir.mkdir(parents=True, exist_ok=True)
+
+    src_st = src_dir / "adapter_model.safetensors"
+    with safe_open(str(src_st), framework="pt") as f:
+        all_keys = list(f.keys())
+        expert_layers = sorted({int(m.group(2)) for k in all_keys if (m := EXPERT_PREFIX_RE.match(k))})
+        print(f"Found {len(expert_layers)} layers with expert LoRA: {expert_layers[:3]}...{expert_layers[-3:]}")
+
+        out_tensors = {}
+        # carry over everything that is NOT an expert-LoRA key unchanged (self_attn etc.)
+        expert_key_prefixes = {f"base_model.model.model.layers.{L}.mlp.experts." for L in expert_layers}
+        for k in all_keys:
+            if any(k.startswith(p) for p in expert_key_prefixes):
+                continue
+            out_tensors[k] = f.get_tensor(k)
+        print(f"Carried over {len(out_tensors)} non-expert tensors unchanged (self_attn etc.)")
+
+        n_pairs = 0
+        for L in expert_layers:
+            layer_prefix = f"base_model.model.model.layers.{L}.mlp.experts."
+            a_to_b = find_layer_expert_keys(all_keys, layer_prefix)
+
+            pairs = [(a_key, f.get_tensor(a_key), b_key, f.get_tensor(b_key)) for a_key, b_key in a_to_b.items()]
+            assert len(pairs) == 2, f"layer {L}: expected 2 expert-LoRA (A,B) pairs, got {len(pairs)}"
+            # Disambiguate gate_up vs down by A's in_features (last dim): whichever
+            # pair has the larger in_features is gate_up (in=hidden), the other is
+            # down (in=inter) -- true for standard transformer FFN shapes (hidden >= inter),
+            # and doesn't depend on PEFT's arbitrary nesting order for the two
+            # target_parameters.
+            pairs.sort(key=lambda p: p[1].shape[-1], reverse=True)
+            (_, gate_up_A, _, gate_up_B), (_, down_A, _, down_B) = pairs
+
+            E_r, hidden = gate_up_A.shape
+            out2, _ = gate_up_B.shape
+            inter = out2 // 2
+            assert E_r == E * r, f"layer {L}: expected E*r={E * r}, got {E_r}"
+            assert gate_up_B.shape[1] == E_r
+            assert down_A.shape == (E_r, inter), f"layer {L}: down_A shape {down_A.shape} != ({E_r},{inter})"
+            assert down_B.shape == (hidden, E_r), f"layer {L}: down_B shape {down_B.shape} != ({hidden},{E_r})"
+
+            for e in range(E):
+                lo, hi = e * r, (e + 1) * r
+                A_e = gate_up_A[lo:hi, :].contiguous()              # [r, hidden] -- shared by gate & up
+                B_e = gate_up_B[:, lo:hi].contiguous()              # [2*inter, r]
+                B_gate = B_e[:inter, :].contiguous()                # [inter, r]
+                B_up = B_e[inter:, :].contiguous()                  # [inter, r]
+                A_dn = down_A[lo:hi, :].contiguous()                # [r, inter]
+                B_dn = down_B[:, lo:hi].contiguous()                # [hidden, r]
+
+                base = f"base_model.model.model.layers.{L}.mlp.experts.{e}"
+                out_tensors[f"{base}.gate_proj.lora_A.weight"] = A_e
+                out_tensors[f"{base}.gate_proj.lora_B.weight"] = B_gate
+                out_tensors[f"{base}.up_proj.lora_A.weight"] = A_e.clone()
+                out_tensors[f"{base}.up_proj.lora_B.weight"] = B_up
+                out_tensors[f"{base}.down_proj.lora_A.weight"] = A_dn
+                out_tensors[f"{base}.down_proj.lora_B.weight"] = B_dn
+                n_pairs += 3
+
+            if L == expert_layers[0]:
+                print(f"  layer {L}: hidden={hidden} inter={inter} E*r={E_r} r={r}")
+
+        print(f"Converted {n_pairs} (layer, expert, projection) tensor pairs across {len(expert_layers)} layers")
+        print(f"Total output tensors: {len(out_tensors)}")
+
+    save_file(out_tensors, str(dst_dir / "adapter_model.safetensors"))
+
+    with open(src_dir / "adapter_config.json") as fh:
+        cfg = json.load(fh)
+    cfg.pop("target_parameters", None)
+    cfg["target_modules"] = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+    cfg["_converted_from"] = str(src_dir)
+    cfg["_conversion_note"] = (
+        "Converted from Unsloth's fused per-layer MoE target_parameters format to "
+        "vLLM's per-expert experts.{e}.{gate_proj,up_proj,down_proj} format by "
+        "moe_lora_convert_vllm.py, per MOE_LORA_LAYOUT_PLAN.md section 3 "
+        "(contiguous-both A/B expert blocking, corrected 2026-09-09)."
+    )
+    with open(dst_dir / "adapter_config.json", "w") as fh:
+        json.dump(cfg, fh, indent=2)
+
+    for extra in ("tokenizer_config.json", "tokenizer.json"):
+        src_f = src_dir / extra
+        if src_f.exists():
+            (dst_dir / extra).write_bytes(src_f.read_bytes())
+
+    print(f"\nWrote converted adapter to {dst_dir}")
+    print("CONVERT_DONE")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print(f"Usage: {sys.argv[0]} <src_adapter_dir> <dst_adapter_dir>")
+        sys.exit(1)
+    convert(sys.argv[1], sys.argv[2])