From f3c8e758e6b1c13e0d74eff4babc0bee03be4236 Mon Sep 17 00:00:00 2001 From: maikzz32 Date: Fri, 4 Sep 2026 12:06:59 +0200 Subject: [PATCH 1/5] feat(moe): cold-owner None for an owner whose cold set lives elsewhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MoeHybridColdBackend has Cpu and Gpu: the experts a rank does not hold hot are evaluated somewhere in this process. There is no way to say "the non-resident experts are not mine at all — their contribution arrives from outside". Expert-parallel inference across processes needs exactly that: each owner evaluates its own experts and a reduction outside ggml sums the partials. Adds MoeHybridColdBackend::None: * storage allocates no cold buffers and never materializes or swaps a cold expert; * the evaluators build no cold graph and never fall back to CPU or streamed evaluation. A route whose id was masked to -1 contributes zero instead of being routed to a cold owner. Purely additive: the enum grows a third value that nothing selects by default, and every Cpu/Gpu path is untouched. Verified by running the existing single-node hybrid configurations unchanged. Also adds eval_moe_shared_expert_batched, the piece such an owner needs that no existing entry point provides: the shared expert alone, batched. An owner that reduces across processes has to keep the shared term OUT of the reduced partial (it is replicated, so summing it would count it once per owner) and add it afterwards, which means evaluating it without the routed experts. Used downstream to run DeepSeek V4 Flash expert-parallel across two hosts, each holding half the routed experts. Co-Authored-By: Claude Opus 5 --- server/src/common/moe_hybrid_ffn_eval.cpp | 75 +++++++++++++++++++++++ server/src/common/moe_hybrid_ffn_eval.h | 16 +++++ server/src/common/moe_hybrid_storage.cpp | 48 +++++++++++---- server/src/common/moe_hybrid_types.h | 6 ++ 4 files changed, 132 insertions(+), 13 deletions(-) diff --git a/server/src/common/moe_hybrid_ffn_eval.cpp b/server/src/common/moe_hybrid_ffn_eval.cpp index d5e80cec0..8b6f30667 100644 --- a/server/src/common/moe_hybrid_ffn_eval.cpp +++ b/server/src/common/moe_hybrid_ffn_eval.cpp @@ -1648,6 +1648,11 @@ bool eval_moe_hybrid_ffn_single( std::vector cold_weights; for (int i = 0; i < n_selected; ++i) { const int32_t gid = selected_ids[i]; + // Cold owner None: routes masked to -1 by the cluster runtime are + // evaluated elsewhere and contribute zero here. + if (gid < 0 && storage.cold_backend_kind == MoeHybridColdBackend::None) { + continue; + } if (gid < 0 || gid >= (int32_t)storage.hot_local_by_global.size()) { if (err) *err = "selected id out of range"; return false; @@ -3982,6 +3987,7 @@ bool eval_moe_hybrid_ffn_gpu_resident( for (int i = 0; i < n_selected; ++i) { const int32_t gid = selected_ids[i]; + if (gid < 0 && storage.cold_backend_kind == MoeHybridColdBackend::None) continue; if (gid < 0 || gid >= (int32_t)storage.hot_local_by_global.size()) return false; const int32_t hot_local = storage.hot_local_by_global[(size_t)gid]; if (hot_local >= 0) { @@ -4232,4 +4238,73 @@ bool eval_moe_hybrid_ffn_gpu_resident( return true; } +// ── Shared expert only ── +// Cluster expert-parallel evaluates the routed partial without the shared +// expert (the MoeLayerDesc handed to the routed path has the shexp tensors +// cleared), all-reduces it, and adds this locally computed term afterwards. +// The graph is cached per n_tokens in storage.shared_batched_graph, which +// release_graph_caches() already frees. +bool eval_moe_shared_expert_batched( + ggml_backend_t gpu_backend, + const MoeHybridConfig & cfg, + const MoeLayerDesc & desc, + MoeHybridLayerStorage & storage, + const float * cur_host, + int n_tokens, + std::vector & out, + std::string * err) { + const int n_embd = cfg.n_embd; + out.assign((size_t)n_embd * (size_t)n_tokens, 0.0f); + if (n_tokens <= 0) return true; + if (!desc.ffn_up_shexp || !desc.ffn_gate_shexp || !desc.ffn_down_shexp) { + return true; + } + if (!cur_host) { + if (err) *err = "shared expert requires a host activation"; + return false; + } + + CachedHotBatchedGraph & g = storage.shared_batched_graph; + if (!g.valid() || g.n_tokens != n_tokens) { + g.free(); + g.n_tokens = n_tokens; + ggml_init_params ip{}; + ip.mem_size = 4 * 1024 * 1024; + ip.mem_buffer = nullptr; + ip.no_alloc = true; + g.ctx = ggml_init(ip); + if (!g.ctx) { + if (err) *err = "shared expert ggml_init failed"; + return false; + } + g.inp = ggml_new_tensor_2d(g.ctx, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_input(g.inp); + g.output = build_shared_expert_subgraph(g.ctx, desc, g.inp, cfg.swiglu_clamp); + if (!g.output) { + g.free(); + if (err) *err = "shared expert subgraph build failed"; + return false; + } + g.gf = ggml_new_graph_custom(g.ctx, 512, false); + ggml_set_output(g.output); + ggml_build_forward_expand(g.gf, g.output); + g.alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(gpu_backend)); + if (!g.alloc || !ggml_gallocr_alloc_graph(g.alloc, g.gf)) { + g.free(); + if (err) *err = "shared expert gallocr failed"; + return false; + } + } + + ggml_backend_tensor_set(g.inp, cur_host, 0, + sizeof(float) * (size_t)n_embd * (size_t)n_tokens); + if (ggml_backend_graph_compute(gpu_backend, g.gf) != GGML_STATUS_SUCCESS) { + if (err) *err = "shared expert compute failed"; + return false; + } + ggml_backend_tensor_get(g.output, out.data(), 0, + sizeof(float) * (size_t)n_embd * (size_t)n_tokens); + return true; +} + } // namespace dflash::common diff --git a/server/src/common/moe_hybrid_ffn_eval.h b/server/src/common/moe_hybrid_ffn_eval.h index 4e7557883..6f195a194 100644 --- a/server/src/common/moe_hybrid_ffn_eval.h +++ b/server/src/common/moe_hybrid_ffn_eval.h @@ -386,6 +386,22 @@ bool build_cached_cold_graph( int n_cold, float swiglu_clamp = 0.0f); +// Shared expert only, batched [n_embd, n_tokens] on the GPU backend. Used by +// the cluster expert-parallel path, which evaluates routed experts without +// the shared term (MoeLayerDesc with shexp tensors cleared), all-reduces the +// routed partial across ranks and adds this local result afterwards. Cached +// per n_tokens in storage.shared_batched_graph. `out` is zero-filled when the +// layer has no shared expert. +bool eval_moe_shared_expert_batched( + ggml_backend_t gpu_backend, + const MoeHybridConfig & cfg, + const MoeLayerDesc & desc, + MoeHybridLayerStorage & storage, + const float * cur_host, + int n_tokens, + std::vector & out, + std::string * err = nullptr); + // Build cached hot-only batched graph for prefill (n_tokens=MMQ_SAFE_SUB_BATCH). bool build_cached_hot_batched_graph( CachedHotBatchedGraph & out, diff --git a/server/src/common/moe_hybrid_storage.cpp b/server/src/common/moe_hybrid_storage.cpp index baa1adca5..eb5808dca 100644 --- a/server/src/common/moe_hybrid_storage.cpp +++ b/server/src/common/moe_hybrid_storage.cpp @@ -289,10 +289,20 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, out.cold_backend_kind = cfg.cold_expert_backend; out.materialized_hot_experts = cfg.materialize_hot_experts; out.materialized_cold_experts = cfg.materialize_cold_experts; - out.cold_backend = cfg.cold_expert_backend == MoeHybridColdBackend::Gpu - ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) - : out.cpu_backend; - if (!out.cold_backend) { + // Cold owner None (cluster expert-parallel): non-resident routes are + // reduced by another process, so there is no cold backend, no cold + // buffer and no cold expert map on this side. + const bool no_cold_owner = + cfg.cold_expert_backend == MoeHybridColdBackend::None; + if (no_cold_owner && cfg.materialize_cold_experts) { + if (err) *err = "cold owner None cannot materialize cold experts"; + return false; + } + out.cold_backend = no_cold_owner ? nullptr + : cfg.cold_expert_backend == MoeHybridColdBackend::Gpu + ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) + : out.cpu_backend; + if (!out.cold_backend && !no_cold_owner) { if (err) *err = "failed to select cold expert backend"; return false; } @@ -331,10 +341,12 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, is_hot[(size_t)expert] = 1; } dst.decode_hot_local_by_global = dst.hot_local_by_global; - for (int expert = 0; expert < cfg.n_expert; ++expert) { - if (duplicate_hot_on_cold || !is_hot[(size_t)expert]) { - dst.cold_local_by_global[(size_t)expert] = (int32_t)dst.cold_expert_ids.size(); - dst.cold_expert_ids.push_back((int32_t)expert); + if (!no_cold_owner) { + for (int expert = 0; expert < cfg.n_expert; ++expert) { + if (duplicate_hot_on_cold || !is_hot[(size_t)expert]) { + dst.cold_local_by_global[(size_t)expert] = (int32_t)dst.cold_expert_ids.size(); + dst.cold_expert_ids.push_back((int32_t)expert); + } } } dst.decode_cold_local_by_global = dst.cold_local_by_global; @@ -503,10 +515,20 @@ bool build_moe_hybrid_storage_from_file( out.cold_backend_kind = cfg.cold_expert_backend; out.materialized_hot_experts = cfg.materialize_hot_experts; out.materialized_cold_experts = cfg.materialize_cold_experts; - out.cold_backend = cfg.cold_expert_backend == MoeHybridColdBackend::Gpu - ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) - : out.cpu_backend; - if (!out.cold_backend) { + // Cold owner None (cluster expert-parallel): non-resident routes are + // reduced by another process, so there is no cold backend, no cold + // buffer and no cold expert map on this side. + const bool no_cold_owner = + cfg.cold_expert_backend == MoeHybridColdBackend::None; + if (no_cold_owner && cfg.materialize_cold_experts) { + if (err) *err = "cold owner None cannot materialize cold experts"; + return false; + } + out.cold_backend = no_cold_owner ? nullptr + : cfg.cold_expert_backend == MoeHybridColdBackend::Gpu + ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) + : out.cpu_backend; + if (!out.cold_backend && !no_cold_owner) { if (err) *err = "failed to select cold expert backend"; return false; } @@ -546,7 +568,7 @@ bool build_moe_hybrid_storage_from_file( is_hot[(size_t)expert] = 1; } dst.decode_hot_local_by_global = dst.hot_local_by_global; - if (allocate_cold) { + if (allocate_cold && !no_cold_owner) { for (int expert = 0; expert < cfg.n_expert; ++expert) { if (duplicate_hot_on_cold || !is_hot[(size_t)expert]) { dst.cold_local_by_global[(size_t)expert] = (int32_t)dst.cold_expert_ids.size(); diff --git a/server/src/common/moe_hybrid_types.h b/server/src/common/moe_hybrid_types.h index bdb22e47e..21f49bdd7 100644 --- a/server/src/common/moe_hybrid_types.h +++ b/server/src/common/moe_hybrid_types.h @@ -21,6 +21,12 @@ int query_gpu_compute_sm(); enum class MoeHybridColdBackend { Cpu, Gpu, + // No cold owner: non-resident routes contribute zero and are never + // materialized; the reduction across owners happens outside this process + // (cluster all-reduce, see server/src/cluster/). Storage allocates no cold + // buffers, evaluators build no cold graph, never fall back to CPU or + // streamed evaluation for non-resident routes and never swap experts. + None, }; // ─── MoE architecture config (model-agnostic) ────────────────────────── From f845dc3b7625b41175087c6166f299a794ac2073 Mon Sep 17 00:00:00 2001 From: maikzz32 Date: Fri, 4 Sep 2026 12:07:28 +0200 Subject: [PATCH 2/5] perf(moe): pack a hot-only prefill batch by expert, not one token at a time An owner with cold-owner None holds a REDUCED expert stack, and mmq_full_batch_ok is false for a reduced stack because MMQ's mul_mat_id illegal-accesses on it. eval_moe_hybrid_ffn_batched therefore falls into its sub-batch loop, whose size is min(mmq_safe_sub_batch(), moe_hybrid_prefill_hot_sub_batch_limit()) which is 1 on gfx1151, where mmq_safe_sub_batch() returns 1 for compute < sm_80. A 1517-token prefill then runs 1517 x 43 = 65k graph computes. With cold-owner None there is exactly one owner and every route that survives masking is resident, so the whole batch can take the packing the heterogeneous prefill already uses: one expert-major graph per layer, via eval_moe_owner_expert_major_batched. That path was written for a reduced stack in the first place, which is why it does not need the MMQ full-batch guarantee. Measured on two Ryzen AI Max 395 (Radeon 8060S, gfx1151, ROCm 10), DeepSeek V4 Flash, --ds4-prefill sparse --chunk 2048, same binary and clocks, prefill wall time: 1517-token prompt: 34.1 s -> 11.45 s (FFN 25.9 s -> 3.7 s) 12017-token prompt: 283.9 s -> 96.9 s Output parity: the completion is unchanged by this patch, and the 128-token greedy benchmark stays byte-identical to a single-node run of the same binary (sha256 87964cbd...), 3 runs. Reachable only with cold-owner None, so no existing configuration changes behaviour. On sm_80+ the sub-batch is 8 rather than 1, so the same shape would have been 8x less bad but still 8k graph computes. Co-Authored-By: Claude Opus 5 --- server/src/common/moe_hybrid_ffn_eval.cpp | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/server/src/common/moe_hybrid_ffn_eval.cpp b/server/src/common/moe_hybrid_ffn_eval.cpp index 8b6f30667..fb63eca69 100644 --- a/server/src/common/moe_hybrid_ffn_eval.cpp +++ b/server/src/common/moe_hybrid_ffn_eval.cpp @@ -3542,6 +3542,51 @@ bool eval_moe_hybrid_ffn_batched( : storage.gate_cold ? (int)storage.gate_cold->ne[2] : 0; const bool cold_on_gpu = storage.cold_backend_kind == MoeHybridColdBackend::Gpu; + // Cold owner None (a cluster rank): every route that survived masking is + // resident here and there is no second owner, so the whole batch can be + // packed by expert into ONE graph per layer. Without this a reduced hot + // stack falls into the sub-batch loop far below, whose size is + // min(mmq_safe_sub_batch(), prefill limit) = 1 on gfx1151 - one graph per + // token per layer. Measured on a 1517-token prompt: 25.9 s of FFN against + // 7.6 s for a single node's whole prefill graph. Expert-major packing is + // also what keeps the reduced stack off the MMQ full-batch path that + // mmq_safe_full_batch=false exists to avoid. + const bool hot_only_expert_major = + !expert_compute && + storage.cold_backend_kind == MoeHybridColdBackend::None && + !storage.gate_cold && !storage.gate_up_cold && !storage.down_cold && + n_hot_stack > 0 && + moe_expert_major_prefill_enabled(n_tokens); + if (hot_only_expert_major) { + static std::once_flag logged; + std::call_once(logged, [n_tokens, n_hot_stack] { + std::fprintf(stderr, + "[hybrid-ffn] hot-only expert-major batch active tokens=%d " + "stack=%d (no cold owner)\n", + n_tokens, n_hot_stack); + }); + const auto wall_t0 = HybridClock::now(); + std::string owner_err; + const bool ok = eval_moe_owner_expert_major_batched( + gpu_backend, cfg, desc, + storage.gate_hot, storage.up_hot, storage.down_hot, + storage.gate_up_hot, storage.hot_local_by_global, + cur_host, selected_ids, selected_weights, n_tokens, + desc.has_shared_expert(), out, &owner_err, + cur_backend, gpu_backend, + /*device_output=*/nullptr, /*device_output_owner=*/nullptr, + p_hot_alloc); + if (!ok) { + if (err) *err = owner_err; + return false; + } + if (telemetry) { + const auto done = HybridClock::now(); + telemetry->hot_us += elapsed_us(wall_t0, done); + telemetry->ffn_wall_us += elapsed_us(wall_t0, done); + } + return true; + } const bool inprocess_expert_major = !expert_compute && moe_expert_major_prefill_enabled(n_tokens) && cold_on_gpu && storage.cold_backend && From a7aafc560687874689cdbc20d373df988a76fcd2 Mon Sep 17 00:00:00 2001 From: maikzz32 Date: Fri, 4 Sep 2026 16:22:32 +0200 Subject: [PATCH 3/5] feat(moe): register mixed-qtype decode tables for hot-only storage Stacked on #701, which adds MoeHybridColdBackend::None. Review that one first; the diff against main shows both. Adaptive (mixed ROCmFPX) experts keep their codebooks out of band and need a decode table registered per resident tensor. register_deepseek4_moe_hybrid_mix_tables demanded a materialized GPU cold owner, so an owner using cold-owner None -- whose resident set is exactly its hot experts -- was refused, even though that is the storage shape the registrar already handles: ds4_register_compact_mix_tensor returns success for a null tensor whose expert-id list is empty, which is how an absent cold owner presents itself. Only the guard was too narrow. It now accepts either a materialized GPU cold owner, as before, or hot-only storage with no cold owner. Nothing else changes, and a Cpu cold owner is still refused. This is what makes an adaptive artifact usable by an owner whose cold set lives outside the process. Measured downstream on two Radeon 8060S nodes under ROCm 10 with DeepSeek V4 Flash, where each node holds half the routed experts with cold owner None: the adaptive ROCMFPX-MIX artifact loads where it previously failed, and against the uniform ROCMFP2 artifact it measures 42.3 against 38.4 tok/s at a verify width of 4, 20.6 against 17.2 on a free-form prompt, and 60/60 against 12/60 on the exact-copy fidelity check from server/docs/DS4.md. Output is byte-identical to a single node. Co-Authored-By: Claude Opus 5 --- server/src/deepseek4/deepseek4_loader.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/server/src/deepseek4/deepseek4_loader.cpp b/server/src/deepseek4/deepseek4_loader.cpp index 2ff7526c8..027368eb3 100644 --- a/server/src/deepseek4/deepseek4_loader.cpp +++ b/server/src/deepseek4/deepseek4_loader.cpp @@ -1346,11 +1346,24 @@ bool register_deepseek4_moe_hybrid_mix_tables( } if (!has_mix_experts) return true; + // Two storage shapes can be decoded. A GPU cold owner needs both halves + // materialized, because the primary and the secondary owner each get their + // own table. Cold owner None has no second owner at all: its resident set + // is exactly the hot experts, and ds4_register_compact_mix_tensor already + // returns success for a null tensor whose expert-id list is empty, which is + // precisely how an absent cold owner presents itself. Only this check stood + // in the way. + const bool hot_only = + storage.cold_backend_kind == MoeHybridColdBackend::None && + !storage.materialized_cold_experts; + const bool gpu_owners = + storage.cold_backend_kind == MoeHybridColdBackend::Gpu && + storage.materialized_cold_experts; if (storage.layers.size() != w.layers.size() || - storage.cold_backend_kind != MoeHybridColdBackend::Gpu || !storage.materialized_hot_experts || - !storage.materialized_cold_experts) { - if (err) *err = "mixed expert qtypes require materialized GPU owners"; + !(hot_only || gpu_owners)) { + if (err) *err = "mixed expert qtypes require materialized hot experts with " + "either a materialized GPU cold owner or no cold owner"; return false; } From 203f8980151ad7a4ad44ed402d2afd1795e0c04e Mon Sep 17 00:00:00 2001 From: mrciffa Date: Wed, 23 Sep 2026 12:57:40 +0200 Subject: [PATCH 4/5] fix(moe): cold-owner None review fixes - None implies no cold materialization (MoeHybridConfig::materializes_cold_experts) instead of failing storage construction when the flag keeps its default. - The routed partial of a None owner never carries the shared expert: every evaluator refuses a desc with shexp tensors rather than letting the cross-owner sum count the replicated term once per owner, and the hot-only expert-major prefill passes include_shared=false. - eval_moe_shared_expert_batched checks n_tokens before sizing the output. - The None comment points at ggml_cluster_allreduce, not a missing directory. - Unit tests for all of the above in test_moe_hybrid_storage. Co-Authored-By: Claude Opus 5.5 (1M context) --- server/src/common/moe_hybrid_ffn_eval.cpp | 27 +++++++++- server/src/common/moe_hybrid_storage.cpp | 20 +++---- server/src/common/moe_hybrid_types.h | 14 ++++- server/test/test_moe_hybrid_storage.cpp | 65 +++++++++++++++++++++++ 4 files changed, 108 insertions(+), 18 deletions(-) diff --git a/server/src/common/moe_hybrid_ffn_eval.cpp b/server/src/common/moe_hybrid_ffn_eval.cpp index 1614b1880..6d6b567a2 100644 --- a/server/src/common/moe_hybrid_ffn_eval.cpp +++ b/server/src/common/moe_hybrid_ffn_eval.cpp @@ -1650,6 +1650,23 @@ static bool build_cached_cold_batched_graph( return true; } +// Cold owner None evaluates a routed partial that the caller sums across +// owners. The shared expert is replicated on every owner, so letting it into +// that partial would count it once per owner. Refuse such a desc instead of +// silently double counting; see MoeHybridColdBackend::None. +static bool none_owner_desc_ok(const MoeHybridLayerStorage & storage, + const MoeLayerDesc & desc, std::string * err) { + if (storage.cold_backend_kind != MoeHybridColdBackend::None || + !desc.has_shared_expert()) { + return true; + } + if (err) { + *err = "cold owner None: pass the routed desc without the shared expert " + "and add eval_moe_shared_expert_batched() after the reduction"; + } + return false; +} + bool eval_moe_hybrid_ffn_single( ggml_backend_t gpu_backend, const MoeHybridConfig & cfg, @@ -1665,6 +1682,7 @@ bool eval_moe_hybrid_ffn_single( std::string * err) { if (telemetry) *telemetry = {}; + if (!none_owner_desc_ok(storage, desc, err)) return false; const auto ffn_wall_t0 = HybridClock::now(); const auto partition_t0 = HybridClock::now(); @@ -3551,6 +3569,7 @@ bool eval_moe_hybrid_ffn_batched( ggml_tensor * cur_backend, const MoeHybridDeviceOutputs * device_outputs) { if (telemetry) *telemetry = {}; + if (!none_owner_desc_ok(storage, desc, err)) return false; const bool materialized_cold = storage.down_cold || storage.gate_up_cold; if (cur_host && compact_materialized_experts_enabled() && materialized_cold && !expert_compute && n_tokens > 0 && n_tokens <= 4) { @@ -3611,7 +3630,7 @@ bool eval_moe_hybrid_ffn_batched( storage.gate_hot, storage.up_hot, storage.down_hot, storage.gate_up_hot, storage.hot_local_by_global, cur_host, selected_ids, selected_weights, n_tokens, - desc.has_shared_expert(), out, &owner_err, + /*include_shared=*/false, out, &owner_err, cur_backend, gpu_backend, /*device_output=*/nullptr, /*device_output_owner=*/nullptr, p_hot_alloc); @@ -4059,6 +4078,7 @@ bool eval_moe_hybrid_ffn_gpu_resident( MoeExpertCompute * expert_compute, const MoeExpertLayer * expert_layer) { + if (!none_owner_desc_ok(storage, desc, nullptr)) return false; const int n_embd = cfg.n_embd; // ── Partition into hot/cold ── @@ -4338,8 +4358,11 @@ bool eval_moe_shared_expert_batched( std::vector & out, std::string * err) { const int n_embd = cfg.n_embd; + if (n_tokens <= 0) { + out.clear(); + return true; + } out.assign((size_t)n_embd * (size_t)n_tokens, 0.0f); - if (n_tokens <= 0) return true; if (!desc.ffn_up_shexp || !desc.ffn_gate_shexp || !desc.ffn_down_shexp) { return true; } diff --git a/server/src/common/moe_hybrid_storage.cpp b/server/src/common/moe_hybrid_storage.cpp index a04dd325b..12d85ae6e 100644 --- a/server/src/common/moe_hybrid_storage.cpp +++ b/server/src/common/moe_hybrid_storage.cpp @@ -268,7 +268,7 @@ bool MoeHybridStorage::matches(const MoeHybridConfig & cfg) const { (int)layers.size() == cfg.n_layer && cold_backend_kind == cfg.cold_expert_backend && materialized_hot_experts == cfg.materialize_hot_experts && - materialized_cold_experts == cfg.materialize_cold_experts && + materialized_cold_experts == cfg.materializes_cold_experts() && mixed_mmq_policy == cfg.mixed_mmq_policy; } @@ -303,16 +303,12 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, out.cold_backend_kind = cfg.cold_expert_backend; out.materialized_hot_experts = cfg.materialize_hot_experts; out.mixed_mmq_policy = cfg.mixed_mmq_policy; - out.materialized_cold_experts = cfg.materialize_cold_experts; + out.materialized_cold_experts = cfg.materializes_cold_experts(); // Cold owner None (cluster expert-parallel): non-resident routes are // reduced by another process, so there is no cold backend, no cold // buffer and no cold expert map on this side. const bool no_cold_owner = cfg.cold_expert_backend == MoeHybridColdBackend::None; - if (no_cold_owner && cfg.materialize_cold_experts) { - if (err) *err = "cold owner None cannot materialize cold experts"; - return false; - } out.cold_backend = no_cold_owner ? nullptr : cfg.cold_expert_backend == MoeHybridColdBackend::Gpu ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) @@ -435,7 +431,7 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, } // Allocate cold expert tensors on the selected cold backend. - if (cold_count > 0 && cfg.materialize_cold_experts) { + if (cold_count > 0 && cfg.materializes_cold_experts()) { ggml_init_params ip{}; ip.mem_size = 16 * ggml_tensor_overhead(); ip.mem_buffer = nullptr; @@ -533,16 +529,12 @@ bool build_moe_hybrid_storage_from_file( out.cold_backend_kind = cfg.cold_expert_backend; out.materialized_hot_experts = cfg.materialize_hot_experts; out.mixed_mmq_policy = cfg.mixed_mmq_policy; - out.materialized_cold_experts = cfg.materialize_cold_experts; + out.materialized_cold_experts = cfg.materializes_cold_experts(); // Cold owner None (cluster expert-parallel): non-resident routes are // reduced by another process, so there is no cold backend, no cold // buffer and no cold expert map on this side. const bool no_cold_owner = cfg.cold_expert_backend == MoeHybridColdBackend::None; - if (no_cold_owner && cfg.materialize_cold_experts) { - if (err) *err = "cold owner None cannot materialize cold experts"; - return false; - } out.cold_backend = no_cold_owner ? nullptr : cfg.cold_expert_backend == MoeHybridColdBackend::Gpu ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) @@ -676,7 +668,7 @@ bool build_moe_hybrid_storage_from_file( } // Allocate cold expert tensors on the selected cold backend. - if (allocate_cold && cold_count > 0 && cfg.materialize_cold_experts) { + if (allocate_cold && cold_count > 0 && cfg.materializes_cold_experts()) { ggml_init_params ip{}; ip.mem_size = 16 * ggml_tensor_overhead(); ip.mem_buffer = nullptr; @@ -734,7 +726,7 @@ bool build_moe_hybrid_storage_from_file( // slice buffers are gone. Retain the mapping for future streaming reads. if (readonly_file_mmap && readonly_file_fd >= 0 && moe_source_pageout_eligible( out.cold_backend_kind == MoeHybridColdBackend::Gpu, - cfg.materialize_hot_experts, cfg.materialize_cold_experts, + cfg.materialize_hot_experts, cfg.materializes_cold_experts(), allocate_cold && cold_count > 0 && dst.cold_buf != nullptr)) { if (dst.fused_gate_up) { advise_copied_source(readonly_file_mmap, readonly_file_mmap_size, fd.gate_up_exps, il, readonly_file_fd); diff --git a/server/src/common/moe_hybrid_types.h b/server/src/common/moe_hybrid_types.h index d872e3abe..8094fb46b 100644 --- a/server/src/common/moe_hybrid_types.h +++ b/server/src/common/moe_hybrid_types.h @@ -22,10 +22,13 @@ enum class MoeHybridColdBackend { Cpu, Gpu, // No cold owner: non-resident routes contribute zero and are never - // materialized; the reduction across owners happens outside this process - // (cluster all-reduce, see server/src/cluster/). Storage allocates no cold + // materialized; the caller reduces the owners' partials outside this + // process (e.g. with ggml_cluster_allreduce). Storage allocates no cold // buffers, evaluators build no cold graph, never fall back to CPU or // streamed evaluation for non-resident routes and never swap experts. + // The routed partial never carries the shared expert, which is replicated + // on every owner: pass a MoeLayerDesc without shexp tensors and add + // eval_moe_shared_expert_batched() once, after the reduction. None, }; @@ -45,6 +48,13 @@ struct MoeHybridConfig { bool materialize_hot_experts = true; bool materialize_cold_experts = true; + // Cold owner None has no cold experts, so nothing to materialize whatever + // materialize_cold_experts says. + bool materializes_cold_experts() const { + return materialize_cold_experts && + cold_expert_backend != MoeHybridColdBackend::None; + } + // When true, MMQ mul_mat_id works correctly with reduced hot stacks // (n_hot < n_expert). Safe on sm_80+ (Ampere/Ada/Hopper/Blackwell). // On sm_75 (Turing) and gfx1151, the kernel has illegal memory accesses diff --git a/server/test/test_moe_hybrid_storage.cpp b/server/test/test_moe_hybrid_storage.cpp index dac508628..1a26a33a0 100644 --- a/server/test/test_moe_hybrid_storage.cpp +++ b/server/test/test_moe_hybrid_storage.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include using namespace luce::common; @@ -77,6 +78,70 @@ TEST_CASE(MoeHybridStorageFixture, storage_identity_includes_mixed_mmq_policy) { REQUIRE(!storage.matches(cfg)); } +TEST_CASE(MoeHybridStorageFixture, cold_owner_none_implies_no_cold_materialization) { + MoeHybridConfig cfg; + cfg.cold_expert_backend = MoeHybridColdBackend::None; + // The flag keeps its default; None must not need it cleared by hand. + REQUIRE(cfg.materialize_cold_experts); + REQUIRE(!cfg.materializes_cold_experts()); + + MoeHybridStorage storage; + cfg.n_layer = storage.placement.n_layer = 1; + cfg.n_expert = storage.placement.n_expert = 2; + cfg.n_expert_used = storage.placement.n_expert_used = 1; + storage.placement.hot_counts = {0}; + storage.placement.hot_expert_ids = {{}}; + storage.layers.resize(1); + storage.cold_backend_kind = MoeHybridColdBackend::None; + // Storage built for None records no cold materialization, and a config + // that kept the default flag still identifies it. + storage.materialized_cold_experts = true; + REQUIRE(!storage.matches(cfg)); + storage.materialized_cold_experts = false; + REQUIRE(storage.matches(cfg)); + + cfg.cold_expert_backend = MoeHybridColdBackend::Gpu; + REQUIRE(cfg.materializes_cold_experts()); +} + +TEST_CASE(MoeHybridStorageFixture, cold_owner_none_refuses_a_shared_expert_in_the_routed_partial) { + auto ctx = std::unique_ptr( + ggml_init({1u << 16, nullptr, true}), ggml_free); + REQUIRE(ctx != nullptr); + MoeHybridConfig cfg; + cfg.n_embd = 4; + cfg.n_expert = 2; + cfg.n_expert_used = 1; + cfg.cold_expert_backend = MoeHybridColdBackend::None; + MoeHybridLayerStorage storage; + storage.cold_backend_kind = MoeHybridColdBackend::None; + MoeLayerDesc desc; + desc.ffn_gate_shexp = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, 4, 8); + desc.ffn_up_shexp = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, 4, 8); + desc.ffn_down_shexp = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, 8, 4); + + // Every owner would add the replicated shared expert into the partial the + // caller sums across owners. The evaluator refuses before touching a + // backend instead of double counting. + const float cur[8] = {}; + const int32_t ids[2] = {0, -1}; + const float weights[2] = {1.0f, 1.0f}; + std::vector out; + std::string err; + CHECK(!eval_moe_hybrid_ffn_batched(nullptr, nullptr, cfg, desc, storage, + cur, ids, weights, 2, out, &err)); + CHECK(err.find("shared expert") != std::string::npos); + err.clear(); + CHECK(!eval_moe_hybrid_ffn_single(nullptr, cfg, desc, storage, nullptr, + cur, ids, weights, 1, out, nullptr, &err)); + CHECK(err.find("shared expert") != std::string::npos); + + // A non-positive batch is empty, not a huge allocation. + out.assign(3, 1.0f); + CHECK(eval_moe_shared_expert_batched(nullptr, cfg, desc, storage, cur, -1, out)); + CHECK(out.empty()); +} + TEST_CASE(MoeHybridStorageFixture, expert_residency_tracks_model_sized_expert_sets) { MoeHybridLayerStorage storage; storage.reset_expert_vram_mask(320); From d4052bbf5a15c28e3974216e1a93434b19ade5ab Mon Sep 17 00:00:00 2001 From: mrciffa Date: Thu, 24 Sep 2026 00:50:25 +0200 Subject: [PATCH 5/5] fix(moe): cold-owner None never sets up cold streaming Second cubic pass on the None review fixes: - MoeHybridStorage::streams_cold_experts(): None is not materialized but has no cold experts, so DS4 no longer initializes the cold stream engine (and its pinned buffers) for it, and the paged-serving check keys on streaming. - The DS4 hybrid log reports cold_backend=none instead of cpu. - eval_moe_shared_expert_batched rejects a null GPU backend. - Tests: streaming predicate per cold owner, null backend, no dead -1 route. Co-Authored-By: Claude Opus 5.5 (1M context) --- server/src/common/moe_hybrid_ffn_eval.cpp | 4 ++++ server/src/common/moe_hybrid_storage.h | 8 ++++++++ server/src/deepseek4/deepseek4_backend.cpp | 8 +++++--- server/test/test_moe_hybrid_storage.cpp | 18 +++++++++++++++++- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/server/src/common/moe_hybrid_ffn_eval.cpp b/server/src/common/moe_hybrid_ffn_eval.cpp index 6d6b567a2..666f0c4de 100644 --- a/server/src/common/moe_hybrid_ffn_eval.cpp +++ b/server/src/common/moe_hybrid_ffn_eval.cpp @@ -4370,6 +4370,10 @@ bool eval_moe_shared_expert_batched( if (err) *err = "shared expert requires a host activation"; return false; } + if (!gpu_backend) { + if (err) *err = "shared expert requires a GPU backend"; + return false; + } CachedHotBatchedGraph & g = storage.shared_batched_graph; if (!g.valid() || g.n_tokens != n_tokens) { diff --git a/server/src/common/moe_hybrid_storage.h b/server/src/common/moe_hybrid_storage.h index e9f775a3e..01555a133 100644 --- a/server/src/common/moe_hybrid_storage.h +++ b/server/src/common/moe_hybrid_storage.h @@ -202,6 +202,14 @@ struct MoeHybridStorage { bool materialized_cold_experts = true; ggml_mixed_mmq_policy mixed_mmq_policy = GGML_MIXED_MMQ_DEFAULT; MoeHybridPlacement placement; + + // Cold experts are streamed from the source file on demand. Cold owner + // None is not materialized either, but it has no cold experts at all, so + // it must not set up a streaming path. + bool streams_cold_experts() const { + return !materialized_cold_experts && + cold_backend_kind != MoeHybridColdBackend::None; + } std::vector layers; // Long heterogeneous prefill uses one routing graph and one owner graph diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index d45d9b762..38652ed66 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1727,7 +1727,7 @@ bool DeepSeek4Backend::init() { return false; } if (cfg_.paged_attention && moe_hybrid_ && - !moe_hybrid_->materialized_cold_experts) { + moe_hybrid_->streams_cold_experts()) { std::fprintf(stderr, "[deepseek4] paged serving requires statically materialized " "expert ownership; enable in-process LUCE_DS4_MOE_TP\n"); @@ -2392,7 +2392,7 @@ bool DeepSeek4Backend::init_hybrid_model() { "[deepseek4] speculative verifier routes all experts " "to the duplicated secondary stack\n"); } - if (hybrid->has_mmap() && !hybrid->materialized_cold_experts) { + if (hybrid->has_mmap() && hybrid->streams_cold_experts()) { size_t max_expert_bytes = 0; for (const auto & layer : hybrid->layers) { const size_t per_expert_bytes = layer.fused_gate_up @@ -2419,7 +2419,9 @@ bool DeepSeek4Backend::init_hybrid_model() { w_.moe_hybrid = true; const int total_cold = w_.n_layer * w_.n_expert - moe_placement_.total_hot; const char * cold_backend = - moe_hybrid_->cold_backend_kind == MoeHybridColdBackend::Gpu ? "gpu" : "cpu"; + moe_hybrid_->cold_backend_kind == MoeHybridColdBackend::Gpu ? "gpu" + : moe_hybrid_->cold_backend_kind == MoeHybridColdBackend::None ? "none" + : "cpu"; std::fprintf(stderr, "[deepseek4] hybrid experts ready: hot=%d cold=%d cold_backend=%s%s\n", moe_placement_.total_hot, total_cold, cold_backend, ""); return true; diff --git a/server/test/test_moe_hybrid_storage.cpp b/server/test/test_moe_hybrid_storage.cpp index 1a26a33a0..342d23b93 100644 --- a/server/test/test_moe_hybrid_storage.cpp +++ b/server/test/test_moe_hybrid_storage.cpp @@ -124,7 +124,7 @@ TEST_CASE(MoeHybridStorageFixture, cold_owner_none_refuses_a_shared_expert_in_th // caller sums across owners. The evaluator refuses before touching a // backend instead of double counting. const float cur[8] = {}; - const int32_t ids[2] = {0, -1}; + const int32_t ids[2] = {0, 1}; const float weights[2] = {1.0f, 1.0f}; std::vector out; std::string err; @@ -140,6 +140,22 @@ TEST_CASE(MoeHybridStorageFixture, cold_owner_none_refuses_a_shared_expert_in_th out.assign(3, 1.0f); CHECK(eval_moe_shared_expert_batched(nullptr, cfg, desc, storage, cur, -1, out)); CHECK(out.empty()); + CHECK(!eval_moe_shared_expert_batched(nullptr, cfg, desc, storage, cur, 2, out, &err)); + CHECK(err.find("GPU backend") != std::string::npos); +} + +TEST_CASE(MoeHybridStorageFixture, cold_owner_none_does_not_stream_cold_experts) { + MoeHybridStorage storage; + storage.materialized_cold_experts = false; + storage.cold_backend_kind = MoeHybridColdBackend::Gpu; + CHECK(storage.streams_cold_experts()); + storage.cold_backend_kind = MoeHybridColdBackend::Cpu; + CHECK(storage.streams_cold_experts()); + storage.cold_backend_kind = MoeHybridColdBackend::None; + CHECK(!storage.streams_cold_experts()); + storage.materialized_cold_experts = true; + storage.cold_backend_kind = MoeHybridColdBackend::Gpu; + CHECK(!storage.streams_cold_experts()); } TEST_CASE(MoeHybridStorageFixture, expert_residency_tracks_model_sized_expert_sets) {