diff --git a/server/src/common/moe_hybrid_ffn_eval.cpp b/server/src/common/moe_hybrid_ffn_eval.cpp index f98b7a98c..666f0c4de 100644 --- a/server/src/common/moe_hybrid_ffn_eval.cpp +++ b/server/src/common/moe_hybrid_ffn_eval.cpp @@ -1650,6 +1650,23 @@ static bool build_cached_cold_batched_graph( return true; } +// Cold owner None evaluates a routed partial that the caller sums across +// owners. The shared expert is replicated on every owner, so letting it into +// that partial would count it once per owner. Refuse such a desc instead of +// silently double counting; see MoeHybridColdBackend::None. +static bool none_owner_desc_ok(const MoeHybridLayerStorage & storage, + const MoeLayerDesc & desc, std::string * err) { + if (storage.cold_backend_kind != MoeHybridColdBackend::None || + !desc.has_shared_expert()) { + return true; + } + if (err) { + *err = "cold owner None: pass the routed desc without the shared expert " + "and add eval_moe_shared_expert_batched() after the reduction"; + } + return false; +} + bool eval_moe_hybrid_ffn_single( ggml_backend_t gpu_backend, const MoeHybridConfig & cfg, @@ -1665,6 +1682,7 @@ bool eval_moe_hybrid_ffn_single( std::string * err) { if (telemetry) *telemetry = {}; + if (!none_owner_desc_ok(storage, desc, err)) return false; const auto ffn_wall_t0 = HybridClock::now(); const auto partition_t0 = HybridClock::now(); @@ -1674,6 +1692,11 @@ bool eval_moe_hybrid_ffn_single( std::vector cold_weights; for (int i = 0; i < n_selected; ++i) { const int32_t gid = selected_ids[i]; + // Cold owner None: routes masked to -1 by the cluster runtime are + // evaluated elsewhere and contribute zero here. + if (gid < 0 && storage.cold_backend_kind == MoeHybridColdBackend::None) { + continue; + } if (gid < 0 || gid >= (int32_t)storage.hot_local_by_global.size()) { if (err) *err = "selected id out of range"; return false; @@ -3546,6 +3569,7 @@ bool eval_moe_hybrid_ffn_batched( ggml_tensor * cur_backend, const MoeHybridDeviceOutputs * device_outputs) { if (telemetry) *telemetry = {}; + if (!none_owner_desc_ok(storage, desc, err)) return false; const bool materialized_cold = storage.down_cold || storage.gate_up_cold; if (cur_host && compact_materialized_experts_enabled() && materialized_cold && !expert_compute && n_tokens > 0 && n_tokens <= 4) { @@ -3576,6 +3600,51 @@ bool eval_moe_hybrid_ffn_batched( : storage.gate_cold ? (int)storage.gate_cold->ne[2] : 0; const bool cold_on_gpu = storage.cold_backend_kind == MoeHybridColdBackend::Gpu; + // Cold owner None (a cluster rank): every route that survived masking is + // resident here and there is no second owner, so the whole batch can be + // packed by expert into ONE graph per layer. Without this a reduced hot + // stack falls into the sub-batch loop far below, whose size is + // min(mmq_safe_sub_batch(), prefill limit) = 1 on gfx1151 - one graph per + // token per layer. Measured on a 1517-token prompt: 25.9 s of FFN against + // 7.6 s for a single node's whole prefill graph. Expert-major packing is + // also what keeps the reduced stack off the MMQ full-batch path that + // mmq_safe_full_batch=false exists to avoid. + const bool hot_only_expert_major = + !expert_compute && + storage.cold_backend_kind == MoeHybridColdBackend::None && + !storage.gate_cold && !storage.gate_up_cold && !storage.down_cold && + n_hot_stack > 0 && + moe_expert_major_prefill_enabled(n_tokens); + if (hot_only_expert_major) { + static std::once_flag logged; + std::call_once(logged, [n_tokens, n_hot_stack] { + std::fprintf(stderr, + "[hybrid-ffn] hot-only expert-major batch active tokens=%d " + "stack=%d (no cold owner)\n", + n_tokens, n_hot_stack); + }); + const auto wall_t0 = HybridClock::now(); + std::string owner_err; + const bool ok = eval_moe_owner_expert_major_batched( + gpu_backend, cfg, desc, + storage.gate_hot, storage.up_hot, storage.down_hot, + storage.gate_up_hot, storage.hot_local_by_global, + cur_host, selected_ids, selected_weights, n_tokens, + /*include_shared=*/false, out, &owner_err, + cur_backend, gpu_backend, + /*device_output=*/nullptr, /*device_output_owner=*/nullptr, + p_hot_alloc); + if (!ok) { + if (err) *err = owner_err; + return false; + } + if (telemetry) { + const auto done = HybridClock::now(); + telemetry->hot_us += elapsed_us(wall_t0, done); + telemetry->ffn_wall_us += elapsed_us(wall_t0, done); + } + return true; + } const bool inprocess_expert_major = !expert_compute && moe_expert_major_prefill_enabled(n_tokens) && cold_on_gpu && storage.cold_backend && @@ -4009,6 +4078,7 @@ bool eval_moe_hybrid_ffn_gpu_resident( MoeExpertCompute * expert_compute, const MoeExpertLayer * expert_layer) { + if (!none_owner_desc_ok(storage, desc, nullptr)) return false; const int n_embd = cfg.n_embd; // ── Partition into hot/cold ── @@ -4021,6 +4091,7 @@ bool eval_moe_hybrid_ffn_gpu_resident( for (int i = 0; i < n_selected; ++i) { const int32_t gid = selected_ids[i]; + if (gid < 0 && storage.cold_backend_kind == MoeHybridColdBackend::None) continue; if (gid < 0 || gid >= (int32_t)storage.hot_local_by_global.size()) return false; const int32_t hot_local = storage.hot_local_by_global[(size_t)gid]; if (hot_local >= 0) { @@ -4271,4 +4342,80 @@ bool eval_moe_hybrid_ffn_gpu_resident( return true; } +// ── Shared expert only ── +// Cluster expert-parallel evaluates the routed partial without the shared +// expert (the MoeLayerDesc handed to the routed path has the shexp tensors +// cleared), all-reduces it, and adds this locally computed term afterwards. +// The graph is cached per n_tokens in storage.shared_batched_graph, which +// release_graph_caches() already frees. +bool eval_moe_shared_expert_batched( + ggml_backend_t gpu_backend, + const MoeHybridConfig & cfg, + const MoeLayerDesc & desc, + MoeHybridLayerStorage & storage, + const float * cur_host, + int n_tokens, + std::vector & out, + std::string * err) { + const int n_embd = cfg.n_embd; + if (n_tokens <= 0) { + out.clear(); + return true; + } + out.assign((size_t)n_embd * (size_t)n_tokens, 0.0f); + if (!desc.ffn_up_shexp || !desc.ffn_gate_shexp || !desc.ffn_down_shexp) { + return true; + } + if (!cur_host) { + if (err) *err = "shared expert requires a host activation"; + return false; + } + if (!gpu_backend) { + if (err) *err = "shared expert requires a GPU backend"; + return false; + } + + CachedHotBatchedGraph & g = storage.shared_batched_graph; + if (!g.valid() || g.n_tokens != n_tokens) { + g.free(); + g.n_tokens = n_tokens; + ggml_init_params ip{}; + ip.mem_size = 4 * 1024 * 1024; + ip.mem_buffer = nullptr; + ip.no_alloc = true; + g.ctx = ggml_init(ip); + if (!g.ctx) { + if (err) *err = "shared expert ggml_init failed"; + return false; + } + g.inp = ggml_new_tensor_2d(g.ctx, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_input(g.inp); + g.output = build_shared_expert_subgraph(g.ctx, desc, g.inp, cfg.swiglu_clamp); + if (!g.output) { + g.free(); + if (err) *err = "shared expert subgraph build failed"; + return false; + } + g.gf = ggml_new_graph_custom(g.ctx, 512, false); + ggml_set_output(g.output); + ggml_build_forward_expand(g.gf, g.output); + g.alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(gpu_backend)); + if (!g.alloc || !ggml_gallocr_alloc_graph(g.alloc, g.gf)) { + g.free(); + if (err) *err = "shared expert gallocr failed"; + return false; + } + } + + ggml_backend_tensor_set(g.inp, cur_host, 0, + sizeof(float) * (size_t)n_embd * (size_t)n_tokens); + if (ggml_backend_graph_compute(gpu_backend, g.gf) != GGML_STATUS_SUCCESS) { + if (err) *err = "shared expert compute failed"; + return false; + } + ggml_backend_tensor_get(g.output, out.data(), 0, + sizeof(float) * (size_t)n_embd * (size_t)n_tokens); + return true; +} + } // namespace luce::common diff --git a/server/src/common/moe_hybrid_ffn_eval.h b/server/src/common/moe_hybrid_ffn_eval.h index 28b4b4629..cef96efc6 100644 --- a/server/src/common/moe_hybrid_ffn_eval.h +++ b/server/src/common/moe_hybrid_ffn_eval.h @@ -388,6 +388,22 @@ bool build_cached_cold_graph( float swiglu_clamp = 0.0f, ggml_mixed_mmq_policy mixed_mmq_policy = GGML_MIXED_MMQ_DEFAULT); +// Shared expert only, batched [n_embd, n_tokens] on the GPU backend. Used by +// the cluster expert-parallel path, which evaluates routed experts without +// the shared term (MoeLayerDesc with shexp tensors cleared), all-reduces the +// routed partial across ranks and adds this local result afterwards. Cached +// per n_tokens in storage.shared_batched_graph. `out` is zero-filled when the +// layer has no shared expert. +bool eval_moe_shared_expert_batched( + ggml_backend_t gpu_backend, + const MoeHybridConfig & cfg, + const MoeLayerDesc & desc, + MoeHybridLayerStorage & storage, + const float * cur_host, + int n_tokens, + std::vector & out, + std::string * err = nullptr); + // Build cached hot-only batched graph for prefill (n_tokens=MMQ_SAFE_SUB_BATCH). bool build_cached_hot_batched_graph( CachedHotBatchedGraph & out, diff --git a/server/src/common/moe_hybrid_storage.cpp b/server/src/common/moe_hybrid_storage.cpp index 7ee0ffaff..12d85ae6e 100644 --- a/server/src/common/moe_hybrid_storage.cpp +++ b/server/src/common/moe_hybrid_storage.cpp @@ -268,7 +268,7 @@ bool MoeHybridStorage::matches(const MoeHybridConfig & cfg) const { (int)layers.size() == cfg.n_layer && cold_backend_kind == cfg.cold_expert_backend && materialized_hot_experts == cfg.materialize_hot_experts && - materialized_cold_experts == cfg.materialize_cold_experts && + materialized_cold_experts == cfg.materializes_cold_experts() && mixed_mmq_policy == cfg.mixed_mmq_policy; } @@ -303,11 +303,17 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, out.cold_backend_kind = cfg.cold_expert_backend; out.materialized_hot_experts = cfg.materialize_hot_experts; out.mixed_mmq_policy = cfg.mixed_mmq_policy; - out.materialized_cold_experts = cfg.materialize_cold_experts; - out.cold_backend = cfg.cold_expert_backend == MoeHybridColdBackend::Gpu - ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) - : out.cpu_backend; - if (!out.cold_backend) { + out.materialized_cold_experts = cfg.materializes_cold_experts(); + // Cold owner None (cluster expert-parallel): non-resident routes are + // reduced by another process, so there is no cold backend, no cold + // buffer and no cold expert map on this side. + const bool no_cold_owner = + cfg.cold_expert_backend == MoeHybridColdBackend::None; + out.cold_backend = no_cold_owner ? nullptr + : cfg.cold_expert_backend == MoeHybridColdBackend::Gpu + ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) + : out.cpu_backend; + if (!out.cold_backend && !no_cold_owner) { if (err) *err = "failed to select cold expert backend"; return false; } @@ -346,10 +352,12 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, is_hot[(size_t)expert] = 1; } dst.decode_hot_local_by_global = dst.hot_local_by_global; - for (int expert = 0; expert < cfg.n_expert; ++expert) { - if (duplicate_hot_on_cold || !is_hot[(size_t)expert]) { - dst.cold_local_by_global[(size_t)expert] = (int32_t)dst.cold_expert_ids.size(); - dst.cold_expert_ids.push_back((int32_t)expert); + if (!no_cold_owner) { + for (int expert = 0; expert < cfg.n_expert; ++expert) { + if (duplicate_hot_on_cold || !is_hot[(size_t)expert]) { + dst.cold_local_by_global[(size_t)expert] = (int32_t)dst.cold_expert_ids.size(); + dst.cold_expert_ids.push_back((int32_t)expert); + } } } dst.decode_cold_local_by_global = dst.cold_local_by_global; @@ -423,7 +431,7 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, } // Allocate cold expert tensors on the selected cold backend. - if (cold_count > 0 && cfg.materialize_cold_experts) { + if (cold_count > 0 && cfg.materializes_cold_experts()) { ggml_init_params ip{}; ip.mem_size = 16 * ggml_tensor_overhead(); ip.mem_buffer = nullptr; @@ -521,11 +529,17 @@ bool build_moe_hybrid_storage_from_file( out.cold_backend_kind = cfg.cold_expert_backend; out.materialized_hot_experts = cfg.materialize_hot_experts; out.mixed_mmq_policy = cfg.mixed_mmq_policy; - out.materialized_cold_experts = cfg.materialize_cold_experts; - out.cold_backend = cfg.cold_expert_backend == MoeHybridColdBackend::Gpu - ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) - : out.cpu_backend; - if (!out.cold_backend) { + out.materialized_cold_experts = cfg.materializes_cold_experts(); + // Cold owner None (cluster expert-parallel): non-resident routes are + // reduced by another process, so there is no cold backend, no cold + // buffer and no cold expert map on this side. + const bool no_cold_owner = + cfg.cold_expert_backend == MoeHybridColdBackend::None; + out.cold_backend = no_cold_owner ? nullptr + : cfg.cold_expert_backend == MoeHybridColdBackend::Gpu + ? (cold_gpu_backend ? cold_gpu_backend : gpu_backend) + : out.cpu_backend; + if (!out.cold_backend && !no_cold_owner) { if (err) *err = "failed to select cold expert backend"; return false; } @@ -565,7 +579,7 @@ bool build_moe_hybrid_storage_from_file( is_hot[(size_t)expert] = 1; } dst.decode_hot_local_by_global = dst.hot_local_by_global; - if (allocate_cold) { + if (allocate_cold && !no_cold_owner) { for (int expert = 0; expert < cfg.n_expert; ++expert) { if (duplicate_hot_on_cold || !is_hot[(size_t)expert]) { dst.cold_local_by_global[(size_t)expert] = (int32_t)dst.cold_expert_ids.size(); @@ -654,7 +668,7 @@ bool build_moe_hybrid_storage_from_file( } // Allocate cold expert tensors on the selected cold backend. - if (allocate_cold && cold_count > 0 && cfg.materialize_cold_experts) { + if (allocate_cold && cold_count > 0 && cfg.materializes_cold_experts()) { ggml_init_params ip{}; ip.mem_size = 16 * ggml_tensor_overhead(); ip.mem_buffer = nullptr; @@ -712,7 +726,7 @@ bool build_moe_hybrid_storage_from_file( // slice buffers are gone. Retain the mapping for future streaming reads. if (readonly_file_mmap && readonly_file_fd >= 0 && moe_source_pageout_eligible( out.cold_backend_kind == MoeHybridColdBackend::Gpu, - cfg.materialize_hot_experts, cfg.materialize_cold_experts, + cfg.materialize_hot_experts, cfg.materializes_cold_experts(), allocate_cold && cold_count > 0 && dst.cold_buf != nullptr)) { if (dst.fused_gate_up) { advise_copied_source(readonly_file_mmap, readonly_file_mmap_size, fd.gate_up_exps, il, readonly_file_fd); diff --git a/server/src/common/moe_hybrid_storage.h b/server/src/common/moe_hybrid_storage.h index e9f775a3e..01555a133 100644 --- a/server/src/common/moe_hybrid_storage.h +++ b/server/src/common/moe_hybrid_storage.h @@ -202,6 +202,14 @@ struct MoeHybridStorage { bool materialized_cold_experts = true; ggml_mixed_mmq_policy mixed_mmq_policy = GGML_MIXED_MMQ_DEFAULT; MoeHybridPlacement placement; + + // Cold experts are streamed from the source file on demand. Cold owner + // None is not materialized either, but it has no cold experts at all, so + // it must not set up a streaming path. + bool streams_cold_experts() const { + return !materialized_cold_experts && + cold_backend_kind != MoeHybridColdBackend::None; + } std::vector layers; // Long heterogeneous prefill uses one routing graph and one owner graph diff --git a/server/src/common/moe_hybrid_types.h b/server/src/common/moe_hybrid_types.h index 3feedb3d6..8094fb46b 100644 --- a/server/src/common/moe_hybrid_types.h +++ b/server/src/common/moe_hybrid_types.h @@ -21,6 +21,15 @@ int query_gpu_compute_sm(); enum class MoeHybridColdBackend { Cpu, Gpu, + // No cold owner: non-resident routes contribute zero and are never + // materialized; the caller reduces the owners' partials outside this + // process (e.g. with ggml_cluster_allreduce). Storage allocates no cold + // buffers, evaluators build no cold graph, never fall back to CPU or + // streamed evaluation for non-resident routes and never swap experts. + // The routed partial never carries the shared expert, which is replicated + // on every owner: pass a MoeLayerDesc without shexp tensors and add + // eval_moe_shared_expert_batched() once, after the reduction. + None, }; // ─── MoE architecture config (model-agnostic) ────────────────────────── @@ -39,6 +48,13 @@ struct MoeHybridConfig { bool materialize_hot_experts = true; bool materialize_cold_experts = true; + // Cold owner None has no cold experts, so nothing to materialize whatever + // materialize_cold_experts says. + bool materializes_cold_experts() const { + return materialize_cold_experts && + cold_expert_backend != MoeHybridColdBackend::None; + } + // When true, MMQ mul_mat_id works correctly with reduced hot stacks // (n_hot < n_expert). Safe on sm_80+ (Ampere/Ada/Hopper/Blackwell). // On sm_75 (Turing) and gfx1151, the kernel has illegal memory accesses diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index d45d9b762..38652ed66 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1727,7 +1727,7 @@ bool DeepSeek4Backend::init() { return false; } if (cfg_.paged_attention && moe_hybrid_ && - !moe_hybrid_->materialized_cold_experts) { + moe_hybrid_->streams_cold_experts()) { std::fprintf(stderr, "[deepseek4] paged serving requires statically materialized " "expert ownership; enable in-process LUCE_DS4_MOE_TP\n"); @@ -2392,7 +2392,7 @@ bool DeepSeek4Backend::init_hybrid_model() { "[deepseek4] speculative verifier routes all experts " "to the duplicated secondary stack\n"); } - if (hybrid->has_mmap() && !hybrid->materialized_cold_experts) { + if (hybrid->has_mmap() && hybrid->streams_cold_experts()) { size_t max_expert_bytes = 0; for (const auto & layer : hybrid->layers) { const size_t per_expert_bytes = layer.fused_gate_up @@ -2419,7 +2419,9 @@ bool DeepSeek4Backend::init_hybrid_model() { w_.moe_hybrid = true; const int total_cold = w_.n_layer * w_.n_expert - moe_placement_.total_hot; const char * cold_backend = - moe_hybrid_->cold_backend_kind == MoeHybridColdBackend::Gpu ? "gpu" : "cpu"; + moe_hybrid_->cold_backend_kind == MoeHybridColdBackend::Gpu ? "gpu" + : moe_hybrid_->cold_backend_kind == MoeHybridColdBackend::None ? "none" + : "cpu"; std::fprintf(stderr, "[deepseek4] hybrid experts ready: hot=%d cold=%d cold_backend=%s%s\n", moe_placement_.total_hot, total_cold, cold_backend, ""); return true; diff --git a/server/src/deepseek4/deepseek4_loader.cpp b/server/src/deepseek4/deepseek4_loader.cpp index 82d227e82..d09195ac6 100644 --- a/server/src/deepseek4/deepseek4_loader.cpp +++ b/server/src/deepseek4/deepseek4_loader.cpp @@ -1376,11 +1376,24 @@ bool register_deepseek4_moe_hybrid_mix_tables( } if (!has_mix_experts) return true; + // Two storage shapes can be decoded. A GPU cold owner needs both halves + // materialized, because the primary and the secondary owner each get their + // own table. Cold owner None has no second owner at all: its resident set + // is exactly the hot experts, and ds4_register_compact_mix_tensor already + // returns success for a null tensor whose expert-id list is empty, which is + // precisely how an absent cold owner presents itself. Only this check stood + // in the way. + const bool hot_only = + storage.cold_backend_kind == MoeHybridColdBackend::None && + !storage.materialized_cold_experts; + const bool gpu_owners = + storage.cold_backend_kind == MoeHybridColdBackend::Gpu && + storage.materialized_cold_experts; if (storage.layers.size() != w.layers.size() || - storage.cold_backend_kind != MoeHybridColdBackend::Gpu || !storage.materialized_hot_experts || - !storage.materialized_cold_experts) { - if (err) *err = "mixed expert qtypes require materialized GPU owners"; + !(hot_only || gpu_owners)) { + if (err) *err = "mixed expert qtypes require materialized hot experts with " + "either a materialized GPU cold owner or no cold owner"; return false; } diff --git a/server/test/test_moe_hybrid_storage.cpp b/server/test/test_moe_hybrid_storage.cpp index dac508628..342d23b93 100644 --- a/server/test/test_moe_hybrid_storage.cpp +++ b/server/test/test_moe_hybrid_storage.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include using namespace luce::common; @@ -77,6 +78,86 @@ TEST_CASE(MoeHybridStorageFixture, storage_identity_includes_mixed_mmq_policy) { REQUIRE(!storage.matches(cfg)); } +TEST_CASE(MoeHybridStorageFixture, cold_owner_none_implies_no_cold_materialization) { + MoeHybridConfig cfg; + cfg.cold_expert_backend = MoeHybridColdBackend::None; + // The flag keeps its default; None must not need it cleared by hand. + REQUIRE(cfg.materialize_cold_experts); + REQUIRE(!cfg.materializes_cold_experts()); + + MoeHybridStorage storage; + cfg.n_layer = storage.placement.n_layer = 1; + cfg.n_expert = storage.placement.n_expert = 2; + cfg.n_expert_used = storage.placement.n_expert_used = 1; + storage.placement.hot_counts = {0}; + storage.placement.hot_expert_ids = {{}}; + storage.layers.resize(1); + storage.cold_backend_kind = MoeHybridColdBackend::None; + // Storage built for None records no cold materialization, and a config + // that kept the default flag still identifies it. + storage.materialized_cold_experts = true; + REQUIRE(!storage.matches(cfg)); + storage.materialized_cold_experts = false; + REQUIRE(storage.matches(cfg)); + + cfg.cold_expert_backend = MoeHybridColdBackend::Gpu; + REQUIRE(cfg.materializes_cold_experts()); +} + +TEST_CASE(MoeHybridStorageFixture, cold_owner_none_refuses_a_shared_expert_in_the_routed_partial) { + auto ctx = std::unique_ptr( + ggml_init({1u << 16, nullptr, true}), ggml_free); + REQUIRE(ctx != nullptr); + MoeHybridConfig cfg; + cfg.n_embd = 4; + cfg.n_expert = 2; + cfg.n_expert_used = 1; + cfg.cold_expert_backend = MoeHybridColdBackend::None; + MoeHybridLayerStorage storage; + storage.cold_backend_kind = MoeHybridColdBackend::None; + MoeLayerDesc desc; + desc.ffn_gate_shexp = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, 4, 8); + desc.ffn_up_shexp = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, 4, 8); + desc.ffn_down_shexp = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, 8, 4); + + // Every owner would add the replicated shared expert into the partial the + // caller sums across owners. The evaluator refuses before touching a + // backend instead of double counting. + const float cur[8] = {}; + const int32_t ids[2] = {0, 1}; + const float weights[2] = {1.0f, 1.0f}; + std::vector out; + std::string err; + CHECK(!eval_moe_hybrid_ffn_batched(nullptr, nullptr, cfg, desc, storage, + cur, ids, weights, 2, out, &err)); + CHECK(err.find("shared expert") != std::string::npos); + err.clear(); + CHECK(!eval_moe_hybrid_ffn_single(nullptr, cfg, desc, storage, nullptr, + cur, ids, weights, 1, out, nullptr, &err)); + CHECK(err.find("shared expert") != std::string::npos); + + // A non-positive batch is empty, not a huge allocation. + out.assign(3, 1.0f); + CHECK(eval_moe_shared_expert_batched(nullptr, cfg, desc, storage, cur, -1, out)); + CHECK(out.empty()); + CHECK(!eval_moe_shared_expert_batched(nullptr, cfg, desc, storage, cur, 2, out, &err)); + CHECK(err.find("GPU backend") != std::string::npos); +} + +TEST_CASE(MoeHybridStorageFixture, cold_owner_none_does_not_stream_cold_experts) { + MoeHybridStorage storage; + storage.materialized_cold_experts = false; + storage.cold_backend_kind = MoeHybridColdBackend::Gpu; + CHECK(storage.streams_cold_experts()); + storage.cold_backend_kind = MoeHybridColdBackend::Cpu; + CHECK(storage.streams_cold_experts()); + storage.cold_backend_kind = MoeHybridColdBackend::None; + CHECK(!storage.streams_cold_experts()); + storage.materialized_cold_experts = true; + storage.cold_backend_kind = MoeHybridColdBackend::Gpu; + CHECK(!storage.streams_cold_experts()); +} + TEST_CASE(MoeHybridStorageFixture, expert_residency_tracks_model_sized_expert_sets) { MoeHybridLayerStorage storage; storage.reset_expert_vram_mask(320);