Skip to content

vllm.models.deepseek_v32.nvidia.glm52_low_latency_gemm

GLM-5.2 decode GEMM selection for unquantized BF16 on SM100/SM103/SM107.

Dispatch is by local (N, K) weight shape and token count M. GLM52_PROJECTIONS was measured on B300 (SM103), GLM52_PROJECTIONS_SM107 on VRNVL72 (SM107) and GLM52_PROJECTIONS_SM100 on B200 (SM100). The SM107 table also covers the TP4 projections (q_b, o_proj, shared expert) and the replicated indexer projections, which run as BF16 GEMMs at decode batch sizes.

Functions:

  • build_glm52_plan –

    Plan for a weight the walk below cannot reach (a plain nn.Linear).

_WarpSplitKGemm

Bases: VllmJitKernel['_WarpSplitKGemm.CompileKey']

Compile keys of FlashInfer's warp split-K: M is dynamic, so one per (N, K, tactic). Compiling fills FlashInfer's own cache, which run_warp_splitk_dense reads, without launching the kernel.

Source code in vllm/models/deepseek_v32/nvidia/glm52_low_latency_gemm.py
class _WarpSplitKGemm(VllmJitKernel["_WarpSplitKGemm.CompileKey"]):
    """Compile keys of FlashInfer's warp split-K: M is dynamic, so one per
    `(N, K, tactic)`. Compiling fills FlashInfer's own cache, which
    `run_warp_splitk_dense` reads, without launching the kernel."""

    @dataclass(frozen=True, slots=True)
    class CompileKey:
        n: int
        k: int
        tactic: WarpSplitKConfig

    def dispatch(  # type: ignore[override]
        self, *, N: int, K: int, tactic: WarpSplitKConfig
    ) -> CompileKey:
        return self.CompileKey(n=N, k=K, tactic=tactic)

    def get_warmup_keys(
        self, *, plans: Iterable[tuple[int, int, WarpSplitKConfig]]
    ) -> list[CompileKey]:
        rows = tuple(dict(N=n, K=k, tactic=tactic) for n, k, tactic in plans)
        if not rows:
            return []
        return self._trace_dispatch(self.dispatch)(zip_inputs(*rows))

    def compile(self, compile_key: CompileKey) -> None:
        if compile_key in self._compiled_cache:
            return
        module = _warp_splitk_module()
        assert module is not None
        # The arguments run_warp_splitk_dense(pdl=True) passes without a bias.
        self._compiled_cache[compile_key] = module._compile(
            torch.accelerator.current_device_index(),
            torch.bfloat16,
            compile_key.n,
            compile_key.k,
            module.WarpSplitKTactic(*compile_key.tactic),
            True,
            False,
        )

_low_latency_table()

Measured dispatch table for the current device, or None if unsupported.

Source code in vllm/models/deepseek_v32/nvidia/glm52_low_latency_gemm.py
def _low_latency_table() -> dict[tuple[int, int], GLM52ProjectionSpec] | None:
    """Measured dispatch table for the current device, or None if unsupported."""
    if current_platform.is_device_capability((10, 7)):
        return GLM52_PROJECTIONS_SM107
    if current_platform.is_device_capability((10, 3)):
        return GLM52_PROJECTIONS
    if current_platform.is_device_capability((10, 0)):
        return GLM52_PROJECTIONS_SM100
    return None

_warp_splitk_module() cached

FlashInfer's CuTe DSL warp split-K BF16 GEMM, called directly so every (N, K, M) runs its measured tactic: flashinfer.mm_bf16 takes no tactic and would use its autotuner or heuristic default instead.

Source code in vllm/models/deepseek_v32/nvidia/glm52_low_latency_gemm.py
@functools.cache
def _warp_splitk_module() -> Any | None:
    """FlashInfer's CuTe DSL warp split-K BF16 GEMM, called directly so every
    (N, K, M) runs its measured tactic: `flashinfer.mm_bf16` takes no tactic
    and would use its autotuner or heuristic default instead."""
    try:
        from flashinfer.gemm.kernels import dense_bf16_gemm_warp_splitk
    except Exception:
        return None
    return dense_bf16_gemm_warp_splitk

_warp_splitk_rows(measured, first_m=1)

Serve every M up to the largest swept M with the tactic swept at the smallest swept M' >= M.

Source code in vllm/models/deepseek_v32/nvidia/glm52_low_latency_gemm.py
def _warp_splitk_rows(
    measured: dict[int, WarpSplitKConfig], first_m: int = 1
) -> tuple[tuple[int, WarpSplitKConfig], ...]:
    """Serve every M up to the largest swept M with the tactic swept at the
    smallest swept M' >= M."""
    swept = sorted(measured)
    return tuple(
        (m, measured[next(s for s in swept if s >= m)])
        for m in range(first_m, swept[-1] + 1)
    )

build_glm52_plan(weight, dtype)

Plan for a weight the walk below cannot reach (a plain nn.Linear).

Source code in vllm/models/deepseek_v32/nvidia/glm52_low_latency_gemm.py
def build_glm52_plan(
    weight: torch.Tensor | None, dtype: torch.dtype
) -> dict[int, ResolvedCall] | None:
    """Plan for a weight the walk below cannot reach (a plain `nn.Linear`)."""
    if dtype != torch.bfloat16:
        return None
    table = _low_latency_table()
    if table is None:
        return None
    if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
        return None
    spec = table.get(tuple(weight.shape))
    if spec is None:
        return None
    _request_warmup(dtype, spec)
    return spec.build_plan()