Skip to content

vllm.models.minimax_m3.nvidia.specialized_decode.draft_norm

The DSpark draft layers' Gemma RMSNorms fused with the MXFP8 quantize of the linear after them.

Each draft layer all-reduces two row-parallel outputs (o_proj, down_proj), normalizes the sum right after (with the residual add), and the next linear (gate_up_proj, the next layer's qkv_proj) quantizes it to MXFP8: the row norm and the quantize are two launches, ten times per draft step.

Here (FusedGemmaNormQuant) the stock all-reduce stays; the residual add, the norm and the quantize run in one launch, and the result is handed to the linear as a QuantizedActivation (also for the context states' norm before the context K/V projection). The norm is exactly that of drafter.GemmaRMSNormRows (the residual add rounded to the residual's dtype and stored, the sum of squares over the same 8192-wide block, x * rstd * (1 + w)), and the quantize is FlashInfer's MXFP8 quantize bit for bit (UE8M0 round-up, RN e4m3, the F8_128x4 scale layout).

Fusing the all-reduce as well (a peer-memory Lamport exchange inside the norm) measured slower at TP4 than FlashInfer's MNNVL all-reduce, which uses NVLS multicast, so it is not done.

Classes:

  • FusedGemmaNormQuant –

    The forward of a draft Gemma RMSNorm after the stock all-reduce, with

Functions:

FusedGemmaNormQuant

The forward of a draft Gemma RMSNorm after the stock all-reduce, with the next linear's MXFP8 quantize in the same launch: residual add, norm and quantize, bit for bit as the row norm kernel followed by FlashInfer's quantize.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/draft_norm.py
class FusedGemmaNormQuant:
    """The forward of a draft Gemma RMSNorm after the stock all-reduce, with
    the next linear's MXFP8 quantize in the same launch: residual add, norm
    and quantize, bit for bit as the row norm kernel followed by FlashInfer's
    quantize."""

    def __init__(self, norm: nn.Module, consumer: nn.Module | None, stock, guard=None):
        self._norm = norm
        self._consumer = consumer
        self._quant: bool | None = None
        self._guard = guard  # () -> bool: the consumer is the norm's only reader
        self.stock = stock

    def __call__(self, x: torch.Tensor, residual: torch.Tensor | None = None):
        if self._quant is None:
            self._quant = _takes_mxfp8(self._consumer) and (
                self._guard is None or self._guard()
            )
        if (
            not self._quant
            or x.dim() != 2
            or not 0 < x.shape[0] <= MAX_ROWS
            or x.dtype != torch.bfloat16
            or x.stride(-1) != 1
            or x.shape[1] % 32
            or x.shape[1] > _BLOCK
            or (
                residual is not None
                and (residual.shape != x.shape or residual.stride(-1) != 1)
            )
        ):
            return self.stock(x, residual)
        scratch = _SCRATCH.get(x.device)
        if scratch is None or scratch.shape[1] != x.shape[1]:
            scratch = _SCRATCH[x.device] = torch.empty(
                MAX_ROWS, x.shape[1], dtype=torch.bfloat16, device=x.device
            )
        quant = _quant_buffers(x)
        norm = self._norm
        res = residual if residual is not None else x  # x is rewritten unchanged
        _norm_quant_kernel[(x.shape[0],)](
            x,
            x.stride(0),
            res,
            res.stride(0),
            norm.weight,
            scratch,
            x.shape[1],
            norm.variance_epsilon,
            quant[0],
            quant[1],
            HAS_RESIDUAL=residual is not None,
            BLOCK=_BLOCK,
            num_warps=8,
        )
        out = _quantized(quant, x)
        if residual is None:
            return out
        return out, residual

attach_fused_norms(model, stock_forward)

Fuse each draft norm with the MXFP8 quantize of the linear it feeds. stock_forward(norm): the (already patched) forward of a norm. Returns the number of fused norms.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/draft_norm.py
def attach_fused_norms(model: nn.Module, stock_forward) -> int:
    """Fuse each draft norm with the MXFP8 quantize of the linear it feeds.
    `stock_forward(norm)`: the (already patched) forward of a norm. Returns
    the number of fused norms."""
    reason = fused_norms_unsupported(model)
    if reason:
        logger.info("DSpark draft norms stay separate: %s.", reason)
        return 0
    inner = model.model
    count = 0
    for layer in inner.layers:
        attn = layer.self_attn
        pairs = [
            # The draft attention reads its input's dtype for the attention output
            # unless it states one (FLASHINFER_TS with an FP8 query): only then may the
            # input norm hand it a QuantizedActivation.
            (
                layer.input_layernorm,
                attn.qkv_proj,
                lambda attn=attn: getattr(attn, "_attn_output_dtype", None) is not None,
            ),
            (layer.post_attention_layernorm, layer.mlp.gate_up_proj, None),
        ]
        for norm, consumer, guard in pairs:
            norm.forward = FusedGemmaNormQuant(
                norm, consumer, stock_forward(norm), guard=guard
            )
            count += 1
    # The context states' norm feeds the fused context K/V projection.
    hidden_norm = getattr(inner, "hidden_norm", None)
    ctx_proj = getattr(inner, "_context_kv_proj", None)
    if hidden_norm is not None and ctx_proj is not None:
        hidden_norm.forward = FusedGemmaNormQuant(
            hidden_norm,
            ctx_proj,
            stock_forward(hidden_norm),
            # A BF16 fused K/V weight takes precedence over the projection.
            guard=lambda: getattr(inner, "_fused_kv_weight", None) is None,
        )
        count += 1
    logger.info("DSpark draft norms: %d with the MXFP8 quantize fused.", count)
    return count

fused_norms_unsupported(model)

Why the draft norms keep their stock forward ("" when they fuse).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/draft_norm.py
def fused_norms_unsupported(model: nn.Module) -> str:
    """Why the draft norms keep their stock forward ("" when they fuse)."""
    layers = getattr(getattr(model, "model", None), "layers", None)
    if not layers:
        return "no draft layers"
    for layer in layers:
        for norm in (layer.input_layernorm, layer.post_attention_layernorm):
            if type(norm).__name__ != "MiniMAXGemmaRMSNorm":
                return f"a {type(norm).__name__} draft norm"
    if type(getattr(model.model, "norm", None)).__name__ != "MiniMAXGemmaRMSNorm":
        return "a final norm that is not a Gemma RMSNorm"
    if model.model.norm.weight.dtype != torch.bfloat16:
        return f"a {model.model.norm.weight.dtype} draft model"
    return ""