Skip to content

vllm.models.mistral.mistral_large4.amd.model

Mistral Large 4: MLA + DSA sparse attention, MoE, NoPE layers, KV sharing.

The text model is hosted by PixtralForConditionalGeneration for the multimodal checkpoint.

Classes:

Functions:

  • defers_allreduce –

    Whether sublayers leave their outputs TP-partial and fuse the all-reduce

MistralLarge4DecoderLayer

Bases: Module

Methods:

  • forward –

    Run the layer on the outputs of input_norm.

  • input_norm –

    Fold the previous layer's output into the residual stream and

  • residual_norm –

    Add a sublayer output to the residual stream and normalize; with

Source code in vllm/models/mistral/mistral_large4/amd/model.py
class MistralLarge4DecoderLayer(nn.Module):
    def __init__(
        self,
        vllm_config: VllmConfig,
        config: PreTrainedConfig,
        spec: MistralLayerSpec,
        prefix: str,
        topk_indices_buffer: torch.Tensor | None,
        index_group_builder: SparseMLAIndexGroupBuilder | None,
        zero_kv_inputs: tuple[torch.Tensor, torch.Tensor],
        quant_config: QuantizationConfig | None = None,
        non_causal: bool = False,
        sliding_window: int | None = None,
    ) -> None:
        super().__init__()
        if quant_config is None:
            quant_config = vllm_config.quant_config
        self.defer_allreduce = defers_allreduce(vllm_config)
        self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        self.self_attn = MistralMLAAttention(
            vllm_config,
            config,
            spec,
            prefix=f"{prefix}.self_attn",
            topk_indices_buffer=topk_indices_buffer,
            index_group_builder=index_group_builder,
            zero_kv_inputs=zero_kv_inputs,
            quant_config=quant_config,
            non_causal=non_causal,
            sliding_window=sliding_window,
            reduce_results=not self.defer_allreduce,
        )
        self.post_attention_layernorm = RMSNorm(
            config.hidden_size, eps=config.rms_norm_eps
        )
        self.mlp: nn.Module
        if spec.is_moe:
            self.mlp = MistralMoE(
                vllm_config,
                config,
                prefix=f"{prefix}.mlp",
                reduce_results=not self.defer_allreduce,
            )
        else:
            self.mlp = MistralMLP(
                config.hidden_size,
                config.intermediate_size,
                quant_config,
                reduce_results=not self.defer_allreduce,
                prefix=f"{prefix}.mlp",
            )

    def residual_norm(
        self, norm: RMSNorm, hidden_states: torch.Tensor, residual: torch.Tensor
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Add a sublayer output to the residual stream and normalize; with
        `defer_allreduce` the output is a TP-partial sum reduced here."""
        if self.defer_allreduce:
            return fused_allreduce_rms_norm(hidden_states, residual, norm)
        return norm(hidden_states, residual)

    def input_norm(
        self, hidden_states: torch.Tensor, residual: torch.Tensor | None
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Fold the previous layer's output into the residual stream and
        normalize; the returned residual is the stream entering this layer."""
        if residual is None:
            return self.input_layernorm(hidden_states), hidden_states
        return self.residual_norm(self.input_layernorm, hidden_states, residual)

    def forward(
        self,
        positions: torch.Tensor,
        hidden_states: torch.Tensor,
        residual: torch.Tensor,
        log_positions: torch.Tensor,
        rotary_emb: MistralRotaryEmbedding,
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Run the layer on the outputs of `input_norm`."""
        hidden_states = self.self_attn(
            positions, hidden_states, log_positions, rotary_emb
        )
        hidden_states, residual = self.residual_norm(
            self.post_attention_layernorm, hidden_states, residual
        )
        return self.mlp(hidden_states), residual

forward(positions, hidden_states, residual, log_positions, rotary_emb)

Run the layer on the outputs of input_norm.

Source code in vllm/models/mistral/mistral_large4/amd/model.py
def forward(
    self,
    positions: torch.Tensor,
    hidden_states: torch.Tensor,
    residual: torch.Tensor,
    log_positions: torch.Tensor,
    rotary_emb: MistralRotaryEmbedding,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Run the layer on the outputs of `input_norm`."""
    hidden_states = self.self_attn(
        positions, hidden_states, log_positions, rotary_emb
    )
    hidden_states, residual = self.residual_norm(
        self.post_attention_layernorm, hidden_states, residual
    )
    return self.mlp(hidden_states), residual

input_norm(hidden_states, residual)

Fold the previous layer's output into the residual stream and normalize; the returned residual is the stream entering this layer.

Source code in vllm/models/mistral/mistral_large4/amd/model.py
def input_norm(
    self, hidden_states: torch.Tensor, residual: torch.Tensor | None
) -> tuple[torch.Tensor, torch.Tensor]:
    """Fold the previous layer's output into the residual stream and
    normalize; the returned residual is the stream entering this layer."""
    if residual is None:
        return self.input_layernorm(hidden_states), hidden_states
    return self.residual_norm(self.input_layernorm, hidden_states, residual)

residual_norm(norm, hidden_states, residual)

Add a sublayer output to the residual stream and normalize; with defer_allreduce the output is a TP-partial sum reduced here.

Source code in vllm/models/mistral/mistral_large4/amd/model.py
def residual_norm(
    self, norm: RMSNorm, hidden_states: torch.Tensor, residual: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
    """Add a sublayer output to the residual stream and normalize; with
    `defer_allreduce` the output is a TP-partial sum reduced here."""
    if self.defer_allreduce:
        return fused_allreduce_rms_norm(hidden_states, residual, norm)
    return norm(hidden_states, residual)

_load_weights(model, mistral_model, weights)

Load a Mistral consolidated checkpoint into model.

model holds mistral_model as model.model and the LM head as model.lm_head.

Source code in vllm/models/mistral/mistral_large4/amd/model.py
def _load_weights(
    model: nn.Module,
    mistral_model: MistralLarge4Model,
    weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
    """Load a Mistral consolidated checkpoint into `model`.

    `model` holds `mistral_model` as `model.model` and the LM head as
    `model.lm_head`.
    """
    config = mistral_model.config
    params = dict(model.named_parameters())
    expert_mapping = fused_moe_make_expert_params_mapping(
        model,
        ckpt_gate_proj_name="w1",
        ckpt_down_proj_name="w2",
        ckpt_up_proj_name="w3",
        num_experts=getattr(config, "n_routed_experts", None) or 0,
    )
    loaded: set[str] = set()
    lerope_alpha: torch.Tensor | None = None

    for ckpt_name, weight in weights:
        if ckpt_name == "lerope.alpha":
            lerope_alpha = weight
            continue
        if ckpt_name.startswith("mtp_"):
            # Speculative-decoding heads are loaded by their draft model.
            continue
        # Layers beyond the configured depth (e.g. a truncated debug model).
        layer = re.match(r"layers\.(\d+)\.", ckpt_name)
        if layer is not None and int(layer.group(1)) >= config.num_hidden_layers:
            continue
        name = _to_vllm_weight_name(ckpt_name)

        if ".mlp.experts." in name:
            is_expert_weight = False
            for param_name, weight_name, expert_id, shard_id in expert_mapping:
                if weight_name not in name:
                    continue
                is_expert_weight = True
                mapped = name.replace(weight_name, param_name)
                param = params[mapped]
                loader = cast(Callable[..., bool], param.weight_loader)
                if loader(
                    param,
                    weight,
                    mapped,
                    shard_id=shard_id,
                    expert_id=expert_id,
                    return_success=True,
                ):
                    loaded.add(mapped)
                    break
            else:
                # A known expert weight that is not local to this EP rank is skipped.
                if not is_expert_weight:
                    raise ValueError(f"Unexpected expert weight: {ckpt_name}")
            continue

        for param_part, ckpt_part, stacked_shard_id in _STACKED_PARAMS:
            if ckpt_part not in name:
                continue
            name = name.replace(ckpt_part, param_part)
            param = params[name]
            if stacked_shard_id < 0:
                param.weight_loader(param, weight)
            else:
                param.weight_loader(param, weight, stacked_shard_id)
            break
        else:
            param = params[name]
            getattr(param, "weight_loader", default_weight_loader)(param, weight)
        loaded.add(name)

    # The default loader skips this check for quantized models.
    if missing := sorted(set(params) - loaded):
        raise ValueError(f"Parameters not loaded from the checkpoint: {missing}")

    if getattr(config, "lerope", False):
        assert lerope_alpha is not None, "Checkpoint is missing lerope.alpha"
        mistral_model.rotary_emb.set_lerope_alpha(lerope_alpha)
    return loaded

defers_allreduce(vllm_config)

Whether sublayers leave their outputs TP-partial and fuse the all-reduce into the next residual RMSNorm.

Source code in vllm/models/mistral/mistral_large4/amd/model.py
def defers_allreduce(vllm_config: VllmConfig) -> bool:
    """Whether sublayers leave their outputs TP-partial and fuse the all-reduce
    into the next residual RMSNorm."""
    parallel_config = vllm_config.parallel_config
    return bool(
        vllm_config.compilation_config.pass_config.fuse_allreduce_rms
        and vllm_config.model_config.dtype == torch.bfloat16
        and not parallel_config.enable_expert_parallel
        and not parallel_config.use_sequence_parallel_moe
    )