Skip to content

vllm.models.qwen3_5.amd.model

Inference-only Qwen3.5 Series, AMD ROCm implementation.

Mirrors ../nvidia/model.py (eager + breakable CUDA graphs) with the ROCm counterparts of its fusions: Triton Gemma RMSNorm instead of FlashInfer's, the row-parallel all-reduce deferred into AITER's fused all-reduce + Gemma RMSNorm, and the fused QK-norm + RoPE + gate kernel. FlashInfer's FP8 / NVFP4 norm-quant fusions have no ROCm counterpart here, so the norms emit unquantized activations.

Classes:

Qwen3_5ForConditionalGeneration

Bases: Qwen3VLForConditionalGeneration, IsHybrid

Methods:

  • forward –

    Run forward pass for Qwen3.5.

Source code in vllm/models/qwen3_5/amd/model.py
@MULTIMODAL_REGISTRY.register_processor(
    Qwen3VLMultiModalProcessor,
    info=Qwen3_5ProcessingInfo,
    dummy_inputs=Qwen3VLDummyInputsBuilder,
)
class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid):
    supports_multimodal_pruning = True

    hf_to_vllm_mapper = (
        Qwen3VLForConditionalGeneration.hf_to_vllm_mapper
        | WeightsMapper(orig_to_new_prefix={"mtp.": None})
    )

    packed_modules_mapping = Qwen3VLForConditionalGeneration.packed_modules_mapping | {
        "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
        "in_proj_ba": ["in_proj_b", "in_proj_a"],
    }

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
        # protocols have not __init__ method, so we need to use nn.Module.__init__
        nn.Module.__init__(self)
        config: Qwen3_5Config = vllm_config.model_config.hf_config
        quant_config = vllm_config.quant_config
        multimodal_config = cast(
            MultiModalConfig, vllm_config.model_config.multimodal_config
        )

        self.config = config
        self.model_config = vllm_config.model_config
        self.multimodal_config = multimodal_config
        self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
        self.is_multimodal_pruning_enabled = (
            multimodal_config.is_multimodal_pruning_enabled()
        )
        self.video_pruning_rate = self.multimodal_config.video_pruning_rate
        self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)

        # attributes needed by EVS-related functions inherited from Qwen3-VL
        self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
        self.deepstack_num_level = (
            len(config.vision_config.deepstack_visual_indexes)
            if self.use_deepstack
            else 0
        )
        self.visual_dim = config.vision_config.out_hidden_size
        self.multiscale_dim = self.visual_dim * self.deepstack_num_level

        with self._mark_tower_model(vllm_config, {"image", "video"}):
            self.visual = Qwen3_VisionTransformer(
                config.vision_config,
                norm_eps=getattr(config, "rms_norm_eps", 1e-6),
                quant_config=quant_config,
                input_norm=build_mm_input_norm(self.model_config),
                prefix=maybe_prefix(prefix, "visual"),
            )

        with self._mark_language_model(vllm_config):
            self.language_model = Qwen3_5ForCausalLM(
                vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
            )

        self.make_empty_intermediate_tensors = (  # type: ignore[method-assign]
            self.language_model.make_empty_intermediate_tensors
        )

    def embed_input_ids(
        self,
        input_ids: torch.Tensor,
        multimodal_embeddings: MultiModalEmbeddings | None = None,
        *,
        is_multimodal: torch.Tensor | None = None,
    ) -> torch.Tensor:
        inputs_embeds = self._embed_text_input_ids(
            input_ids,
            self.language_model.embed_input_ids,
            is_multimodal=is_multimodal,
        )

        if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
            return inputs_embeds

        is_multimodal = _require_is_multimodal(is_multimodal)

        inputs_embeds = _merge_multimodal_embeddings(
            inputs_embeds=inputs_embeds,
            multimodal_embeddings=multimodal_embeddings,
            is_multimodal=is_multimodal,
        )

        return inputs_embeds

    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
        intermediate_tensors: IntermediateTensors | None = None,
        inputs_embeds: torch.Tensor | None = None,
        **kwargs: object,
    ) -> torch.Tensor | IntermediateTensors:
        """Run forward pass for Qwen3.5.

        With mrope enabled (default for Qwen3VL-class models), `positions`
        has shape `(3, seq_len)`, otherwise `(seq_len,)`.
        """
        if intermediate_tensors is not None:
            inputs_embeds = None

        hidden_states = self.language_model.model(
            input_ids=input_ids,
            positions=positions,
            intermediate_tensors=intermediate_tensors,
            inputs_embeds=inputs_embeds,
        )

        return hidden_states

    def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
        return self.language_model.compute_logits_local(hidden_states)

    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        loader = AutoWeightsLoader(self)
        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

    @classmethod
    def get_mamba_state_dtype_from_config(
        cls,
        vllm_config: "VllmConfig",
    ) -> tuple[torch.dtype, torch.dtype]:
        return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
            vllm_config.model_config.dtype,
            vllm_config.cache_config.mamba_cache_dtype,
            vllm_config.cache_config.mamba_ssm_cache_dtype,
        )

    @classmethod
    def get_mamba_state_shape_from_config(
        cls, vllm_config: "VllmConfig"
    ) -> tuple[tuple[int, int], tuple[int, int, int]]:
        parallel_config = vllm_config.parallel_config
        hf_config = vllm_config.model_config.hf_text_config
        tp_size = parallel_config.tensor_parallel_size
        num_spec = (
            vllm_config.speculative_config.num_speculative_tokens
            if vllm_config.speculative_config
            else 0
        )
        return MambaStateShapeCalculator.gated_delta_net_state_shape(
            tp_size,
            hf_config.linear_num_key_heads,
            hf_config.linear_num_value_heads,
            hf_config.linear_key_head_dim,
            hf_config.linear_value_head_dim,
            hf_config.linear_conv_kernel_dim,
            num_spec,
        )

    @classmethod
    def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
        return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func()

forward(input_ids, positions, intermediate_tensors=None, inputs_embeds=None, **kwargs)

Run forward pass for Qwen3.5.

With mrope enabled (default for Qwen3VL-class models), positions has shape (3, seq_len), otherwise (seq_len,).

Source code in vllm/models/qwen3_5/amd/model.py
def forward(
    self,
    input_ids: torch.Tensor,
    positions: torch.Tensor,
    intermediate_tensors: IntermediateTensors | None = None,
    inputs_embeds: torch.Tensor | None = None,
    **kwargs: object,
) -> torch.Tensor | IntermediateTensors:
    """Run forward pass for Qwen3.5.

    With mrope enabled (default for Qwen3VL-class models), `positions`
    has shape `(3, seq_len)`, otherwise `(seq_len,)`.
    """
    if intermediate_tensors is not None:
        inputs_embeds = None

    hidden_states = self.language_model.model(
        input_ids=input_ids,
        positions=positions,
        intermediate_tensors=intermediate_tensors,
        inputs_embeds=inputs_embeds,
    )

    return hidden_states

Qwen3_5RMSNorm

Bases: Module

Gemma-style RMS normalization backed by Triton kernels.

When residual is given, the fused add + norm returns the updated (x, residual) pair.

reduce_inputs: in forward_reduced, an x that comes with a residual is the un-reduced TP partial of a row-parallel linear, and the norm absorbs its all-reduce. An x without a residual (first decoder layer, MTP layer) is always complete.

Methods:

  • forward_reduced –

    forward with the preceding TP all-reduce fused in when

Source code in vllm/models/qwen3_5/amd/model.py
class Qwen3_5RMSNorm(nn.Module):
    """Gemma-style RMS normalization backed by Triton kernels.

    When `residual` is given, the fused add + norm returns the updated
    `(x, residual)` pair.

    `reduce_inputs`: in `forward_reduced`, an `x` that comes with a
    residual is the un-reduced TP partial of a row-parallel linear, and the
    norm absorbs its all-reduce. An `x` without a residual (first decoder
    layer, MTP layer) is always complete.
    """

    def __init__(
        self, hidden_size: int, eps: float = 1e-6, reduce_inputs: bool = False
    ) -> None:
        super().__init__()
        self.weight = nn.Parameter(torch.zeros(hidden_size))
        set_weight_attrs(self.weight, {"dummy_weight_value": 0.0})
        self.variance_epsilon = eps
        self.reduce_inputs = reduce_inputs

    def forward(
        self,
        x: torch.Tensor,
        residual: torch.Tensor | None = None,
    ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
        if residual is None:
            return gemma_rmsnorm(x, self.weight, self.variance_epsilon)
        return gemma_fused_add_rmsnorm(x, residual, self.weight, self.variance_epsilon)

    def forward_reduced(
        self, x: torch.Tensor, residual: torch.Tensor | None
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """`forward` with the preceding TP all-reduce fused in when
        `reduce_inputs`. Returns the normed activation and the residual (x
        when `residual` is None)."""
        if residual is None:
            return self(x), x
        if self.reduce_inputs:
            return fused_allreduce_gemma_rms_norm(x, residual, self)
        return self(x, residual)

forward_reduced(x, residual)

forward with the preceding TP all-reduce fused in when reduce_inputs. Returns the normed activation and the residual (x when residual is None).

Source code in vllm/models/qwen3_5/amd/model.py
def forward_reduced(
    self, x: torch.Tensor, residual: torch.Tensor | None
) -> tuple[torch.Tensor, torch.Tensor]:
    """`forward` with the preceding TP all-reduce fused in when
    `reduce_inputs`. Returns the normed activation and the residual (x
    when `residual` is None)."""
    if residual is None:
        return self(x), x
    if self.reduce_inputs:
        return fused_allreduce_gemma_rms_norm(x, residual, self)
    return self(x, residual)