Skip to content

vllm.model_executor.layers.fused_moe.experts.xpu_moe

Classes:

XPUExperts

Bases: LoRAExpertsMixin, FusedMoEExpertsModular

Source code in vllm/model_executor/layers/fused_moe/experts/xpu_moe.py
class XPUExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
    def __init__(
        self,
        moe_config: FusedMoEConfig,
        quant_config: FusedMoEQuantConfig,
        max_num_tokens: int | None = None,
        num_dispatchers: int | None = None,
    ):
        super().__init__(
            moe_config,
            quant_config,
            max_num_tokens,
            num_dispatchers,
        )
        self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
        self.fused_moe_impl: XpuFusedMoe | None = None
        self._lora_ones = torch.empty(0)
        is_xe2_or_xe3 = torch.ops._xpu_C.is_xe2_arch() or torch.ops._xpu_C.is_xe3_arch()
        self._expects_unquantized_inputs = is_xe2_or_xe3

    @property
    def expects_unquantized_inputs(self) -> bool:
        return self._expects_unquantized_inputs

    @staticmethod
    def activation_format() -> mk.FusedMoEActivationFormat:
        return mk.FusedMoEActivationFormat.Standard

    @staticmethod
    def _supports_current_device() -> bool:
        return current_platform.is_xpu()

    @staticmethod
    def _supports_no_act_and_mul() -> bool:
        return True

    @staticmethod
    def _supports_activation(activation: MoEActivation) -> bool:
        return activation in [
            MoEActivation.SILU,
            MoEActivation.GELU,
            MoEActivation.GELU_TANH,
            MoEActivation.SWIGLUOAI,
            MoEActivation.RELU2_NO_MUL,
        ]

    @staticmethod
    def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
        return True

    @staticmethod
    def _supports_quant_scheme(
        weight_key: QuantKey | None,
        activation_key: QuantKey | None,
    ) -> bool:
        SUPPORTED_W_A = [
            (None, None),
            (kFp8StaticTensorSym, None),
            (kFp8StaticTensorSym, kFp8DynamicTensorSym),
        ]
        return (weight_key, activation_key) in SUPPORTED_W_A

    def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
        return TopKWeightAndReduceNoOP()

    def workspace_shapes(
        self,
        M: int,
        N: int,
        K: int,
        topk: int,
        global_num_experts: int,
        local_num_experts: int,
        expert_tokens_meta: mk.ExpertTokensMetadata | None,
        activation: MoEActivation,
    ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
        workspace1 = (0,)
        workspace2 = (0,)
        output = (M, K)
        return (workspace1, workspace2, output)

    def apply(
        self,
        output: torch.Tensor,
        hidden_states: torch.Tensor,
        w1: torch.Tensor,
        w2: torch.Tensor,
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
        activation: MoEActivation,
        global_num_experts: int,
        expert_map: torch.Tensor | None,
        a1q_scale: torch.Tensor | None,
        a2_scale: torch.Tensor | None,
        workspace13: torch.Tensor,
        workspace2: torch.Tensor,
        expert_tokens_meta: mk.ExpertTokensMetadata | None,
        apply_router_weight_on_input: bool,
    ):
        if self.moe_config.is_lora_enabled and apply_router_weight_on_input:
            raise NotImplementedError(
                "XPU MoE LoRA requires routing weights on the output."
            )
        if self.fused_moe_impl is None:
            topk = topk_ids.size(-1)
            if (
                self.quant_config is not None
                and self.quant_config.weight_quant_dtype == "mxfp4"
            ):
                w1 = w1.view(torch.float4_e2m1fn_x2)
                w2 = w2.view(torch.float4_e2m1fn_x2)
            # XpuFusedMoe relayouts the loaded [E, N, K] weights and scales
            # in place for this device's grouped GEMM.
            self.fused_moe_impl = XpuFusedMoe(
                w13=w1,
                w13_scales=self.w1_scale,
                w13_bias=self.w1_bias,
                w2=w2,
                w2_scales=self.w2_scale,
                w2_bias=self.w2_bias,
                n_experts_per_token=topk,
                activation=activation.value,
                num_experts=self.moe_config.num_local_experts,
                ep_rank=self.moe_config.ep_rank,
                ep_size=self.moe_config.ep_size,
                expert_map=expert_map,
                gemm1_clamp_limit=self.gemm1_clamp_limit,
            )
        assert self.fused_moe_impl is not None
        if self.moe_config.ep_size > 1:
            # Avoid topk_ids = -1, remap them to an
            # expert not owned by this rank so they are skipped.
            sentinel = global_num_experts - 1 if self.moe_config.ep_rank == 0 else 0
            topk_ids = torch.where(topk_ids < 0, sentinel, topk_ids)
        self._apply_experts(
            output,
            hidden_states,
            w1,
            w2,
            topk_weights,
            topk_ids,
            expert_map,
            a1q_scale,
        )

    def _apply_experts(
        self,
        output: torch.Tensor,
        hidden_states: torch.Tensor,
        w1: torch.Tensor,
        w2: torch.Tensor,
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
        expert_map: torch.Tensor | None,
        a1q_scale: torch.Tensor | None,
    ) -> None:
        assert self.fused_moe_impl is not None
        hooks = {}
        if self._lora_context is not None:
            hooks = self._lora_hooks(
                self._lora_context,
                hidden_states,
                w1,
                w2,
                topk_weights,
                topk_ids,
                expert_map,
            )
        self.fused_moe_impl.apply(
            output=output,
            hidden_states=hidden_states,
            topk_weights=topk_weights,
            topk_ids=topk_ids,
            a1q_scale=a1q_scale,
            **hooks,
        )

    def _lora_hooks(
        self,
        ctx: MoELoRAContext,
        hidden_states: torch.Tensor,
        w1: torch.Tensor,
        w2: torch.Tensor,
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
        expert_map: torch.Tensor | None,
    ) -> dict[str, Callable[..., None]]:
        """Builds XpuFusedMoe hooks that add the LoRA deltas in place.

        gemm1_output / act_output / gemm2_output hold one row per
        (token, topk) pair grouped by expert; `row_map` gives each pair's
        row (-1 for skipped pairs), so the LoRA kernels read and accumulate
        into those rows directly.
        """
        num_tokens, top_k = topk_ids.shape
        # XpuFusedMoe stores bf16 weights as [E, K, N]; the LoRA config
        # lookup expects the loaded [E, N, K] shapes.
        w1 = w1.transpose(-1, -2)
        w2 = w2.transpose(-1, -2)
        # moe_gather applies the routing weights to base and LoRA outputs.
        if self._lora_ones.shape[0] < num_tokens:
            self._lora_ones = torch.ones_like(topk_weights)
        unweighted = self._lora_ones[:num_tokens]
        w13_meta: tuple = ()

        def activation_hook(gemm1_output: torch.Tensor, row_map: torch.Tensor):
            nonlocal w13_meta
            w13_meta = self.apply_w13_lora(
                ctx,
                y=gemm1_output,
                x=hidden_states,
                topk_ids=topk_ids,
                topk_weights=unweighted,
                expert_map=expert_map,
                w1=w1,
                w2=w2,
                num_tokens=num_tokens,
                top_k_num=top_k,
                row_map=row_map.view(-1),
            )

        def output_hook(
            gemm2_output: torch.Tensor,
            act_output: torch.Tensor,
            row_map: torch.Tensor,
        ):
            sorted_token_ids, expert_ids, num_tokens_post_padded, token_lora_map = (
                w13_meta
            )
            self.apply_w2_lora(
                ctx,
                y=gemm2_output.view(num_tokens, top_k, -1),
                x=act_output,
                topk_weights=unweighted,
                sorted_token_ids_lora=sorted_token_ids,
                expert_ids_lora=expert_ids,
                num_tokens_post_padded_lora=num_tokens_post_padded,
                token_lora_mapping=token_lora_map,
                num_tokens=num_tokens,
                w1=w1,
                w2=w2,
                top_k_num=top_k,
                row_map=row_map.view(-1),
            )

        return dict(activation_hook=activation_hook, output_hook=output_hook)

_lora_hooks(ctx, hidden_states, w1, w2, topk_weights, topk_ids, expert_map)

Builds XpuFusedMoe hooks that add the LoRA deltas in place.

gemm1_output / act_output / gemm2_output hold one row per (token, topk) pair grouped by expert; row_map gives each pair's row (-1 for skipped pairs), so the LoRA kernels read and accumulate into those rows directly.

Source code in vllm/model_executor/layers/fused_moe/experts/xpu_moe.py
def _lora_hooks(
    self,
    ctx: MoELoRAContext,
    hidden_states: torch.Tensor,
    w1: torch.Tensor,
    w2: torch.Tensor,
    topk_weights: torch.Tensor,
    topk_ids: torch.Tensor,
    expert_map: torch.Tensor | None,
) -> dict[str, Callable[..., None]]:
    """Builds XpuFusedMoe hooks that add the LoRA deltas in place.

    gemm1_output / act_output / gemm2_output hold one row per
    (token, topk) pair grouped by expert; `row_map` gives each pair's
    row (-1 for skipped pairs), so the LoRA kernels read and accumulate
    into those rows directly.
    """
    num_tokens, top_k = topk_ids.shape
    # XpuFusedMoe stores bf16 weights as [E, K, N]; the LoRA config
    # lookup expects the loaded [E, N, K] shapes.
    w1 = w1.transpose(-1, -2)
    w2 = w2.transpose(-1, -2)
    # moe_gather applies the routing weights to base and LoRA outputs.
    if self._lora_ones.shape[0] < num_tokens:
        self._lora_ones = torch.ones_like(topk_weights)
    unweighted = self._lora_ones[:num_tokens]
    w13_meta: tuple = ()

    def activation_hook(gemm1_output: torch.Tensor, row_map: torch.Tensor):
        nonlocal w13_meta
        w13_meta = self.apply_w13_lora(
            ctx,
            y=gemm1_output,
            x=hidden_states,
            topk_ids=topk_ids,
            topk_weights=unweighted,
            expert_map=expert_map,
            w1=w1,
            w2=w2,
            num_tokens=num_tokens,
            top_k_num=top_k,
            row_map=row_map.view(-1),
        )

    def output_hook(
        gemm2_output: torch.Tensor,
        act_output: torch.Tensor,
        row_map: torch.Tensor,
    ):
        sorted_token_ids, expert_ids, num_tokens_post_padded, token_lora_map = (
            w13_meta
        )
        self.apply_w2_lora(
            ctx,
            y=gemm2_output.view(num_tokens, top_k, -1),
            x=act_output,
            topk_weights=unweighted,
            sorted_token_ids_lora=sorted_token_ids,
            expert_ids_lora=expert_ids,
            num_tokens_post_padded_lora=num_tokens_post_padded,
            token_lora_mapping=token_lora_map,
            num_tokens=num_tokens,
            w1=w1,
            w2=w2,
            top_k_num=top_k,
            row_map=row_map.view(-1),
        )

    return dict(activation_hook=activation_hook, output_hook=output_hook)

XPUExpertsWNA16

Bases: XPUExperts

W4A16 INT4-symmetric MoE backed by xpu_fused_moe(is_int4=True).

Weight layout when is_int4=True (per xpu_fused_moe docstring): w13: [num_experts, 2inter_size, hidden_size] contiguous int4-packed w13_scales: [num_experts, 2inter_size, hidden_size // group_size] w2: [num_experts, hidden_size, inter_size] contiguous int4-packed w2_scales: [num_experts, hidden_size, inter_size // group_size]

Pairs with INCXPULinearMethod for the linear layers; together they cover full-attn + MoE on Intel XPU end-to-end without IPEX.

Source code in vllm/model_executor/layers/fused_moe/experts/xpu_moe.py
class XPUExpertsWNA16(XPUExperts):
    """W4A16 INT4-symmetric MoE backed by `xpu_fused_moe(is_int4=True)`.

    Weight layout when `is_int4=True` (per `xpu_fused_moe` docstring):
        w13: [num_experts, 2*inter_size, hidden_size]   contiguous int4-packed
        w13_scales: [num_experts, 2*inter_size, hidden_size // group_size]
        w2:  [num_experts, hidden_size, inter_size]     contiguous int4-packed
        w2_scales:  [num_experts, hidden_size, inter_size // group_size]

    Pairs with `INCXPULinearMethod` for the linear layers; together they
    cover full-attn + MoE on Intel XPU end-to-end without IPEX.
    """

    @staticmethod
    def supports_lora() -> bool:
        return False

    def __init__(
        self,
        moe_config: FusedMoEConfig,
        quant_config: FusedMoEQuantConfig,
        max_num_tokens: int | None = None,
        num_dispatchers: int | None = None,
    ):
        super().__init__(
            moe_config,
            quant_config,
            max_num_tokens,
            num_dispatchers,
        )

    @staticmethod
    def _supports_quant_scheme(
        weight_key: QuantKey | None,
        activation_key: QuantKey | None,
    ) -> bool:
        return (weight_key, activation_key) in (
            (kInt4Static, None),
            (kInt4Static32, None),
        )