Skip to content

vllm.model_executor.kernels.linear.mxfp8.humming

Classes:

HummingMxfp8LinearKernel

Bases: Mxfp8LinearKernel

Humming GEMM Kernel for MXFP8.

Source code in vllm/model_executor/kernels/linear/mxfp8/humming.py
class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
    """Humming GEMM Kernel for MXFP8."""

    weight_quant_config = {
        "quant_method": "humming",
        "dtype": "float8e4m3",
        "scale_dtype": "float8e8m0",
        "group_size": 32,
        "weight_scale_type": "group",
    }

    @classmethod
    def is_supported(
        cls, compute_capability: int | None = None
    ) -> tuple[bool, str | None]:
        if not current_platform.is_cuda():
            return False, "Humming only supported on CUDA"

        if not has_humming():
            return False, "Humming is not installed"

        if not current_platform.has_device_capability(75):
            return False, "Humming only supported on SM75+"

        return True, None

    @classmethod
    def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
        return True, None

    @classmethod
    def executed_activation_key(
        cls, activation_quant_key: QuantKey | None
    ) -> QuantKey | None:
        # Always converts for MXFP8 activations, whatever the layer requests.
        return humming_executed_input_quant_key(cls.weight_quant_config, kMxfp8Dynamic)

    @classmethod
    def supports_batch_invariance(cls) -> bool:
        return True

    def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
        layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
        name_map = {"weight": "weight", "weight_scale": "weight_scale"}

        convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
        input_schema = quant_key_to_input_schema(kMxfp8Dynamic)
        self.layer_config = prepare_humming_linear_layer_config(
            layer, self.weight_quant_config, input_schema=input_schema
        )
        self.compute_config = get_humming_linear_compute_config()
        self.locks = create_humming_locks(layer.weight.device)

    def apply_weights(
        self,
        layer: torch.nn.Module,
        x: torch.Tensor,
        bias: torch.Tensor | None = None,
    ) -> torch.Tensor:
        return apply_humming_linear(
            layer,
            x,
            skip_bias_add=bias is None,
            layer_config=self.layer_config,
            compute_config=self.compute_config,
            locks=self.locks,
        )