Skip to content

vllm.models.deepseek_v41.nvidia.specialized_decode.ampere.attention

The DeepSeek-V4.1 attention layer on SM80.

DeepSeek-V4.1 attends each query token to two key sets: the last sliding_window tokens of its request (the SWA cache, 528-byte MXFP8 records) and the index_topk compressed tokens its indexer selected (the compressed cache, 288-byte NVFP4 records here). On SM90 and SM100 FlashMLA runs that sparse MLA; on SM80 this layer does, with the same records:

  • the SWA records are written by the stock fused Q/KV kernel, which runs on SM80;
  • the compressor writes its NVFP4 records with AmpereCompressor;
  • forward_mqa attends decode and prefill tokens alike, each over its own window and selected compressed keys, and writes the output already inverse-RoPE'd, so _o_proj is a BF16 grouped wo_a and wo_b.

The metadata is the stock one, minus FlashMLA's tile scheduler and DeepGEMM's paged-logits schedule: the SWA and indexer builders here plan neither. Prefill context parallelism is not supported.

Classes:

AmpereCompressor

Bases: DeepseekCompressor

The compressor, writing its 288-byte NVFP4 records on SM80.

Methods:

  • insert_cache –

    Publish the group-boundary latents as NVFP4 compressed records.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
class AmpereCompressor(DeepseekCompressor):
    """The compressor, writing its 288-byte NVFP4 records on SM80."""

    def insert_cache(
        self,
        latent: torch.Tensor | None,
        positions: torch.Tensor,
        rotary_emb,
    ) -> None:
        """Publish the group-boundary latents as NVFP4 compressed records."""
        if latent is None:
            return
        attn_metadata = get_forward_context().attn_metadata
        assert isinstance(attn_metadata, dict)
        slot_mapping = cast(Any, attn_metadata[self.k_cache_prefix]).slot_mapping
        kv_cache = self._static_forward_context[self.k_cache_prefix].kv_cache
        store_compressed_records(
            latent,
            positions,
            slot_mapping,
            rope_table_f32(rotary_emb),
            self.compress_ratio,
            kv_cache,
        )

insert_cache(latent, positions, rotary_emb)

Publish the group-boundary latents as NVFP4 compressed records.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
def insert_cache(
    self,
    latent: torch.Tensor | None,
    positions: torch.Tensor,
    rotary_emb,
) -> None:
    """Publish the group-boundary latents as NVFP4 compressed records."""
    if latent is None:
        return
    attn_metadata = get_forward_context().attn_metadata
    assert isinstance(attn_metadata, dict)
    slot_mapping = cast(Any, attn_metadata[self.k_cache_prefix]).slot_mapping
    kv_cache = self._static_forward_context[self.k_cache_prefix].kv_cache
    store_compressed_records(
        latent,
        positions,
        slot_mapping,
        rope_table_f32(rotary_emb),
        self.compress_ratio,
        kv_cache,
    )

AmpereSWABackend

Bases: DeepseekSparseSWAFlashMLABackend

The SWA cache backend of the SM80 layer.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
class AmpereSWABackend(DeepseekSparseSWAFlashMLABackend):
    """The SWA cache backend of the SM80 layer."""

    @classmethod
    def supports_pcp(cls) -> bool:
        return False

    @staticmethod
    def get_builder_cls() -> type[AmpereSWAMetadataBuilder]:
        return AmpereSWAMetadataBuilder

AmpereSWAMetadataBuilder

Bases: DeepseekSparseSWAFlashMLAMetadataBuilder

SWA metadata without FlashMLA's tile-scheduler plans.

Methods:

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
class AmpereSWAMetadataBuilder(DeepseekSparseSWAFlashMLAMetadataBuilder):
    """SWA metadata without FlashMLA's tile-scheduler plans."""

    def build_tile_scheduler(self, num_decode_tokens: int) -> dict[str, Any]:
        """No plans: the base builds none for a step without decode tokens."""
        return super().build_tile_scheduler(0)

build_tile_scheduler(num_decode_tokens)

No plans: the base builds none for a step without decode tokens.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
def build_tile_scheduler(self, num_decode_tokens: int) -> dict[str, Any]:
    """No plans: the base builds none for a step without decode tokens."""
    return super().build_tile_scheduler(0)

AmpereSparseMLABackend

Bases: DeepseekV4FlashMLABackend

The compressed-cache backend of the SM80 layer: FlashMLA's metadata and paging, with the NVFP4 compressed record.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
class AmpereSparseMLABackend(DeepseekV4FlashMLABackend):
    """The compressed-cache backend of the SM80 layer: FlashMLA's metadata
    and paging, with the NVFP4 compressed record."""

    supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
        "auto",
        "fp8",
        "nvfp4_ds_mla",
    ]

    @classmethod
    def supports_pcp(cls) -> bool:
        return False

    @staticmethod
    def get_name() -> str:
        return "AMPERE_SPARSE_MLA_DSV41"

    @classmethod
    def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
        return capability.to_int() == AMPERE_CAPABILITY

DeepseekV41AmpereAttention

Bases: DeepseekV4Attention

DeepSeek-V4.1 attention on SM80 (A100).

Methods:

Attributes:

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
class DeepseekV41AmpereAttention(DeepseekV4Attention):
    """DeepSeek-V4.1 attention on SM80 (A100)."""

    backend_cls = AmpereSparseMLABackend
    swa_backend_cls = AmpereSWABackend
    compressor_cls = AmpereCompressor

    @property
    def packed_kv_cache_dtype(self) -> CacheDType:
        """The NVFP4 compressed record, as on SM100's mega attention."""
        return "nvfp4_ds_mla"

    @classmethod
    def get_padded_num_q_heads(cls, num_heads: int) -> int:
        """No padding: the kernel takes any head count."""
        return num_heads

    def _indexer_cls(
        self, k_cache: DeepseekV4IndexerCache | None
    ) -> type[DeepseekV4Indexer]:
        """The indexer with SM80 query quantization and logits."""
        return AmpereIndexer

    def _indexer_cache_cls(self) -> type[DeepseekV4IndexerCache]:
        """The index K cache with the SM80 indexer metadata."""
        return AmpereIndexerCache

    def _o_proj(
        self,
        attn_out: "torch.Tensor | QuantizedActivation",
        positions: torch.Tensor,
    ) -> torch.Tensor:
        """`wo_a` per head group as a BF16 batched GEMM, then `wo_b`.

        `forward_mqa` already applied the inverse RoPE. `wo_a` is the
        MXFP8 emulation kernel's BF16 weight `[groups * o_lora_rank, K]`.
        """
        assert isinstance(attn_out, torch.Tensor)
        weight = self.wo_a.weight
        assert isinstance(weight, torch.Tensor)
        assert weight.dtype == torch.bfloat16, (
            "the SM80 wo_a needs the BF16 weight the MXFP8 emulation kernel "
            "dequantizes at load (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=1)"
        )
        num_tokens = attn_out.shape[0]
        groups = self.n_local_groups
        o = attn_out.view(num_tokens, groups, -1).transpose(0, 1)
        z = torch.bmm(o, weight.view(groups, self.o_lora_rank, -1).transpose(1, 2))
        return self._wo_b_proj(z.transpose(0, 1).reshape(num_tokens, -1))

    def forward_mqa(
        self,
        q: torch.Tensor,
        kv: torch.Tensor,
        positions: torch.Tensor,
        output: torch.Tensor,
    ) -> None:
        """Sparse MLA of every token over its window and selected keys."""
        attn_metadata = get_forward_context().attn_metadata
        if not isinstance(attn_metadata, dict):
            # Profile run: no metadata and nothing to read.
            output.zero_()
            return
        swa_metadata = cast(
            "DeepseekSparseSWAMetadata", attn_metadata[self.swa_cache_layer.prefix]
        )
        compressed_metadata = (
            cast(
                DeepseekV4FlashMLAMetadata, attn_metadata[self.compressed_cache_prefix]
            )
            if self.compressed_cache_prefix is not None
            else None
        )
        num_decode = swa_metadata.num_decode_tokens
        num_prefill = swa_metadata.num_prefill_tokens
        output[num_decode + num_prefill :].zero_()
        segments = []
        if num_decode > 0:
            assert swa_metadata.decode_swa_indices is not None
            assert swa_metadata.decode_swa_lens is not None
            segments.append(
                (
                    0,
                    num_decode,
                    swa_metadata.decode_swa_indices.view(num_decode, -1),
                    swa_metadata.decode_swa_lens,
                )
            )
        if num_prefill > 0:
            assert swa_metadata.prefill_swa_indices is not None
            assert swa_metadata.prefill_swa_lens is not None
            segments.append(
                (
                    num_decode,
                    num_prefill,
                    swa_metadata.prefill_swa_indices.view(num_prefill, -1),
                    swa_metadata.prefill_swa_lens,
                )
            )
        for start, count, swa_indices, swa_lens in segments:
            rows = slice(start, start + count)
            compressed_cache = compressed_indices = compressed_lens = None
            if compressed_metadata is not None:
                assert self.topk_indices_buffer is not None
                assert swa_metadata.token_to_req_indices is not None
                assert swa_metadata.is_valid_token is not None
                compressed_indices, compressed_lens = (
                    compute_global_topk_indices_and_lens(
                        self.topk_indices_buffer[rows],
                        swa_metadata.token_to_req_indices[rows],
                        compressed_metadata.block_table,
                        compressed_metadata.block_size // self.compress_ratio,
                        swa_metadata.is_valid_token[rows],
                    )
                )
                compressed_cache = self._compressed_kv_cache()
            sparse_mla(
                q[rows],
                self.swa_cache_layer.kv_cache,
                swa_indices,
                swa_lens,
                compressed_cache,
                compressed_indices,
                compressed_lens,
                self.attn_sink[: self.n_local_heads],
                self.scale,
                positions[rows],
                rope_table_f32(self.rotary_emb),
                output[rows],
            )

packed_kv_cache_dtype property

The NVFP4 compressed record, as on SM100's mega attention.

_indexer_cache_cls()

The index K cache with the SM80 indexer metadata.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
def _indexer_cache_cls(self) -> type[DeepseekV4IndexerCache]:
    """The index K cache with the SM80 indexer metadata."""
    return AmpereIndexerCache

_indexer_cls(k_cache)

The indexer with SM80 query quantization and logits.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
def _indexer_cls(
    self, k_cache: DeepseekV4IndexerCache | None
) -> type[DeepseekV4Indexer]:
    """The indexer with SM80 query quantization and logits."""
    return AmpereIndexer

_o_proj(attn_out, positions)

wo_a per head group as a BF16 batched GEMM, then wo_b.

forward_mqa already applied the inverse RoPE. wo_a is the MXFP8 emulation kernel's BF16 weight [groups * o_lora_rank, K].

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
def _o_proj(
    self,
    attn_out: "torch.Tensor | QuantizedActivation",
    positions: torch.Tensor,
) -> torch.Tensor:
    """`wo_a` per head group as a BF16 batched GEMM, then `wo_b`.

    `forward_mqa` already applied the inverse RoPE. `wo_a` is the
    MXFP8 emulation kernel's BF16 weight `[groups * o_lora_rank, K]`.
    """
    assert isinstance(attn_out, torch.Tensor)
    weight = self.wo_a.weight
    assert isinstance(weight, torch.Tensor)
    assert weight.dtype == torch.bfloat16, (
        "the SM80 wo_a needs the BF16 weight the MXFP8 emulation kernel "
        "dequantizes at load (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=1)"
    )
    num_tokens = attn_out.shape[0]
    groups = self.n_local_groups
    o = attn_out.view(num_tokens, groups, -1).transpose(0, 1)
    z = torch.bmm(o, weight.view(groups, self.o_lora_rank, -1).transpose(1, 2))
    return self._wo_b_proj(z.transpose(0, 1).reshape(num_tokens, -1))

forward_mqa(q, kv, positions, output)

Sparse MLA of every token over its window and selected keys.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
def forward_mqa(
    self,
    q: torch.Tensor,
    kv: torch.Tensor,
    positions: torch.Tensor,
    output: torch.Tensor,
) -> None:
    """Sparse MLA of every token over its window and selected keys."""
    attn_metadata = get_forward_context().attn_metadata
    if not isinstance(attn_metadata, dict):
        # Profile run: no metadata and nothing to read.
        output.zero_()
        return
    swa_metadata = cast(
        "DeepseekSparseSWAMetadata", attn_metadata[self.swa_cache_layer.prefix]
    )
    compressed_metadata = (
        cast(
            DeepseekV4FlashMLAMetadata, attn_metadata[self.compressed_cache_prefix]
        )
        if self.compressed_cache_prefix is not None
        else None
    )
    num_decode = swa_metadata.num_decode_tokens
    num_prefill = swa_metadata.num_prefill_tokens
    output[num_decode + num_prefill :].zero_()
    segments = []
    if num_decode > 0:
        assert swa_metadata.decode_swa_indices is not None
        assert swa_metadata.decode_swa_lens is not None
        segments.append(
            (
                0,
                num_decode,
                swa_metadata.decode_swa_indices.view(num_decode, -1),
                swa_metadata.decode_swa_lens,
            )
        )
    if num_prefill > 0:
        assert swa_metadata.prefill_swa_indices is not None
        assert swa_metadata.prefill_swa_lens is not None
        segments.append(
            (
                num_decode,
                num_prefill,
                swa_metadata.prefill_swa_indices.view(num_prefill, -1),
                swa_metadata.prefill_swa_lens,
            )
        )
    for start, count, swa_indices, swa_lens in segments:
        rows = slice(start, start + count)
        compressed_cache = compressed_indices = compressed_lens = None
        if compressed_metadata is not None:
            assert self.topk_indices_buffer is not None
            assert swa_metadata.token_to_req_indices is not None
            assert swa_metadata.is_valid_token is not None
            compressed_indices, compressed_lens = (
                compute_global_topk_indices_and_lens(
                    self.topk_indices_buffer[rows],
                    swa_metadata.token_to_req_indices[rows],
                    compressed_metadata.block_table,
                    compressed_metadata.block_size // self.compress_ratio,
                    swa_metadata.is_valid_token[rows],
                )
            )
            compressed_cache = self._compressed_kv_cache()
        sparse_mla(
            q[rows],
            self.swa_cache_layer.kv_cache,
            swa_indices,
            swa_lens,
            compressed_cache,
            compressed_indices,
            compressed_lens,
            self.attn_sink[: self.n_local_heads],
            self.scale,
            positions[rows],
            rope_table_f32(self.rotary_emb),
            output[rows],
        )

get_padded_num_q_heads(num_heads) classmethod

No padding: the kernel takes any head count.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/ampere/attention.py
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
    """No padding: the kernel takes any head count."""
    return num_heads