Skip to content

vllm.models.qwen4_exp.nvidia.ops.qsa_indexer

Triton kernels for Qwen4Exp QSA index selection.

Functions:

expand_qsa_block_indices(block_indices, query_positions, visible_blocks, compress_ratio, token_topk, out)

Expand compressed blocks and compact the causal tail of the open group.

Source code in vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
def expand_qsa_block_indices(
    block_indices: torch.Tensor,
    query_positions: torch.Tensor,
    visible_blocks: torch.Tensor,
    compress_ratio: int,
    token_topk: int,
    out: torch.Tensor,
) -> None:
    """Expand compressed blocks and compact the causal tail of the open group."""
    assert token_topk % compress_ratio == 0
    block_topk = token_topk // compress_ratio
    output_width = token_topk + compress_ratio - 1
    assert block_indices.shape == (query_positions.numel(), block_topk)
    assert visible_blocks.shape == query_positions.shape
    # +1: the packed buffer's trailing column holds each row's valid-entry
    # count (never a token index); the index region below only writes
    # columns [0, output_width).
    assert out.shape == (block_indices.shape[0], output_width + 1)
    column_block = 256
    grid = (block_indices.shape[0], triton.cdiv(output_width, column_block))
    _expand_qsa_indices_kernel[grid](
        block_indices,
        query_positions,
        visible_blocks,
        out,
        *block_indices.stride(),
        *out.stride(),
        BLOCK_TOPK=block_topk,
        COMPRESS_RATIO=compress_ratio,
        COLUMN_BLOCK=column_block,
        num_warps=4,
    )

qsa_prefill_logits_width(max_seq_len, compress_ratio, max_columns)

Compressed columns any prefill row can score, rounded up to 64 for the top-k kernels' row alignment.

Source code in vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
def qsa_prefill_logits_width(
    max_seq_len: int, compress_ratio: int, max_columns: int
) -> int:
    """Compressed columns any prefill row can score, rounded up to 64 for
    the top-k kernels' row alignment."""
    width = round_up(triton.cdiv(max_seq_len, compress_ratio), 64)
    return min(max(64, width), max_columns)

qsa_select_paged_decode(q, k_cache, page_table, visible_blocks, token_topk, compress_ratio, decode_query_len, block_indices, topk_backend='auto')

Score and select compressed blocks for a request-major decode batch.

Parameters:

  • q

    (Tensor) –

    Query tensor shaped [num_requests * decode_query_len, heads, head_dim].

  • k_cache

    (Tensor) –

    Compressed key cache shaped [blocks, page_size, 1, head_dim].

  • page_table

    (Tensor) –

    Request block table shaped [num_requests, max_pages].

  • visible_blocks

    (Tensor) –

    Number of visible compressed blocks per query.

  • token_topk

    (int) –

    Number of logical tokens selected per query.

  • compress_ratio

    (int) –

    Number of logical tokens represented by a cache row.

  • decode_query_len

    (int) –

    Number of query tokens per request.

  • block_indices

    (Tensor) –

    Compressed-index output buffer.

  • topk_backend

    (str, default: 'auto' ) –

    One of QSA_TOPK_BACKENDS.

Source code in vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
def qsa_select_paged_decode(
    q: torch.Tensor,
    k_cache: torch.Tensor,
    page_table: torch.Tensor,
    visible_blocks: torch.Tensor,
    token_topk: int,
    compress_ratio: int,
    decode_query_len: int,
    block_indices: torch.Tensor,
    topk_backend: str = "auto",
) -> None:
    """Score and select compressed blocks for a request-major decode batch.

    Args:
        q: Query tensor shaped `[num_requests * decode_query_len, heads,
            head_dim]`.
        k_cache: Compressed key cache shaped `[blocks, page_size, 1,
            head_dim]`.
        page_table: Request block table shaped `[num_requests, max_pages]`.
        visible_blocks: Number of visible compressed blocks per query.
        token_topk: Number of logical tokens selected per query.
        compress_ratio: Number of logical tokens represented by a cache row.
        decode_query_len: Number of query tokens per request.
        block_indices: Compressed-index output buffer.
        topk_backend: One of QSA_TOPK_BACKENDS.

    """
    assert token_topk % compress_ratio == 0
    assert block_indices.shape == (q.shape[0], token_topk // compress_ratio)
    assert decode_query_len > 0 and q.shape[0] % decode_query_len == 0
    assert q.dtype == k_cache.dtype, "Q and the compressed K cache must match"
    num_requests = q.shape[0] // decode_query_len
    assert page_table.shape[0] == num_requests
    assert visible_blocks.shape == (q.shape[0],)

    columns = page_table.shape[1] * k_cache.shape[1]
    logits = torch.empty((q.shape[0], columns), dtype=torch.float32, device=q.device)
    tiles_per_program = _decode_tiles_per_program(num_requests, columns)
    grid = (
        num_requests,
        triton.cdiv(columns, _DECODE_BLOCK_N * tiles_per_program),
    )
    _qsa_mqa_paged_uniform_kernel[grid](
        q,
        k_cache,
        page_table,
        visible_blocks,
        logits,
        *q.stride()[:-1],
        *k_cache.stride()[:2],
        *page_table.stride()[:-1],
        *logits.stride()[:-1],
        PAGE_SIZE=k_cache.shape[1],
        PAGE_TABLE_WIDTH=page_table.shape[1],
        NUM_HEADS=q.shape[1],
        HEAD_DIM=q.shape[2],
        DECODE_QUERY_LEN=decode_query_len,
        BLOCK_N=_DECODE_BLOCK_N,
        TILES_PER_PROG=tiles_per_program,
        STAGES=2,
        # tuned on GB300
        num_warps=1 if k_cache.dtype == torch.float8_e4m3fn else 2,
    )
    _topk(
        logits,
        visible_blocks,
        token_topk,
        compress_ratio,
        block_indices,
        torch.empty((_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device),
        topk_backend,
        prefill=False,
    )

qsa_select_paged_prefill(q, k_cache, page_table, query_start_loc, visible_blocks, token_topk, compress_ratio, max_query_len, block_indices, max_seq_len, topk_backend='auto')

Score and select compressed prefill blocks in bounded chunks.

Parameters:

  • q

    (Tensor) –

    Packed prefill query tensor shaped [num_tokens, heads, head_dim].

  • k_cache

    (Tensor) –

    Compressed key cache shaped [blocks, page_size, 1, head_dim].

  • page_table

    (Tensor) –

    Block table shaped [num_requests, max_pages].

  • query_start_loc

    (Tensor) –

    Packed prefill query offsets with a terminal offset. Offsets may share the base of a larger backing tensor.

  • visible_blocks

    (Tensor) –

    Number of visible compressed blocks per query.

  • token_topk

    (int) –

    Number of logical tokens selected per query.

  • compress_ratio

    (int) –

    Number of logical tokens represented by a cache row.

  • max_query_len

    (int) –

    Maximum number of query tokens in one request.

  • block_indices

    (Tensor) –

    Compressed-index output buffer.

  • max_seq_len

    (int) –

    Longest context length in the batch this step.

  • topk_backend

    (str, default: 'auto' ) –

    One of QSA_TOPK_BACKENDS.

Source code in vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
def qsa_select_paged_prefill(
    q: torch.Tensor,
    k_cache: torch.Tensor,
    page_table: torch.Tensor,
    query_start_loc: torch.Tensor,
    visible_blocks: torch.Tensor,
    token_topk: int,
    compress_ratio: int,
    max_query_len: int,
    block_indices: torch.Tensor,
    max_seq_len: int,
    topk_backend: str = "auto",
) -> None:
    """Score and select compressed prefill blocks in bounded chunks.

    Args:
        q: Packed prefill query tensor shaped `[num_tokens, heads, head_dim]`.
        k_cache: Compressed key cache shaped `[blocks, page_size, 1,
            head_dim]`.
        page_table: Block table shaped `[num_requests, max_pages]`.
        query_start_loc: Packed prefill query offsets with a terminal offset.
            Offsets may share the base of a larger backing tensor.
        visible_blocks: Number of visible compressed blocks per query.
        token_topk: Number of logical tokens selected per query.
        compress_ratio: Number of logical tokens represented by a cache row.
        max_query_len: Maximum number of query tokens in one request.
        block_indices: Compressed-index output buffer.
        max_seq_len: Longest context length in the batch this step.
        topk_backend: One of QSA_TOPK_BACKENDS.

    """
    assert token_topk % compress_ratio == 0
    assert block_indices.shape == (q.shape[0], token_topk // compress_ratio)
    assert q.dtype == k_cache.dtype, "Q and the compressed K cache must match"
    rows = q.shape[0]
    logits_width = qsa_prefill_logits_width(
        max_seq_len, compress_ratio, page_table.shape[1] * k_cache.shape[1]
    )

    # chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB
    # always allocate the worst case to avoid memory fragmentation.
    max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
    rows_per_chunk = max(1, max_logits_bytes // (logits_width * 4))

    budget_bytes = max(max_logits_bytes, page_table.shape[1] * k_cache.shape[1] * 4)
    logits_workspace = q.new_empty(budget_bytes // 4, dtype=torch.float32)

    topk_workspace = torch.empty(
        (_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device
    )

    for query_start in range(0, rows, rows_per_chunk):
        query_end = min(query_start + rows_per_chunk, rows)
        query_slice = slice(query_start, query_end)
        logits = _prefill_logits(
            q,
            k_cache,
            page_table,
            query_start_loc,
            visible_blocks,
            logits_workspace,
            max_query_len,
            logits_width,
            query_offset=query_start,
            num_queries=query_end - query_start,
        )
        _topk(
            logits,
            visible_blocks[query_slice],
            token_topk,
            compress_ratio,
            block_indices[query_slice],
            topk_workspace,
            topk_backend,
            prefill=True,
        )

use_gvr2_topk(topk_backend, block_topk, width)

Whether _topk uses gvr_2. QSA logits are contiguous fp32, so only k and the width can rule it out.

Source code in vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
def use_gvr2_topk(topk_backend: str, block_topk: int, width: int) -> bool:
    """Whether _topk uses gvr_2. QSA logits are contiguous fp32, so only k
    and the width can rule it out."""
    if topk_backend not in ("auto", "flashinfer_gvr2"):
        return False
    supported = (
        current_platform.is_device_capability_family(100)
        and block_topk in (512, 1024, 2048)
        and width % 4 == 0
        and has_flashinfer_topk_varlen_gvr2()
    )
    if topk_backend == "flashinfer_gvr2" and not supported:
        raise RuntimeError(
            "sparse_indexer_topk_backend='flashinfer_gvr2' needs SM100, "
            "FlashInfer gvr_2, k in (512, 1024, 2048) and a width divisible "
            f"by 4; got k={block_topk}, width={width}"
        )
    return supported

warmup_gvr2_topk(topk_backend, block_topk, decode_width, max_decode_rows, prefill_width)

Pre-compile gvr_2 for every serving geometry. Decode compiles per exact width, prefill per pow2 width bucket, so prefill_width covers every narrower chunk.

Source code in vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
def warmup_gvr2_topk(
    topk_backend: str,
    block_topk: int,
    decode_width: int,
    max_decode_rows: int,
    prefill_width: int,
) -> None:
    """Pre-compile gvr_2 for every serving geometry. Decode compiles per
    exact width, prefill per pow2 width bucket, so prefill_width covers
    every narrower chunk."""
    from flashinfer.topk_varlen.kernels.gvr2_topk_host import (
        warmup_prefill,
        warmup_varlen,
    )

    if use_gvr2_topk(topk_backend, block_topk, decode_width):
        warmup_varlen(
            top_k=block_topk,
            max_seq_len=decode_width,
            num_rows_list=range(1, max_decode_rows + 1),
            row_stride=decode_width,
        )
    if use_gvr2_topk(topk_backend, block_topk, prefill_width):
        warmup_prefill(top_k=block_topk, max_cols=prefill_width)

warmup_qsa_mqa_paged_decode(k_cache, page_table, *, num_heads, head_dim, max_decode_query_len, max_num_reqs, max_num_batched_tokens)

Compile every reachable decode specialization without launching it.

Source code in vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
def warmup_qsa_mqa_paged_decode(
    k_cache: torch.Tensor,
    page_table: torch.Tensor,
    *,
    num_heads: int,
    head_dim: int,
    max_decode_query_len: int,
    max_num_reqs: int,
    max_num_batched_tokens: int,
) -> tuple[tuple[int, int], ...]:
    """Compile every reachable decode specialization without launching it."""
    page_size = k_cache.shape[1]
    page_table_width = page_table.shape[1]
    columns = page_table_width * page_size
    profiles = _qsa_decode_warmup_profiles(
        max_decode_query_len,
        max_num_reqs,
        max_num_batched_tokens,
        columns,
    )
    if not profiles:
        return ()

    k_cache_ptr = TritonWarmupTensor(k_cache.dtype, shape=tuple(k_cache.shape))
    page_table_ptr = TritonWarmupTensor(
        page_table.dtype,
        shape=(max_num_reqs, page_table_width),
    )
    visible_blocks_ptr = TritonWarmupTensor(torch.int32)

    for decode_query_len, num_requests in profiles:
        num_rows = decode_query_len * num_requests
        q_ptr = TritonWarmupTensor(
            k_cache.dtype,
            shape=(num_rows, num_heads, head_dim),
        )
        logits_ptr = TritonWarmupTensor(
            torch.float32,
            shape=(num_rows, columns),
        )
        tiles_per_program = _decode_tiles_per_program(num_requests, columns)
        _qsa_mqa_paged_uniform_kernel.warmup(
            q_ptr,
            k_cache_ptr,
            page_table_ptr,
            visible_blocks_ptr,
            logits_ptr,
            num_heads * head_dim,
            head_dim,
            k_cache.stride(0),
            k_cache.stride(1),
            page_table.stride(0),
            columns,
            PAGE_SIZE=page_size,
            PAGE_TABLE_WIDTH=page_table_width,
            NUM_HEADS=num_heads,
            HEAD_DIM=head_dim,
            DECODE_QUERY_LEN=decode_query_len,
            BLOCK_N=_DECODE_BLOCK_N,
            TILES_PER_PROG=tiles_per_program,
            STAGES=2,
            # tuned on GB300
            num_warps=1 if k_cache.dtype == torch.float8_e4m3fn else 2,
            grid=(
                num_requests,
                triton.cdiv(columns, _DECODE_BLOCK_N * tiles_per_program),
            ),
        )
    return profiles