Skip to content

vllm.models.minimax_m3.nvidia.specialized_decode.kernels.o_proj.kernel

MXFP8 decode GEMV on tcgen05: y[M, N] = x[M, K] @ W[N, K]^T for M <= 8, block-scaled E4M3 x E4M3.

The weight is the MMA's A operand (128 output features per CTA), the activation is B (8 token columns), fp32 accumulation in TMEM. Both operands come in their MXFP8 storage: E4M3 values (K contiguous) and UE8M0 scales in the 128x4-swizzled layout, whose 512-byte atom per (128 rows, 128 K) is exactly the source of a tcgen05.cp 32x128b.warpx4 scale copy, so both scale tensors are bulk-copied unchanged. For an activation of at most 32 rows the atom of k-tile k sits at byte 512 k.

A cluster of SPLIT CTAs shares one 128-row weight tile; rank r streams the interleaved k-tiles r, r + SPLIT, ... through a RING-stage ring. Before the grid dependency each CTA issues its first RING weight tiles (and their scale atoms) and prefetches the rest into L2; after it, the activation slice of the CTA (values and scale atoms) lands in one buffer. Rank r owns rows [r * 128 / SPLIT, (r + 1) * 128 / SPLIT) of the tile: the other ranks store their fp32 partials of those rows into its shared memory and arrive on its barrier (release, cluster scope); the owner adds the SPLIT partials in rank order and stores bf16.

Warps: 0 weight TMA, 1 activation TMA, 2 TMEM allocation + scale copies + MMA, 3 idle, 4-7 epilogue (warp 4 + i reads TMEM lanes 32 i .. 32 i + 31).

_absmax(acc, v, *, loc=None, ip=None)

max(acc, |v|).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _absmax(acc, v, *, loc=None, ip=None):
    """max(acc, |v|)."""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    return cutlass.Float32(
        _llvm.inline_asm(
            _T.f32(), [cutlass.Float32(acc).ir_value(loc=loc, ip=ip), cutlass.Float32(v).ir_value(loc=loc, ip=ip)],
            "{\n\t.reg .f32 a;\n\tabs.f32 a, $2;\n\tmax.f32 $0, $1, a;\n\t}", "=f,f,f", has_side_effects=False,
            is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
        )
    )  # fmt: skip

_bf16_bits(x, *, loc=None, ip=None)

bf16(x) (RN) as the fp32 bit pattern of the rounded value.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _bf16_bits(x, *, loc=None, ip=None):
    """bf16(x) (RN) as the fp32 bit pattern of the rounded value."""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    return cutlass.Int32(
        _llvm.inline_asm(
            _T.i32(), [cutlass.Float32(x).ir_value(loc=loc, ip=ip)],
            "{\n\t.reg .b16 h, z;\n\tcvt.rn.bf16.f32 h, $1;\n\tmov.b16 z, 0;\n\tmov.b32 $0, {z, h};\n\t}", "=r,f",
            has_side_effects=False, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
        )
    )  # fmt: skip

_bf16x2_word(lo, hi, *, loc=None, ip=None)

bf16(lo) | bf16(hi) << 16 (RN), with -0.0 halves turned into +0.0 (0x80000000 marks an empty exchange word).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _bf16x2_word(lo, hi, *, loc=None, ip=None):
    """bf16(lo) | bf16(hi) << 16 (RN), with -0.0 halves turned into +0.0 (0x80000000 marks an empty exchange word)."""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    return cutlass.Int32(
        _llvm.inline_asm(
            _T.i32(), [cutlass.Float32(hi).ir_value(loc=loc, ip=ip), cutlass.Float32(lo).ir_value(loc=loc, ip=ip)],
            "{\n\t.reg .b32 w, l, h;\n\t.reg .pred pl, ph;\n\tcvt.rn.bf16x2.f32 w, $1, $2;\n\t"
            "and.b32 l, w, 0xFFFF;\n\tsetp.eq.u32 pl, l, 0x8000;\n\t@pl and.b32 w, w, 0xFFFF0000;\n\t"
            "shr.u32 h, w, 16;\n\tsetp.eq.u32 ph, h, 0x8000;\n\t@ph and.b32 w, w, 0x0000FFFF;\n\t"
            "mov.b32 $0, w;\n\t}", "=r,f,f",
            has_side_effects=False, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
        )
    )  # fmt: skip

_din_load4(table, hidden, hidden_half, word_idx)

Four int32 words at word_idx of the hidden rows (hidden_half == 1) or the embedding table: one CTA-uniform base-address select instead of a branch.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
def _din_load4(table, hidden, hidden_half, word_idx):
    """Four int32 words at word_idx of the hidden rows (hidden_half == 1) or the embedding table: one CTA-uniform
    base-address select instead of a branch."""
    addr = cutlass.select_(
        hidden_half == cutlass.Int32(1),
        hidden.data_ptr(0).toint(),
        table.data_ptr(0).toint(),
    )
    ptr = cutlass.inttoptr(
        addr + cutlass.Int64(word_idx) * cutlass.Int64(4),
        mem_space=1,
        dtype=cutlass.Int32,
    )
    return ptr.load(count=4, alignment=16)

_e4m3x4(v0, v1, v2, v3, *, loc=None, ip=None)

Four f32 to E4M3 (RN, satfinite), packed little-endian into one int32 word (v0 in the low byte).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _e4m3x4(v0, v1, v2, v3, *, loc=None, ip=None):
    """Four f32 to E4M3 (RN, satfinite), packed little-endian into one int32 word (v0 in the low byte)."""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    return cutlass.Int32(
        _llvm.inline_asm(
            _T.i32(), [cutlass.Float32(v).ir_value(loc=loc, ip=ip) for v in (v0, v1, v2, v3)],
            "{\n\t.reg .b16 lo, hi;\n\tcvt.rn.satfinite.e4m3x2.f32 lo, $2, $1;\n\t"
            "cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3;\n\tmov.b32 $0, {lo, hi};\n\t}", "=r,f,f,f,f",
            has_side_effects=False, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
        )
    )  # fmt: skip

_ld_shared_v4_f32(smem_ptr, *, loc=None, ip=None)

ld.shared.v4.f32 of four fp32 at a 16-byte-aligned shared-memory pointer.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ld_shared_v4_f32(smem_ptr, *, loc=None, ip=None):
    """ld.shared.v4.f32 of four fp32 at a 16-byte-aligned shared-memory pointer."""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    out = _llvm.inline_asm(
        _llvm.StructType.get_literal([_T.f32()] * 4), [smem_ptr.toint(loc=loc, ip=ip).ir_value()],
        "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", "=f,=f,=f,=f,r", has_side_effects=True,
        is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
    )  # fmt: skip
    return [
        cutlass.Float32(_llvm.extractvalue(_T.f32(), out, [i], loc=loc, ip=ip))
        for i in range(4)
    ]

_mapa_u32(smem_ptr, peer, *, loc=None, ip=None)

The shared::cluster address of this CTA's shared-memory location in cluster CTA peer.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _mapa_u32(smem_ptr, peer, *, loc=None, ip=None):
    """The shared::cluster address of this CTA's shared-memory location in cluster CTA `peer`."""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    return cutlass.Int32(
        _llvm.inline_asm(
            _T.i32(), [smem_ptr.toint(loc=loc, ip=ip).ir_value(), cutlass.Int32(peer).ir_value(loc=loc, ip=ip)],
            "mapa.shared::cluster.u32 $0, $1, $2;", "=r,r,r", has_side_effects=False,
            is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
        )
    )  # fmt: skip

_p5_e8m0(x, *, loc=None, ip=None)

UE8M0 of x, rounded up, satfinite (trtllm mxfp8_quantize scale rule).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _p5_e8m0(x, *, loc=None, ip=None):
    """UE8M0 of x, rounded up, satfinite (trtllm mxfp8_quantize scale rule)."""
    from cutlass._mlir.extras import types as _T

    return cutlass.Int32(
        _asm(
            _T.i32(),
            [cutlass.Float32(x)],
            "{\n\t.reg .b16 h;\n\t.reg .b32 w;\n\tcvt.rp.satfinite.ue8m0x2.f32 h, 0f00000000, $1;\n\t"
            "cvt.u32.u16 w, h;\n\tand.b32 $0, w, 255;\n\t}",
            "=r,f",
            side=False,
            loc=loc,
            ip=ip,
        )
    )

_p5_fix(addr, tag, w, active, *, loc=None, ip=None)

W was loaded from addr; while active and its tag is stale, re-poll (stragglers only).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _p5_fix(addr, tag, w, active, *, loc=None, ip=None):
    """W was loaded from addr; while active and its tag is stale, re-poll (stragglers only)."""
    from cutlass._mlir.extras import types as _T

    return cutlass.Int32(
        _asm(
            _T.i32(),
            [
                cutlass.Int64(addr),
                cutlass.Int32(tag),
                cutlass.Int32(w),
                cutlass.Int32(active),
            ],
            "{\n\t.reg .b32 w, g;\n\t.reg .pred p, a;\n\tmov.b32 w, $3;\n\tsetp.ne.u32 a, $4, 0;\n\t"
            "@!a bra P5FD;\n\tP5FL:\n\tshr.u32 g, w, 16;\n\tsetp.ne.u32 p, g, $2;\n\t@!p bra P5FD;\n\t"
            "ld.volatile.global.b32 w, [$1];\n\tbra P5FL;\n\tP5FD:\n\tmov.b32 $0, w;\n\t}",
            "=r,l,r,r,r",
            loc=loc,
            ip=ip,
        )
    )

_p5_fix64(addr, tag, w, active, *, loc=None, ip=None)

Tagged rowsq word: re-poll while active and stale; returns the float payload.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _p5_fix64(addr, tag, w, active, *, loc=None, ip=None):
    """Tagged rowsq word: re-poll while active and stale; returns the float payload."""
    from cutlass._mlir.extras import types as _T

    return cutlass.Float32(
        _asm(
            _T.f32(),
            [
                cutlass.Int64(addr),
                cutlass.Int32(tag),
                cutlass.Int64(w),
                cutlass.Int32(active),
            ],
            "{\n\t.reg .b64 w;\n\t.reg .b32 lo, hi;\n\t.reg .pred p, a;\n\tmov.b64 w, $3;\n\t"
            "setp.ne.u32 a, $4, 0;\n\tP5GL:\n\tmov.b64 {lo, hi}, w;\n\t@!a bra P5GD;\n\t"
            "setp.ne.u32 p, hi, $2;\n\t@!p bra P5GD;\n\tld.relaxed.gpu.global.b64 w, [$1];\n\tbra P5GL;\n\t"
            "P5GD:\n\tmov.b32 $0, lo;\n\t}",
            "=f,l,r,l,r",
            loc=loc,
            ip=ip,
        )
    )

_p5_spin(addr, tag, *, loc=None, ip=None)

Poll a Lamport word until its tag (high 16 bits) matches; returns the word.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _p5_spin(addr, tag, *, loc=None, ip=None):
    """Poll a Lamport word until its tag (high 16 bits) matches; returns the word."""
    from cutlass._mlir.extras import types as _T

    return cutlass.Int32(
        _asm(
            _T.i32(),
            [cutlass.Int64(addr), cutlass.Int32(tag)],
            "{\n\t.reg .b32 w, g;\n\t.reg .pred p;\n\tP5SPIN:\n\tld.volatile.global.b32 w, [$1];\n\t"
            "shr.u32 g, w, 16;\n\tsetp.ne.u32 p, g, $2;\n\t@p bra P5SPIN;\n\tmov.b32 $0, w;\n\t}",
            "=r,l,r",
            loc=loc,
            ip=ip,
        )
    )

_p5_st_bf16(addr, x, *, loc=None, ip=None)

bf16(x) RN stored to addr (x is already bf16-exact here).

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _p5_st_bf16(addr, x, *, loc=None, ip=None):
    """bf16(x) RN stored to addr (x is already bf16-exact here)."""
    _asm(
        None,
        [cutlass.Int64(addr), cutlass.Float32(x)],
        "{\n\t.reg .b16 h;\n\tcvt.rn.bf16.f32 h, $1;\n\tst.global.b16 [$0], h;\n\t}",
        "l,f",
        loc=loc,
        ip=ip,
    )

_px_poll(addr, *, loc=None, ip=None)

Poll mode: a bf16-pair word of the decode output, re-polled while it is the 0xFFFFFFFF sentinel.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _px_poll(addr, *, loc=None, ip=None):
    """Poll mode: a bf16-pair word of the decode output, re-polled while it is the 0xFFFFFFFF sentinel."""
    from cutlass._mlir.extras import types as _T

    return cutlass.Int32(
        _asm(
            _T.i32(),
            [cutlass.Int64(addr)],
            "{\n\t.reg .b32 w;\n\t.reg .pred p;\n\tPXPL:\n\tld.volatile.global.b32 w, [$1];\n\t"
            "setp.eq.u32 p, w, 0xFFFFFFFF;\n\t@p bra PXPL;\n\tmov.b32 $0, w;\n\t}",
            "=r,l",
            loc=loc,
            ip=ip,
        )
    )

_ra3_bf4(p, *, loc=None, ip=None)

2 packed bf16 words -> 4 fp32 (exact)

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra3_bf4(p, *, loc=None, ip=None):
    """2 packed bf16 words -> 4 fp32 (exact)"""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    out = _asm(
        _llvm.StructType.get_literal([_T.f32()] * 4),
        [cutlass.Int32(p[0]), cutlass.Int32(p[1])],
        "{\n\tshl.b32 $0, $4, 16;\n\tand.b32 $1, $4, 0xFFFF0000;\n\tshl.b32 $2, $5, 16;\n\t"
        "and.b32 $3, $5, 0xFFFF0000;\n\t}",
        "=f,=f,=f,=f,r,r",
        loc=loc,
        ip=ip,
    )  # side: no CSE across passes
    return _f4(out, loc, ip)

_ra3_e4m3x4(w, *, loc=None, ip=None)

_ra_e4m3x4 kept per use (side effects: a CSE across the halves would hold 16 T fp32 live)

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra3_e4m3x4(w, *, loc=None, ip=None):
    """_ra_e4m3x4 kept per use (side effects: a CSE across the halves would hold 16 T fp32 live)"""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    out = _asm(
        _llvm.StructType.get_literal([_T.f32()] * 4),
        [cutlass.Int32(w)],
        "{\n\t.reg .b16 lo, hi, a, b, c, d;\n\t.reg .b32 h0, h1;\n\tmov.b32 {lo, hi}, $4;\n\t"
        "cvt.rn.f16x2.e4m3x2 h0, lo;\n\tcvt.rn.f16x2.e4m3x2 h1, hi;\n\tmov.b32 {a, b}, h0;\n\t"
        "mov.b32 {c, d}, h1;\n\tcvt.f32.f16 $0, a;\n\tcvt.f32.f16 $1, b;\n\tcvt.f32.f16 $2, c;\n\t"
        "cvt.f32.f16 $3, d;\n\t}",
        "=f,=f,=f,=f,r",
        loc=loc,
        ip=ip,
    )
    return _f4(out, loc, ip)

_ra3_entry(tid, dst, mb_r, mb_a, src_r, src_a, *, loc=None, ip=None)

Thread 0: init both barriers, then W' (mb_r) and W'' (mb_a) by 16 KB bulk copies in one burst

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra3_entry(tid, dst, mb_r, mb_a, src_r, src_a, *, loc=None, ip=None):
    """Thread 0: init both barriers, then W' (mb_r) and W'' (mb_a) by 16 KB bulk copies in one burst"""
    cpr = "".join(
        f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{RA_R_OFF + 16384 * i}], "
        f"[$4+{16384 * i}], 16384, [$2];\n"
        for i in range(RA_R_BYTES // 16384)
    )
    cpa = "".join(
        f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{16384 * i}], "
        f"[$5+{16384 * i}], 16384, [$3];\n"
        for i in range(RA_A_BYTES // 16384)
    )
    _asm(
        None,
        [
            cutlass.Int32(tid),
            cutlass.Int32(dst),
            cutlass.Int32(mb_r),
            cutlass.Int32(mb_a),
            cutlass.Int64(src_r),
            cutlass.Int64(src_a),
        ],
        "{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p mbarrier.init.shared::cta.b64 [$2], 1;\n\t"
        "@p mbarrier.init.shared::cta.b64 [$3], 1;\n\t@p fence.mbarrier_init.release.cluster;\n\t"
        "@p fence.proxy.async.shared::cta;\n\t"
        f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$2], {RA_R_BYTES};\n"
        + cpr
        + f"\t@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$3], {RA_A_BYTES};\n"
        + cpa
        + "\t}",
        "r,r,r,r,l,l,~{memory}",
        loc=loc,
        ip=ip,
    )

_ra3_ld2_cg(addr, *, loc=None, ip=None)

4 bf16 (8 B) at L2, kept packed

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra3_ld2_cg(addr, *, loc=None, ip=None):
    """4 bf16 (8 B) at L2, kept packed"""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    out = _asm(
        _llvm.StructType.get_literal([_T.i32()] * 2),
        [cutlass.Int64(addr)],
        "ld.global.cg.v2.b32 {$0, $1}, [$2];",
        "=r,=r,l",
        loc=loc,
        ip=ip,
    )
    return [
        cutlass.Int32(_llvm.extractvalue(_T.i32(), out, [i], loc=loc, ip=ip))
        for i in range(2)
    ]

_ra3_wred(v, lane)

Warp reduce-scatter of len(v) (power of 2, <= 32) per-lane values: returns the lane's sum of output lane >> (5 - log2 len(v)) over all 32 lanes (halving exchange, then plain butterflies)

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
def _ra3_wred(v, lane):
    """Warp reduce-scatter of len(v) (power of 2, <= 32) per-lane values: returns the lane's sum of output
    lane >> (5 - log2 len(v)) over all 32 lanes (halving exchange, then plain butterflies)"""
    d = 16
    while len(v) > 1:
        half = len(v) // 2
        up = (lane & cutlass.Int32(d)) != cutlass.Int32(0)
        nv = []
        for i in range(half):
            snd = cutlass.select_(up, v[i], v[i + half])
            kp = cutlass.select_(up, v[i + half], v[i])
            nv.append(_ra_f("add.rn", kp, _ra_bfly(snd, d)))
        v = nv
        d //= 2
    x = v[0]
    while d >= 1:
        x = _ra_f("add.rn", x, _ra_bfly(x, d))
        d //= 2
    return x

_ra_e4m3x4(w, *, loc=None, ip=None)

4 packed E4M3 (byte 0 first) -> 4 fp32 (exact)

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra_e4m3x4(w, *, loc=None, ip=None):
    """4 packed E4M3 (byte 0 first) -> 4 fp32 (exact)"""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    out = _asm(
        _llvm.StructType.get_literal([_T.f32()] * 4),
        [cutlass.Int32(w)],
        "{\n\t.reg .b16 lo, hi, a, b, c, d;\n\t.reg .b32 h0, h1;\n\tmov.b32 {lo, hi}, $4;\n\t"
        "cvt.rn.f16x2.e4m3x2 h0, lo;\n\tcvt.rn.f16x2.e4m3x2 h1, hi;\n\tmov.b32 {a, b}, h0;\n\t"
        "mov.b32 {c, d}, h1;\n\tcvt.f32.f16 $0, a;\n\tcvt.f32.f16 $1, b;\n\tcvt.f32.f16 $2, c;\n\t"
        "cvt.f32.f16 $3, d;\n\t}",
        "=f,=f,=f,=f,r",
        side=False,
        loc=loc,
        ip=ip,
    )
    return _f4(out, loc, ip)

_ra_entry(tid, mb0, mb1, dst, src_a, src_r, *, loc=None, ip=None)

Thread 0: init both W'' barriers, the first half (experts 0-3, 64 KB) by 4 bulk copies on mb0; L2 prefetch of the CTA's r blob

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra_entry(tid, mb0, mb1, dst, src_a, src_r, *, loc=None, ip=None):
    """Thread 0: init both W'' barriers, the first half (experts 0-3, 64 KB) by 4 bulk copies on mb0; L2 prefetch of the
    CTA's r blob"""
    cp = "".join(
        f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$3+{16384 * i}], "
        f"[$4+{16384 * i}], 16384, [$1];\n"
        for i in range(RA_A_BYTES // 2 // 16384)
    )
    _asm(
        None,
        [
            cutlass.Int32(tid),
            cutlass.Int32(mb0),
            cutlass.Int32(mb1),
            cutlass.Int32(dst),
            cutlass.Int64(src_a),
            cutlass.Int64(src_r),
        ],
        "{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p mbarrier.init.shared::cta.b64 [$1], 1;\n\t"
        "@p mbarrier.init.shared::cta.b64 [$2], 1;\n\t"
        "@p fence.mbarrier_init.release.cluster;\n\t@p fence.proxy.async.shared::cta;\n\t"
        f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$1], {RA_A_BYTES // 2};\n"
        + cp
        + f"\t@p cp.async.bulk.prefetch.L2.global [$5], {RA_R_BYTES};\n\t}}",
        "r,r,r,r,l,l,~{memory}",
        loc=loc,
        ip=ip,
    )

_ra_ld2bf_cg(addr, *, loc=None, ip=None)

4 bf16 at addr (8 B) as 4 fp32

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra_ld2bf_cg(addr, *, loc=None, ip=None):
    """4 bf16 at addr (8 B) as 4 fp32"""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    out = _asm(
        _llvm.StructType.get_literal([_T.f32()] * 4),
        [cutlass.Int64(addr)],
        "{\n\t.reg .b32 a, b, c, d;\n\tld.global.cg.v2.b32 {a, c}, [$4];\n\tshl.b32 $0, a, 16;\n\t"
        "and.b32 $1, a, 0xFFFF0000;\n\tshl.b32 $2, c, 16;\n\tand.b32 $3, c, 0xFFFF0000;\n\t}",
        "=f,=f,=f,=f,l",
        loc=loc,
        ip=ip,
    )
    return _f4(out, loc, ip)

_ra_ld4b_cg(addr, *, loc=None, ip=None)

4 words at L2 (the decode's output: never an L1 copy)

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra_ld4b_cg(addr, *, loc=None, ip=None):
    """4 words at L2 (the decode's output: never an L1 copy)"""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    out = _asm(
        _llvm.StructType.get_literal([_T.i32()] * 4),
        [cutlass.Int64(addr)],
        "ld.global.cg.v4.b32 {$0, $1, $2, $3}, [$4];",
        "=r,=r,=r,=r,l",
        loc=loc,
        ip=ip,
    )
    return [
        cutlass.Int32(_llvm.extractvalue(_T.i32(), out, [i], loc=loc, ip=ip))
        for i in range(4)
    ]

_ra_load_a1(tid, mb1, dst, src_a, *, loc=None, ip=None)

Thread 0, after the CTA barrier that ends the generic-proxy use of the region: the second W'' half on mb1

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra_load_a1(tid, mb1, dst, src_a, *, loc=None, ip=None):
    """Thread 0, after the CTA barrier that ends the generic-proxy use of the region: the second W'' half on mb1"""
    cp = "".join(
        f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$2+{16384 * i}], "
        f"[$3+{16384 * i}], 16384, [$1];\n"
        for i in range(RA_A_BYTES // 2 // 16384)
    )
    _asm(
        None,
        [
            cutlass.Int32(tid),
            cutlass.Int32(mb1),
            cutlass.Int32(dst),
            cutlass.Int64(src_a),
        ],
        "{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p fence.proxy.async.shared::cta;\n\t"
        f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$1], {RA_A_BYTES // 2};\n"
        + cp
        + "\t}",
        "r,r,r,l,~{memory}",
        loc=loc,
        ip=ip,
    )

_ra_scale_cg(addr, *, loc=None, ip=None)

UE8M0 byte at addr -> fp32 2^(s - 127) (s = 0: 2^-127)

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra_scale_cg(addr, *, loc=None, ip=None):
    """UE8M0 byte at addr -> fp32 2^(s - 127) (s = 0: 2^-127)"""
    from cutlass._mlir.extras import types as _T

    return cutlass.Float32(
        _asm(
            _T.f32(),
            [cutlass.Int64(addr)],
            "{\n\t.reg .b32 s, b;\n\t.reg .pred z;\n\tld.global.cg.u8 s, [$1];\n\t"
            "shl.b32 b, s, 23;\n\tsetp.eq.u32 z, s, 0;\n\tselp.b32 $0, 0x00400000, b, z;\n\t}",
            "=f,l",
            loc=loc,
            ip=ip,
        )
    )

_ra_st_tag1(addr, tag, z, on, *, loc=None, ip=None)

TP4: one rx word (part 0 = a + r) of one (token, src, expert): tag << 32 | fp32, relaxed sys

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra_st_tag1(addr, tag, z, on, *, loc=None, ip=None):
    """TP4: one rx word (part 0 = a + r) of one (token, src, expert): tag << 32 | fp32, relaxed sys"""
    _asm(
        None,
        [
            cutlass.Int64(addr),
            cutlass.Int32(tag),
            cutlass.Float32(z),
            cutlass.Int32(on),
        ],
        "{\n\t.reg .pred p;\n\t.reg .b32 f;\n\t.reg .b64 q;\n\tsetp.ne.u32 p, $3, 0;\n\tmov.b32 f, $2;\n\t"
        "mov.b64 q, {f, $1};\n\t@p st.relaxed.sys.global.b64 [$0], q;\n\t}",
        "l,r,f,r",
        loc=loc,
        ip=ip,
    )

_ra_st_tag2(addr, tag, za, zr, on, *, loc=None, ip=None)

Rx words (part 0 = a, part 1 = r) of one (token, src, expert): tag << 32 | fp32, relaxed sys (peer memory)

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _ra_st_tag2(addr, tag, za, zr, on, *, loc=None, ip=None):
    """Rx words (part 0 = a, part 1 = r) of one (token, src, expert): tag << 32 | fp32, relaxed sys (peer memory)"""
    _asm(
        None,
        [
            cutlass.Int64(addr),
            cutlass.Int32(tag),
            cutlass.Float32(za),
            cutlass.Float32(zr),
            cutlass.Int32(on),
        ],
        "{\n\t.reg .pred p;\n\t.reg .b32 f, g;\n\t.reg .b64 q, q1;\n\tsetp.ne.u32 p, $4, 0;\n\tmov.b32 f, $2;\n\t"
        "mov.b32 g, $3;\n\tmov.b64 q, {f, $1};\n\tmov.b64 q1, {g, $1};\n\t"
        f"@p st.relaxed.sys.global.b64 [$0], q;\n\t@p st.relaxed.sys.global.b64 [$0+{8 * 128}], q1;\n\t}}",
        "l,r,f,f,r",
        loc=loc,
        ip=ip,
    )

_rc_entry(tid, dst, mb0, mb1, mb2, src_r, src_a, *, loc=None, ip=None)

Thread 0: init 3 barriers; W' (96 KB, mb0) at 0 and W'' experts 0..RC_A0-1 (mb1) after it, one burst

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _rc_entry(tid, dst, mb0, mb1, mb2, src_r, src_a, *, loc=None, ip=None):
    """Thread 0: init 3 barriers; W' (96 KB, mb0) at 0 and W'' experts 0..RC_A0-1 (mb1) after it, one burst"""
    cpr = "".join(
        f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{16384 * i}], "
        f"[$5+{16384 * i}], 16384, [$2];\n"
        for i in range(RA_R_BYTES // 16384)
    )
    cpa = "".join(
        f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{RA_R_BYTES + 16384 * i}], "
        f"[$6+{16384 * i}], 16384, [$3];\n"
        for i in range(RC_A0 * RC_E // 16384)
    )
    _asm(
        None,
        [
            cutlass.Int32(tid),
            cutlass.Int32(dst),
            cutlass.Int32(mb0),
            cutlass.Int32(mb1),
            cutlass.Int32(mb2),
            cutlass.Int64(src_r),
            cutlass.Int64(src_a),
        ],
        "{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p mbarrier.init.shared::cta.b64 [$2], 1;\n\t"
        "@p mbarrier.init.shared::cta.b64 [$3], 1;\n\t@p mbarrier.init.shared::cta.b64 [$4], 1;\n\t"
        "@p fence.mbarrier_init.release.cluster;\n\t@p fence.proxy.async.shared::cta;\n\t"
        f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$2], {RA_R_BYTES};\n"
        + cpr
        + f"\t@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$3], {RC_A0 * RC_E};\n"
        + cpa
        + "\t}",
        "r,r,r,r,r,l,l,~{memory}",
        loc=loc,
        ip=ip,
    )

_rc_load_a(tid, dst, mb2, src_a, *, loc=None, ip=None)

Thread 0, after the CTA barrier that ends the generic reads of W': W'' experts RC_A0..7 over it (mb2)

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _rc_load_a(tid, dst, mb2, src_a, *, loc=None, ip=None):
    """Thread 0, after the CTA barrier that ends the generic reads of W': W'' experts RC_A0..7 over it (mb2)"""
    n = (RA_E - RC_A0) * RC_E
    cp = "".join(
        f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{16384 * i}], "
        f"[$3+{RC_A0 * RC_E + 16384 * i}], 16384, [$2];\n"
        for i in range(n // 16384)
    )
    _asm(
        None,
        [
            cutlass.Int32(tid),
            cutlass.Int32(dst),
            cutlass.Int32(mb2),
            cutlass.Int64(src_a),
        ],
        "{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p fence.proxy.async.shared::cta;\n\t"
        f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$2], {n};\n" + cp + "\t}",
        "r,r,r,l,~{memory}",
        loc=loc,
        ip=ip,
    )

_rope_bits(c, a, s, b, *, loc=None, ip=None)

bf16(c * a + s * b) with separately rounded products (no FMA), as fp32 bits: the stock RoPE arithmetic.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _rope_bits(c, a, s, b, *, loc=None, ip=None):
    """bf16(c * a + s * b) with separately rounded products (no FMA), as fp32 bits: the stock RoPE arithmetic."""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    return cutlass.Int32(
        _llvm.inline_asm(
            _T.i32(), [cutlass.Float32(v).ir_value(loc=loc, ip=ip) for v in (c, a, s, b)],
            "{\n\t.reg .f32 p, q, r;\n\t.reg .b16 h, z;\n\tmul.rn.f32 p, $1, $2;\n\tmul.rn.f32 q, $3, $4;\n\t"
            "add.rn.f32 r, p, q;\n\tcvt.rn.bf16.f32 h, r;\n\tmov.b16 z, 0;\n\tmov.b32 $0, {z, h};\n\t}", "=r,f,f,f,f",
            has_side_effects=False, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
        )
    )  # fmt: skip

_st_async_v4(dst, w0, w1, w2, w3, mbar, *, loc=None, ip=None)

st.async of four int32 words to a shared::cluster address, completing mbar there by 16 bytes.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _st_async_v4(dst, w0, w1, w2, w3, mbar, *, loc=None, ip=None):
    """st.async of four int32 words to a shared::cluster address, completing `mbar` there by 16 bytes."""
    from cutlass._mlir.dialects import llvm as _llvm

    _llvm.inline_asm(
        None,
        [cutlass.Int32(dst).ir_value(loc=loc, ip=ip), cutlass.Int32(w0).ir_value(loc=loc, ip=ip),
         cutlass.Int32(w1).ir_value(loc=loc, ip=ip), cutlass.Int32(w2).ir_value(loc=loc, ip=ip),
         cutlass.Int32(w3).ir_value(loc=loc, ip=ip), cutlass.Int32(mbar).ir_value(loc=loc, ip=ip)],
        "st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [$0], {$1, $2, $3, $4}, [$5];", "r,r,r,r,r,r",
        has_side_effects=True, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
    )  # fmt: skip

_test_wait_cluster(mbar, parity, *, loc=None, ip=None)

mbarrier.test_wait.parity.acquire.cluster on a barrier of this CTA completed by other CTAs' st.async.

Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
@dsl_user_op
def _test_wait_cluster(mbar, parity, *, loc=None, ip=None):
    """mbarrier.test_wait.parity.acquire.cluster on a barrier of this CTA completed by other CTAs' st.async."""
    from cutlass._mlir.dialects import llvm as _llvm
    from cutlass._mlir.extras import types as _T

    done = cutlass.Int32(
        _llvm.inline_asm(
            _T.i32(), [mbar.toint(loc=loc, ip=ip).ir_value(), cutlass.Int32(parity).ir_value(loc=loc, ip=ip)],
            "{\n\t.reg .pred p;\n\tmbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 p, [$1], $2;\n\t"
            "selp.u32 $0, 1, 0, p;\n\t}", "=r,r,r,~{memory}", has_side_effects=True,
            is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
        )
    )  # fmt: skip
    return done != cutlass.Int32(0)