Skip to content

vllm.distributed.pp_transport.rocm_sdma

The flag ops and peer copy of copy_engine.CopyEngineTransport on ROCm (gfx950): HIP stream memory ops and SDMA engines.

On HIP 7.16 a HIP peer copy runs as a blit kernel on the sender's CUs (hipMemcpyFlagExtPreferCE does not apply to device-to-device copies, and hipMemcpyDeviceToDeviceNoCU picks engines whose peer copy fails on MI355X while returning success). The copies therefore go through ROCr's hsa_amd_memory_async_copy_on_engine on explicit engines, kept in order with the side stream by HSA signals. Per step N:

  • the host queues the data copies behind a dependency signal, which the side stream opens (writes 0) after the transport's empty >= N - 1 wait;
  • two 8-byte copies of N, queued behind the data copies, publish full = N to the receiver and sent = N to this rank;
  • the side stream waits for sent >= N, so the transport's event recorded on it covers the copies, as on CUDA.

The copies sit on their engines' queues, blocked on the dependency signal, until the side stream opens it. ROCr gives each engine one queue per process, so only other SDMA peer copies from this GPU on the same engines would wait behind them: host copies run on engines 0 and 1, and HIP peer copies on blit kernels.

The shared buffers come from allocate_shared_buffer_and_handle, which on ROCm allocates uncached memory, so a stream wait never reads a stale L2 line.

Classes:

  • SdmaCopyEngines –

    ROCm's flag ops and, for a sender, its peer copy to the next stage.

SdmaCopyEngines

ROCm's flag ops and, for a sender, its peer copy to the next stage.

Methods:

  • __init__ –

    words: this rank's uncached shared-buffer bytes for sent and

  • copy –

    Queue, behind stream's work, the copy of each key's first rows

Source code in vllm/distributed/pp_transport/rocm_sdma.py
class SdmaCopyEngines:
    """ROCm's flag ops and, for a sender, its peer copy to the next stage."""

    @staticmethod
    def check_device(device: int) -> None:
        from vllm.platforms.rocm import on_gfx950

        if not on_gfx950():
            raise RuntimeError("PP SDMA copies need gfx950 (MI355X)")

    @staticmethod
    def wait_flag(stream: torch.cuda.Stream, addr: int, at_least: int) -> None:
        mask = 0xFFFF_FFFF_FFFF_FFFF
        gte = _HIP_STREAM_WAIT_VALUE_GTE
        if err := _hip().hipStreamWaitValue64(
            stream.cuda_stream, addr, at_least, gte, mask
        ):
            raise RuntimeError(f"hipStreamWaitValue64 failed: {err}")

    @staticmethod
    def write_flag(stream: torch.cuda.Stream, addr: int, value: int) -> None:
        if err := _hip().hipStreamWriteValue64(stream.cuda_stream, addr, value, 0):
            raise RuntimeError(f"hipStreamWriteValue64 failed: {err}")

    def __init__(
        self,
        send_buf: int,
        peer_recv_buf: int,
        peer_full: int,
        words: int,
        peer_pci: tuple[int, int, int],
        cross_node: bool,
    ) -> None:
        """`words`: this rank's uncached shared-buffer bytes for `sent` and
        the step-number sources; `peer_pci`: the next stage's PCI address.
        The peer must be on this node (`cross_node` false)."""
        assert not cross_node, "PP SDMA copies need one node"
        props = torch.cuda.get_device_properties(
            torch.accelerator.current_device_index()
        )
        self.agent = _agent(
            (props.pci_domain_id, props.pci_bus_id, props.pci_device_id)
        )
        self.peer_agent = _agent(peer_pci)
        hsa = _hsa()
        free = hsa.free_engines(self.peer_agent, self.agent)
        if any(e & ~free for e in _ENGINES):
            raise RuntimeError(
                f"SDMA engines {[e.bit_length() - 1 for e in _ENGINES]} are not "
                f"free for xGMI copies to the next stage (free mask {free:#x})"
            )
        self.send_buf = send_buf
        self.peer_recv_buf = peer_recv_buf
        self.peer_full = peer_full
        self.sent, self.seq_sources = words, words + _SEQ_SOURCES
        self.ring = [
            _StepSignals(
                hsa.signal(),
                tuple(hsa.signal() for _ in _ENGINES),
                hsa.signal(),
                hsa.signal(),
            )
            for _ in range(_RING)
        ]

    def copy(
        self, stream: torch.cuda.Stream, layout: dict, rows: dict, seq: int
    ) -> None:
        """Queue, behind `stream`'s work, the copy of each key's first `rows`
        rows into the peer's receive region and the publish of `full = seq`,
        and make `stream` wait for them."""
        hsa = _hsa()
        i = seq % _RING
        sig = self.ring[i]
        if seq > _RING:
            # Resetting a signal that a queued copy still uses deadlocks it.
            # `sent` is the entry's last copy, queued after `full` on the same
            # engine. This blocks only if the host is _RING steps ahead of the
            # GPU, more than vLLM keeps batches in flight.
            hsa.wait_zero(sig.sent)

        # Each key's rows split across the engines, in 256-byte pieces.
        copies: list[list[tuple[int, int]]] = [[] for _ in _ENGINES]
        for key, n in rows.items():
            r = layout[key]
            size = n * r.row_bytes
            piece = -(-size // len(_ENGINES) // _ALIGN) * _ALIGN
            for e in range(len(_ENGINES)):
                lo = e * piece
                if lo < size:
                    copies[e].append((r.offset + lo, min(piece, size - lo)))

        hsa.store(sig.dep, 1)
        for done, engine_copies in zip(sig.data, copies):
            hsa.store(done, len(engine_copies))
        hsa.store(sig.full, 1)
        hsa.store(sig.sent, 1)
        for engine, done, engine_copies in zip(_ENGINES, sig.data, copies):
            for offset, size in engine_copies:
                hsa.copy(
                    self.peer_recv_buf + offset,
                    self.peer_agent,
                    self.send_buf + offset,
                    self.agent,
                    size,
                    [sig.dep],
                    done,
                    engine,
                )
        # Behind `dep` too: with no rows the data signals start at 0.
        seq_source = self.seq_sources + 8 * i
        after_data = [sig.dep, *sig.data]
        for dst, agent, done in (
            (self.peer_full, self.peer_agent, sig.full),
            (self.sent, self.agent, sig.sent),
        ):
            hsa.copy(
                dst, agent, seq_source, self.agent, 8, after_data, done, _ENGINES[0]
            )

        self.write_flag(stream, seq_source, seq)
        self.write_flag(stream, hsa.value_pointer(sig.dep), 0)
        self.wait_flag(stream, self.sent, seq)

__init__(send_buf, peer_recv_buf, peer_full, words, peer_pci, cross_node)

words: this rank's uncached shared-buffer bytes for sent and the step-number sources; peer_pci: the next stage's PCI address. The peer must be on this node (cross_node false).

Source code in vllm/distributed/pp_transport/rocm_sdma.py
def __init__(
    self,
    send_buf: int,
    peer_recv_buf: int,
    peer_full: int,
    words: int,
    peer_pci: tuple[int, int, int],
    cross_node: bool,
) -> None:
    """`words`: this rank's uncached shared-buffer bytes for `sent` and
    the step-number sources; `peer_pci`: the next stage's PCI address.
    The peer must be on this node (`cross_node` false)."""
    assert not cross_node, "PP SDMA copies need one node"
    props = torch.cuda.get_device_properties(
        torch.accelerator.current_device_index()
    )
    self.agent = _agent(
        (props.pci_domain_id, props.pci_bus_id, props.pci_device_id)
    )
    self.peer_agent = _agent(peer_pci)
    hsa = _hsa()
    free = hsa.free_engines(self.peer_agent, self.agent)
    if any(e & ~free for e in _ENGINES):
        raise RuntimeError(
            f"SDMA engines {[e.bit_length() - 1 for e in _ENGINES]} are not "
            f"free for xGMI copies to the next stage (free mask {free:#x})"
        )
    self.send_buf = send_buf
    self.peer_recv_buf = peer_recv_buf
    self.peer_full = peer_full
    self.sent, self.seq_sources = words, words + _SEQ_SOURCES
    self.ring = [
        _StepSignals(
            hsa.signal(),
            tuple(hsa.signal() for _ in _ENGINES),
            hsa.signal(),
            hsa.signal(),
        )
        for _ in range(_RING)
    ]

copy(stream, layout, rows, seq)

Queue, behind stream's work, the copy of each key's first rows rows into the peer's receive region and the publish of full = seq, and make stream wait for them.

Source code in vllm/distributed/pp_transport/rocm_sdma.py
def copy(
    self, stream: torch.cuda.Stream, layout: dict, rows: dict, seq: int
) -> None:
    """Queue, behind `stream`'s work, the copy of each key's first `rows`
    rows into the peer's receive region and the publish of `full = seq`,
    and make `stream` wait for them."""
    hsa = _hsa()
    i = seq % _RING
    sig = self.ring[i]
    if seq > _RING:
        # Resetting a signal that a queued copy still uses deadlocks it.
        # `sent` is the entry's last copy, queued after `full` on the same
        # engine. This blocks only if the host is _RING steps ahead of the
        # GPU, more than vLLM keeps batches in flight.
        hsa.wait_zero(sig.sent)

    # Each key's rows split across the engines, in 256-byte pieces.
    copies: list[list[tuple[int, int]]] = [[] for _ in _ENGINES]
    for key, n in rows.items():
        r = layout[key]
        size = n * r.row_bytes
        piece = -(-size // len(_ENGINES) // _ALIGN) * _ALIGN
        for e in range(len(_ENGINES)):
            lo = e * piece
            if lo < size:
                copies[e].append((r.offset + lo, min(piece, size - lo)))

    hsa.store(sig.dep, 1)
    for done, engine_copies in zip(sig.data, copies):
        hsa.store(done, len(engine_copies))
    hsa.store(sig.full, 1)
    hsa.store(sig.sent, 1)
    for engine, done, engine_copies in zip(_ENGINES, sig.data, copies):
        for offset, size in engine_copies:
            hsa.copy(
                self.peer_recv_buf + offset,
                self.peer_agent,
                self.send_buf + offset,
                self.agent,
                size,
                [sig.dep],
                done,
                engine,
            )
    # Behind `dep` too: with no rows the data signals start at 0.
    seq_source = self.seq_sources + 8 * i
    after_data = [sig.dep, *sig.data]
    for dst, agent, done in (
        (self.peer_full, self.peer_agent, sig.full),
        (self.sent, self.agent, sig.sent),
    ):
        hsa.copy(
            dst, agent, seq_source, self.agent, 8, after_data, done, _ENGINES[0]
        )

    self.write_flag(stream, seq_source, seq)
    self.write_flag(stream, hsa.value_pointer(sig.dep), 0)
    self.wait_flag(stream, self.sent, seq)

_Hsa

The ROCr calls of an SDMA copy, over ctypes.

Methods:

  • copy –

    Queue a copy on engine that starts once every signal in deps is

  • free_engines –

    The mask of SDMA engines free for copies from src to dst.

Source code in vllm/distributed/pp_transport/rocm_sdma.py
class _Hsa:
    """The ROCr calls of an SDMA copy, over ctypes."""

    def __init__(self) -> None:
        lib = ctypes.CDLL("libhsa-runtime64.so.1")
        self._visit_type = ctypes.CFUNCTYPE(ctypes.c_int, _Agent, ctypes.c_void_p)
        lib.hsa_iterate_agents.argtypes = [self._visit_type, ctypes.c_void_p]
        lib.hsa_agent_get_info.argtypes = [_Agent, ctypes.c_int, ctypes.c_void_p]
        lib.hsa_amd_signal_create.argtypes = [
            ctypes.c_int64,
            ctypes.c_uint32,
            ctypes.c_void_p,
            ctypes.c_uint64,
            ctypes.POINTER(_Signal),
        ]
        lib.hsa_amd_signal_value_pointer.argtypes = [
            _Signal,
            ctypes.POINTER(ctypes.c_void_p),
        ]
        lib.hsa_amd_memory_copy_engine_status.argtypes = [
            _Agent,
            _Agent,
            ctypes.POINTER(ctypes.c_uint32),
        ]
        lib.hsa_signal_store_relaxed.argtypes = [_Signal, ctypes.c_int64]
        lib.hsa_signal_wait_scacquire.argtypes = [
            _Signal,
            ctypes.c_int,
            ctypes.c_int64,
            ctypes.c_uint64,
            ctypes.c_int,
        ]
        lib.hsa_signal_wait_scacquire.restype = ctypes.c_int64
        lib.hsa_amd_memory_async_copy_on_engine.argtypes = [
            ctypes.c_void_p,
            _Agent,
            ctypes.c_void_p,
            _Agent,
            ctypes.c_size_t,
            ctypes.c_uint32,
            ctypes.POINTER(_Signal),
            _Signal,
            ctypes.c_uint32,
            ctypes.c_bool,
        ]
        if lib.hsa_init():
            raise RuntimeError("hsa_init failed")
        self.lib = lib
        self.agents: dict[tuple[int, int], _Agent] = {}

        def visit(agent, _) -> int:
            kind = ctypes.c_uint32()
            lib.hsa_agent_get_info(agent, _HSA_AGENT_INFO_DEVICE, ctypes.byref(kind))
            if kind.value == _HSA_DEVICE_TYPE_GPU:
                domain, bdf = ctypes.c_uint32(), ctypes.c_uint32()
                for attr, out in (
                    (_HSA_AMD_AGENT_INFO_DOMAIN, domain),
                    (_HSA_AMD_AGENT_INFO_BDFID, bdf),
                ):
                    lib.hsa_agent_get_info(agent, attr, ctypes.byref(out))
                key = (domain.value, bdf.value & ~0x7)
                self.agents[key] = _Agent(agent.handle)
            return 0

        lib.hsa_iterate_agents(self._visit_type(visit), None)

    def agent(self, domain: int, bdf: int) -> _Agent:
        if (domain, bdf) not in self.agents:
            raise RuntimeError(f"no HSA GPU agent at PCI {domain:#x}:{bdf:#x}")
        return self.agents[(domain, bdf)]

    def free_engines(self, dst: _Agent, src: _Agent) -> int:
        """The mask of SDMA engines free for copies from `src` to `dst`."""
        mask = ctypes.c_uint32()
        if self.lib.hsa_amd_memory_copy_engine_status(dst, src, ctypes.byref(mask)):
            return 0
        return mask.value

    def signal(self) -> _Signal:
        s = _Signal()
        if self.lib.hsa_amd_signal_create(
            1, 0, None, _HSA_AMD_SIGNAL_AMD_GPU_ONLY, ctypes.byref(s)
        ):
            raise RuntimeError("hsa_amd_signal_create failed")
        return s

    def value_pointer(self, signal: _Signal) -> int:
        p = ctypes.c_void_p()
        if self.lib.hsa_amd_signal_value_pointer(signal, ctypes.byref(p)):
            raise RuntimeError("hsa_amd_signal_value_pointer failed")
        assert p.value is not None
        return p.value

    def store(self, signal: _Signal, value: int) -> None:
        self.lib.hsa_signal_store_relaxed(signal, value)

    def wait_zero(self, signal: _Signal) -> None:
        while (
            self.lib.hsa_signal_wait_scacquire(
                signal, _HSA_SIGNAL_CONDITION_LT, 1, 1 << 40, _HSA_WAIT_STATE_ACTIVE
            )
            > 0
        ):
            pass

    def copy(
        self,
        dst: int,
        dst_agent: _Agent,
        src: int,
        src_agent: _Agent,
        size: int,
        deps: list[_Signal],
        done: _Signal,
        engine: int,
    ) -> None:
        """Queue a copy on `engine` that starts once every signal in `deps` is
        0 and decrements `done` when it finishes."""
        dep_array = (_Signal * len(deps))(*deps)
        if self.lib.hsa_amd_memory_async_copy_on_engine(
            dst,
            dst_agent,
            src,
            src_agent,
            size,
            len(deps),
            dep_array,
            done,
            engine,
            True,
        ):
            raise RuntimeError("hsa_amd_memory_async_copy_on_engine failed")

copy(dst, dst_agent, src, src_agent, size, deps, done, engine)

Queue a copy on engine that starts once every signal in deps is 0 and decrements done when it finishes.

Source code in vllm/distributed/pp_transport/rocm_sdma.py
def copy(
    self,
    dst: int,
    dst_agent: _Agent,
    src: int,
    src_agent: _Agent,
    size: int,
    deps: list[_Signal],
    done: _Signal,
    engine: int,
) -> None:
    """Queue a copy on `engine` that starts once every signal in `deps` is
    0 and decrements `done` when it finishes."""
    dep_array = (_Signal * len(deps))(*deps)
    if self.lib.hsa_amd_memory_async_copy_on_engine(
        dst,
        dst_agent,
        src,
        src_agent,
        size,
        len(deps),
        dep_array,
        done,
        engine,
        True,
    ):
        raise RuntimeError("hsa_amd_memory_async_copy_on_engine failed")

free_engines(dst, src)

The mask of SDMA engines free for copies from src to dst.

Source code in vllm/distributed/pp_transport/rocm_sdma.py
def free_engines(self, dst: _Agent, src: _Agent) -> int:
    """The mask of SDMA engines free for copies from `src` to `dst`."""
    mask = ctypes.c_uint32()
    if self.lib.hsa_amd_memory_copy_engine_status(dst, src, ctypes.byref(mask)):
        return 0
    return mask.value

_agent(pci)

The HSA agent at PCI (domain, bus, device).

Source code in vllm/distributed/pp_transport/rocm_sdma.py
def _agent(pci: tuple[int, int, int]) -> _Agent:
    """The HSA agent at PCI (domain, bus, device)."""
    domain, bus, device = pci
    return _hsa().agent(domain, (bus << 8) | (device << 3))