Skip to content

vllm.v1.core.sched.prefix_lm_scheduler

Prefix caching for PrefixLM requests, which send their prefix length in extra_args["prefix_lm_len"]: salts each prefix's blocks by its digest, as the KV of every prefix token depends on the whole prefix, and caps hits at the prefix. The model turns chunked prefill off, so the rest of the prompt is prefilled in one step.

Classes:

Functions:

  • prefix_lm_len –

    The valid PrefixLM prefix length a request sends, else None.

  • prefix_lm_salt –

    PrefixLM cache_salt: namespace, prefix digest, then the caller's salt.

Attributes:

  • PREFIX_LM_LEN –

    SamplingParams.extra_args key of the length of a prompt's PrefixLM prefix.

  • PREFIX_LM_SALT –

    Start of the cache_salt of every PrefixLM request, and of no other.

PREFIX_LM_LEN = 'prefix_lm_len' module-attribute

SamplingParams.extra_args key of the length of a prompt's PrefixLM prefix.

PREFIX_LM_SALT = 'prefix-lm:' module-attribute

Start of the cache_salt of every PrefixLM request, and of no other.

PrefixLMScheduler

Bases: Scheduler

Source code in vllm/v1/core/sched/prefix_lm_scheduler.py
class PrefixLMScheduler(Scheduler):
    def __new__(cls, *args, **kwargs):
        # Model hooks pick this class before async scheduling is decided, so
        # build the async twin here when the engine runs async.
        vllm_config = kwargs["vllm_config"] if "vllm_config" in kwargs else args[0]
        if cls is PrefixLMScheduler and vllm_config.scheduler_config.async_scheduling:
            cls = PrefixLMAsyncScheduler
        return super().__new__(cls)

    def add_request(self, request: Request) -> None:
        params = request.sampling_params
        if (
            (request.resumable or request.mm_features)
            and params is not None
            and PREFIX_LM_LEN in (params.extra_args or {})
        ):
            # Later inputs and encoder budgets can cut such a prompt short.
            logger.warning_once(
                "Requests with streamed or multimodal inputs ignore %s",
                PREFIX_LM_LEN,
            )
            request.sampling_params = params = copy.copy(params)
            params.extra_args = dict(params.extra_args or {})
            del params.extra_args[PREFIX_LM_LEN]
        # A streaming update re-adds an existing request.
        if request.request_id not in self.requests:
            self._salt_by_kind(request)
        super().add_request(request)

    def _salt_by_kind(self, request: Request) -> None:
        """Salt PrefixLM requests by prefix; keep causal requests out."""
        prefix_len = _request_prefix_lm_len(request)
        salt = request.cache_salt
        if prefix_len is None:
            if not (salt and salt.startswith(PREFIX_LM_SALT)):
                return
            salt = f"causal:{salt}"
        else:
            assert request.prompt_token_ids is not None
            own = prefix_lm_salt(request.prompt_token_ids[:prefix_len], None)
            if salt == own or (salt and salt.startswith(f"{own}:")):
                return
            salt = f"{own}:{salt}" if salt else own
        request.cache_salt = salt
        request.block_hashes.clear()
        request.update_block_hashes()

    def _get_local_prefix_cache_hit(
        self, request: Request, max_cache_hit_length: int | None = None
    ) -> tuple[KVCacheBlocks, int, int, bool]:
        prefix_len = _request_prefix_lm_len(request)
        if prefix_len is None:
            return super()._get_local_prefix_cache_hit(request, max_cache_hit_length)
        # KV past the prefix depends on the rest of the prompt, and a hit keeps
        # the last prompt token to compute.
        cap = min(prefix_len, request.num_tokens - 1)
        if max_cache_hit_length is not None:
            cap = min(cap, max_cache_hit_length)
        blocks, num_tokens, _, diverged = super()._get_local_prefix_cache_hit(
            request, cap
        )
        # No shared-prefix junction: Mamba alignment would end a step there,
        # inside the prefix.
        return blocks, num_tokens, 0, diverged

_salt_by_kind(request)

Salt PrefixLM requests by prefix; keep causal requests out.

Source code in vllm/v1/core/sched/prefix_lm_scheduler.py
def _salt_by_kind(self, request: Request) -> None:
    """Salt PrefixLM requests by prefix; keep causal requests out."""
    prefix_len = _request_prefix_lm_len(request)
    salt = request.cache_salt
    if prefix_len is None:
        if not (salt and salt.startswith(PREFIX_LM_SALT)):
            return
        salt = f"causal:{salt}"
    else:
        assert request.prompt_token_ids is not None
        own = prefix_lm_salt(request.prompt_token_ids[:prefix_len], None)
        if salt == own or (salt and salt.startswith(f"{own}:")):
            return
        salt = f"{own}:{salt}" if salt else own
    request.cache_salt = salt
    request.block_hashes.clear()
    request.update_block_hashes()

prefix_lm_len(params, prompt_token_ids)

The valid PrefixLM prefix length a request sends, else None.

Source code in vllm/v1/core/sched/prefix_lm_scheduler.py
def prefix_lm_len(
    params: SamplingParams | None, prompt_token_ids: Sequence[int] | None
) -> int | None:
    """The valid PrefixLM prefix length a request sends, else None."""
    extra_args = params.extra_args if params is not None else None
    prefix_len = extra_args.get(PREFIX_LM_LEN) if extra_args else None
    if (
        type(prefix_len) is int
        and prompt_token_ids is not None
        and 0 < prefix_len <= len(prompt_token_ids)
    ):
        return prefix_len
    return None

prefix_lm_salt(prefix_token_ids, cache_salt)

PrefixLM cache_salt: namespace, prefix digest, then the caller's salt.

Source code in vllm/v1/core/sched/prefix_lm_scheduler.py
def prefix_lm_salt(prefix_token_ids: Sequence[int], cache_salt: str | None) -> str:
    """PrefixLM `cache_salt`: namespace, prefix digest, then the caller's salt."""
    digest = hashlib.sha256(
        np.asarray(prefix_token_ids, dtype="<i8").tobytes()
    ).hexdigest()
    salt = PREFIX_LM_SALT + digest
    return f"{salt}:{cache_salt}" if cache_salt else salt