Prefix caching for PrefixLM requests, which send their prefix length in
extra_args["prefix_lm_len"]: salts each prefix's blocks by its digest, as the KV
of every prefix token depends on the whole prefix, and caps hits at the prefix.
The model turns chunked prefill off, so the rest of the prompt is prefilled in
one step.
Classes:
Functions:
-
prefix_lm_len
–
The valid PrefixLM prefix length a request sends, else None.
-
prefix_lm_salt
–
PrefixLM cache_salt: namespace, prefix digest, then the caller's salt.
Attributes:
-
PREFIX_LM_LEN
–
SamplingParams.extra_args key of the length of a prompt's PrefixLM prefix.
-
PREFIX_LM_SALT
–
Start of the cache_salt of every PrefixLM request, and of no other.
PREFIX_LM_LEN = 'prefix_lm_len'
module-attribute
SamplingParams.extra_args key of the length of a prompt's PrefixLM prefix.
PREFIX_LM_SALT = 'prefix-lm:'
module-attribute
Start of the cache_salt of every PrefixLM request, and of no other.
PrefixLMScheduler
Bases: Scheduler
Source code in vllm/v1/core/sched/prefix_lm_scheduler.py
| class PrefixLMScheduler(Scheduler):
def __new__(cls, *args, **kwargs):
# Model hooks pick this class before async scheduling is decided, so
# build the async twin here when the engine runs async.
vllm_config = kwargs["vllm_config"] if "vllm_config" in kwargs else args[0]
if cls is PrefixLMScheduler and vllm_config.scheduler_config.async_scheduling:
cls = PrefixLMAsyncScheduler
return super().__new__(cls)
def add_request(self, request: Request) -> None:
params = request.sampling_params
if (
(request.resumable or request.mm_features)
and params is not None
and PREFIX_LM_LEN in (params.extra_args or {})
):
# Later inputs and encoder budgets can cut such a prompt short.
logger.warning_once(
"Requests with streamed or multimodal inputs ignore %s",
PREFIX_LM_LEN,
)
request.sampling_params = params = copy.copy(params)
params.extra_args = dict(params.extra_args or {})
del params.extra_args[PREFIX_LM_LEN]
# A streaming update re-adds an existing request.
if request.request_id not in self.requests:
self._salt_by_kind(request)
super().add_request(request)
def _salt_by_kind(self, request: Request) -> None:
"""Salt PrefixLM requests by prefix; keep causal requests out."""
prefix_len = _request_prefix_lm_len(request)
salt = request.cache_salt
if prefix_len is None:
if not (salt and salt.startswith(PREFIX_LM_SALT)):
return
salt = f"causal:{salt}"
else:
assert request.prompt_token_ids is not None
own = prefix_lm_salt(request.prompt_token_ids[:prefix_len], None)
if salt == own or (salt and salt.startswith(f"{own}:")):
return
salt = f"{own}:{salt}" if salt else own
request.cache_salt = salt
request.block_hashes.clear()
request.update_block_hashes()
def _get_local_prefix_cache_hit(
self, request: Request, max_cache_hit_length: int | None = None
) -> tuple[KVCacheBlocks, int, int, bool]:
prefix_len = _request_prefix_lm_len(request)
if prefix_len is None:
return super()._get_local_prefix_cache_hit(request, max_cache_hit_length)
# KV past the prefix depends on the rest of the prompt, and a hit keeps
# the last prompt token to compute.
cap = min(prefix_len, request.num_tokens - 1)
if max_cache_hit_length is not None:
cap = min(cap, max_cache_hit_length)
blocks, num_tokens, _, diverged = super()._get_local_prefix_cache_hit(
request, cap
)
# No shared-prefix junction: Mamba alignment would end a step there,
# inside the prefix.
return blocks, num_tokens, 0, diverged
|
_salt_by_kind(request)
Salt PrefixLM requests by prefix; keep causal requests out.
Source code in vllm/v1/core/sched/prefix_lm_scheduler.py
| def _salt_by_kind(self, request: Request) -> None:
"""Salt PrefixLM requests by prefix; keep causal requests out."""
prefix_len = _request_prefix_lm_len(request)
salt = request.cache_salt
if prefix_len is None:
if not (salt and salt.startswith(PREFIX_LM_SALT)):
return
salt = f"causal:{salt}"
else:
assert request.prompt_token_ids is not None
own = prefix_lm_salt(request.prompt_token_ids[:prefix_len], None)
if salt == own or (salt and salt.startswith(f"{own}:")):
return
salt = f"{own}:{salt}" if salt else own
request.cache_salt = salt
request.block_hashes.clear()
request.update_block_hashes()
|
prefix_lm_len(params, prompt_token_ids)
The valid PrefixLM prefix length a request sends, else None.
Source code in vllm/v1/core/sched/prefix_lm_scheduler.py
| def prefix_lm_len(
params: SamplingParams | None, prompt_token_ids: Sequence[int] | None
) -> int | None:
"""The valid PrefixLM prefix length a request sends, else None."""
extra_args = params.extra_args if params is not None else None
prefix_len = extra_args.get(PREFIX_LM_LEN) if extra_args else None
if (
type(prefix_len) is int
and prompt_token_ids is not None
and 0 < prefix_len <= len(prompt_token_ids)
):
return prefix_len
return None
|
prefix_lm_salt(prefix_token_ids, cache_salt)
PrefixLM cache_salt: namespace, prefix digest, then the caller's salt.
Source code in vllm/v1/core/sched/prefix_lm_scheduler.py
| def prefix_lm_salt(prefix_token_ids: Sequence[int], cache_salt: str | None) -> str:
"""PrefixLM `cache_salt`: namespace, prefix digest, then the caller's salt."""
digest = hashlib.sha256(
np.asarray(prefix_token_ids, dtype="<i8").tobytes()
).hexdigest()
salt = PREFIX_LM_SALT + digest
return f"{salt}:{cache_salt}" if cache_salt else salt
|