class MuseSparkDSparkForCausalLM(Qwen3DSparkForCausalLM):
"""Muse DSpark head with the target's output-distribution transform."""
has_own_lm_head = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".self_attn.q_proj.": (".self_attn.qkv_proj.", "q"),
".self_attn.k_proj.": (".self_attn.qkv_proj.", "k"),
".self_attn.v_proj.": (".self_attn.qkv_proj.", "v"),
".self_attn.o_gate_proj.": (".self_attn.qkv_proj.", "o_gate"),
".feed_forward.gate_proj.": (".feed_forward.gate_up_proj.", 0),
".feed_forward.up_proj.": (".feed_forward.gate_up_proj.", 1),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
nn.Module.__init__(self)
assert vllm_config.speculative_config is not None
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.model = MuseSparkDSparkModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
start_layer_id=target_layer_num,
)
self.lm_head = ParallelLMHead(
self.config.draft_vocab_size,
self.config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(self.config.draft_vocab_size)
self.target_vocab_size = vllm_config.model_config.get_vocab_size()
self.draft_id_to_target_id = nn.Parameter(
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
requires_grad=False,
)
metap = self.config.metap
if metap.use_metap and metap.router_has_one_inf_dim:
width_mult = self.config.hidden_size / metap.base_width
if metap.metap_mode == "mup":
self.output_multiplier = width_mult**-1
elif metap.metap_mode == "sp":
self.output_multiplier = width_mult**-0.5
else:
raise ValueError(f"Unknown Metap mode {metap.metap_mode!r}")
else:
self.output_multiplier = 1.0
self.output_soft_cap_temp = self.config.output_soft_cap_temp
topk = getattr(self.config, "dspark_draft_topk", None)
if self.model.specialized_decode and topk is not None:
# The speculator's top-k loop never calls sample_sequential.
raise ValueError(
"The Muse Spark specialized decode runs the fused DSpark Markov "
"loop, which does not run here: dspark_draft_topk is set"
)
# The fused Markov loop under the specialized decode (sample_sequential).
self._markov_sampler: MarkovSampler | None = None
self._ordinary_markov_chain: MarkovChain | None = None
self._ordinary_markov_chain_initialized = False
if self.model.specialized_decode:
# The hook exists only here: without it the speculator may
# batch-shard the draft logits and the sequential sampling.
self.sample_sequential = self._sample_sequential_fused
def apply_output_logit_transform(self, logits: torch.Tensor) -> torch.Tensor:
logits = logits.float() * self.output_multiplier
if self.output_soft_cap_temp is not None:
logits = self.output_soft_cap_temp * torch.tanh(
logits / self.output_soft_cap_temp
)
return logits
def sample_markov_chain(
self,
speculator,
base_logits,
prev,
idx_map,
sample_pos,
out_tokens,
*,
prev_indices=None,
) -> torch.Tensor | None:
"""Fuse sampling tails after the ordinary base-logits projection.
Returns each step's Markov embedding, `[rows, steps, rank]`, or None
when the speculator must run its own loop."""
if (
self.model.specialized_decode
or self.model._compilation_mode != CompilationMode.NONE
or not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100)
):
return None
if not self._ordinary_markov_chain_initialized:
from .specialized_decode.runtime.markov_chain import build_markov_chain
self._ordinary_markov_chain = build_markov_chain(self, speculator)
self._ordinary_markov_chain_initialized = True
if self._ordinary_markov_chain is not None:
logger.info_once(
"Using the fused DSpark Markov chain (confidence head: %s).",
speculator.use_confidence_head,
scope="process",
)
if self._ordinary_markov_chain is None:
return None
return self._ordinary_markov_chain.run(
speculator,
base_logits,
prev,
idx_map,
sample_pos,
out_tokens,
prev_indices=prev_indices,
)
def output_logit_transform_params(self) -> tuple[float, float | None]:
"""`(multiplier, soft_cap)` of `apply_output_logit_transform`.
The fused DSpark Markov loop computes the transform from these values
(`logits * multiplier`, then `soft_cap * tanh(x / soft_cap)` unless
`soft_cap` is None) instead of calling it, so a subclass that changes
the transform must override this too.
"""
return self.output_multiplier, self.output_soft_cap_temp
def _sample_sequential_fused(
self,
speculator: "DSparkSpeculator",
num_reqs: int,
head_hidden: torch.Tensor,
) -> bool:
"""The `sample_sequential` hook under the specialized decode, which
DSparkSpeculator._sample_sequential asks first: the fused Markov
sampling loop samples the drafts, and this returns True."""
if self._markov_sampler is None:
# Built on the first call (the memory-profiling run), from the
# speculator's sampling state.
self._markov_sampler = self._build_markov_sampler(speculator)
draft_logits = _draft_logits_values(speculator)
assert draft_logits is not None
self._markov_sampler.sample(
num_reqs,
head_hidden,
sample_indices=speculator.sample_indices,
sample_idx_mapping=speculator.sample_idx_mapping,
sample_pos=speculator.sample_pos,
input_ids=speculator.input_buffers.input_ids,
anchor_idx=speculator._anchor_idx,
seeds=speculator.seeds,
temperature=speculator.temperature,
draft_tokens=speculator.draft_tokens,
draft_logits=draft_logits,
)
return True
def _build_markov_sampler(self, speculator: "DSparkSpeculator") -> "MarkovSampler":
if speculator.use_confidence_head or speculator.use_acceptance_estimator:
raise ValueError(
"The Muse Spark specialized decode's fused DSpark Markov loop "
"computes neither the confidence head's outputs nor the "
"acceptance estimates: it does not run with adaptive verification."
)
from vllm.models.muse_spark.nvidia.specialized_decode.runtime.markov import (
build_markov_sampler,
)
return build_markov_sampler(
self,
draft_logits=_draft_logits_values(speculator),
draft_topk=speculator._draft_topk,
watermarked=speculator.draft_watermarker is not None,
use_fp64_gumbel=speculator.use_fp64_gumbel,
sample_from_anchor=speculator.sample_from_anchor,
max_num_reqs=speculator.max_num_reqs,
num_steps=speculator.num_speculative_steps,
)
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
# Identical full-vocabulary heads may be shared with an FP32 target head.
return self.logits_processor(
self.lm_head, hidden_states.to(self.lm_head.weight.dtype)
)
def combine_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
# Target residuals are FP32; the training collator casts features before FC.
hidden_states = hidden_states.to(self.model.fc.weight.dtype)
out = super().combine_hidden_states(hidden_states)
if isinstance(self.model.fc, ColumnParallelLinear):
# Each rank holds its block of output columns for every token.
out = gather_columns(out.view(-1, out.shape[-1])).view(*out.shape[:-1], -1)
return out
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
model_weights = {}
for name, loaded_weight in weights:
if name == "t2d":
continue
if name == "d2t":
name = "draft_id_to_target_id"
elif "lm_head" not in name:
name = "model." + name
model_weights[name] = loaded_weight
mapper = WeightsMapper(
orig_to_new_substr={
"model.embed_tokens": None,
"model.mask_embedding": None,
},
orig_to_new_stacked=self.hf_to_vllm_mapper.orig_to_new_stacked,
)
return AutoWeightsLoader(self).load_weights(
model_weights.items(), mapper=mapper
)
def process_weights_after_loading(self) -> None:
if self.model.specialized_draft is not None or self._markov_sampler is not None:
# Runs again after a weight reload; the captured draft steps read
# the kernels' constants and buffers built from the loaded weights.
raise RuntimeError(
"The Muse Spark specialized decode does not support reloading "
"the DSpark draft's weights"
)
# Builds the stacked K/V weights that the fused context chain reads.
super().process_weights_after_loading()
materialize_residual_path_constants(self.model)
if self.model.specialized_decode:
from vllm.models.muse_spark.nvidia.specialized_decode.runtime.draft import (
build_draft_decode,
)
self.model.specialized_draft = build_draft_decode(
self.model, self.model._compilation_mode
)
elif (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and self.model._compilation_mode == CompilationMode.NONE
):
from vllm.models.muse_spark.nvidia.specialized_decode.runtime.draft import (
build_draft_norms,
)
self.model._fused_draft_norms = build_draft_norms(self.model)