class PeerModelLoader(BaseModelLoader):
"""Loads a daemon's model from a source replica's weight cache daemon."""
def __init__(self, load_config: LoadConfig):
super().__init__(load_config)
extra_config = copy(load_config.model_loader_extra_config or {})
self.is_draft = bool(extra_config.pop("is_draft", False))
if extra_config:
raise ValueError(
f"Unexpected extra config keys for load format "
f"{load_config.load_format}: {sorted(extra_config)}"
)
def download_model(self, model_config: ModelConfig) -> None:
return None
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
raise NotImplementedError(
"Standalone weight reload is not supported for load format 'peer_cache' yet"
)
@instrument(span_name="Load model")
def load_model(
self,
vllm_config: VllmConfig,
model_config: ModelConfig,
prefix: str = "",
*,
payload: WeightsSeedPayload,
transport: SeedTransport,
) -> nn.Module:
check_weight_cache_loader_support(
vllm_config, model_config, is_draft=self.is_draft
)
manifest = payload["manifest"]
mover = get_mover(payload["seed"]["backend"])
device = torch.device(
current_platform.device_type, torch.accelerator.current_device_index()
)
tensors = mover.fill(manifest, payload["seed"], device)
acknowledge_lease(transport)
kinds: dict[str, TensorKind] = {
name: "param" if view["is_param"] else "buffer"
for name, view in manifest["tensors"].items()
}
model = build_model_from_processed_weights(
vllm_config,
model_config,
prefix,
tensors,
kinds,
aliases=dict(payload["aliases"]),
attrs=dict(payload["attrs"]),
load_config=self.load_config,
)
logger.info(
"Loaded %d tensors from a peer daemon over %s",
len(tensors),
payload["seed"]["backend"],
)
return model