class Lfm3OmniMultiModalProcessor(Qwen3VLMultiModalProcessor):
info: Lfm3OmniProcessingInfo
def _get_mm_fields_config(
self, hf_inputs: BatchFeature, hf_processor_mm_kwargs: Mapping[str, object]
) -> Mapping[str, MultiModalFieldConfig]:
return {
**super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
"input_features": MultiModalFieldConfig.batched("audio"),
"audio_embeds": MultiModalFieldConfig.batched("audio"),
}
def _postprocess_hf_mm_data(
self,
mm_inputs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
return split_audio_features(processed_data)
def _span_token_ids(self) -> tuple[int, int]:
processor = self.info.get_hf_processor()
return processor.vision_start_token_id, processor.vision_end_token_id
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
# Each placeholder gets exactly one span pair, so drop the pair a chat
# template already put around it.
processor = self.info.get_hf_processor()
start, end = self._span_token_ids()
placeholders = {
processor.image_token_id,
processor.video_token_id,
processor.audio_token_id,
}
stripped: list[int] = []
index = 0
while index < len(prompt):
if (
prompt[index] == start
and index + 2 < len(prompt)
and prompt[index + 1] in placeholders
and prompt[index + 2] == end
):
stripped.append(prompt[index + 1])
index += 3
else:
stripped.append(prompt[index])
index += 1
return stripped
def _wrap(self, tokens: list[int]) -> list[int]:
start, end = self._span_token_ids()
return [start, *tokens, end]
def _with_soundtrack(
self, video: _VideoGroups, num_tokens: int
) -> PromptUpdateDetails:
"""Splits the soundtrack's encoder tokens at the first frame of each
group, after the group."""
processor = self.info.get_hf_processor()
extractor = processor.feature_extractor
seconds_per_token = (
extractor.hop_length * extractor.encoder_stride / extractor.sampling_rate
)
bounds = [0] + [
min(num_tokens, round(start / seconds_per_token))
for start in video.group_starts[1:]
]
bounds = [max(bounds[: i + 1]) for i in range(len(bounds))] + [num_tokens]
tokens: list[int] = []
for group, first, last in zip(video.groups, bounds, bounds[1:]):
tokens += group + [processor.audio_token_id] * (last - first)
return PromptUpdateDetails.select_token_id(
self._wrap(tokens), processor.video_token_id
)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
extractor = processor.feature_extractor
start, end = self._span_token_ids()
temporal_patch_size = processor.video_processor.temporal_patch_size
merge_length = processor.image_processor.merge_size**2
use_audio_in_video = bool(hf_processor_mm_kwargs.get("use_audio_in_video"))
def wrapped(token_id: int, count: int) -> PromptUpdateDetails:
return PromptUpdateDetails.select_token_id(
self._wrap([token_id] * count), token_id
)
def image_replacement(item_idx: int) -> PromptUpdateDetails:
grid_thw = out_mm_kwargs["image"][item_idx]["image_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
return wrapped(
processor.image_token_id, int(grid_thw.prod()) // merge_length
)
def audio_replacement(item_idx: int) -> PromptUpdateDetails:
item = out_mm_kwargs["audio"][item_idx]
if (embeds := item.get("audio_embeds")) is not None:
count = len(embeds.data)
else:
features = item["input_features"].data
assert isinstance(features, torch.Tensor)
count = extractor.num_encoder_tokens(features.shape[-1])
return wrapped(processor.audio_token_id, count)
def video_replacement(item_idx: int) -> PromptUpdateDetails:
item = out_mm_kwargs["video"][item_idx]
grid_thw = item["video_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
timestamps = item["timestamps"].data
assert isinstance(timestamps, (list, torch.Tensor))
tokens_per_group = int(grid_thw[1:].prod()) // merge_length
groups = [
self.get_video_repl(
tokens_per_frame=[tokens_per_group],
timestamps=[timestamp],
tokenizer=tokenizer,
vision_start_token_id=start,
vision_end_token_id=end,
video_token_id=processor.video_token_id,
).full
for timestamp in timestamps
]
details = PromptUpdateDetails.select_token_id(
self._wrap([token for group in groups for token in group]),
processor.video_token_id,
)
if not use_audio_in_video:
return details
# The soundtrack splits at the first frame of each group, which the
# averaged timestamps lose, so take the frame indices the loader
# sampled; frames resampled by the HF processor change the count.
videos = mm_items.get("video")
video = (
videos.get(item_idx)
if isinstance(videos, VideoProcessorItems)
else None
)
metadata = video[1] if isinstance(video, tuple) else {}
indices = metadata.get("frames_indices", [])
if -(-len(indices) // temporal_patch_size) != len(groups):
raise ValueError(
"`use_audio_in_video` needs videos sampled by the video loader"
)
return _VideoGroups(
full=details.full,
is_embed=details.is_embed,
groups=groups,
group_starts=[
index / metadata["fps"] for index in indices[::temporal_patch_size]
],
)
return [
PromptReplacement("image", [processor.image_token_id], image_replacement),
PromptReplacement("video", [processor.video_token_id], video_replacement),
PromptReplacement("audio", [processor.audio_token_id], audio_replacement),
]
def _with_soundtrack_placeholders(
self, prompt: list[int], num_audios: int
) -> list[int]:
"""The chat parser adds each video's decoded soundtrack as the audio
item right after it; the openai content format puts no placeholder for
it in the prompt. When the audio items outnumber the placeholders by
exactly one per video, put that placeholder right after every video,
as HF's `load_audio_from_video` does; otherwise the prompt is kept."""
processor = self.info.get_hf_processor()
video_token_id = processor.video_token_id
audio_token_id = processor.audio_token_id
num_videos = prompt.count(video_token_id)
if num_audios != prompt.count(audio_token_id) + num_videos:
return prompt
with_tracks: list[int] = []
for token in prompt:
with_tracks.append(token)
if token == video_token_id:
with_tracks.append(audio_token_id)
return with_tracks
def _soundtracks(self, prompt: list[int]) -> dict[int, int]:
"""The audio item each video takes as its soundtrack: the first clip
after the video that no earlier video took."""
processor = self.info.get_hf_processor()
clips = [i for i, t in enumerate(prompt) if t == processor.audio_token_id]
taken: list[int] = []
for video in (i for i, t in enumerate(prompt) if t == processor.video_token_id):
clip_idx = next(
(i for i, clip in enumerate(clips) if clip > video and i not in taken),
None,
)
if clip_idx is None:
raise ValueError(
"`use_audio_in_video` needs an audio clip after each video"
)
taken.append(clip_idx)
return dict(enumerate(taken))
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
if not any(
isinstance(update.content, _VideoGroups)
for item in mm_res.prompt_updates.get("video", [])
for update in item
):
return super()._maybe_apply_prompt_updates(mm_items, mm_res)
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
prompt = self._with_soundtrack_placeholders(
mm_res.prompt_ids, mm_item_counts.get("audio", 0)
)
# A soundtrack's own placeholder expands to nothing; its tokens follow
# its video's frame groups instead.
soundtracks = self._soundtracks(prompt)
updates: dict[str, list[list[ResolvedPromptUpdate]]] = {
modality: [list(item) for item in items]
for modality, items in mm_res.prompt_updates.items()
}
audio_token_id = self.info.get_hf_processor().audio_token_id
for video_idx, audio_idx in soundtracks.items():
(video,) = updates["video"][video_idx]
(audio,) = updates["audio"][audio_idx]
assert isinstance(video.content, _VideoGroups)
content = self._with_soundtrack(
video.content, audio.content.full.count(audio_token_id)
)
updates["video"][video_idx] = [replace(video, content=content)]
updates["audio"][audio_idx] = [
replace(audio, content=PromptUpdateDetails.from_seq([]))
]
prompt_ids, placeholders = self._apply_prompt_updates(prompt, updates)
audio_placeholders = list(placeholders.get("audio", []))
for video_placeholder in placeholders["video"]:
audio_idx = soundtracks[video_placeholder.item_idx]
audio_placeholders.append(
replace(
video_placeholder,
modality="audio",
item_idx=audio_idx,
is_embed=torch.tensor(video_placeholder.tokens) == audio_token_id,
)
)
placeholders = {
**placeholders,
"audio": sorted(audio_placeholders, key=lambda info: info.item_idx),
}
self._validate_mm_placeholders(placeholders, mm_item_counts)
return prompt_ids, placeholders