From 6277c214f67d2ffaf37fdea9369841293e1ec63b Mon Sep 17 00:00:00 2001 From: craftingmod Date: Sat, 26 Sep 2026 20:35:02 +0900 Subject: [PATCH 1/9] feat(mtmd): support `prefill` mode Assisted-By: codex --- llama_cpp/llama_chat_format.py | 18 ++- llama_cpp/llama_multimodal.py | 189 +++++++++++++++++++++++-------- tests/test_media.py | 198 +++++++++++++++++++++++++++------ 3 files changed, 319 insertions(+), 86 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 96314f8f4..dd0b05b2e 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -19,6 +19,7 @@ Union, Protocol, cast, + overload, ) import jinja2 @@ -72,8 +73,12 @@ class LlamaChatCompletionHandler(Protocol): Very generic protocol that can be used to implement any chat format. The only hard requirement is that it must return a ChatCompletion when - stream=False and an iterator of ChatCompletionChunks when stream=True.""" + stream=False and an iterator of ChatCompletionChunks when stream=True. + ``prefill_only`` is MTMD-specific; other handlers may ignore it, so this + Protocol leaves its result type unspecified. + """ + @overload def __call__( self, *, @@ -133,12 +138,23 @@ def __call__( reasoning_budget_message: Optional[str] = None, reasoning_start_in_prompt: bool = False, reasoning_start_max_tokens: Optional[int] = 32, + prefill_only: Literal[False] = False, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, Iterator[llama_types.CreateChatCompletionStreamResponse], ]: ... + @overload + def __call__( + self, + *, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + prefill_only: bool, + **kwargs: Any, + ) -> Any: ... + class LlamaChatCompletionHandlerNotFoundException(Exception): pass diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index 15752e11f..9de1a2375 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -26,11 +26,14 @@ Protocol, TYPE_CHECKING, cast, + overload, ) import urllib.request from urllib.error import URLError, HTTPError +import numpy as np + import llama_cpp.llama_cpp as llama_cpp_lib import llama_cpp.llama_types as llama_types import llama_cpp.llama_grammar as llama_grammar @@ -698,7 +701,6 @@ def _create_audio_sampler(self, llama, *, seed, temperature, top_k, top_p, min_p @staticmethod def _validate_audio(data: bytes, response_format: str, rate: int, samples: int) -> None: """Reject malformed buffers and clearly invalid signals, not low volume speech.""" - import numpy as np if samples <= 0: raise RuntimeError("TTS returned empty audio") @@ -907,6 +909,15 @@ def create_speech( self._request_lock.release() +@dataclass(frozen=True) +class MTMDPrefillResult: + """Prompt, owned final logits, and token count from a multimodal prefill.""" + + prompt: List[int] + logits: np.ndarray + n_tokens: int + + class MTMDChatHandler(MTMDBaseHandler): DEFAULT_SYSTEM_MESSAGE: Optional[str] = ( "You are an exceptionally capable, precise, and helpful multimodal AI assistant that excels at deeply understanding and richly describing images, charts, diagrams, text in images, scenes, and any visual content, " @@ -1695,7 +1706,7 @@ def _create_bitmap_func(idx: int, item: dict): self._free_mtmd_resources(chunks, bitmap_cleanup, video_cleanup) raise - def __call__( + def _prefill_mtmd( self, *, llama: llama_core.Llama, @@ -1704,53 +1715,13 @@ def __call__( function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, tools: Optional[List[llama_types.ChatCompletionTool]] = None, tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, - temperature: float = 0.2, - top_p: float = 0.95, - top_k: int = 40, - min_p: float = 0.05, - typical_p: float = 1.0, - stream: bool = False, - stop: Optional[Union[str, List[str]]] = [], - seed: Optional[int] = None, - response_format: Optional[ - llama_types.ChatCompletionRequestResponseFormat - ] = None, - max_tokens: Optional[int] = None, - present_penalty: float = 0.0, - frequency_penalty: float = 0.0, - repeat_penalty: float = 1.1, - top_n_sigma: float = -1.00, - mirostat_mode: int = 0, - mirostat_tau: float = 5.0, - mirostat_eta: float = 0.1, - xtc_threshold: float = 0.1, - xtc_probability: float = 0.0, - dry_multiplier: float = 0.0, - dry_base: float = 1.75, - dry_allowed_length: int = 2, - dry_penalty_last_n:int = 64, - dry_seq_breakers: list[str] = ["\n", ":", "\"", "*"], - adaptive_target : float = -1.0, - adaptive_decay : float = 0.9, - use_infill: bool = False, - model: Optional[str] = None, - logits_processor: Optional[llama_core.LogitsProcessorList] = None, - grammar: Optional[llama_grammar.LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, - logprobs: Optional[bool] = None, - top_logprobs: Optional[int] = None, add_generation_prompt: bool = True, - reasoning_budget: int = -1, - reasoning_start: str = "", - reasoning_end: str = "", - reasoning_budget_message: Optional[str] = None, - reasoning_start_in_prompt: bool = False, - reasoning_start_max_tokens: Optional[int] = 32, - **kwargs, # type: ignore - ) -> Union[ - llama_types.CreateChatCompletionResponse, - Iterator[llama_types.CreateChatCompletionStreamResponse], - ]: + ) -> MTMDPrefillResult: + """Evaluate a multimodal chat prompt without sampling or generating tokens. + + The returned logits are an owned copy, and the Llama KV state remains + available for a subsequent generation call. + """ # 1. Initialize mtmd context self._init_mtmd_context(llama) assert self.mtmd_ctx is not None @@ -1977,6 +1948,14 @@ def __call__( tokens=prompt, seq_id=0 ) + logits = llama._restored_logits + if logits is None: + raise RuntimeError("MTMD prefill did not produce final logits") + return MTMDPrefillResult( + prompt=prompt, + logits=logits.copy(), + n_tokens=llama.n_tokens, + ) except BaseException: # A helper can commit earlier ubatches before reporting failure. if prefill_started: @@ -1986,6 +1965,120 @@ def __call__( # Generation no longer needs these resources once prompt evaluation ends. self._free_mtmd_resources(chunks, bitmap_cleanup) + @overload + def __call__( + self, + *, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + prefill_only: Literal[True], + **kwargs: Any, + ) -> MTMDPrefillResult: ... + + @overload + def __call__( + self, + *, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + prefill_only: Literal[False] = False, + **kwargs: Any, + ) -> Union[ + llama_types.CreateChatCompletionResponse, + Iterator[llama_types.CreateChatCompletionStreamResponse], + ]: ... + + @overload + def __call__( + self, + *, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + prefill_only: bool, + **kwargs: Any, + ) -> Union[ + llama_types.CreateChatCompletionResponse, + Iterator[llama_types.CreateChatCompletionStreamResponse], + MTMDPrefillResult, + ]: ... + + def __call__( + self, + *, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + temperature: float = 0.2, + top_p: float = 0.95, + top_k: int = 40, + min_p: float = 0.05, + typical_p: float = 1.0, + stream: bool = False, + stop: Optional[Union[str, List[str]]] = [], + seed: Optional[int] = None, + response_format: Optional[ + llama_types.ChatCompletionRequestResponseFormat + ] = None, + max_tokens: Optional[int] = None, + present_penalty: float = 0.0, + frequency_penalty: float = 0.0, + repeat_penalty: float = 1.1, + top_n_sigma: float = -1.00, + mirostat_mode: int = 0, + mirostat_tau: float = 5.0, + mirostat_eta: float = 0.1, + xtc_threshold: float = 0.1, + xtc_probability: float = 0.0, + dry_multiplier: float = 0.0, + dry_base: float = 1.75, + dry_allowed_length: int = 2, + dry_penalty_last_n:int = 64, + dry_seq_breakers: list[str] = ["\n", ":", "\"", "*"], + adaptive_target : float = -1.0, + adaptive_decay : float = 0.9, + use_infill: bool = False, + model: Optional[str] = None, + logits_processor: Optional[llama_core.LogitsProcessorList] = None, + grammar: Optional[llama_grammar.LlamaGrammar] = None, + logit_bias: Optional[Dict[str, float]] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, + add_generation_prompt: bool = True, + prefill_only: bool = False, + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, + **kwargs, # type: ignore + ) -> Union[ + llama_types.CreateChatCompletionResponse, + Iterator[llama_types.CreateChatCompletionStreamResponse], + MTMDPrefillResult, + ]: + """Call the handler, or return the MTMD prompt state when requested. + + ``prefill_only=True`` is supported by direct MTMD handler calls. It still + runs subclass ``__call__`` preprocessing, then returns before sampling. + """ + prefill = MTMDChatHandler._prefill_mtmd( + self, + llama=llama, + messages=messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + add_generation_prompt=add_generation_prompt, + ) + if prefill_only: + return prefill + prompt = prefill.prompt + # Handle response format and tools (same as before) if response_format is not None and response_format["type"] == "json_object": grammar = _grammar_for_response_format(response_format) diff --git a/tests/test_media.py b/tests/test_media.py index a375264b8..51490f673 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -11,51 +11,175 @@ @pytest.fixture -def chat_prefill(tmp_path, monkeypatch): +def chat_prefill_builder(tmp_path, monkeypatch): import numpy as np from llama_cpp import Llama from llama_cpp import llama_multimodal as multimodal - handler = multimodal.MTMDChatHandler(mmproj_path=str(tmp_path), verbose=False) - monkeypatch.setattr(handler, "mtmd_ctx", object()) - monkeypatch.setattr(handler, "_init_mtmd_context", Mock()) - monkeypatch.setattr(handler, "_free_mtmd_resources", Mock()) - monkeypatch.setattr(handler, "_is_text_chunk", lambda kind: False) - monkeypatch.setattr(handler, "_is_image_chunk", lambda kind: True) - monkeypatch.setattr(multimodal, "_convert_completion_to_chat", lambda result, **kw: result) - handler._process_mtmd_prompt = Mock(return_value=( - [1, 2, -9, -9], [(2, 4, object(), 1, -9)], object(), [] - )) - ctx = SimpleNamespace( - ctx=object(), memory_seq_rm=Mock(return_value=True), - memory_seq_add=Mock(), memory_can_shift=lambda: True, - memory_clear=Mock(), - ) - llm = SimpleNamespace( - n_tokens=3, input_ids=np.array([1, 2, 3, 0, 0, 0]), - _n_ctx=6, n_ctx=lambda: 6, n_batch=1, n_keep=0, - _ctx=ctx, is_hybrid=False, _hybrid_cache_mgr=None, - speculative=None, verbose=False, - context_params=SimpleNamespace(no_perf=True), - _prefilled_prompt=(1, 2, 3), _restored_logits=object(), - _last_eval_output_start=2, _last_eval_output_count=1, - longest_token_prefix=Llama.longest_token_prefix, - create_completion=Mock(return_value="completed"), + def build(handler_class=multimodal.MTMDChatHandler, **handler_kwargs): + handler_kwargs.setdefault("mmproj_path", str(tmp_path)) + handler_kwargs.setdefault("verbose", False) + handler = handler_class(**handler_kwargs) + monkeypatch.setattr(handler, "mtmd_ctx", object()) + monkeypatch.setattr(handler, "_init_mtmd_context", Mock()) + monkeypatch.setattr(handler, "_free_mtmd_resources", Mock()) + monkeypatch.setattr(handler, "_is_text_chunk", lambda kind: False) + monkeypatch.setattr(handler, "_is_image_chunk", lambda kind: True) + monkeypatch.setattr(multimodal, "_convert_completion_to_chat", lambda result, **kw: result) + handler._process_mtmd_prompt = Mock(return_value=( + [1, 2, -9, -9], [(2, 4, object(), 1, -9)], object(), [] + )) + ctx = SimpleNamespace( + ctx=object(), memory_seq_rm=Mock(return_value=True), + memory_seq_add=Mock(), memory_can_shift=lambda: True, + memory_clear=Mock(), + ) + llm = SimpleNamespace( + n_tokens=3, input_ids=np.array([1, 2, 3, 0, 0, 0]), + _n_ctx=6, n_ctx=lambda: 6, n_vocab=lambda: 3, n_batch=1, n_keep=0, + _ctx=ctx, is_hybrid=False, _hybrid_cache_mgr=None, + speculative=None, verbose=False, + context_params=SimpleNamespace(no_perf=True), + _prefilled_prompt=(1, 2, 3), _restored_logits=object(), + _last_eval_output_start=2, _last_eval_output_count=1, + longest_token_prefix=Llama.longest_token_prefix, + create_completion=Mock(return_value="completed"), + ) + llm.reset = Mock(side_effect=lambda: Llama.reset(llm)) + llm._memory_seq_rm_or_raise = lambda *args: Llama._memory_seq_rm_or_raise(llm, *args) + llm._mark_prefilled_prompt = Mock( + side_effect=lambda: setattr( + llm, "_restored_logits", np.array([0.25, -0.5, 1.0], dtype=np.float32) + ) + ) + + def evaluate(mtmd, ctx, chunk, pos, seq, batch, logits, output): + output._obj.value = pos.value + 2 + return 0 + + backend = SimpleNamespace( + mtmd_input_chunk_get_n_tokens=lambda chunk: 2, + mtmd_helper_eval_chunk_single=Mock(side_effect=evaluate), + ) + monkeypatch.setattr(handler, "_mtmd_cpp", backend) + return handler, llm, backend + + return build + + +@pytest.fixture +def chat_prefill(chat_prefill_builder): + return chat_prefill_builder() + + +def test_chat_prefill_returns_owned_logits_without_generation(chat_prefill): + import numpy as np + + handler, llm, _ = chat_prefill + result = handler(llama=llm, messages=[], prefill_only=True) + + assert result.prompt == [1, 2, -9, -9] + assert result.n_tokens == llm.n_tokens == 4 + assert result.logits.shape == (llm.n_vocab(),) + assert result.logits.dtype == np.float32 + assert not np.shares_memory(result.logits, llm._restored_logits) + np.testing.assert_array_equal(result.logits, [0.25, -0.5, 1.0]) + llm.create_completion.assert_not_called() + + saved_logits = result.logits.copy() + llm.reset() + np.testing.assert_array_equal(result.logits, saved_logits) + + +def test_chat_completion_uses_prefill_and_preserves_generation_args(chat_prefill): + handler, llm, _ = chat_prefill + + assert handler( + llama=llm, messages=[], add_generation_prompt=False, temperature=0.7, top_p=0.8 + ) == "completed" + + completion_args = llm.create_completion.call_args.kwargs + assert completion_args["prompt"] == [1, 2, -9, -9] + assert completion_args["temperature"] == 0.7 + assert completion_args["top_p"] == 0.8 + + +def test_chat_completion_ignores_legacy_subclass_prefill_name(chat_prefill_builder): + from llama_cpp import llama_multimodal as multimodal + + class LegacyChatHandler(multimodal.MTMDChatHandler): + def prefill(self, *args, **kwargs): + raise AssertionError("legacy prefill override must not intercept generation") + + handler, llm, _ = chat_prefill_builder(LegacyChatHandler) + + assert handler(llama=llm, messages=[]) == "completed" + assert llm.create_completion.call_args.kwargs["prompt"] == [1, 2, -9, -9] + + +def test_external_handler_forwards_prefill_only_through_call(chat_prefill_builder): + from llama_cpp import llama_multimodal as multimodal + + class WrappedChatHandler(multimodal.MTMDChatHandler): + def __call__(self, **kwargs): + self.wrapper_ran = True + return super().__call__(**kwargs) + + handler, llm, _ = chat_prefill_builder(WrappedChatHandler) + result = handler(llama=llm, messages=[], prefill_only=True) + + assert handler.wrapper_ran + assert isinstance(result, multimodal.MTMDPrefillResult) + llm.create_completion.assert_not_called() + + +def test_generic_chat_prefill_resolves_model_template(chat_prefill_builder): + from llama_cpp import llama_multimodal as multimodal + + handler, llm, _ = chat_prefill_builder( + multimodal.GenericMTMDChatHandler, chat_format=None ) - llm.reset = Mock(side_effect=lambda: Llama.reset(llm)) - llm._memory_seq_rm_or_raise = lambda *args: Llama._memory_seq_rm_or_raise(llm, *args) - llm._mark_prefilled_prompt = Mock() + template = "{% for message in messages %}{{ message.content }}{% endfor %}<|image|>" + llm._model = SimpleNamespace(model_chat_template=Mock(return_value=template)) - def evaluate(mtmd, ctx, chunk, pos, seq, batch, logits, output): - output._obj.value = pos.value + 2 - return 0 + result = handler(llama=llm, messages=[], prefill_only=True) - backend = SimpleNamespace( - mtmd_input_chunk_get_n_tokens=lambda chunk: 2, - mtmd_helper_eval_chunk_single=Mock(side_effect=evaluate), + assert isinstance(result, multimodal.MTMDPrefillResult) + llm._model.model_chat_template.assert_called_once_with(None) + assert handler._template_initialized + assert handler._chat_format_parser_tags == ["<|image|>"] + + +def test_minicpmv45_prefill_prepares_prompt_and_keeps_generation_stops( + chat_prefill_builder, +): + from llama_cpp import llama_multimodal as multimodal + + handler, llm, _ = chat_prefill_builder( + multimodal.MiniCPMv45ChatHandler, enable_thinking=False ) - monkeypatch.setattr(handler, "_mtmd_cpp", backend) - yield handler, llm, backend + llm.input_ids[:] = [1, 2, 3, 4, 5, 6] + observed = [] + + def process_prompt(**kwargs): + observed.append((llm.input_ids.copy(), handler.extra_template_arguments.copy())) + return [1, 2, -9, -9], [(2, 4, object(), 1, -9)], object(), [] + + handler._process_mtmd_prompt.side_effect = process_prompt + result = handler(llama=llm, messages=[], prefill_only=True) + + assert isinstance(result, multimodal.MTMDPrefillResult) + llm.create_completion.assert_not_called() + assert observed[0][0].tolist() == [0, 0, 0, 0, 0, 0] + assert observed[0][1]["enable_thinking"] is False + + llm.reset() + handler(llama=llm, messages=[], stop=["caller-stop"]) + assert llm.create_completion.call_args.kwargs["stop"] == [ + handler.MINICPMV_EOS_TOKEN, + handler.MINICPMV_PAD_TOKEN, + ] + llm.create_completion.assert_called_once() @pytest.mark.parametrize("failure", ["rollback", "shift", "helper", "position", "interrupt"]) From 8ba5579bb70e00f855df138e7a5021674efc5719 Mon Sep 17 00:00:00 2001 From: craftingmod Date: Sat, 26 Sep 2026 21:21:03 +0900 Subject: [PATCH 2/9] feat(mtmd): add `create_chat_prefill` public API with immutable result Assisted-By: codex --- llama_cpp/llama.py | 52 +++++++++++++++++++--- llama_cpp/llama_multimodal.py | 30 ++++++++++--- tests/test_media.py | 82 ++++++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 13 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 7a72afdf2..4c209bade 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -4361,6 +4361,15 @@ def __call__( presence_penalty=presence_penalty, ) + def _get_chat_completion_handler( + self, + ) -> llama_chat_format.LlamaChatCompletionHandler: + return ( + self.chat_handler + or self._chat_handlers.get(self.chat_format) + or llama_chat_format.get_chat_completion_handler(self.chat_format) + ) + def create_chat_completion( self, messages: List[ChatCompletionRequestMessage], @@ -4489,11 +4498,7 @@ def create_chat_completion( if presence_penalty is not None and present_penalty == 0.0: present_penalty = presence_penalty - handler = ( - self.chat_handler - or self._chat_handlers.get(self.chat_format) - or llama_chat_format.get_chat_completion_handler(self.chat_format) - ) + handler = self._get_chat_completion_handler() return handler( llama=self, messages=messages, @@ -4551,6 +4556,43 @@ def create_chat_completion( reasoning_start_max_tokens=reasoning_start_max_tokens, ) + def create_chat_prefill( + self, + messages: List[ChatCompletionRequestMessage], + functions: Optional[List[ChatCompletionFunction]] = None, + function_call: Optional[ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[ChatCompletionTool]] = None, + tool_choice: Optional[ChatCompletionToolChoiceOption] = None, + add_generation_prompt: bool = True, + ) -> llama_multimodal.MTMDPrefillResult: + """Prefill a chat prompt with the selected MTMD handler, without generation. + + Raises: + TypeError: If the selected chat handler is not an MTMDChatHandler. + """ + handler = self._get_chat_completion_handler() + if not isinstance(handler, llama_multimodal.MTMDChatHandler): + raise TypeError( + "create_chat_prefill requires an MTMDChatHandler; " + f"the selected handler is {type(handler).__name__}." + ) + + result = handler( + llama=self, + messages=messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + add_generation_prompt=add_generation_prompt, + prefill_only=True, + ) + if not isinstance(result, llama_multimodal.MTMDPrefillResult): + raise TypeError( + "The selected MTMDChatHandler did not return an MTMDPrefillResult." + ) + return result + def create_chat_completion_openai_v1( self, *args: Any, diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index 9de1a2375..45a028741 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -908,12 +908,20 @@ def create_speech( finally: self._request_lock.release() +@dataclass +class _MTMDPrefillInternal: + """Internal mutable state produced by a multimodal prefill.""" + prompt: List[int] + logits: np.ndarray + n_tokens: int @dataclass(frozen=True) class MTMDPrefillResult: - """Prompt, owned final logits, and token count from a multimodal prefill.""" - - prompt: List[int] + """Immutable multimodal prefill result. + + `logits` is a read-only NumPy view. Call `.copy()` to obtain a mutable array. + """ + prompt: tuple[int, ...] logits: np.ndarray n_tokens: int @@ -1716,7 +1724,7 @@ def _prefill_mtmd( tools: Optional[List[llama_types.ChatCompletionTool]] = None, tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, add_generation_prompt: bool = True, - ) -> MTMDPrefillResult: + ) -> _MTMDPrefillInternal: """Evaluate a multimodal chat prompt without sampling or generating tokens. The returned logits are an owned copy, and the Llama KV state remains @@ -1951,9 +1959,9 @@ def _prefill_mtmd( logits = llama._restored_logits if logits is None: raise RuntimeError("MTMD prefill did not produce final logits") - return MTMDPrefillResult( + return _MTMDPrefillInternal( prompt=prompt, - logits=logits.copy(), + logits=logits, n_tokens=llama.n_tokens, ) except BaseException: @@ -2076,8 +2084,16 @@ def __call__( add_generation_prompt=add_generation_prompt, ) if prefill_only: - return prefill + logits = prefill.logits.view() + logits.flags.writeable = False + return MTMDPrefillResult( + prompt=tuple(prefill.prompt), + logits=logits, + n_tokens=prefill.n_tokens, + ) + prompt = prefill.prompt + del prefill # Handle response format and tools (same as before) if response_format is not None and response_format["type"] == "json_object": diff --git a/tests/test_media.py b/tests/test_media.py index 51490f673..dc2e8ba02 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -1,7 +1,7 @@ import ctypes import importlib import os -from types import SimpleNamespace +from types import MethodType, SimpleNamespace from unittest.mock import Mock import threading import struct @@ -150,6 +150,86 @@ def test_generic_chat_prefill_resolves_model_template(chat_prefill_builder): assert handler._chat_format_parser_tags == ["<|image|>"] +@pytest.mark.parametrize( + "resolution", ["chat_handler", "chat_format", "global_registry"] +) +def test_llama_create_chat_prefill_uses_selected_mtmd_handler( + chat_prefill_builder, resolution, monkeypatch +): + import numpy as np + from llama_cpp import Llama + from llama_cpp import llama_multimodal as multimodal + + handler, llm, backend = chat_prefill_builder( + multimodal.GenericMTMDChatHandler, chat_format=None + ) + template = "{% for message in messages %}{{ message.content }}{% endfor %}<|image|>" + llm._model = SimpleNamespace(model_chat_template=Mock(return_value=template)) + llm.create_chat_prefill = MethodType(Llama.create_chat_prefill, llm) + llm.chat_handler = handler if resolution == "chat_handler" else None + llm.chat_format = "test-prefill" + llm._chat_handlers = ( + {"test-prefill": handler} if resolution == "chat_format" else {} + ) + registry_lookup = Mock(return_value=handler) + monkeypatch.setattr( + "llama_cpp.llama.llama_chat_format.get_chat_completion_handler", + registry_lookup, + ) + + messages = [{"role": "user", "content": "inspect this"}] + functions = [{ + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }] + result = llm.create_chat_prefill( + messages=messages, + functions=functions, + function_call="auto", + tools=[], + tool_choice="none", + add_generation_prompt=False, + ) + + assert isinstance(result, multimodal.MTMDPrefillResult) + assert result.n_tokens == llm.n_tokens + assert llm._restored_logits is not None + assert not llm.reset.called + llm._ctx.memory_clear.assert_not_called() + backend.mtmd_helper_eval_chunk_single.assert_called_once() + assert not np.shares_memory(result.logits, llm._restored_logits) + llm.create_completion.assert_not_called() + llm._model.model_chat_template.assert_called_once_with(None) + processed = handler._process_mtmd_prompt.call_args.kwargs + assert processed["messages"] is messages + assert processed["functions"] is functions + assert processed["function_call"] == "auto" + assert processed["tools"] == [] + assert processed["tool_choice"] == "none" + assert processed["add_generation_prompt"] is False + if resolution == "global_registry": + registry_lookup.assert_called_once_with("test-prefill") + else: + registry_lookup.assert_not_called() + + +def test_llama_create_chat_prefill_rejects_non_mtmd_handler(): + from llama_cpp import Llama + + handler = Mock() + llm = SimpleNamespace( + chat_handler=handler, + _chat_handlers={}, + chat_format="unused", + ) + llm.create_chat_prefill = MethodType(Llama.create_chat_prefill, llm) + + with pytest.raises(TypeError, match="requires an MTMDChatHandler"): + llm.create_chat_prefill(messages=[]) + + handler.assert_not_called() + + def test_minicpmv45_prefill_prepares_prompt_and_keeps_generation_stops( chat_prefill_builder, ): From d9677255c0bbd0cbba0899f9885c23f678cc5778 Mon Sep 17 00:00:00 2001 From: craftingmod Date: Sat, 26 Sep 2026 21:51:29 +0900 Subject: [PATCH 3/9] chore: add `chat completion handler` method in test Assisted-by: codex --- llama_cpp/llama_multimodal.py | 7 ++++--- tests/test_media.py | 10 +++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index 45a028741..deae0d4df 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -918,8 +918,9 @@ class _MTMDPrefillInternal: @dataclass(frozen=True) class MTMDPrefillResult: """Immutable multimodal prefill result. - - `logits` is a read-only NumPy view. Call `.copy()` to obtain a mutable array. + + `logits` is an owned, read-only NumPy array. Call `.copy()` to obtain a + mutable array. """ prompt: tuple[int, ...] logits: np.ndarray @@ -2084,7 +2085,7 @@ def __call__( add_generation_prompt=add_generation_prompt, ) if prefill_only: - logits = prefill.logits.view() + logits = prefill.logits.copy() logits.flags.writeable = False return MTMDPrefillResult( prompt=tuple(prefill.prompt), diff --git a/tests/test_media.py b/tests/test_media.py index dc2e8ba02..b76d7a18c 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -78,10 +78,12 @@ def test_chat_prefill_returns_owned_logits_without_generation(chat_prefill): handler, llm, _ = chat_prefill result = handler(llama=llm, messages=[], prefill_only=True) - assert result.prompt == [1, 2, -9, -9] + assert result.prompt == (1, 2, -9, -9) assert result.n_tokens == llm.n_tokens == 4 assert result.logits.shape == (llm.n_vocab(),) assert result.logits.dtype == np.float32 + assert result.logits.flags.owndata + assert not result.logits.flags.writeable assert not np.shares_memory(result.logits, llm._restored_logits) np.testing.assert_array_equal(result.logits, [0.25, -0.5, 1.0]) llm.create_completion.assert_not_called() @@ -165,6 +167,9 @@ def test_llama_create_chat_prefill_uses_selected_mtmd_handler( ) template = "{% for message in messages %}{{ message.content }}{% endfor %}<|image|>" llm._model = SimpleNamespace(model_chat_template=Mock(return_value=template)) + llm._get_chat_completion_handler = MethodType( + Llama._get_chat_completion_handler, llm + ) llm.create_chat_prefill = MethodType(Llama.create_chat_prefill, llm) llm.chat_handler = handler if resolution == "chat_handler" else None llm.chat_format = "test-prefill" @@ -222,6 +227,9 @@ def test_llama_create_chat_prefill_rejects_non_mtmd_handler(): _chat_handlers={}, chat_format="unused", ) + llm._get_chat_completion_handler = MethodType( + Llama._get_chat_completion_handler, llm + ) llm.create_chat_prefill = MethodType(Llama.create_chat_prefill, llm) with pytest.raises(TypeError, match="requires an MTMDChatHandler"): From 803f448521d64c5dd887f3a111eb64643ca52676 Mon Sep 17 00:00:00 2001 From: craftingmod Date: Sun, 27 Sep 2026 00:25:38 +0900 Subject: [PATCH 4/9] feat: support prefill both MTMD and text. Assisted-by: codex --- llama_cpp/llama.py | 62 ++++++++---- llama_cpp/llama_chat_format.py | 149 +++++++++++++++++++++------- llama_cpp/llama_multimodal.py | 61 ++++++------ tests/test_formats.py | 175 +++++++++++++++++++++++++++++++++ tests/test_media.py | 29 +++--- 5 files changed, 383 insertions(+), 93 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 4c209bade..3e8c06810 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -46,6 +46,7 @@ import llama_cpp.llama_cpp as llama_cpp_lib import llama_cpp.llama_chat_format as llama_chat_format import llama_cpp.llama_multimodal as llama_multimodal +from .llama_chat_format import PrefillResult from llama_cpp.llama_speculative import ( LlamaDraftModel, @@ -1813,6 +1814,41 @@ def eval( logits_view = np.ctypeslib.as_array(logits_ptr, shape=(self._n_vocab,)) self.scores[0, :] = logits_view + def prefill( + self, + prompt: Union[str, Sequence[int]], + *, + reset: bool = True, + add_bos: bool = True, + special: bool = True, + active_loras: Optional[List[Dict[str, Union[str, float]]]] = None, + control_vector: Optional[Dict[str, Any]] = None, + ) -> PrefillResult: + """Evaluate a text prompt and return its final next-token logits.""" + tokens = ( + self.tokenize(prompt.encode("utf-8"), add_bos=add_bos, special=special) + if isinstance(prompt, str) + else list(prompt) + ) + if not tokens: + raise ValueError("Prefill requires at least one token") + if reset: + self.reset() + + self.eval( + tokens, + active_loras=active_loras, + control_vector=control_vector, + copy_logits=True, + ) + + logits = ( + self.scores[self.n_tokens - 1] + if self._logits_all + else self.scores[0] + ) + return PrefillResult(logits=logits) + # Helper method: Convert dict logit_bias to List[llama_logit_bias] def _convert_logit_bias(self, logit_bias: Optional[Dict[int, float]]) -> List[llama_cpp_lib.llama_logit_bias]: if not logit_bias: @@ -4564,20 +4600,17 @@ def create_chat_prefill( tools: Optional[List[ChatCompletionTool]] = None, tool_choice: Optional[ChatCompletionToolChoiceOption] = None, add_generation_prompt: bool = True, - ) -> llama_multimodal.MTMDPrefillResult: - """Prefill a chat prompt with the selected MTMD handler, without generation. - - Raises: - TypeError: If the selected chat handler is not an MTMDChatHandler. - """ + assistant_prefill: bool = False, + ) -> PrefillResult: + """Prefill a chat prompt through its handler without generating a token.""" handler = self._get_chat_completion_handler() - if not isinstance(handler, llama_multimodal.MTMDChatHandler): - raise TypeError( - "create_chat_prefill requires an MTMDChatHandler; " - f"the selected handler is {type(handler).__name__}." + prefill = getattr(handler, "prefill", None) + if not callable(prefill): + raise NotImplementedError( + "The selected chat handler does not support prefill" ) - result = handler( + return prefill( llama=self, messages=messages, functions=functions, @@ -4585,13 +4618,8 @@ def create_chat_prefill( tools=tools, tool_choice=tool_choice, add_generation_prompt=add_generation_prompt, - prefill_only=True, + assistant_prefill=assistant_prefill, ) - if not isinstance(result, llama_multimodal.MTMDPrefillResult): - raise TypeError( - "The selected MTMDChatHandler did not return an MTMDPrefillResult." - ) - return result def create_chat_completion_openai_v1( self, diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index dd0b05b2e..510737e6b 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -224,6 +224,27 @@ class ChatFormatterResponse: added_special: bool = False +@dataclasses.dataclass(frozen=True) +class PrefillResult: + """owned, read-only final next-token logits.""" + + logits: npt.NDArray[np.single] + + def __post_init__(self) -> None: + logits = np.array(self.logits, copy=True) + if logits.ndim != 1: + raise ValueError("Prefill logits must be a one-dimensional vocabulary vector") + logits.flags.writeable = False + object.__setattr__(self, "logits", logits) + + +@dataclasses.dataclass +class _PreparedChatPrompt: + prompt: List[int] + stop: Optional[Union[str, List[str]]] + stopping_criteria: Optional[llama_core.StoppingCriteriaList] + + class ChatFormatter(Protocol): """Base Protocol for a chat formatter. A chat formatter is a function that takes a list of messages and returns a chat format response which can be used @@ -803,6 +824,64 @@ def _stream_response_to_function_stream( def chat_formatter_to_chat_completion_handler( chat_formatter: ChatFormatter, ) -> LlamaChatCompletionHandler: + def prepare_chat_prompt( + *, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]], + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall], + tools: Optional[List[llama_types.ChatCompletionTool]], + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption], + stop: Optional[Union[str, List[str]]], + assistant_prefill: bool, + add_generation_prompt: Optional[bool], + ) -> _PreparedChatPrompt: + partial_assistant_text = "" + if assistant_prefill: + if not messages: + if llama.verbose: + print("Llama.create_chat_completion: Warning! 'assistant_prefill=True' but messages list is empty. Ignoring prefill.", file=sys.stderr) + elif messages[-1].get("role") != "assistant": + if llama.verbose: + print(f"Llama.create_chat_completion: Warning! 'assistant_prefill=True' but last message role is '{messages[-1].get('role')}'. Expected 'assistant'. Ignoring prefill.", file=sys.stderr) + else: + messages = messages.copy() + partial_message = messages.pop() + partial_assistant_text = partial_message.get("content", "") or "" + if not partial_assistant_text and llama.verbose: + print("Llama.create_chat_completion: Warning! 'assistant_prefill=True' but the assistant message has no content.", file=sys.stderr) + + format_kwargs = { + "messages": messages, + "functions": functions, + "function_call": function_call, + "tools": tools, + "tool_choice": tool_choice, + } + if add_generation_prompt is not None: + format_kwargs["add_generation_prompt"] = add_generation_prompt + result = chat_formatter( + **format_kwargs, + ) + if partial_assistant_text: + result.prompt += partial_assistant_text + + prompt = llama.tokenize( + result.prompt.encode("utf-8"), + add_bos=not result.added_special, + special=True, + ) + if result.stop is not None: + stop = [] if stop is None else [stop] if isinstance(stop, str) else stop + rstop = result.stop if isinstance(result.stop, list) else [result.stop] + stop = stop + rstop + + return _PreparedChatPrompt( + prompt=prompt, + stop=stop, + stopping_criteria=result.stopping_criteria, + ) + def chat_completion_handler( *, llama: llama_core.Llama, @@ -858,54 +937,27 @@ def chat_completion_handler( reasoning_budget_message: Optional[str] = None, reasoning_start_in_prompt: bool = False, reasoning_start_max_tokens: Optional[int] = 32, + add_generation_prompt: Optional[bool] = None, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, Iterator[llama_types.CreateChatCompletionStreamResponse], ]: - # JIT Interception for Assistant Prefill (Continue Generation) - partial_assistant_text = "" - if assistant_prefill: - if not messages: - if llama.verbose: - print("Llama.create_chat_completion: Warning! 'assistant_prefill=True' but messages list is empty. Ignoring prefill.", file=sys.stderr) - elif messages[-1].get("role") != "assistant": - if llama.verbose: - print(f"Llama.create_chat_completion: Warning! 'assistant_prefill=True' but last message role is '{messages[-1].get('role')}'. Expected 'assistant'. Ignoring prefill.", file=sys.stderr) - else: - # Safe to prefill: pop the last message without mutating the user's original list - messages = messages.copy() - partial_message = messages.pop() - partial_assistant_text = partial_message.get("content", "") or "" - if not partial_assistant_text and llama.verbose: - print("Llama.create_chat_completion: Warning! 'assistant_prefill=True' but the assistant message has no content.", file=sys.stderr) - - result = chat_formatter( + prepared = prepare_chat_prompt( + llama=llama, messages=messages, functions=functions, function_call=function_call, tools=tools, tool_choice=tool_choice, + stop=stop, + assistant_prefill=assistant_prefill, + add_generation_prompt=add_generation_prompt, ) - - # Seamlessly append the partial assistant text to the standard generated Jinja template - if partial_assistant_text: - result.prompt += partial_assistant_text - - prompt = llama.tokenize( - result.prompt.encode("utf-8"), - add_bos=not result.added_special, - special=True, - ) - if result.stop is not None: - stop = [] if stop is None else [stop] if isinstance(stop, str) else stop - rstop = result.stop if isinstance(result.stop, list) else [result.stop] - stop = stop + rstop - - stopping_criteria = None - if result.stopping_criteria is not None: - stopping_criteria = result.stopping_criteria + prompt = prepared.prompt + stop = prepared.stop + stopping_criteria = prepared.stopping_criteria if response_format is not None and response_format["type"] == "json_object": grammar = _grammar_for_response_format( @@ -1007,6 +1059,31 @@ def chat_completion_handler( ) return _convert_completion_to_chat(completion_or_chunks, stream=stream) + def prefill_handler( + *, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + assistant_prefill: bool = False, + add_generation_prompt: bool = True, + ) -> PrefillResult: + prepared = prepare_chat_prompt( + llama=llama, + messages=messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + stop=[], + assistant_prefill=assistant_prefill, + add_generation_prompt=add_generation_prompt, + ) + return llama.prefill(prepared.prompt, reset=True) + + chat_completion_handler.prefill = prefill_handler # type: ignore[attr-defined] return chat_completion_handler diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index deae0d4df..d4dc1070a 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -44,6 +44,7 @@ from ._logger import ggml_log_callback from llama_cpp.llama_chat_format import ( + PrefillResult, _convert_completion_to_chat, _convert_completion_to_chat_function, _grammar_for_response_format, @@ -909,22 +910,11 @@ def create_speech( self._request_lock.release() @dataclass -class _MTMDPrefillInternal: +class _MTMDPrefillResult: """Internal mutable state produced by a multimodal prefill.""" prompt: List[int] logits: np.ndarray - n_tokens: int -@dataclass(frozen=True) -class MTMDPrefillResult: - """Immutable multimodal prefill result. - - `logits` is an owned, read-only NumPy array. Call `.copy()` to obtain a - mutable array. - """ - prompt: tuple[int, ...] - logits: np.ndarray - n_tokens: int class MTMDChatHandler(MTMDBaseHandler): @@ -1725,7 +1715,7 @@ def _prefill_mtmd( tools: Optional[List[llama_types.ChatCompletionTool]] = None, tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, add_generation_prompt: bool = True, - ) -> _MTMDPrefillInternal: + ) -> _MTMDPrefillResult: """Evaluate a multimodal chat prompt without sampling or generating tokens. The returned logits are an owned copy, and the Llama KV state remains @@ -1960,10 +1950,9 @@ def _prefill_mtmd( logits = llama._restored_logits if logits is None: raise RuntimeError("MTMD prefill did not produce final logits") - return _MTMDPrefillInternal( + return _MTMDPrefillResult( prompt=prompt, logits=logits, - n_tokens=llama.n_tokens, ) except BaseException: # A helper can commit earlier ubatches before reporting failure. @@ -1974,6 +1963,26 @@ def _prefill_mtmd( # Generation no longer needs these resources once prompt evaluation ends. self._free_mtmd_resources(chunks, bitmap_cleanup) + def prefill( + self, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + assistant_prefill: bool = False, + **kwargs: Any, + ) -> PrefillResult: + """Evaluate a multimodal chat prompt and retain its native KV state. + + Public calls go through ``__call__`` so model-specific subclasses can + prepare template arguments and input state. The base handler uses the + private prepared path after that setup and before generation. + """ + if assistant_prefill: + raise NotImplementedError( + "assistant_prefill is not supported by MTMD chat handlers" + ) + + return self(llama=llama, messages=messages, prefill_only=True, **kwargs) + @overload def __call__( self, @@ -1982,7 +1991,7 @@ def __call__( messages: List[llama_types.ChatCompletionRequestMessage], prefill_only: Literal[True], **kwargs: Any, - ) -> MTMDPrefillResult: ... + ) -> PrefillResult: ... @overload def __call__( @@ -2008,7 +2017,7 @@ def __call__( ) -> Union[ llama_types.CreateChatCompletionResponse, Iterator[llama_types.CreateChatCompletionStreamResponse], - MTMDPrefillResult, + PrefillResult, ]: ... def __call__( @@ -2067,15 +2076,15 @@ def __call__( ) -> Union[ llama_types.CreateChatCompletionResponse, Iterator[llama_types.CreateChatCompletionStreamResponse], - MTMDPrefillResult, + PrefillResult, ]: - """Call the handler, or return the MTMD prompt state when requested. + """Call the handler, or return final next-token logits when requested. ``prefill_only=True`` is supported by direct MTMD handler calls. It still - runs subclass ``__call__`` preprocessing, then returns before sampling. + runs subclass ``__call__`` preprocessing, then returns a ``PrefillResult`` + before sampling. """ - prefill = MTMDChatHandler._prefill_mtmd( - self, + prefill = self._prefill_mtmd( llama=llama, messages=messages, functions=functions, @@ -2085,12 +2094,8 @@ def __call__( add_generation_prompt=add_generation_prompt, ) if prefill_only: - logits = prefill.logits.copy() - logits.flags.writeable = False - return MTMDPrefillResult( - prompt=tuple(prefill.prompt), - logits=logits, - n_tokens=prefill.n_tokens, + return PrefillResult( + logits=prefill.logits, ) prompt = prefill.prompt diff --git a/tests/test_formats.py b/tests/test_formats.py index 3f4e5d828..ad86c8f3e 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -528,6 +528,181 @@ def test_formatter_stop_token_boundary(): assert criterion(np.array([1, 2], dtype=np.intc), logits) is False +@pytest.mark.parametrize( + "logits_all, logits_index", [(False, 0), (True, 1)] +) +def test_text_prefill_uses_eval_and_owns_final_logits(logits_all, logits_index): + from llama_cpp import Llama, PrefillResult + + llama = Llama.__new__(Llama) + llama.tokenize = Mock(return_value=[11, 12]) + llama.reset = Mock() + llama.eval = Mock() + llama.n_tokens = 2 + llama._logits_all = logits_all + llama.scores = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float32) + llama.create_completion = Mock() + + result = llama.prefill("hello") + + assert isinstance(result, PrefillResult) + assert not hasattr(result, "prompt") + assert not hasattr(result, "n_tokens") + assert result.logits.shape == (3,) + assert result.logits.flags.owndata and not result.logits.flags.writeable + np.testing.assert_array_equal(result.logits, [1.0, 2.0, 3.0] if not logits_all else [4.0, 5.0, 6.0]) + llama.tokenize.assert_called_once_with(b"hello", add_bos=True, special=True) + llama.reset.assert_called_once_with() + llama.eval.assert_called_once_with( + [11, 12], active_loras=None, control_vector=None, copy_logits=True + ) + llama.create_completion.assert_not_called() + + saved = result.logits.copy() + llama.scores[logits_index, 0] = 99.0 + np.testing.assert_array_equal(result.logits, saved) + + +@pytest.mark.parametrize( + "prompt, kwargs", + [([], {}), ("", {"add_bos": False})], +) +def test_text_prefill_rejects_empty_tokens_without_reset(prompt, kwargs): + from llama_cpp import Llama + + llama = Llama.__new__(Llama) + llama.tokenize = Mock(return_value=[]) + llama.reset = Mock() + llama.eval = Mock() + + with pytest.raises(ValueError, match="at least one token"): + llama.prefill(prompt, **kwargs) + + llama.reset.assert_not_called() + llama.eval.assert_not_called() + + +def test_standard_chat_prefill_shares_completion_preparation(monkeypatch): + from llama_cpp import PrefillResult + from llama_cpp.llama_chat_format import ( + ChatFormatterResponse, + chat_formatter_to_chat_completion_handler, + ) + + messages = [ + {"role": "user", "content": "question"}, + {"role": "assistant", "content": "partial"}, + ] + original = copy.deepcopy(messages) + formatter_calls = [] + stopping_criteria = object() + + def format_chat(**kwargs): + formatter_calls.append(kwargs) + return ChatFormatterResponse( + prompt="rendered:", + stop=[""], + stopping_criteria=stopping_criteria, + added_special=True, + ) + + tokenized = Mock(return_value=[21, 22]) + prefill_result = PrefillResult(np.array([1.0, 2.0])) + prefill = Mock(return_value=prefill_result) + create_completion = Mock(return_value=object()) + llama = SimpleNamespace( + verbose=False, + tokenize=tokenized, + prefill=prefill, + create_completion=create_completion, + ) + handler = chat_formatter_to_chat_completion_handler(format_chat) + monkeypatch.setattr( + "llama_cpp.llama_chat_format._convert_completion_to_chat", + lambda completion, stream=False: completion, + ) + + assert handler.prefill( + llama=llama, messages=messages, assistant_prefill=True + ) is prefill_result + assert handler( + llama=llama, + messages=messages, + assistant_prefill=True, + add_generation_prompt=True, + ) is create_completion.return_value + + assert messages == original + assert len(formatter_calls) == 2 + for call in formatter_calls: + assert call["messages"] == [original[0]] + assert call["add_generation_prompt"] is True + assert tokenized.call_args_list == [ + ((b"rendered:partial",), {"add_bos": False, "special": True}), + ((b"rendered:partial",), {"add_bos": False, "special": True}), + ] + prefill.assert_called_once_with([21, 22], reset=True) + completion_args = create_completion.call_args.kwargs + assert completion_args["prompt"] == [21, 22] + assert completion_args["stop"] == [""] + assert completion_args["stopping_criteria"] is stopping_criteria + + +def test_standard_completion_preserves_formatter_generation_prompt_default(monkeypatch): + from llama_cpp.llama_chat_format import Jinja2ChatFormatter + + formatter = Jinja2ChatFormatter( + template="{% if add_generation_prompt %}generation{% else %}configured-default{% endif %}", + eos_token="", + bos_token="", + add_generation_prompt=False, + ) + tokenized = Mock(return_value=[31]) + llama = SimpleNamespace( + verbose=False, + tokenize=tokenized, + create_completion=Mock(return_value=object()), + ) + handler = formatter.to_chat_handler() + monkeypatch.setattr( + "llama_cpp.llama_chat_format._convert_completion_to_chat", + lambda completion, stream=False: completion, + ) + + handler(llama=llama, messages=[{"role": "user", "content": "question"}]) + + assert tokenized.call_args.args[0] == b"configured-default" + + +def test_create_chat_prefill_dispatches_by_capability(): + from types import MethodType + from llama_cpp import Llama, PrefillResult + + result = PrefillResult(np.array([0.0, 1.0])) + handler = SimpleNamespace(prefill=Mock(return_value=result)) + llama = SimpleNamespace( + chat_handler=handler, + _chat_handlers={}, + chat_format="unused", + ) + llama._get_chat_completion_handler = MethodType( + Llama._get_chat_completion_handler, llama + ) + llama.create_chat_prefill = MethodType(Llama.create_chat_prefill, llama) + + assert llama.create_chat_prefill(messages=[]) is result + handler.prefill.assert_called_once_with( + llama=llama, + messages=[], + functions=None, + function_call=None, + tools=None, + tool_choice=None, + add_generation_prompt=True, + assistant_prefill=False, + ) + + def test_formatter_preserves_inputs_and_exposes_hf_template_context(): messages = [{"role": "user", "content": [{"type": "text", "text": "<&中文"}]}] tools = [{"type": "function", "function": {"name": "lookup"}}] diff --git a/tests/test_media.py b/tests/test_media.py index b76d7a18c..f5c013813 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -74,12 +74,15 @@ def chat_prefill(chat_prefill_builder): def test_chat_prefill_returns_owned_logits_without_generation(chat_prefill): import numpy as np + from llama_cpp import PrefillResult handler, llm, _ = chat_prefill result = handler(llama=llm, messages=[], prefill_only=True) - assert result.prompt == (1, 2, -9, -9) - assert result.n_tokens == llm.n_tokens == 4 + assert isinstance(result, PrefillResult) + assert not hasattr(result, "prompt") + assert not hasattr(result, "n_tokens") + assert llm.n_tokens == 4 assert result.logits.shape == (llm.n_vocab(),) assert result.logits.dtype == np.float32 assert result.logits.flags.owndata @@ -121,6 +124,7 @@ def prefill(self, *args, **kwargs): def test_external_handler_forwards_prefill_only_through_call(chat_prefill_builder): from llama_cpp import llama_multimodal as multimodal + from llama_cpp import PrefillResult class WrappedChatHandler(multimodal.MTMDChatHandler): def __call__(self, **kwargs): @@ -131,12 +135,13 @@ def __call__(self, **kwargs): result = handler(llama=llm, messages=[], prefill_only=True) assert handler.wrapper_ran - assert isinstance(result, multimodal.MTMDPrefillResult) + assert isinstance(result, PrefillResult) llm.create_completion.assert_not_called() def test_generic_chat_prefill_resolves_model_template(chat_prefill_builder): from llama_cpp import llama_multimodal as multimodal + from llama_cpp import PrefillResult handler, llm, _ = chat_prefill_builder( multimodal.GenericMTMDChatHandler, chat_format=None @@ -146,7 +151,7 @@ def test_generic_chat_prefill_resolves_model_template(chat_prefill_builder): result = handler(llama=llm, messages=[], prefill_only=True) - assert isinstance(result, multimodal.MTMDPrefillResult) + assert isinstance(result, PrefillResult) llm._model.model_chat_template.assert_called_once_with(None) assert handler._template_initialized assert handler._chat_format_parser_tags == ["<|image|>"] @@ -161,6 +166,7 @@ def test_llama_create_chat_prefill_uses_selected_mtmd_handler( import numpy as np from llama_cpp import Llama from llama_cpp import llama_multimodal as multimodal + from llama_cpp import PrefillResult handler, llm, backend = chat_prefill_builder( multimodal.GenericMTMDChatHandler, chat_format=None @@ -196,8 +202,8 @@ def test_llama_create_chat_prefill_uses_selected_mtmd_handler( add_generation_prompt=False, ) - assert isinstance(result, multimodal.MTMDPrefillResult) - assert result.n_tokens == llm.n_tokens + assert isinstance(result, PrefillResult) + assert llm.n_tokens == 4 assert llm._restored_logits is not None assert not llm.reset.called llm._ctx.memory_clear.assert_not_called() @@ -221,7 +227,7 @@ def test_llama_create_chat_prefill_uses_selected_mtmd_handler( def test_llama_create_chat_prefill_rejects_non_mtmd_handler(): from llama_cpp import Llama - handler = Mock() + handler = lambda **kwargs: None llm = SimpleNamespace( chat_handler=handler, _chat_handlers={}, @@ -232,16 +238,15 @@ def test_llama_create_chat_prefill_rejects_non_mtmd_handler(): ) llm.create_chat_prefill = MethodType(Llama.create_chat_prefill, llm) - with pytest.raises(TypeError, match="requires an MTMDChatHandler"): + with pytest.raises(NotImplementedError, match="handler does not support prefill"): llm.create_chat_prefill(messages=[]) - handler.assert_not_called() - def test_minicpmv45_prefill_prepares_prompt_and_keeps_generation_stops( chat_prefill_builder, ): from llama_cpp import llama_multimodal as multimodal + from llama_cpp import PrefillResult handler, llm, _ = chat_prefill_builder( multimodal.MiniCPMv45ChatHandler, enable_thinking=False @@ -254,9 +259,9 @@ def process_prompt(**kwargs): return [1, 2, -9, -9], [(2, 4, object(), 1, -9)], object(), [] handler._process_mtmd_prompt.side_effect = process_prompt - result = handler(llama=llm, messages=[], prefill_only=True) + result = handler.prefill(llama=llm, messages=[]) - assert isinstance(result, multimodal.MTMDPrefillResult) + assert isinstance(result, PrefillResult) llm.create_completion.assert_not_called() assert observed[0][0].tolist() == [0, 0, 0, 0, 0, 0] assert observed[0][1]["enable_thinking"] is False From 3c3cab24c752b6689d3849a9020c85bb4bd4e549 Mon Sep 17 00:00:00 2001 From: craftingmod Date: Sun, 27 Sep 2026 01:58:09 +0900 Subject: [PATCH 5/9] feat(chat): add `chat_template_kwargs` option to chat completion & prefill Assisted-by: codex --- llama_cpp/llama.py | 7 +++++- llama_cpp/llama_chat_format.py | 15 ++++++++++++ llama_cpp/llama_multimodal.py | 42 +++++++++++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 3e8c06810..c4a92500e 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -4456,6 +4456,7 @@ def create_chat_completion( top_logprobs: Optional[int] = None, assistant_prefill: bool = False, add_generation_prompt: bool = True, + chat_template_kwargs: Optional[Dict[str, Any]] = None, # Reasoning Budget Params reasoning_budget: int = -1, reasoning_start: str = "", @@ -4502,7 +4503,7 @@ def create_chat_completion( dry_base`: Set the DRY repetition penalty base value. Default: `1.75` dry_allowed_length: Tokens that extend repetition beyond this receive exponentially increasing penalty: multiplier * base ^ (length of repeating sequence before token - allowed length). Default: `2` dry_penalty_last_n: How many tokens to scan for repetitions. Default: `64`; `0` disables scanning and `-1` uses the context size. - dry_seq_breakers: Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted. Default: `['\n', ':', '"', '*']` + dry_seq_breakers: Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted. Default: `['\\n', ':', '"', '*']` adaptive-target: Adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) (default: %.2f) [(more info)](https://github.com/ggml-org/llama.cpp/pull/17927) adaptive-decay: Adaptive-p: decay rate for target adaptation over time. lower values are more reactive, higher values are more stable. (valid range 0.0 to 0.99) (default: %.2f) use_infill: Determines whether to activate the specialized fill-in-the-middle sampler that consolidates probabilities of tokens sharing common prefixes to ensure the generated text coherently bridges the gap between the prefix and suffix. @@ -4511,6 +4512,7 @@ def create_chat_completion( logits_processor: A list of logits processors to use. grammar: A grammar to use. grammar_lazy: If True, enables lazy evaluation. + chat_template_kwargs: Optional keyword arguments passed to the Jinja chat template at render time. These values override matching handler-level template defaults for the current request only. reasoning_budget: Token budget for the first visible reasoning block. -1 disables the sampler, 0 forces an immediate end after reasoning starts, and N > 0 allows at most N generated tokens inside the block. @@ -4584,6 +4586,7 @@ def create_chat_completion( control_vector=control_vector, assistant_prefill=assistant_prefill, add_generation_prompt=add_generation_prompt, + chat_template_kwargs=chat_template_kwargs, reasoning_budget=reasoning_budget, reasoning_start=reasoning_start, reasoning_end=reasoning_end, @@ -4601,6 +4604,7 @@ def create_chat_prefill( tool_choice: Optional[ChatCompletionToolChoiceOption] = None, add_generation_prompt: bool = True, assistant_prefill: bool = False, + chat_template_kwargs: Optional[Dict[str, Any]] = None, ) -> PrefillResult: """Prefill a chat prompt through its handler without generating a token.""" handler = self._get_chat_completion_handler() @@ -4619,6 +4623,7 @@ def create_chat_prefill( tool_choice=tool_choice, add_generation_prompt=add_generation_prompt, assistant_prefill=assistant_prefill, + chat_template_kwargs=chat_template_kwargs, ) def create_chat_completion_openai_v1( diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 510737e6b..34df4f3b5 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -835,6 +835,7 @@ def prepare_chat_prompt( stop: Optional[Union[str, List[str]]], assistant_prefill: bool, add_generation_prompt: Optional[bool], + chat_template_kwargs: Optional[Dict[str, Any]], ) -> _PreparedChatPrompt: partial_assistant_text = "" if assistant_prefill: @@ -858,8 +859,20 @@ def prepare_chat_prompt( "tools": tools, "tool_choice": tool_choice, } + if add_generation_prompt is not None: format_kwargs["add_generation_prompt"] = add_generation_prompt + + if chat_template_kwargs is not None: + reserved = format_kwargs.keys() & chat_template_kwargs.keys() + if reserved: + raise ValueError( + "chat_template_kwargs contains reserved keys: " + f"{sorted(reserved)}" + ) + + format_kwargs.update(chat_template_kwargs) + result = chat_formatter( **format_kwargs, ) @@ -926,6 +939,7 @@ def chat_completion_handler( logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, assistant_prefill: bool = False, + chat_template_kwargs: Optional[Dict[str, Any]] = None, # Reasoning Budget Params # # Generic first-reasoning-block budget control. These parameters are @@ -954,6 +968,7 @@ def chat_completion_handler( stop=stop, assistant_prefill=assistant_prefill, add_generation_prompt=add_generation_prompt, + chat_template_kwargs=chat_template_kwargs, ) prompt = prepared.prompt stop = prepared.stop diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index d4dc1070a..e274fdb09 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -1268,6 +1268,7 @@ def _render_mtmd_prompt( tools: Optional[List[llama_types.ChatCompletionTool]] = None, tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, add_generation_prompt: bool = True, + chat_template_kwargs: Optional[Dict[str, Any]] = None, ) -> str: """ Render the chat template into plain prompt text. @@ -1275,6 +1276,37 @@ def _render_mtmd_prompt( This stage only renders the Jinja template. It does not normalize media placeholders or replace media URLs with the MTMD runtime marker. """ + + template_kwargs = dict(getattr(self, "extra_template_arguments", {})) + if chat_template_kwargs: + # `_render_mtmd_prompt` arguments + reserved = { + "messages", + "add_generation_prompt", + "eos_token", + "bos_token", + "functions", + "function_call", + "tools", + "tool_choice", + } + invalid = reserved.intersection(chat_template_kwargs) + if invalid: + raise ValueError( + f"{self.log_prefix}(_render_mtmd_prompt): chat_template_kwargs contains reserved keys - {sorted(invalid)}" + ) + + if self.verbose: + overrides = { + key: value + for key, value in chat_template_kwargs.items() + if key in template_kwargs and template_kwargs[key] != value + } + if overrides: + print(f"{self.log_prefix}(_render_mtmd_prompt): Template kwargs override - {overrides}") + + template_kwargs.update(chat_template_kwargs) + return self.chat_template.render( messages=messages, add_generation_prompt=add_generation_prompt, @@ -1284,7 +1316,7 @@ def _render_mtmd_prompt( function_call=function_call, tools=tools, tool_choice=tool_choice, - **getattr(self, "extra_template_arguments", {}), + **template_kwargs, ) def _replace_media_placeholders( @@ -1351,6 +1383,7 @@ def _render_and_replace_media( tools: Optional[List[llama_types.ChatCompletionTool]] = None, tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, add_generation_prompt: bool = True, + chat_template_kwargs: Optional[Dict[str, Any]] = None, ) -> str: """ Render chat messages and normalize rendered media placeholders into MTMD markers. @@ -1362,6 +1395,7 @@ def _render_and_replace_media( tools=tools, tool_choice=tool_choice, add_generation_prompt=add_generation_prompt, + chat_template_kwargs=chat_template_kwargs, ) return self._replace_media_placeholders( @@ -1520,6 +1554,7 @@ def _process_mtmd_prompt( tools: Optional[List[llama_types.ChatCompletionTool]] = None, tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, add_generation_prompt: bool = True, + chat_template_kwargs: Optional[Dict[str, Any]] = None, ) -> Tuple[List[int], List[tuple], Any, List[Any]]: """ Core multimodal preprocessing pipeline. @@ -1552,6 +1587,7 @@ def _process_mtmd_prompt( tools=tools, tool_choice=tool_choice, add_generation_prompt=add_generation_prompt, + chat_template_kwargs=chat_template_kwargs, ) if self.verbose: @@ -1715,6 +1751,7 @@ def _prefill_mtmd( tools: Optional[List[llama_types.ChatCompletionTool]] = None, tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, add_generation_prompt: bool = True, + chat_template_kwargs: Optional[Dict[str, Any]] = None, ) -> _MTMDPrefillResult: """Evaluate a multimodal chat prompt without sampling or generating tokens. @@ -1734,6 +1771,7 @@ def _prefill_mtmd( tools=tools, tool_choice=tool_choice, add_generation_prompt=add_generation_prompt, + chat_template_kwargs=chat_template_kwargs, ) prefill_started = False @@ -2065,6 +2103,7 @@ def __call__( logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, add_generation_prompt: bool = True, + chat_template_kwargs: Optional[Dict[str, Any]] = None, prefill_only: bool = False, reasoning_budget: int = -1, reasoning_start: str = "", @@ -2092,6 +2131,7 @@ def __call__( tools=tools, tool_choice=tool_choice, add_generation_prompt=add_generation_prompt, + chat_template_kwargs=chat_template_kwargs, ) if prefill_only: return PrefillResult( From 5712f6771065255095b851ffe918f3c6020e5200 Mon Sep 17 00:00:00 2001 From: craftingmod Date: Sun, 27 Sep 2026 02:00:59 +0900 Subject: [PATCH 6/9] chore(prefill): add example for prefill feature Assisted-by: codex --- .../high_level_api/high_level_api_prefill.py | 49 +++++++++ .../high_level_api_prefill_vision.py | 96 ++++++++++++++++++ examples/high_level_api/media/apple.jpg | Bin 0 -> 11626 bytes 3 files changed, 145 insertions(+) create mode 100644 examples/high_level_api/high_level_api_prefill.py create mode 100644 examples/high_level_api/high_level_api_prefill_vision.py create mode 100644 examples/high_level_api/media/apple.jpg diff --git a/examples/high_level_api/high_level_api_prefill.py b/examples/high_level_api/high_level_api_prefill.py new file mode 100644 index 000000000..e7642966c --- /dev/null +++ b/examples/high_level_api/high_level_api_prefill.py @@ -0,0 +1,49 @@ +"""Decision one token without sampling +""" +import argparse +import numpy as np +from llama_cpp import Llama + +parser = argparse.ArgumentParser() +parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-model.bin") +args = parser.parse_args() + +llm = Llama( + model_path=args.model, + n_ctx=2048, +) + +prompt = """\ +Question: Is the following statement true? + +2 + 2 = 4 + +Answer with exactly Y or N. +Answer: +""" + +result = llm.prefill(prompt) + +y_tokens = llm.tokenize(b"Y", add_bos=False, special=False) +n_tokens = llm.tokenize(b"N", add_bos=False, special=False) + +assert len(y_tokens) == 1 +assert len(n_tokens) == 1 + +y_token = y_tokens[0] +n_token = n_tokens[0] + +# pull Y/N logit from vocab +y_logit = float(result.logits[y_token]) +n_logit = float(result.logits[n_token]) + +print("Y logit:", y_logit) +print("N logit:", n_logit) + +# softmax from Y/N only +candidate_logits = np.array([y_logit, n_logit]) +candidate_probs = np.exp(candidate_logits - candidate_logits.max()) +candidate_probs /= candidate_probs.sum() + +print("Y:", candidate_probs[0]) +print("N:", candidate_probs[1]) \ No newline at end of file diff --git a/examples/high_level_api/high_level_api_prefill_vision.py b/examples/high_level_api/high_level_api_prefill_vision.py new file mode 100644 index 000000000..af193f0e3 --- /dev/null +++ b/examples/high_level_api/high_level_api_prefill_vision.py @@ -0,0 +1,96 @@ +"""Decision one token without sampling (with vision). +Gemma4 model only. +""" +import numpy as np +import argparse +from pathlib import Path +from llama_cpp import Llama +from llama_cpp.llama_chat_format import Gemma4ChatHandler + +BASE_DIR = Path(__file__).resolve().parent +IMAGE_PATH = BASE_DIR / "media" / "apple.jpg" + +parser = argparse.ArgumentParser() +parser.add_argument("-m", "--model", type=str) +parser.add_argument("-mm", "--mmproj", type=str) +parser.add_argument("-i", "--image", type=str, default=IMAGE_PATH) +args = parser.parse_args() + +# Use gemma4 handler +chat_handler = Gemma4ChatHandler( + clip_model_path=args.mmproj, + enable_thinking=False, +) + +llm = Llama( + model_path=args.model, + chat_handler=chat_handler, + n_ctx=4096, +) + +messages = [ + { + "role": "system", + "content": "Answer with exactly Y or N." + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": args.image.as_uri(), + }, + }, + { + "type": "text", + "text": "Is there any visible written word in this image? " + "Answer with exactly one character: Y or N." + }, + ], + }, +] + +result = llm.create_chat_prefill( + messages=messages, +) + +result_chat = llm.create_chat_completion( + messages=messages, +) + +print(result_chat) + +print("Top 20 tokens: ") +top_k = 20 +top_ids = np.argsort(result.logits)[-top_k:][::-1] + +for token_id in top_ids: + token = llm.detokenize([int(token_id)]) + print( + repr(token), + float(result.logits[token_id]), + ) + +y_tokens = llm.tokenize(b"Y", add_bos=False, special=False) +n_tokens = llm.tokenize(b"N", add_bos=False, special=False) + +assert len(y_tokens) == 1 +assert len(n_tokens) == 1 + +y_token = y_tokens[0] +n_token = n_tokens[0] + +y_logit = float(result.logits[y_token]) +n_logit = float(result.logits[n_token]) + +print("Y logit:", y_logit) +print("N logit:", n_logit) + +# Softmax in Y/N +candidate_logits = np.array([y_logit, n_logit], dtype=np.float32) +candidate_probs = np.exp(candidate_logits - candidate_logits.max()) +candidate_probs /= candidate_probs.sum() + +print("P(Y | {Y,N}) =", float(candidate_probs[0])) +print("P(N | {Y,N}) =", float(candidate_probs[1])) diff --git a/examples/high_level_api/media/apple.jpg b/examples/high_level_api/media/apple.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cb01d2ca6ea5bfc26ea811366f2aa470a6449378 GIT binary patch literal 11626 zcmeHs2UHX5y7q(s0Td#LAfN;(0)o;(6cVrjB7#^bQdCfyfFMXu0;qHm5ZqWuP!R+a z=~X%eMd{L80O`?$5(8=f=)ULdwa;4j{GNUO|E_h{I+K+(nas>L^StGGpYQ$HgX~FQ z+sP9qCjbZp03hH8U=so4@j$n00AOOW58wfQ{H+Gz03^U);^6nkcI2{`v)cu?s{v96 z52#8h`1oJG?&cSCNa~=vBETL2jso1AoLroo++18-+}zwSUIAWS9v)sHettfI9YP`^ zJA{OVMaB2*65WXq6BgbrzZ)SfBPS;(vP)4}K~`yxteotRK_K9*yga;uyu5<4JB4@3 z{>vM?9uVY#NO1sAh#K@$Kdlk<}#1QGP)CiR{;aMf@dXe+Tvlu3=y+ z6aqRADhQx}&6ij{KmovW{Ac$c6Z~(B0^mNkNxd-8b}e>FO;%ktCs~!z(Ua!)_8);iijUNTQ@QRLAH=j{}t_~1T;-MbX2~JkmDBO9H={bj= zEp&fu=WD3m_gHuu%WZzP!{ieX;8n^uPvoi6AiF+!=25$s=(VzIs zg(*Nux&K0fl2u~rO8s$7-w5+w_mb3Dkx6Ut(SQvrn}1@4-Y7+ ziLxk;SQ#rn>LcDR;UsoJ47Q>P(faF_gRH8MQYZ(^P-ews$)~C|Dt-3uT;|eJ6V>Ll z?TYfu?!hjYK3$3-XMbU3ju71PHpKJ>wJOwCK}n-1HK|SC;A}}qA&UeAskAx*b__*Y zV#v!8UShG(mUr4{L$&Rx4PVcFF-=c=mRkLd7%JSNTL$V7E({C@A zV%IJk?Ed`AAP`RwVk&ZQNBN4uo0ef z9=d74^{;s1&!n1{>wfFefye%@COXQfX-S%^C&vVZopEgzusM-eL3s>&TB>aBrvsS< z?i>e?-A>whYd9|BLtOZ>2o`mefgr}IaM2u3k*OJ|?h=~Fj_a2y5kZ~RYRX~n@o+_E zJ_887NS<8MPMb12YIxI+2ccFjBa~Rrg`q+4a4}{gOFH{5!_9|yafoARso|JhBCju& zgdj_uq~9-rP_usBT#tr**+8Qe<@AvGH&m&AODEmId72Hx;^94y{)H(0vuzZhjTjF$ zAP$bKPafCj<`uU0D2}#vnccQ`cN^oDN_PSf6t(O$dH!?g*;j$JUv~uh<^Of7&6INJyLpNFNLFd% zxLdIcPN8gmQ1XlM2=2Tlv3ya1@sFIb|3#G z`#@5rNLKVMjgLBB$20G?hCB&gPI$Zt`j-vV+pDc(;?dj76LcQbnMUY8<356s)r4&B z|2G&abUWQL@FT5!>Jzg>{abPO12-}@aI~fkUEL0+@!;Y47?*F*LZ z_S?Xx`_Ra*d4-rsO~`x+U;*oNEK_M}DhNC>&_eZ@{dUarA;d_#us7pA6=o* z%y_iacmx1j=`kEOs^LpWPtbLH1`!&I%d9W#Z9hZeq zBf7EGW*4jtQ*^Z)M=3B(aYG(2{g}SS>^*>?eCx*|DSx>X31FsRv6*C}h|&VVQy-o+ zex5NH6F+rXl;ZoGf`^Brw*{VI$uiD$uT}T-w%qjdDzK@X5Wi-kju~zp-F%COcfWb? zqjc=h3ftFf-&3HON4wo0@b+CzH*V^LB@ZJ3K>NCDR}mV%#Uvj~PdwA51yDp6q%Mr! zo~GH~2y3!{&1c^&rK5ytn9~EwyYyc(tL#qP>{9K3SImdC0B*lhm0(XZ&8`%0L4YOH#mNT9Lyr7)b0f_t8ZUwk?83uwb*=@`NIu%r0_(uYCP8;q z$DqCAYxV|(F&u0l@6yp-_qRYZ8o<(6$jo2^Zv0z_b&)RQxmcOLS!*xbi`u0L&RZnn zi*~(jm5^s~x@i6Q(&MGzhTx_;q2(_)p*5 zsN06rqA3y6GCi_sahAzS)v2o@J1@K0$aLtGb=S8c=U^OI725MD+-*GsG0$AcefZ;< zg@+AhFADRFy>t!`3(pA!&goZo!S*=O&xOj@q_lernqChPTXq3hdjwG8z>+2+E(1c* z%M(bM8k3&7cF@|a{@w}Xhe7!jyg=38^68*y-Mu<~Wrwf243`&CC7wmOZPs%K8?gdx zfF|Gym8)~>ydBHS25#4CB$_JWSULtf*Z?sXErjX%q{7eK}|>dvoi(Fb-qklw~g9mf!CpDX5bK-YaX>;<&zy+tgBk&SC}!SSQB_%m^1Y z(4Axf8|ecq0Ar}^Bp)!zI8W&%4n#hE_F3=j6)#cUh`z@$kTq%5Tevq0(D#hvbWa+( zuP6}HBz_McBzbkF-K)+*G)QJC7MKNJu)^PaQ)n4O)&gO~0&4}g9ZUCyQxKMkM?YV$ zd+Q%`Z^YlNrp?Rz`+2e|p(v^Iin%a^1U~)O#T;y|jwCkCMZvYabD!l69WmW29##pH zxo)Qo&kUObIUuJ-i>?(Y^t~RvtB|4+mF#)C=l!u&g7W?Mvq<|U;mg%qKXNaFNIU!3 zfRlHWPR+2LM^(^V^qcbp7V>$$2lEsy6{=Zf8nWMZ_M)+b$3oRz=V-|1Tjh#u03wY=w$Kh! zMw-}wsPacu_%g|hbId3E@3vakmixy|<7&n|xVx;A6XBdMti8>z*;^Jz**y?(8^NJjH8q`ZY02))(2W zgV^B1D&q!mtt0AshQuBky?VA4U26~AV5u@6qh)E@>r&}A&j>a=H*}bXYbBfUt3`_Dv%l@72w*2GH23##q zDz;x#TW(*NZ<5+v|2czBwgH>H>`AvoHQ z=W#a9MnI(TW02|-N<>~eSeWC)fl)GMxsIaLk|`G^r+LvzZ2My37tD<67M42~5d1v~ zb!ES{RB~WSOMsPp-qGy(LI~;IcNf-%Fl2xYG$BIOM#|g!Z^nDkT~LnI28ey3b~JAc z8AGsia%?O%}qk!KhuWM(~*x`wN)TamK zM{W$$NLARwAAVW->8iv@s(_@mo!I|witAGG@bKGm)Y?#+F&JC3j*VC(f{Ck}swvcz?!Z7gZVg;P2o5z{wmTk`{cYF& z50_BPHNNu7{RWa>=!$`iz$W4g+w;v?>-<2Ru0~&@?`3cL49Iv`3s}idVhL5FJF$kPe`4=KhEa~xcf7zGtgd;S&NTuN6Xyjx*K>ffuBM9Uz#%>O32^w^jy)ICFG}KE?qk#ypx=Jl9xhHJfSh)7N^|>Su%3nMxO6K~&$LV;Z7tR4 zCX>?n#wQlo*mA0`<}w*YxbyjH&-zncsu|~#7dbahmL2rF9GSFi*uf8=BLd;AB1k0p zUiy^1-j0aP-LvYxS8f>lAs6ct8?ib$P%r}B>NfV&DzE|H2{v%@3LXwt;45_-`wrol z;aOm{=<0%fW&_)`78okz@6ExV^KMrb_s*t?pqlnJm9T*cYdjpw5(dzFNupv(%DN7l_7O-&#D9wD54cw)=dE;Hl1XYCoOudPZ;<_78=cKAW z7vR74qbMK_vxd=P_rScUvWgax^3oickbB}Ucrsg;4TMXhuVHmN2QhYG zQOkgAB(VWh$_gHyJO9hlX(Nt0L~)Be*%d0k^|sc>*O(ibb>ZB7oWVjJ=vM}CW~|<> z{U!F^ExmhI4}1|3nV}}Ab{G|2JHrM5JpAP!5)H1y%+!Oifh|xX`qfp=6Q*=Idf0DJ zW#@RIb0I}(j<19vE12aL3RmbBigy2&S9}Y@UE-mv*yW~nOv77^K@AJogz!(;ThU-H zuULO!18>9XR@{Njyh#;CQYSBFP#=|!7oh%75zslzh*JHT#c_cW%AFD=>O1K({7rW@ z9*rLjPnqyYCD2I*yAqerOv^`}?`I%1aa9%Z?jnmJ#uX-N>y}Y0(6|>ZV2@edU^eL|Qcc-FveHY;z^6`Df+l3y4>H3BN)M|u_$dNCCv5w6 z1t8pFv3l}U`Iw7S+NFvJb+@gsX-yl}Imp^T>tkMj)a`NQGVHaZFHcVflPm|>0KpYH zT{q-t6K6#0exBdF&w-$icN%)6Znb(93YPq0E%P`u&|O0eo8HeYe>*i^Wxs9He0hQ>M}!X7Zcc!t(`12J(3MZ%vEN%5bpt)1h~q0w-VuuZmQ)R_3|0E_2!{ zA(WH4qVIEc5)BZi&~2!#p#I^#xP+7*SlcL>&XK)g+i>H~UWFHF*Uo;3xHYZ!WP7g| zu!;Pe7zDlf&w~}-X+A8|;yv{(u4=vhz6sZpi6JUI2qL|48u_}0r3snvr9<}o3rD?!^B5YLO+F(5{D~)oA)`N%^OvIZ0j)6m zcrHVYa{k4H&sttuXQuT~ID)OJb8HMS$x|FB)Q^{8O)zH^|1dIO!P0koLu-O`;l z-F|r&Uu#nWj{B-)o|h65F!ZdC;E@q)1842xM$6_?blm<@$8Gavd`$=5#>o3oTZO>_ zahNGel_%fHh2{rF+4RX07;gnfUZQX7GF_q7U0`As+42u)X zQUH5dW;Nf{lC)BK4@z9m>?O^L zm>BJ~uG=q{(duDrh0-i)i_gtH{NxE8Fn0%N6IA<>jJfHksEnOM{g9i=xBZUO^&;gd zmr#NR2#f`+15-A(d3-VqYOWF^-}kXW|H!d=Tzwnr5YWMlx?biwuKaF!O%mymn zV5q^s=B=Of{HYchY7t3{!hU^orQ&?Q%IDW5^MvtLp4n5uQo5aQy2r6d111hBS>Bu& zlO{5t;M|*UKJqPY>)m5AJect-&*x1@28c|s9>DIQ`yV2W8thN&53YXoeBOG9wWf`t z3Qe=q)0$SFY@38l7Ec{$rB^6 zX411zC2_k`#~yu}xkb8HYm27Toj0U)!G1$Gvv%ra>>CI*k!F2u?DmwA^$)rzn|PxT z6a*Z2_*>UMG3?#V4-=L}6-CuCSIa9BGSjr@=bJbYIeW1v-CC(YtFttH%DyGD^2%i0 zwUGyvg;fq>DNzZmiq^0OJiHFQoAzO33Kmg<6B@?Gg#m6dFr?k{FMf=jsvw0t!^0yn z9@-aZ1G#{qlIL}J;mQLx0QS<@z+^dsS8oq)lXClezWae7CY0pldc~@49^-rEU zpI|g)EDRxd;C9tY1P1GbOo+J&SU3|rbcPWNW%{jHJ;&CGvUmB0uNk6vy{`huf!xqn zv=dSCi`HPXWNtz@aG;e9?DVH`XWb0RcW4{gpZ|JVZoB^Tm=mzd*3-IsvB)asV-~kg zBXh64%f)lvg6?W}l7u6arzSTHzF#p=4!!{_BtuB&AS*ahyH9$k#C1M$d6D1*P8L4n z3U*`SNX?FX1zFSTi>s0uNyikRBmM~iC&}y29X!NSGoD_5ceN_r$?;P(9mS6XhlNo@zC%zQN z>IKiPf=eqkHi$<9<5K$8Dib5^?n;(Ql}dfuxG79W~81@9jRLMthG)RP{Sa|5-KMnW2C$zi((_oWP| zrhrr98?QwD?7z!m?#GYumNZ1)eacg z>VDW%09lyeuaR*gzE+z_Q@Ug`n{xNj30)Utx_tKK@L}vB+y9%^__UN>~ffr`^W2SxqTeroAmLkmW<7hj8 zkgNaDQ4bxXUklZ@H*ZRF&?zybChWiSdK@+$KEi{UJmA9yJmFcuO-x}xGk1BPKMFc- zPR~o#>EAc@;!MoBFe?uy=3THA`q~vb={TItGJP108ZBi5J+AZ6fT@D!Z-_vTW8~8@ zS?&Rw2kwRHTvk@MRf`@iyc3WjH5b|16}U8x)f+fX4ggn4?6y3%mvJ1BMI-wt@12UnOQ&}IORd=(3Tp|66ON504sa#7Uv-IRBs96WuLhQneq-S|L=cHLCQ58%P!sF@sI5kJ4l9G3R;F2l=e8(*S+vb6Jf|wh9o&7oTzlIv`f>2|A0ut)zWaM@i6P~_un-k zc|XJxWc4Ab;N_wYTA=F<>Gg}Wj-K^T?ayY8*#JkzBfo83doTG6yC2;uiKyveKyuv! zZWS3+rIJ-e?ScxQK`Me1TJDA=0AaA7Lp?4CbU8*b%N|_+w3l$UoV2@7Ny^bDso}dY zWNtHRxrK~7la%GnmoG3z`Z^a6e-<^2+6TOUOg%=kSK>&seIX?f6WomG)P&qLVWy&Y zEcgZ`W_F_MXTsimJppBLynxao4J5N)E|l5o$7_rGzYw`~|3Di)oI_d#4-GdE`7_u5 z_u3kzi`MWcrnOJaMSnXdCT?c<`qG&(8KkAt;x@D&WD`1B2QRpzC-dBO-#IqmVsrHR z$9H0>z%|Bc^JxPK%|9s{VaUU&ryL_K4ibH9N^f8C9~m~iCF|n#MM_)Wut9PPi_)u= zaJL$V(F!t0RoZVl<;vCMX6LAUX{-Eh*#u@(k2pXRrM?1{=9ek>_x+f)M~48q3ei7b z6t$&h_gs^3wPuL6{i1~1idE7hU}^JbV*{+fpbVE`jiOd|k+Yd0c9?lLgWcK71(V0dUN)_u7suOEPp058Mi{6`ybw&9}UYNN{<>kH8fV|rSZM8l~hT( zrvUAn|5HW&=_Nf4me}DYM{*I0sKV;6q05gTCz$D9;)ANor$2Vw@*>4g$9KUhyI>9A zgz|OvP_5>o`ZU6=wCv$hbR%Z@0c5?de&ll<&5)>=^!-lnmFOpP4E*KA+zM1Kww-zh|coUoUroAm#G52*S{XP1d>QT8=&h0 zA^d-TIAj$@BJh=^62*!FWWO833avX?9~7W2RQdGB#Qsu28Q#2N$ii<|*Z?1}DgJ8` z>%UKxe?CRsNTg17zj%C%Tvn1tul7hM*}t1Pe-&&OnHh+pu9)4hqoJC`_t5*T-unt* z?w%V-?AX511?&04j<8R-vXw}p3V#_l5hDew)vIHb0xHYL*?@FnG}!OngJ8u8L)KFN zj{dLO{GZ(inVKm5^2{z&yegHo{%*bI)w=T84K+Mmi5V@Q6ly*daszXVYEcchEZT-z zlQBRzbp1Uu5bEerQS!?>L{B~3gsH#YWbnRck((9xEp>&vgj6DG40RpHYG34J_eDRz zF?lRt-F^S;MgHw`FkQ-)`r&|P46kFL$GM;@<<)%03Y|hv=ZjgpbcGXqk_>h|Z1Bcz zIyN-qm~&?%^DyWgl0njf`%b*LpNr3!OE#H5WC6?hgAfpV(F3)dl!wuJo-Y<{FC^qF zR=Esb7)BM=_Xb(U?0S0%pLDMnD*ylh literal 0 HcmV?d00001 From dc5f5280162fbfae684726948dccb80e8897a6c2 Mon Sep 17 00:00:00 2001 From: craftingmod Date: Sun, 27 Sep 2026 02:10:58 +0900 Subject: [PATCH 7/9] test: add `chat_template_kwargs` override test Assisted-by: codex --- llama_cpp/llama_chat_format.py | 14 +++++++++++++- tests/test_media.py | 23 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 34df4f3b5..46ecfa6fa 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -859,12 +859,24 @@ def prepare_chat_prompt( "tools": tools, "tool_choice": tool_choice, } + # Write `format_kwargs` keys from upon + _RESERVED_CHAT_TEMPLATE_KWARGS = { + "messages", + "functions", + "function_call", + "tools", + "tool_choice", + "add_generation_prompt", + } if add_generation_prompt is not None: format_kwargs["add_generation_prompt"] = add_generation_prompt if chat_template_kwargs is not None: - reserved = format_kwargs.keys() & chat_template_kwargs.keys() + reserved = ( + _RESERVED_CHAT_TEMPLATE_KWARGS + & chat_template_kwargs.keys() + ) if reserved: raise ValueError( "chat_template_kwargs contains reserved keys: " diff --git a/tests/test_media.py b/tests/test_media.py index f5c013813..fe0bf3450 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -7,6 +7,7 @@ import struct import pytest +from jinja2 import Template from jinja2.exceptions import TemplateError @@ -571,6 +572,28 @@ class CustomChatHandler(multimodal.MTMDChatHandler): handler.close() +def test_mtmd_render_prompt_applies_request_template_kwargs_without_mutating_defaults(): + multimodal = importlib.import_module("llama_cpp.llama_multimodal") + template_arguments = {"enable_thinking": False} + handler = SimpleNamespace( + chat_template=Template("{{ 'enabled' if enable_thinking else 'disabled' }}"), + extra_template_arguments=template_arguments, + verbose=False, + log_prefix="test", + mtmd_eos_token="", + mtmd_bos_token="", + ) + + prompt = multimodal.MTMDChatHandler._render_mtmd_prompt( + handler, + messages=[], + chat_template_kwargs={"enable_thinking": True}, + ) + + assert prompt == "enabled" + assert template_arguments == {"enable_thinking": False} + + def test_mtmd_chat_template_raise_exception_preserves_message(tmp_path): multimodal = importlib.import_module("llama_cpp.llama_multimodal") handler = multimodal.MTMDChatHandler( From 33d0d2ca393887877d6df6f8710d791b19d78e20 Mon Sep 17 00:00:00 2001 From: craftingmod Date: Sun, 27 Sep 2026 02:45:21 +0900 Subject: [PATCH 8/9] chore: fix `chat_template_kwargs` parameter error Assisted-by: codex --- llama_cpp/llama.py | 26 +++++++++++++++----------- llama_cpp/llama_chat_format.py | 2 ++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index c4a92500e..a06733d21 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -4614,17 +4614,21 @@ def create_chat_prefill( "The selected chat handler does not support prefill" ) - return prefill( - llama=self, - messages=messages, - functions=functions, - function_call=function_call, - tools=tools, - tool_choice=tool_choice, - add_generation_prompt=add_generation_prompt, - assistant_prefill=assistant_prefill, - chat_template_kwargs=chat_template_kwargs, - ) + prefill_kwargs: Dict[str, Any] = { + "llama": self, + "messages": messages, + "functions": functions, + "function_call": function_call, + "tools": tools, + "tool_choice": tool_choice, + "add_generation_prompt": add_generation_prompt, + "assistant_prefill": assistant_prefill, + } + # For compatibility + if chat_template_kwargs is not None: + prefill_kwargs["chat_template_kwargs"] = chat_template_kwargs + + return prefill(**prefill_kwargs) def create_chat_completion_openai_v1( self, diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 46ecfa6fa..1ff3430ff 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1096,6 +1096,7 @@ def prefill_handler( tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, assistant_prefill: bool = False, add_generation_prompt: bool = True, + chat_template_kwargs: Optional[Dict[str, Any]] = None, ) -> PrefillResult: prepared = prepare_chat_prompt( llama=llama, @@ -1107,6 +1108,7 @@ def prefill_handler( stop=[], assistant_prefill=assistant_prefill, add_generation_prompt=add_generation_prompt, + chat_template_kwargs=chat_template_kwargs, ) return llama.prefill(prepared.prompt, reset=True) From 399fbb1c005e4358d027504d6676c91d49663c35 Mon Sep 17 00:00:00 2001 From: craftingmod Date: Sun, 27 Sep 2026 22:45:59 +0900 Subject: [PATCH 9/9] chore(prefill): add `n_tokens` field again Assisted-by: codex --- llama_cpp/llama.py | 5 ++++- llama_cpp/llama_chat_format.py | 5 ++++- llama_cpp/llama_multimodal.py | 1 + tests/test_formats.py | 12 ++++++++---- tests/test_media.py | 2 -- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index a06733d21..5222dcc58 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1847,7 +1847,10 @@ def prefill( if self._logits_all else self.scores[0] ) - return PrefillResult(logits=logits) + return PrefillResult( + n_tokens=len(tokens), + logits=logits + ) # Helper method: Convert dict logit_bias to List[llama_logit_bias] def _convert_logit_bias(self, logit_bias: Optional[Dict[int, float]]) -> List[llama_cpp_lib.llama_logit_bias]: diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 1ff3430ff..559f64c62 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -226,8 +226,11 @@ class ChatFormatterResponse: @dataclasses.dataclass(frozen=True) class PrefillResult: - """owned, read-only final next-token logits.""" + """owned, read-only final next-token logits. + + n_tokens is the number of token IDs in the final prompt passed to the prefill/completion path.""" + n_tokens: int logits: npt.NDArray[np.single] def __post_init__(self) -> None: diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index e274fdb09..372788359 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -2135,6 +2135,7 @@ def __call__( ) if prefill_only: return PrefillResult( + n_tokens=len(prefill.prompt), logits=prefill.logits, ) diff --git a/tests/test_formats.py b/tests/test_formats.py index ad86c8f3e..ca7c7808c 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -546,8 +546,6 @@ def test_text_prefill_uses_eval_and_owns_final_logits(logits_all, logits_index): result = llama.prefill("hello") assert isinstance(result, PrefillResult) - assert not hasattr(result, "prompt") - assert not hasattr(result, "n_tokens") assert result.logits.shape == (3,) assert result.logits.flags.owndata and not result.logits.flags.writeable np.testing.assert_array_equal(result.logits, [1.0, 2.0, 3.0] if not logits_all else [4.0, 5.0, 6.0]) @@ -607,7 +605,10 @@ def format_chat(**kwargs): ) tokenized = Mock(return_value=[21, 22]) - prefill_result = PrefillResult(np.array([1.0, 2.0])) + prefill_result = PrefillResult( + n_tokens=2, + logits=np.array([1.0, 2.0]) + ) prefill = Mock(return_value=prefill_result) create_completion = Mock(return_value=object()) llama = SimpleNamespace( @@ -678,7 +679,10 @@ def test_create_chat_prefill_dispatches_by_capability(): from types import MethodType from llama_cpp import Llama, PrefillResult - result = PrefillResult(np.array([0.0, 1.0])) + result = PrefillResult( + n_tokens=2, + logits=np.array([0.0, 1.0]) + ) handler = SimpleNamespace(prefill=Mock(return_value=result)) llama = SimpleNamespace( chat_handler=handler, diff --git a/tests/test_media.py b/tests/test_media.py index fe0bf3450..eaa2c0ca9 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -81,8 +81,6 @@ def test_chat_prefill_returns_owned_logits_without_generation(chat_prefill): result = handler(llama=llm, messages=[], prefill_only=True) assert isinstance(result, PrefillResult) - assert not hasattr(result, "prompt") - assert not hasattr(result, "n_tokens") assert llm.n_tokens == 4 assert result.logits.shape == (llm.n_vocab(),) assert result.logits.dtype == np.float32