Feat/OpenAI agent compat

python/infinilm/llm/llm.py

@@ -35,6 +35,31 @@
 logger = logging.getLogger(__name__)
 
 
+def validate_request_capacity(
+    config: EngineConfig, prompt_tokens: int, output_tokens: int
+) -> None:
+    """Reject requests that can never fit in the configured KV cache."""
+    if config.cache_type == "paged":
+        capacity = config.num_blocks * config.block_size
+    else:
+        capacity = config.max_cache_len
+
+    if prompt_tokens > capacity:
+        raise ValueError(
+            f"The maximum context length is {capacity} tokens. "
+            f"The prompt contains {prompt_tokens} tokens."
+        )
+
+    total_tokens = prompt_tokens + output_tokens
+    if total_tokens > capacity:
+        available_output_tokens = max(capacity - prompt_tokens, 0)
+        raise ValueError(
+            f"The maximum context length is {capacity} tokens. "
+            f"The prompt contains {prompt_tokens} tokens, so max_tokens must be "
+            f"at most {available_output_tokens}; received {output_tokens}."
+        )
+
+
 class LLMEngine:
     """Low-level LLM engine that handles inference execution."""
 
@@ -784,6 +809,7 @@ def add_request(
         request_id: Optional[str] = None,
         # For server use
         request_data: Optional[dict] = None,
+        chat_template_kwargs: Optional[dict] = None,
     ) -> InferenceRequest:
         """Add a request to the engine.
 
@@ -829,17 +855,27 @@ def add_request(
         elif prompt is not None:
             prompt_token_ids = self.engine.tokenize(prompt)
         else:
-            assert messages is not None, (
-                "Either messages or prompt/prompt_token_ids must be provided"
-            )
+            assert (
+                messages is not None
+            ), "Either messages or prompt/prompt_token_ids must be provided"
 
-            assert apply_chat_template, (
-                "apply_chat_template needs to be true for multi-role conversation"
-            )
+            assert (
+                apply_chat_template
+            ), "apply_chat_template needs to be true for multi-role conversation"
 
             prompt = self.engine.apply_chat_template(
-                messages, add_generation_prompt=add_generation_prompt
+                messages,
+                add_generation_prompt=add_generation_prompt,
+                chat_template_kwargs=chat_template_kwargs,
+            )
+
+            forced_tool_prefix = (
+                request_data.get("_infinilm_forced_tool_prefix")
+                if request_data
+                else None
             )
+            if forced_tool_prefix:
+                prompt += forced_tool_prefix
 
             mm_inputs = resolve_multimodal_inputs(messages)
 
@@ -868,6 +904,12 @@ def add_request(
             sampling_params = sampling_params.clone()
             sampling_params.max_tokens = self.config.max_tokens
 
+        validate_request_capacity(
+            self.config,
+            prompt_tokens=len(prompt_token_ids),
+            output_tokens=sampling_params.max_tokens,
+        )
+
         request = InferenceRequest(
             request_id=request_id,
             prompt=prompt,
@@ -897,6 +939,7 @@ def add_chat_request(
         request_id: Optional[str] = None,
         request_data: Optional[dict] = None,
         add_generation_prompt: bool = True,
+        chat_template_kwargs: Optional[dict] = None,
         **kwargs,
     ) -> InferenceRequest:
         """Add a chat request to the engine.
@@ -918,6 +961,7 @@ def add_chat_request(
             sampling_params=sampling_params,
             request_id=request_id,
             request_data=request_data,
+            chat_template_kwargs=chat_template_kwargs,
         )
 
     async def stream_request(

python/infinilm/processors/basic_llm_processor.py

@@ -3,7 +3,11 @@
 
 from ..llm.scheduler import SchedulerOutput
 from ..llm.static_scheduler import StaticSchedulerOutput
-from .processor import InfinilmProcessor, register_processor
+from .processor import (
+    InfinilmProcessor,
+    normalize_openai_messages,
+    register_processor,
+)
 
 
 @register_processor("default")
@@ -44,8 +48,8 @@ def apply_chat_template(
         **kwargs,
     ):
         normalized_conversation = []
-        for message in conversation:
-            if isinstance(message["content"], list):
+        for message in normalize_openai_messages(conversation):
+            if isinstance(message.get("content"), list):
                 assert len(message["content"]) == 1, (
                     "Only one content item supported in list"
                 )

python/infinilm/processors/processor.py

@@ -1,3 +1,6 @@
+import json
+
+
 class InfinilmProcessor:
     def __init__(self, model_dir_path: str):
         """Initialize the processor with the model directory path."""
@@ -43,6 +46,50 @@ def get_mm_token_index_list(
         raise NotImplementedError("get_mm_token_index_list is not implemented yet")
 
 
+def normalize_openai_messages(messages: list[dict]) -> list[dict]:
+    """Convert OpenAI JSON-string tool arguments to template mappings."""
+    normalized = []
+    for message in messages:
+        if not isinstance(message, dict):
+            normalized.append(message)
+            continue
+
+        normalized_message = message.copy()
+        if "content" not in normalized_message and normalized_message.get("tool_calls"):
+            normalized_message["content"] = None
+
+        tool_calls = normalized_message.get("tool_calls")
+        if isinstance(tool_calls, list):
+            normalized_calls = []
+            for tool_call in tool_calls:
+                if not isinstance(tool_call, dict):
+                    normalized_calls.append(tool_call)
+                    continue
+                normalized_call = tool_call.copy()
+                function = normalized_call.get("function")
+                if isinstance(function, dict):
+                    normalized_function = function.copy()
+                    arguments = normalized_function.get("arguments")
+                    if isinstance(arguments, str):
+                        try:
+                            arguments = json.loads(arguments)
+                        except json.JSONDecodeError as exc:
+                            raise ValueError(
+                                "tool call function.arguments must be valid JSON"
+                            ) from exc
+                        if not isinstance(arguments, dict):
+                            raise ValueError(
+                                "tool call function.arguments must decode to an object"
+                            )
+                        normalized_function["arguments"] = arguments
+                    normalized_call["function"] = normalized_function
+                normalized_calls.append(normalized_call)
+            normalized_message["tool_calls"] = normalized_calls
+
+        normalized.append(normalized_message)
+    return normalized
+
+
 # Global registry mapping model_type strings to their Processor classes
 _PROCESSOR_REGISTRY = {}
 

python/infinilm/processors/qwen3_5_processor.py

@@ -7,7 +7,7 @@
 from ..llm.scheduler import SchedulerOutput
 from ..llm.static_scheduler import StaticSchedulerOutput
 from .basic_llm_processor import BasicLLMProcessor
-from .processor import register_processor
+from .processor import normalize_openai_messages, register_processor
 
 
 @register_processor("qwen3_5_moe")
@@ -134,8 +134,8 @@ def apply_chat_template(
         **kwargs,
     ):
         normalized_conversation = []
-        for message in conversation:
-            content = message["content"]
+        for message in normalize_openai_messages(conversation):
+            content = message.get("content")
             if not isinstance(content, list):
                 normalized_conversation.append(message)
                 continue
@@ -156,9 +156,10 @@ def apply_chat_template(
                         f"Unsupported Qwen3.5 content type: {item_type}"
                     )
 
-            normalized_conversation.append(
-                {"role": message.get("role", "user"), "content": normalized_content}
-            )
+            normalized_message = message.copy()
+            normalized_message["role"] = message.get("role", "user")
+            normalized_message["content"] = normalized_content
+            normalized_conversation.append(normalized_message)
 
         template_owner = (
             self.processor if self.processor is not None else self.tokenizer