diff --git a/README.md b/README.md index 76a6d994d..a015e1ab8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Inspired by **KoboldAI** and built on **llama.cpp** One executable file, no installation required. Ready-to-run downloads are available for Windows, Linux, and macOS. -**[Download KoboldCpp](https://github.com/LostRuins/koboldcpp/releases/latest) | [Documentation and FAQ](https://github.com/LostRuins/koboldcpp/wiki) | [API reference](https://lite.koboldai.net/koboldcpp_api) | [Discord community](https://koboldai.org/discord)** +**[Download KoboldCpp](https://github.com/LostRuins/koboldcpp/releases/latest) | [Quick Start](#quick-start) | [Documentation and FAQ](https://github.com/LostRuins/koboldcpp/wiki) | [API reference](https://lite.koboldai.net/koboldcpp_api) | [Discord community](https://koboldai.org/discord)** ![Integrated Web UI](media/preview.png) ![Roleplay Chat Mode](media/preview2.png) @@ -99,8 +99,11 @@ KoboldCpp also retains backward compatibility with legacy GGML `.bin` models, th - Lightweight and Fast: [Gemma3-4B](https://huggingface.co/ggml-org/gemma-3-4b-it-GGUF/resolve/main/gemma-3-4b-it-Q4_K_M.gguf) - Add [optional Gemma3-4B MMproj file](https://huggingface.co/koboldcpp/mmproj/resolve/main/gemma3-4b-mmproj-q8.gguf) for vision recognition capabilities. - Image Generation: [PicX Real](https://huggingface.co/koboldcpp/imgmodel/resolve/main/picx_real_q5_1.gguf) -- Speech Recognition: [Whisper models for Speech-To-Text](https://huggingface.co/koboldcpp/whisper/tree/main) -- Text-To-Speech: [TTS models for Narration](https://huggingface.co/koboldcpp/tts/tree/main) + - Or browse [More image generation models](https://huggingface.co/koboldcpp/imgmodel/tree/main) +- Speech Recognition: [Whisper Base](https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin) + - Or browse [More Whisper models for Speech-To-Text](https://huggingface.co/koboldcpp/whisper/tree/main) +- Text-To-Speech: [Kokoro TTS](https://huggingface.co/koboldcpp/tts/resolve/main/Kokoro_no_espeak_Q4.gguf) + - Or browse [More TTS models for Narration](https://huggingface.co/koboldcpp/tts/tree/main) - This is just a list for noobs to get started! There are hundreds more GGUFs out there! - [More newbie templates](https://huggingface.co/koboldcpp/newbie-templates) - Contains premade KoboldCpp quick launch templates curated for newbies. - [More popular templates](https://huggingface.co/koboldcpp/popular-templates) - Contains premade KoboldCpp quick launch templates curated for popularity. diff --git a/koboldcpp.py b/koboldcpp.py index d90dc16a7..79c98b54f 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -4301,6 +4301,9 @@ def remove_outer_tags(inputstr): def normalize_anthropic_tools_input(tools): #Convert Anthropic-format tool definitions to OpenAI format. normalized = [] for tool in tools: + if not isinstance(tool, dict): + print(f"Dropped unsupported tool: {tool}") + continue if tool.get("type") == "function" and "function" in tool: # Already in OpenAI format — leave it alone normalized.append(tool) continue @@ -4316,6 +4319,22 @@ def normalize_anthropic_tools_input(tools): #Convert Anthropic-format tool defin # normalized.append(tool) return normalized +def normalize_openai_tools_input(tools): + """Keep only function tools that KoboldCpp can execute.""" + normalized = [] + if not isinstance(tools, list): + print(f"Dropped unsupported tools value: {tools}") + return normalized + for tool in tools: + function = tool.get("function") if isinstance(tool, dict) else None + if isinstance(tool, dict) and tool.get("type") == "function" and isinstance(function, dict) and function.get("name"): + normalized.append(tool) + else: + # OpenAI built-in tools (for example web_search) require provider-side + # implementations and cannot be exposed to the local model as functions. + print(f"Dropped unsupported tool: {tool}") + return normalized + def normalize_tool_call_resp(obj): # Normalize various tool call formats to OpenAI format if "type" in obj and "function" in obj: # Already in OpenAI format return obj @@ -4634,16 +4653,28 @@ def determine_tool_json_to_use(genparams, curr_ctx, assistant_message_start, is_ def compress_tools_array(tools_array): tools_array_filtered = [] for tool_dict in tools_array: + if not isinstance(tool_dict, dict): + continue tool_data = tool_dict if 'function' in tool_dict: tool_data = tool_dict['function'] + if not isinstance(tool_data, dict) or not tool_data.get("name"): + continue tool_props = {} params = tool_data.get("parameters", {}) + if not isinstance(params, dict): + params = {} props = params.get("properties", {}) + if not isinstance(props, dict): + props = {} for prop_name, prop_data in props.items(): + if not isinstance(prop_data, dict): + continue prop_type = prop_data.get("type") if prop_type is None and "anyOf" in prop_data: for option in prop_data["anyOf"]: + if not isinstance(option, dict): + continue option_type = option.get("type") if option_type and option_type != "null": prop_type = option_type @@ -4652,7 +4683,7 @@ def compress_tools_array(tools_array): prop_type = "string" tool_props[prop_name] = prop_type tools_array_filtered.append({ - "name": tool_data['name'], + "name": tool_data["name"], "description": tool_data.get("description", ""), "properties": tool_props }) @@ -4806,6 +4837,8 @@ ws ::= | " " | "\n" [ \t]{0,20} genparams["mirostat"] = genparams.get('mirostat_mode', 0) if api_format==4 or api_format==7: #handle ollama chat here too + if genparams.get("tools"): + genparams["tools"] = normalize_openai_tools_input(genparams["tools"]) # translate openai chat completion messages format into one big string. messages_array = genparams.get('messages', []) messages_string = adapter_obj.get("chat_start", "")