feat(bench): detect llama-server as a first-class provider

Previously `llmfit bench` only probed Ollama, vLLM, and MLX. A running
llama-server was either missed entirely (custom port) or silently
benchmarked under the MLX label when on the shared default port 8080 —
which would have polluted community benchmark data submitted via --share.

- New BenchTarget::LlamaCpp, positively identified via llama.cpp's
  /props endpoint (MLX and vLLM 404 it), probed before the MLX check.
- Respects LLAMA_SERVER_HOST (full URL) and LLAMA_SERVER_PORT
  (default 8080, matching the existing providers.rs convention).
- `--provider llamacpp` (aliases: llama.cpp, llama-server) selects it
  explicitly for both performance and quality benchmarks.
- discover_all_targets skips the MLX probe when llama-server already
  claimed the same URL, so models aren't double-counted.
- Results are labeled "llamacpp"; community schema enum updated and the
  share payload test now covers it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

docs/cli.md

@@ -98,7 +98,10 @@ python3 scripts/test_api.py --base-url http://127.0.0.1:8787
 ### Contributing benchmarks (`bench --share`)
 
 `llmfit bench` measures inference performance against a running provider
-(Ollama, vLLM, or MLX). Add `--share` to contribute your results back to the
+(Ollama, vLLM, MLX, or llama-server). llama-server is auto-detected on port
+8080 via its `/props` endpoint (override with `LLAMA_SERVER_HOST` for a full
+URL, or `LLAMA_SERVER_PORT`), or select it explicitly with
+`--provider llamacpp`. Add `--share` to contribute your results back to the
 project as a pull request — **no `gh` CLI and no account on a third-party
 service required**:
 

llmfit-core/data/community/schema.json

@@ -43,7 +43,7 @@
         "required": ["model", "provider", "numRuns", "avgTps", "minTps", "maxTps", "avgTotalMs", "avgOutputTokens"],
         "properties": {
           "model": { "type": "string", "minLength": 1 },
-          "provider": { "type": "string", "enum": ["ollama", "vllm", "mlx"] },
+          "provider": { "type": "string", "enum": ["ollama", "vllm", "mlx", "llamacpp"] },
           "numRuns": { "type": "integer", "minimum": 1 },
           "avgTps": { "type": "number", "minimum": 0, "maximum": 100000 },
           "minTps": { "type": "number", "minimum": 0, "maximum": 100000 },

llmfit-core/src/bench.rs

@@ -339,6 +339,32 @@ pub enum BenchTarget {
     Ollama { url: String, model: String },
     VLlm { url: String, model: String },
     Mlx { url: String, model: String },
+    LlamaCpp { url: String, model: String },
+}
+
+/// Base URL for a running llama-server instance.
+/// `LLAMA_SERVER_HOST` (full URL) wins; otherwise localhost with
+/// `LLAMA_SERVER_PORT` (default 8080, llama-server's own default).
+pub fn llamacpp_url() -> String {
+    if let Ok(host) = std::env::var("LLAMA_SERVER_HOST")
+        && !host.trim().is_empty()
+    {
+        return host.trim().trim_end_matches('/').to_string();
+    }
+    let port = std::env::var("LLAMA_SERVER_PORT").unwrap_or_else(|_| "8080".to_string());
+    format!("http://localhost:{}", port)
+}
+
+/// Positively identify a llama-server instance via its `/props` endpoint.
+/// llama.cpp serves it; MLX and vLLM return 404, which disambiguates
+/// llama-server from `mlx_lm.server` on the shared default port 8080.
+pub fn probe_llamacpp(base_url: &str) -> bool {
+    ureq::get(&format!("{}/props", base_url.trim_end_matches('/')))
+        .config()
+        .timeout_global(Some(Duration::from_secs(2)))
+        .build()
+        .call()
+        .is_ok()
 }
 
 /// Auto-detect available providers and pick the best one to benchmark.
@@ -370,6 +396,18 @@ pub fn auto_detect_target(model_hint: Option<&str>) -> Result<BenchTarget, Strin
         });
     }
 
+    // Check llama-server before MLX: both default to port 8080, but only
+    // llama.cpp answers /props, so it can be identified positively.
+    let llama_url = llamacpp_url();
+    if probe_llamacpp(&llama_url)
+        && let Ok(model_name) = detect_openai_model(&llama_url, model_hint)
+    {
+        return Ok(BenchTarget::LlamaCpp {
+            url: llama_url,
+            model: model_name,
+        });
+    }
+
     // Check MLX
     let mlx_url =
         std::env::var("MLX_LM_HOST").unwrap_or_else(|_| "http://localhost:8080".to_string());
@@ -387,7 +425,7 @@ pub fn auto_detect_target(model_hint: Option<&str>) -> Result<BenchTarget, Strin
         });
     }
 
-    Err("No inference provider found. Start Ollama, vLLM, or MLX first.".to_string())
+    Err("No inference provider found. Start Ollama, vLLM, MLX, or llama-server first.".to_string())
 }
 
 /// Discover all available models across all providers.
@@ -418,10 +456,26 @@ pub fn discover_all_targets() -> Vec<BenchTarget> {
         }
     }
 
-    // Check MLX
+    // Check llama-server before MLX: both default to port 8080, but only
+    // llama.cpp answers /props, so it can be identified positively.
+    let llama_url = llamacpp_url();
+    let llamacpp_found = probe_llamacpp(&llama_url);
+    if llamacpp_found && let Ok(models) = list_openai_models(&llama_url) {
+        for model in models {
+            targets.push(BenchTarget::LlamaCpp {
+                url: llama_url.clone(),
+                model,
+            });
+        }
+    }
+
+    // Check MLX (skip if the llama-server probe already claimed this URL,
+    // e.g. both on the default port 8080)
     let mlx_url =
         std::env::var("MLX_LM_HOST").unwrap_or_else(|_| "http://localhost:8080".to_string());
-    if let Ok(models) = list_openai_models(&mlx_url) {
+    if !(llamacpp_found && mlx_url.trim_end_matches('/') == llama_url)
+        && let Ok(models) = list_openai_models(&mlx_url)
+    {
         for model in models {
             targets.push(BenchTarget::Mlx {
                 url: mlx_url.clone(),

llmfit-core/src/share.rs

@@ -688,8 +688,26 @@ mod tests {
                 avg_output_tokens: 104.0,
             },
         };
+        // llama-server results are labeled "llamacpp" — must be schema-valid too.
+        let llamacpp_result = BenchResult {
+            model: "qwen2.5-7b-q4_k_m".to_string(),
+            provider: "llamacpp".to_string(),
+            runs: vec![],
+            summary: BenchSummary {
+                num_runs: 3,
+                avg_ttft_ms: None,
+                avg_tps: 42.5,
+                min_tps: 40.0,
+                max_tps: 45.0,
+                avg_total_ms: 2400.0,
+                avg_output_tokens: 100.0,
+            },
+        };
 
-        let submission = build_submission(&[result], &specs_with_gpu("NVIDIA GeForce RTX 4090"));
+        let submission = build_submission(
+            &[result, llamacpp_result],
+            &specs_with_gpu("NVIDIA GeForce RTX 4090"),
+        );
         let value = serde_json::to_value(&submission).unwrap();
 
         let schema_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))

llmfit-tui/src/main.rs

@@ -785,7 +785,7 @@ AGENT USAGE:
         /// Model name to benchmark (auto-detects provider if omitted)
         model: Option<String>,
 
-        /// Provider to benchmark (auto, ollama, vllm, mlx)
+        /// Provider to benchmark (auto, ollama, vllm, mlx, llamacpp)
         #[arg(long, default_value = "auto")]
         provider: String,
 
@@ -2017,6 +2017,7 @@ fn target_info(target: &bench::BenchTarget) -> (&str, &str, &str) {
         bench::BenchTarget::Ollama { url, model } => ("Ollama", url.as_str(), model.as_str()),
         bench::BenchTarget::VLlm { url, model } => ("vLLM", url.as_str(), model.as_str()),
         bench::BenchTarget::Mlx { url, model } => ("MLX", url.as_str(), model.as_str()),
+        bench::BenchTarget::LlamaCpp { url, model } => ("llama.cpp", url.as_str(), model.as_str()),
     }
 }
 
@@ -2049,7 +2050,7 @@ fn run_bench(
     if all {
         let targets = bench::discover_all_targets();
         if targets.is_empty() {
-            eprintln!("No providers or models found. Start Ollama, vLLM, or MLX first.");
+            eprintln!("No providers or models found. Start Ollama, vLLM, MLX, or llama-server first.");
             std::process::exit(1);
         }
 
@@ -2089,6 +2090,9 @@ fn run_bench(
                 bench::BenchTarget::Mlx { url, model } => {
                     bench::bench_openai_compat(url, model, "mlx", runs, &progress)
                 }
+                bench::BenchTarget::LlamaCpp { url, model } => {
+                    bench::bench_openai_compat(url, model, "llamacpp", runs, &progress)
+                }
             };
 
             if !json {
@@ -2176,6 +2180,28 @@ fn run_bench(
                 model: model_name,
             }
         }
+        "llamacpp" | "llama.cpp" | "llama-server" => {
+            let url = url_override.clone().unwrap_or_else(bench::llamacpp_url);
+            match bench::detect_model_from_url(&url, model.as_deref()) {
+                Ok(model_name) => bench::BenchTarget::LlamaCpp {
+                    url,
+                    model: model_name,
+                },
+                Err(_) => {
+                    let model_name = model.unwrap_or_else(|| {
+                        eprintln!(
+                            "Error: could not detect model from llama-server at {}. Use --model",
+                            url
+                        );
+                        std::process::exit(1);
+                    });
+                    bench::BenchTarget::LlamaCpp {
+                        url,
+                        model: model_name,
+                    }
+                }
+            }
+        }
         _ => match bench::auto_detect_target(model.as_deref()) {
             Ok(t) => t,
             Err(e) => {
@@ -2209,6 +2235,9 @@ fn run_bench(
         bench::BenchTarget::Mlx { url, model } => {
             bench::bench_openai_compat(url, model, "mlx", runs, &progress)
         }
+        bench::BenchTarget::LlamaCpp { url, model } => {
+            bench::bench_openai_compat(url, model, "llamacpp", runs, &progress)
+        }
     };
 
     if !json {
@@ -2326,7 +2355,7 @@ fn run_quality_bench(
     let targets: Vec<bench::BenchTarget> = if all {
         let all_targets = bench::discover_all_targets();
         if all_targets.is_empty() {
-            eprintln!("No providers or models found. Start Ollama, vLLM, or MLX first.");
+            eprintln!("No providers or models found. Start Ollama, vLLM, MLX, or llama-server first.");
             std::process::exit(1);
         }
         if skip_patterns.is_empty() {
@@ -2396,6 +2425,28 @@ fn run_quality_bench(
                     model: model_name,
                 }
             }
+            "llamacpp" | "llama.cpp" | "llama-server" => {
+                let url = url_override.clone().unwrap_or_else(bench::llamacpp_url);
+                match bench::detect_model_from_url(&url, model.as_deref()) {
+                    Ok(model_name) => bench::BenchTarget::LlamaCpp {
+                        url,
+                        model: model_name,
+                    },
+                    Err(_) => {
+                        let model_name = model.unwrap_or_else(|| {
+                            eprintln!(
+                                "Error: could not detect model from llama-server at {}. Use --model",
+                                url
+                            );
+                            std::process::exit(1);
+                        });
+                        bench::BenchTarget::LlamaCpp {
+                            url,
+                            model: model_name,
+                        }
+                    }
+                }
+            }
             _ => match bench::auto_detect_target(model.as_deref()) {
                 Ok(t) => t,
                 Err(e) => {
@@ -2434,7 +2485,9 @@ fn run_quality_bench(
             bench::BenchTarget::Ollama { url, model } => {
                 quality::bench_quality_ollama(url, model, &config, rf)
             }
-            bench::BenchTarget::VLlm { url, model } | bench::BenchTarget::Mlx { url, model } => {
+            bench::BenchTarget::VLlm { url, model }
+            | bench::BenchTarget::Mlx { url, model }
+            | bench::BenchTarget::LlamaCpp { url, model } => {
                 quality::bench_quality_openai_compat(url, model, provider_name, &config, rf)
             }
         };