From 5a9b06adfefdb8ff559c41e3efc0cf55c92be2ea Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:24:12 +0800 Subject: [PATCH] cleanup unused examples --- examples/diffusion/README.md | 59 - examples/diffusion/diffusion-cli.cpp | 266 --- examples/diffusion/diffusion.cpp | 406 ----- examples/diffusion/diffusion.h | 57 - examples/llama-eval/README.md | 26 - examples/llama-eval/llama-eval.py | 1591 ----------------- examples/llama-eval/llama-server-simulator.py | 376 ---- examples/llama-eval/test-simulator.sh | 86 - examples/test-cmake/.gitignore | 4 - examples/test-cmake/CMakeLists.txt | 22 - examples/test-cmake/README.md | 45 - examples/test-cmake/build-install.sh | 19 - examples/test-cmake/build.sh | 18 - examples/test-cmake/test-cmake.cpp | 17 - 14 files changed, 2992 deletions(-) delete mode 100644 examples/diffusion/README.md delete mode 100644 examples/diffusion/diffusion-cli.cpp delete mode 100644 examples/diffusion/diffusion.cpp delete mode 100644 examples/diffusion/diffusion.h delete mode 100644 examples/llama-eval/README.md delete mode 100644 examples/llama-eval/llama-eval.py delete mode 100644 examples/llama-eval/llama-server-simulator.py delete mode 100644 examples/llama-eval/test-simulator.sh delete mode 100644 examples/test-cmake/.gitignore delete mode 100644 examples/test-cmake/CMakeLists.txt delete mode 100644 examples/test-cmake/README.md delete mode 100755 examples/test-cmake/build-install.sh delete mode 100755 examples/test-cmake/build.sh delete mode 100644 examples/test-cmake/test-cmake.cpp diff --git a/examples/diffusion/README.md b/examples/diffusion/README.md deleted file mode 100644 index 6d2fffd64..000000000 --- a/examples/diffusion/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Diffusion Text Generation - -This directory contains implementations for Diffusion LLMs (DLLMs) - -More Info: -- https://github.com/ggml-org/llama.cpp/pull/14644 -- https://github.com/ggml-org/llama.cpp/pull/14771 - -## Parameters -The diffusion CLI supports various parameters to control the generation process: - -### Core Diffusion Parameters -- `--diffusion-steps`: Number of diffusion steps (default: 256) -- `--diffusion-algorithm`: Algorithm for token selection - - `0`: DIFFUSION_ALGORITHM_ORIGIN - Token will be generated in a purely random order from https://arxiv.org/abs/2107.03006. - - `1`: DIFFUSION_ALGORITHM_ENTROPY_BASED - Entropy-based selection - - `2`: DIFFUSION_ALGORITHM_MARGIN_BASED - Margin-based selection - - `3`: DIFFUSION_ALGORITHM_RANDOM - Random selection - - `4`: DIFFUSION_ALGORITHM_CONFIDENCE_BASED - Confidence-based selection (default) - - More documentation here https://github.com/DreamLM/Dream -- `--diffusion-visual`: Enable live visualization during generation - -### Scheduling Parameters -Choose one of the following scheduling methods: - -**Timestep-based scheduling:** -- `--diffusion-eps`: Epsilon value for timestep scheduling (e.g., 0.001) - -**Block-based scheduling:** -- `--diffusion-block-length`: Block size for block-based scheduling (e.g., 32) - -### Sampling Parameters -- `--temp`: Temperature for sampling (0.0 = greedy/deterministic, higher = more random) -- `--top-k`: Top-k filtering for sampling -- `--top-p`: Top-p (nucleus) filtering for sampling -- `--seed`: Random seed for reproducibility - -### Model Parameters -- `-m`: Path to the GGUF model file -- `-p`: Input prompt text -- `-ub`: Maximum sequence length (ubatch size) -- `-c`: Context size -- `-b`: Batch size - -### Examples -#### Dream architecture: -``` -llama-diffusion-cli -m dream7b.gguf -p "write code to train MNIST in pytorch" -ub 512 --diffusion-eps 0.001 --diffusion-algorithm 3 --diffusion-steps 256 --diffusion-visual -``` - -#### LLaDA architecture: -``` -llama-diffusion-cli -m llada-8b.gguf -p "write code to train MNIST in pytorch" -ub 512 --diffusion-block-length 32 --diffusion-steps 256 --diffusion-visual -``` - -#### RND1 architecture: -``` -llama-diffusion-cli -m RND1-Base-0910.gguf -p "write code to train MNIST in pytorch" -ub 512 --diffusion-algorithm 1 --diffusion-steps 256 --diffusion-visual --temp 0.5 --diffusion-eps 0.001 -``` diff --git a/examples/diffusion/diffusion-cli.cpp b/examples/diffusion/diffusion-cli.cpp deleted file mode 100644 index d58d22eff..000000000 --- a/examples/diffusion/diffusion-cli.cpp +++ /dev/null @@ -1,266 +0,0 @@ -#include "arg.h" -#include "chat.h" -#include "common.h" -#include "diffusion.h" -#include "llama.h" -#include "log.h" - -#include - -#include -#include -#include -#include - -struct callback_data { - diffusion_params * diff_params; - const llama_vocab * vocab; - int32_t n_input; -}; - -static bool diffusion_step_callback(int32_t step, - int32_t total_steps, - const llama_token * tokens, - int32_t n_tokens, - void * user_data) { - (void) user_data; - - callback_data * data = static_cast(user_data); - - auto print_progress_bar = [](int32_t step, int32_t total_steps) { - int progress_percent = (step * 100) / total_steps; - int progress_bars = (step * 50) / total_steps; - LOG_INF("\rdiffusion step: %d/%d [%s%s] %d%%", - step, - total_steps, - std::string(progress_bars, '=').c_str(), - std::string(50 - progress_bars, ' ').c_str(), - progress_percent); - }; - - if (data->diff_params->visual_mode) { - // Visual mode: clear - LOG_INF("\033[2J\033[H"); // Clear screen and move cursor to top-left - - print_progress_bar(step, total_steps); - - LOG_INF("\n"); - - std::string current_text = " "; - - for (int32_t i = data->n_input; i < n_tokens; i++) { - std::string token_str; - if (tokens[i] != llama_vocab_mask(data->vocab)) { - char piece[256]; - int n_chars = llama_token_to_piece(data->vocab, tokens[i], piece, sizeof(piece), 0, false); - if (n_chars > 0) { - piece[n_chars] = '\0'; - token_str = piece; - } - } else { - token_str = " "; - } - - current_text += token_str; - } - - LOG_INF("%s\n", current_text.c_str()); - } else { - print_progress_bar(step, total_steps); - } - - return true; -} - -static std::string format_input_text(const std::string & prompt, const std::string & system_prompt, bool use_chat_template, llama_model * model) { - if (!use_chat_template) { - return prompt; - } - - auto chat_templates = common_chat_templates_init(model, ""); - common_chat_templates_inputs inputs; - common_chat_msg system_msg; - - if (!system_prompt.empty()) { - system_msg.role = "system"; - system_msg.content = system_prompt; - inputs.messages.push_back(system_msg); - } - - common_chat_msg user_msg; - user_msg.role = "user"; - user_msg.content = prompt; - - inputs.messages.push_back(user_msg); - inputs.add_generation_prompt = true; - - auto result = common_chat_templates_apply(chat_templates.get(), inputs); - - return result.prompt; -} - -int main(int argc, char ** argv) { - std::setlocale(LC_NUMERIC, "C"); - - ggml_time_init(); - - common_params params; - - common_init(); - - if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_DIFFUSION)) { - return 1; - } - - llama_backend_init(); - - llama_model_params model_params = llama_model_default_params(); - model_params.n_gpu_layers = params.n_gpu_layers; - model_params.devices = params.devices.data(); - model_params.load_mode = params.load_mode; - model_params.check_tensors = params.check_tensors; - - llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params); - if (!model) { - LOG_ERR("error: failed to load model '%s'\n", params.model.path.c_str()); - return 1; - } - - if (!llama_model_is_diffusion(model)) { - LOG_ERR("error: unsupported model for diffusion"); - llama_model_free(model); - return 1; - } - - llama_context_params ctx_params = llama_context_default_params(); - ctx_params.n_ctx = params.n_ctx; - ctx_params.n_batch = params.n_batch; - ctx_params.n_ubatch = params.n_ubatch; - ctx_params.flash_attn_type = params.flash_attn_type; - ctx_params.no_perf = params.no_perf; - ctx_params.type_k = params.cache_type_k; - ctx_params.type_v = params.cache_type_v; - - llama_context * ctx = llama_init_from_model(model, ctx_params); - if (!ctx) { - LOG_ERR("error: failed to create context\n"); - llama_model_free(model); - return 1; - } - - llama_set_n_threads(ctx, params.cpuparams.n_threads, params.cpuparams_batch.n_threads); - - const llama_vocab * vocab = llama_model_get_vocab(model); - - std::string formatted_prompt = format_input_text(params.prompt, params.system_prompt, params.enable_chat_template, model); - - std::vector input_tokens = common_tokenize(vocab, - formatted_prompt, - /*add special tokens*/ true, - /*parse special*/ true); - - int n_input = input_tokens.size(); - - if (static_cast(n_input) >= llama_n_ctx(ctx)) { - LOG_ERR("error: input too long (%d tokens), max context is %d\n", n_input, llama_n_ctx(ctx)); - llama_free(ctx); - llama_model_free(model); - return 1; - } - - llama_token mask_token_id = llama_vocab_mask(vocab); - - GGML_ASSERT(mask_token_id != LLAMA_TOKEN_NULL); - - bool visual_mode = params.diffusion.visual_mode; - - int32_t n_generated = 0; - std::vector output_tokens(params.n_ubatch); - - struct diffusion_params diff_params; - - char shift_logits_str[8]; - if (llama_model_meta_val_str(model, "diffusion.shift_logits", shift_logits_str, sizeof(shift_logits_str)) >= 0) { - diff_params.shift_logits = (strcmp(shift_logits_str, "true") == 0); - } else { - diff_params.shift_logits = true; - } - - //Use either eps or block length, but not both - GGML_ASSERT((params.diffusion.eps == 0) ^ (params.diffusion.block_length == 0)); - - if (params.diffusion.eps) { - diff_params.schedule = DIFFUSION_TRANSFER_SCHEDULE_TIMESTEP_BASED; - diff_params.eps = params.diffusion.eps; - } else if (params.diffusion.block_length) { - diff_params.schedule = DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED; - diff_params.block_length = params.diffusion.block_length; - } - - diff_params.mask_token_id = mask_token_id; - diff_params.seed = params.sampling.seed; - diff_params.temperature = params.sampling.temp; - diff_params.steps = params.diffusion.steps; - diff_params.algorithm = static_cast(params.diffusion.algorithm); - diff_params.max_length = params.n_ubatch; - diff_params.top_p = params.sampling.top_p; - diff_params.top_k = params.sampling.top_k; - diff_params.visual_mode = params.diffusion.visual_mode; - diff_params.add_gumbel_noise = params.diffusion.add_gumbel_noise; - - diff_params.step_callback = diffusion_step_callback; - callback_data cb_data = { &diff_params, vocab, n_input }; - diff_params.step_callback_user_data = &cb_data; - - const char * alg_names[] = { - "DIFFUSION_ALGORITHM_ORIGIN", - "DIFFUSION_ALGORITHM_ENTROPY_BASED", - "DIFFUSION_ALGORITHM_MARGIN_BASED", - "DIFFUSION_ALGORITHM_RANDOM", - "DIFFUSION_ALGORITHM_CONFIDENCE_BASED", - }; - const char * sched_names[] = { - "DIFFUSION_TRANSFER_SCHEDULE_TIMESTEP_BASED", - "DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED", - }; - const char * alg_name = - (diff_params.algorithm >= 0 && diff_params.algorithm <= 4) ? alg_names[diff_params.algorithm] : "UNKNOWN"; - const char * sched_name = - (diff_params.schedule >= 0 && diff_params.schedule <= 1) ? sched_names[diff_params.schedule] : "UNKNOWN"; - - LOG_INF("diffusion_params: - %-25s llama_token = %d\n", "mask_token_id", mask_token_id); - LOG_INF("diffusion_params: - %-25s u32 = %d\n", "steps", diff_params.steps); - LOG_INF("diffusion_params: - %-25s u32 = %d\n", "max_length", diff_params.max_length); - LOG_INF("diffusion_params: - %-25s enum = %d (%s)\n", "algorithm", diff_params.algorithm, alg_name); - LOG_INF("diffusion_params: - %-25s enum = %d (%s)\n", "schedule", diff_params.schedule, sched_name); - LOG_INF("diffusion_params: - %-25s f32 = %.3f\n", "temperature", diff_params.temperature); - if (diff_params.schedule == DIFFUSION_TRANSFER_SCHEDULE_TIMESTEP_BASED) { - LOG_INF("diffusion_params: - %-25s f32 = %.6f\n", "eps", diff_params.eps); - LOG_INF("diffusion_params: - %-25s f32 = %.3f\n", "alg_temp", diff_params.alg_temp); - } - if (diff_params.schedule == DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED) { - LOG_INF("diffusion_params: - %-25s u32 = %d\n", "block_length", diff_params.block_length); - LOG_INF("diffusion_params: - %-25s f32 = %.3f\n", "cfg_scale", diff_params.cfg_scale); - } - - diffusion_generate(ctx, input_tokens.data(), output_tokens.data(), n_input, diff_params, n_generated); - - if (n_generated > 0) { - if (visual_mode) { - //clear screen and move cursor to top-left - LOG_INF("\033[2J\033[H"); - } - - output_tokens.erase(output_tokens.begin(), output_tokens.begin() + n_input); - std::string output_data = common_detokenize(vocab, output_tokens, false); - LOG_INF("\n%s\n", output_data.c_str()); - } else { - LOG_INF("Error: diffusion generation failed\n"); - } - - llama_free(ctx); - llama_model_free(model); - llama_backend_free(); - - return 0; -} diff --git a/examples/diffusion/diffusion.cpp b/examples/diffusion/diffusion.cpp deleted file mode 100644 index 0ff57d953..000000000 --- a/examples/diffusion/diffusion.cpp +++ /dev/null @@ -1,406 +0,0 @@ -#include "diffusion.h" - -#include "common.h" - -#include "log.h" - -#include -#include -#include -#include -#include -#include -#include - -static float calculate_confidence(const llama_token_data_array & cur_p, - diffusion_algorithm algorithm, - std::mt19937 & rng) { - switch (algorithm) { - case DIFFUSION_ALGORITHM_CONFIDENCE_BASED: - return cur_p.data[cur_p.selected].p; // Selected token probability - - case DIFFUSION_ALGORITHM_ENTROPY_BASED: - { - float entropy = 0.0f; - const float epsilon = 1e-10f; - for (size_t i = 0; i < cur_p.size; i++) { - float prob = cur_p.data[i].p; - entropy += prob * logf(prob + epsilon); - } - return -entropy; // Higher entropy = lower confidence - } - - case DIFFUSION_ALGORITHM_MARGIN_BASED: - return (cur_p.size > 1) ? cur_p.data[0].p - cur_p.data[1].p : cur_p.data[0].p; - - case DIFFUSION_ALGORITHM_RANDOM: - { - std::uniform_real_distribution uniform(0.0f, 1.0f); - return uniform(rng); // Random confidence - } - - case DIFFUSION_ALGORITHM_ORIGIN: - return cur_p.data[cur_p.selected].p; - - default: - return 0.0f; - } -} - -// Unified transfer count calculation function -static int32_t calculate_transfer_count(int32_t step, - int32_t total_steps, - int32_t remaining_masked, - diffusion_transfer_schedule schedule, - float eps, - const std::vector & num_transfer_tokens = {}) { - switch (schedule) { - case DIFFUSION_TRANSFER_SCHEDULE_TIMESTEP_BASED: - { - float t = 1.0f - (float) step / total_steps * (1.0f - eps); - float s = 1.0f - (float) (step + 1) / total_steps * (1.0f - eps); - float p_transfer = (step < total_steps - 1) ? (1.0f - s / t) : 1.0f; - return (int32_t) (remaining_masked * p_transfer); - } - - case DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED: - if (!num_transfer_tokens.empty() && step < (int32_t) num_transfer_tokens.size()) { - return num_transfer_tokens[step]; - } - return remaining_masked / (total_steps - step); // Fallback - - default: - return remaining_masked / (total_steps - step); - } -} - -static void add_gumbel_noise(float * logits, int32_t n_vocab, float temperature, std::mt19937 & rng) { - if (temperature == 0.0f) { - return; - } - - std::uniform_real_distribution uniform(0.0, 1.0); - for (int32_t i = 0; i < n_vocab; i++) { - double noise = uniform(rng); - // Prevent log(0) - noise = std::max(noise, 1e-20); - double gumbel_noise = std::pow(-std::log(noise), temperature); - logits[i] = std::exp(logits[i]) / gumbel_noise; - } -} - -static std::vector get_num_transfer_tokens(int32_t mask_count, int32_t steps) { - std::vector num_transfer_tokens(steps); - - int32_t base = mask_count / steps; - int32_t remainder = mask_count % steps; - - for (int32_t i = 0; i < steps; i++) { - num_transfer_tokens[i] = base + (i < remainder ? 1 : 0); - } - - return num_transfer_tokens; -} - -void diffusion_generate(llama_context * ctx, - const llama_token * input_tokens, - llama_token * output_tokens, - int32_t n_input, - const diffusion_params & params, - int32_t & n_generated) { - n_generated = 0; - if (!ctx || !input_tokens || !output_tokens || n_input <= 0 || params.max_length <= n_input) { - return; - } - - const llama_model * model = llama_get_model(ctx); - - // Initialize with input and pad with mask tokens - std::copy(input_tokens, input_tokens + n_input, output_tokens); - std::fill(output_tokens + n_input, output_tokens + params.max_length, params.mask_token_id); - - std::mt19937 rng(params.seed); - - llama_set_causal_attn(ctx, false); - - int32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); - - std::vector candidates(n_vocab); - std::vector conf_candidates; - conf_candidates.reserve(params.max_length); - std::vector mask_positions; - mask_positions.reserve(params.max_length); - - // Setup sampler chain - struct llama_sampler * sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()); - if (params.top_k > 0) { - llama_sampler_chain_add(sampler, llama_sampler_init_top_k(params.top_k)); - } - if (params.top_p < 1.0f) { - llama_sampler_chain_add(sampler, llama_sampler_init_top_p(params.top_p, 1)); - } - if (params.temperature > 0.0f) { - llama_sampler_chain_add(sampler, llama_sampler_init_temp(params.temperature)); - } - llama_sampler_chain_add(sampler, llama_sampler_init_dist(params.seed)); - - struct llama_sampler * dist_sampler = llama_sampler_init_dist(params.seed); - - common_batch batch(ctx); - - // Pre-allocate buffers for CFG if needed - int32_t logits_size = n_vocab * params.max_length; - std::vector cond_logits_buffer; - std::vector un_x_buffer; - if (params.cfg_scale > 0.0f) { - cond_logits_buffer.resize(logits_size); - un_x_buffer.resize(params.max_length); - } - - // For block-based processing - std::vector num_transfer_tokens; - int32_t num_blocks = 1; - int32_t steps_per_block = params.steps; - - if (params.schedule == DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED) { - GGML_ASSERT(params.max_length % params.block_length == 0); - num_blocks = params.max_length / params.block_length; - GGML_ASSERT(params.steps % num_blocks == 0); - steps_per_block = params.steps / num_blocks; - } - - std::vector confidence(params.max_length); - - int64_t total_sampling_time = 0; - int64_t total_time = 0; - int64_t time_start = ggml_time_us(); - - for (int block_num = 0; block_num < num_blocks; block_num++) { - int32_t block_start = (params.schedule == DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED) ? n_input + block_num * params.block_length : 0; - int32_t block_end = (params.schedule == DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED) ? - std::min(n_input + (block_num + 1) * params.block_length, params.max_length) : - params.max_length; - - // Count masked tokens in current block for block-based processing - if (params.schedule == DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED) { - int32_t block_mask_count = 0; - for (int i = block_start; i < block_end; i++) { - if (output_tokens[i] == params.mask_token_id) { - block_mask_count++; - } - } - num_transfer_tokens = get_num_transfer_tokens(block_mask_count, steps_per_block); - } - - for (int32_t step = 0; step < steps_per_block; step++) { - int32_t global_step = block_num * steps_per_block + step; - - if (params.step_callback) { - if (!params.step_callback( - global_step, params.steps, output_tokens, params.max_length, params.step_callback_user_data)) { - break; - } - } - - // Setup batch - batch.clear(); - for (int32_t i = 0; i < params.max_length; i++) { - batch.add(output_tokens[i], i, 0, true); - } - - float * logits = nullptr; - - if (params.cfg_scale > 0.0f) { - int ret = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()); - if (ret != 0) { - LOG_ERR("Failed to generate conditional"); - break; - } - float * cond_logits_ptr = llama_get_logits(ctx); - std::memcpy(cond_logits_buffer.data(), cond_logits_ptr, logits_size * sizeof(float)); - - // Unconditional generation (mask input) - std::copy(output_tokens, output_tokens + params.max_length, un_x_buffer.begin()); - for (int32_t i = 0; i < n_input; i++) { - un_x_buffer[i] = params.mask_token_id; - } - - batch.clear(); - for (int32_t i = 0; i < params.max_length; i++) { - batch.add(un_x_buffer[i], i, 0, true); - } - ret = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()); - if (ret != 0) { - LOG_ERR("Failed to generate unconditional"); - break; - } - float * uncond_logits = llama_get_logits(ctx); - - // Apply CFG - for (int32_t i = 0; i < logits_size; i++) { - cond_logits_buffer[i] = - uncond_logits[i] + (params.cfg_scale + 1.0f) * (cond_logits_buffer[i] - uncond_logits[i]); - } - logits = cond_logits_buffer.data(); - } else { - int ret = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()); - if (ret != 0) { - LOG_ERR("%s: failed to decode at step %d, ret = %d\n", __func__, global_step, ret); - break; - } - logits = llama_get_logits(ctx); - } - - if (!logits) { - LOG_ERR("%s: failed to get logits at step %d\n", __func__, global_step); - break; - } - - auto get_logits_for_pos = [&](int32_t pos) -> const float * { - if (params.shift_logits) { - return pos == 0 ? logits : logits + (pos - 1) * n_vocab; - } - return logits + pos * n_vocab; - }; - - int64_t time_start_sampling = ggml_time_us(); - - mask_positions.clear(); - for (int32_t i = 0; i < params.max_length; i++) { - if (output_tokens[i] == params.mask_token_id) { - // For block-based, only consider current block - if (params.schedule != DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED || (i >= block_start && i < block_end)) { - mask_positions.push_back(i); - } - } - } - - if (mask_positions.empty()) { - break; - } - - if (params.add_gumbel_noise && params.temperature > 0.0f) { - add_gumbel_noise(logits, n_vocab, params.temperature, rng); - } - - if (params.algorithm == DIFFUSION_ALGORITHM_ORIGIN) { - int32_t transfer_count = calculate_transfer_count( - step, steps_per_block, mask_positions.size(), params.schedule, params.eps, num_transfer_tokens); - float p_transfer = (float) transfer_count / mask_positions.size(); - - for (int32_t pos : mask_positions) { - if (std::uniform_real_distribution(0.0f, 1.0f)(rng) < p_transfer) { - const float * pos_logits = get_logits_for_pos(pos); - for (int32_t token_id = 0; token_id < n_vocab; token_id++) { - candidates[token_id].id = token_id; - candidates[token_id].logit = pos_logits[token_id]; - candidates[token_id].p = 0.0f; - } - - llama_token_data_array cur_p = { - candidates.data(), - (size_t) n_vocab, - -1, - false, - }; - - llama_sampler_apply(sampler, &cur_p); - output_tokens[pos] = cur_p.data[cur_p.selected].id; - } - } - } else { - std::vector> confidences; - std::vector sampled_tokens(mask_positions.size()); - - for (size_t i = 0; i < mask_positions.size(); i++) { - int32_t pos = mask_positions[i]; - const float * pos_logits = get_logits_for_pos(pos); - - for (int32_t token_id = 0; token_id < n_vocab; token_id++) { - candidates[token_id].logit = pos_logits[token_id]; - candidates[token_id].p = 0.0f; - candidates[token_id].id = token_id; - } - - llama_token_data_array cur_p = { - candidates.data(), - candidates.size(), - -1, - false, - }; - - llama_sampler_apply(sampler, &cur_p); - llama_token sampled_token = cur_p.data[cur_p.selected].id; - - float conf = calculate_confidence(cur_p, params.algorithm, rng); - - sampled_tokens[i] = sampled_token; - confidences.emplace_back(conf, i); - } - - int32_t transfer_count = calculate_transfer_count( - step, steps_per_block, mask_positions.size(), params.schedule, params.eps, num_transfer_tokens); - - if (transfer_count > 0) { - if (params.alg_temp == 0.0f) { - std::partial_sort(confidences.begin(), - confidences.begin() + std::min(transfer_count, (int32_t) confidences.size()), - confidences.end(), - [](const std::pair & a, const std::pair & b) { - if (a.first != b.first) { - return a.first > b.first; - } - return a.second < b.second; - }); - - for (int32_t i = 0; i < std::min(transfer_count, (int32_t) confidences.size()); i++) { - int32_t mask_idx = confidences[i].second; - int32_t pos = mask_positions[mask_idx]; - output_tokens[pos] = sampled_tokens[mask_idx]; - } - } else { - conf_candidates.clear(); - for (size_t i = 0; i < confidences.size(); i++) { - float conf_logit = confidences[i].first / params.alg_temp; - conf_candidates.emplace_back(llama_token_data{ (int32_t) i, conf_logit, 0.0f }); - } - - llama_token_data_array conf_array = { - conf_candidates.data(), - conf_candidates.size(), - -1, - false, - }; - - for (int32_t i = 0; i < std::min(transfer_count, (int32_t) confidences.size()); i++) { - llama_sampler_apply(dist_sampler, &conf_array); - int32_t selected_idx = conf_array.selected; - int32_t mask_idx = selected_idx; - int32_t pos = mask_positions[mask_idx]; - output_tokens[pos] = sampled_tokens[mask_idx]; - - conf_candidates[selected_idx].p = 0.0f; - conf_array.selected = -1; - } - } - } - } - - int64_t time_end_sampling = ggml_time_us(); - total_sampling_time += time_end_sampling - time_start_sampling; - } - } - - int64_t time_end = ggml_time_us(); - total_time += time_end - time_start; - - LOG_INF("\ntotal time: %0.2fms, time per step: %0.2fms, sampling time per step: %0.2fms\n", - total_time / 1000.0, - total_time / 1000.0 / params.steps, - total_sampling_time / 1000.0 / params.steps); - - llama_sampler_free(sampler); - llama_sampler_free(dist_sampler); - - n_generated = params.max_length; -} diff --git a/examples/diffusion/diffusion.h b/examples/diffusion/diffusion.h deleted file mode 100644 index 783144522..000000000 --- a/examples/diffusion/diffusion.h +++ /dev/null @@ -1,57 +0,0 @@ -#pragma once - -#include "llama.h" - -#include - -enum diffusion_algorithm { - DIFFUSION_ALGORITHM_ORIGIN = 0, - DIFFUSION_ALGORITHM_ENTROPY_BASED = 1, - DIFFUSION_ALGORITHM_MARGIN_BASED = 2, - DIFFUSION_ALGORITHM_RANDOM = 3, - DIFFUSION_ALGORITHM_CONFIDENCE_BASED = 4, -}; - -// Unified transfer scheduling methods -enum diffusion_transfer_schedule { - DIFFUSION_TRANSFER_SCHEDULE_TIMESTEP_BASED = 0, // Dream-style: (1.0 - s/t) * remaining - DIFFUSION_TRANSFER_SCHEDULE_BLOCK_BASED = 1, // LLaDA-style: process in blocks with get_num_transfer_tokens -}; - -typedef bool (*diffusion_step_callback_t)(int32_t step, - int32_t total_steps, - const llama_token * tokens, - int32_t n_tokens, - void * user_data); - -struct diffusion_params { - int32_t steps = 0; - float temperature = 0; - llama_token mask_token_id = LLAMA_TOKEN_NULL; - diffusion_step_callback_t step_callback = nullptr; - void * step_callback_user_data = nullptr; - int32_t seed = 0; - bool visual_mode = false; - bool shift_logits = false; // Shift logits by -1 after decode - - float top_p = 0.; - int32_t top_k = 0.; - - diffusion_algorithm algorithm = DIFFUSION_ALGORITHM_CONFIDENCE_BASED; - diffusion_transfer_schedule schedule = DIFFUSION_TRANSFER_SCHEDULE_TIMESTEP_BASED; - - float cfg_scale = 0.; // Config scale for classifier-free guidance - float eps = 0.; // Timestep scheduling - int32_t block_length = 0; // Block size (for block scheduling) - float alg_temp = 0; // algorithm temperature (0.0 = deterministic) - bool add_gumbel_noise = false; // Add gumbel noise to the logits if temp > 0.0 - - int32_t max_length = 0; // Maximum sequence length -}; - -void diffusion_generate(llama_context * ctx, - const llama_token * input_tokens, - llama_token * output_tokens, - int32_t n_input, - const diffusion_params & params, - int32_t & n_generated); diff --git a/examples/llama-eval/README.md b/examples/llama-eval/README.md deleted file mode 100644 index 3c5c35f78..000000000 --- a/examples/llama-eval/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# llama-eval - -Simple evaluation tool for llama.cpp with support for multiple datasets. - -For a full description, usage examples, and sample results, see: - -- [PR 21152](https://github.com/ggml-org/llama.cpp/pull/21152) - -## Quick start - -```bash -# Single server -python3 llama-eval.py \ - --server http://localhost:8033 \ - --model my-model \ - --dataset gsm8k --n_cases 100 \ - --grader-type regex --threads 32 - -# Multiple servers (comma-separated URLs and thread counts) -python3 llama-eval.py \ - --server http://server1:8033,http://server2:8033 \ - --server-name server1,server2 \ - --threads 16,16 \ - --dataset aime2025 --n_cases 240 \ - --grader-type regex -``` diff --git a/examples/llama-eval/llama-eval.py b/examples/llama-eval/llama-eval.py deleted file mode 100644 index 61bdbddd8..000000000 --- a/examples/llama-eval/llama-eval.py +++ /dev/null @@ -1,1591 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json -import os -import re -import subprocess -import sys -import threading -import time -from abc import ABC, abstractmethod -from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass, asdict, field -from pathlib import Path -from queue import Queue -from typing import Dict, List, Optional, Any, Tuple -import requests -from tqdm import tqdm -import random -from math import sqrt - - -@dataclass -class ServerConfig: - url: str - threads: int - name: str = "" - -def wilson_interval(correct: int, total: int, z: float = 1.96) -> Tuple[float, float]: - """Wilson score confidence interval for a proportion.""" - if total == 0: - return (0.0, 1.0) - p = correct / total - z2 = z * z / total - center = (p + z2 / 2) / (1 + z2) - margin = z * sqrt((p * (1 - p) + z2 / 4) / total) / (1 + z2) - return (center - margin, center + margin) - -cache_dir = Path.home() / ".cache" / "huggingface" / "datasets" -cache_dir.mkdir(parents=True, exist_ok=True) -os.environ["HF_DATASETS_CACHE"] = str(cache_dir) -os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1" - -GRADER_PATTERNS = { - "aime": r'\boxed{(\d+)}|\b(\d+)\b', - "aime2025": r'\boxed{(\d+)}|\b(\d+)\b', - "aime2026": r'\boxed{(\d+)}|\b(\d+)\b', - "gsm8k": r'\b(\d+)\b', -} - -SAMPLE_ANSWERS = { - "aime": [ - "42", - "-123", - "999" - ], - "aime2025": [ - "42", - "-123", - "999" - ], - "aime2026": [ - "42", - "-123", - "999" - ], - "gsm8k": [ - "42", - "-123", - "999" - ], - "gpqa": [ - "A", - "D", - "C" - ], -} - -TEMPLATE_REGISTRY = { - "aime": """Solve the following math problem step by step. Put your answer inside \\boxed{{}}. - -{question} - -Remember to put your answer inside \\boxed{{}}. -""", - "aime2025": """Solve the following math problem step by step. Put your answer inside \\boxed{{}}. - -{question} - -Remember to put your answer inside \\boxed{{}}. -""", - "aime2026": """Solve the following math problem step by step. Put your answer inside \\boxed{{}}. - -{question} - -Remember to put your answer inside \\boxed{{}}. -""", - "gsm8k": """{question} -Please reason step by step, and put your final numeric answer within \\boxed{{}} without any extra characters. -""", - "gpqa": """Answer the following multiple choice question. The last line of your response should be in the following format: 'Answer: A/B/C/D' (e.g. 'Answer: A'). - -{Question} - -A) {A} -B) {B} -C) {C} -D) {D} -""", -} - - -class BaseDataset(ABC): - questions: List[Dict] - - @abstractmethod - def get_question(self, index: int) -> Dict: - pass - - @abstractmethod - def get_question_text(self, question: Dict) -> str: - pass - - @abstractmethod - def get_answer(self, question: Dict) -> str: - pass - - @abstractmethod - def get_prompt(self, question: Dict) -> str: - pass - - def __len__(self) -> int: - return len(self.questions) - - -@dataclass -class TaskState: - task_id: str - prompt: str - expected: str - question_text: str = "" - response: Optional[str] = None - answer: Optional[str] = None - grader_log: Dict[str, Any] = field(default_factory=dict) - correct: bool = False - status: str = "pending" - tokens: Optional[int] = None - tps_gen: Optional[float] = None - t_gen_ms: Optional[float] = None - reasoning_content: Optional[str] = None - server_name: Optional[str] = None - chunk_idx: int = 0 - problem_idx: int = 0 - - -class EvalState: - def __init__( - self, - dataset_type: str, - sampling_config: Dict[str, Any], - output_file: Path = Path("llama-eval-state.json"), - model_name: Optional[str] = None - ): - self.dataset_type = dataset_type - self.sampling_config = sampling_config - self.output_file = output_file - self.model_name = model_name - self.dataset: Optional[BaseDataset] = None - self.tasks: List[Tuple[int, str]] = [] - self.all_tasks: List[Tuple[int, str]] = [] - self.task_states: Dict[str, Any] = {} - self.total = 0 - self.correct = 0 - self.processed = 0 - self.total_time: float = 0.0 - self._lock = threading.Lock() - - def load_dataset(self, seed: int = 1234): - if self.dataset_type == "aime": - self.dataset = AimeDataset() - elif self.dataset_type == "aime2025": - self.dataset = Aime2025Dataset() - elif self.dataset_type == "aime2026": - self.dataset = Aime2026Dataset() - elif self.dataset_type == "gsm8k": - self.dataset = Gsm8kDataset() - elif self.dataset_type == "gpqa": - self.dataset = GpqaDataset(variant="diamond", seed=seed) - else: - raise ValueError(f"Unknown dataset type: {self.dataset_type}") - - def setup_tasks(self, n_cases: Optional[int] = None, seed: int = 1234): - if self.dataset is None: - raise ValueError("Dataset not loaded. Call load_dataset() first.") - - if n_cases is None: - n_cases = len(self.dataset) - - dataset_size = len(self.dataset) - rng = random.Random(seed) - - self.tasks = [] - for chunk_idx in range((n_cases + dataset_size - 1) // dataset_size): - chunk_size = min(dataset_size, n_cases - chunk_idx * dataset_size) - indices = list(range(dataset_size)) - rng.shuffle(indices) - chunk_indices = indices[:chunk_size] - - for i in chunk_indices: - task_id = f"{self.dataset_type}_{chunk_idx:03d}_{i:03d}" - self.tasks.append((i, task_id)) - - self.all_tasks = list(self.tasks) - - def get_case(self, index: int) -> Tuple[str, str, str]: - if self.dataset is None: - raise ValueError("Dataset not loaded.") - question = self.dataset.get_question(index) - question_text = self.dataset.get_question_text(question) - prompt = self.dataset.get_prompt(question) - expected = self.dataset.get_answer(question) - return question_text, prompt, expected - - def add_result( - self, - task_id: str, - prompt: str, - expected: str, - response: Optional[str], - answer: Optional[str], - grader_log: Dict[str, Any], - correct: bool, - status: str, - tokens: Optional[int] = None, - tps_gen: Optional[float] = None, - t_gen_ms: Optional[float] = None, - reasoning_content: Optional[str] = None, - server_name: Optional[str] = None, - chunk_idx: int = 0, - problem_idx: int = 0, - ): - with self._lock: - if "cases" not in self.task_states: - self.task_states["cases"] = {} - - self.task_states["cases"][task_id] = { - "task_id": task_id, - "prompt": prompt, - "expected": expected, - "response": response, - "answer": answer, - "grader_log": grader_log, - "correct": correct, - "status": status, - "tokens": tokens, - "tps_gen": tps_gen, - "t_gen_ms": t_gen_ms, - "reasoning_content": reasoning_content, - "server_name": server_name, - "chunk_idx": chunk_idx, - "problem_idx": problem_idx, - } - - self.correct = sum(1 for c in self.task_states.get("cases", {}).values() if c.get("correct", False)) - - def print_progress(self, task_state: TaskState, total_tasks: int, n_correct: int = 0): - display_answer = task_state.answer if task_state.answer else "N/A" - display_tokens = str(task_state.tokens) if task_state.tokens is not None else "N/A" - display_tps = f"{task_state.tps_gen:.1f}" if task_state.tps_gen is not None else "N/A" - display_t_gen = f"{task_state.t_gen_ms/1000:.1f}" if task_state.t_gen_ms is not None else "N/A" - display_server = task_state.server_name if task_state.server_name else "N/A" - success_ratio = n_correct / self.processed if self.processed > 0 else 0.0 - first_line = task_state.question_text.split('\n')[0] - truncated_question = first_line[:43] - if len(first_line) > 43: - truncated_question += "..." - else: - truncated_question = truncated_question.ljust(43) + "..." - print(f"{self.processed:3}/{total_tasks:3} {task_state.task_id:<20} {self.dataset_type.upper()} {truncated_question:<40} {task_state.expected:<10} {display_answer:<10} {display_tokens:<6} {display_tps:<6} {display_t_gen:<8} {'✓' if task_state.correct else '✗'} [{n_correct:3}/{self.processed:3}, {success_ratio:.3f}] {display_server}") - - def print_summary(self): - if self.total == 0: - print(f"\n{'='*60}") - print(f"Results: 0/0 correct (0.0%)") - print(f"{'='*60}") - else: - ci_lower, ci_upper = self.accuracy_ci() - print(f"\n{'='*60}") - print(f"Results: {self.correct}/{self.total} correct ({self.correct/self.total*100:.1f}%) [{ci_lower*100:.1f}%, {ci_upper*100:.1f}%]") - print(f"{'='*60}") - - def dump(self): - with self._lock: - tasks_to_save = self.all_tasks if self.all_tasks else self.tasks - all_cases = {} - for i, task_id in tasks_to_save: - question_text, prompt, expected = self.get_case(i) - # Extract chunk_idx from task_id for pending cases - _parts = task_id.rsplit("_", 2) - _chunk_idx = int(_parts[-2]) if len(_parts) >= 3 else 0 - if task_id in self.task_states.get("cases", {}): - all_cases[task_id] = self.task_states["cases"][task_id] - else: - all_cases[task_id] = { - "task_id": task_id, - "prompt": prompt, - "expected": expected, - "question_text": question_text, - "response": None, - "answer": None, - "grader_log": {}, - "correct": False, - "status": "pending", - "tokens": None, - "tps_gen": None, - "t_gen_ms": None, - "reasoning_content": None, - "server_name": None, - "chunk_idx": _chunk_idx, - "problem_idx": i, - } - - ci_lower, ci_upper = self.accuracy_ci() - data = { - "id": self.dataset_type, - "model_name": self.model_name, - "tasks": [tid for _, tid in tasks_to_save], - "task_states": { - "total": self.total, - "correct": self.correct, - "total_time": self.total_time, - "ci_lower": ci_lower, - "ci_upper": ci_upper, - "cases": all_cases, - }, - "sampling_config": self.sampling_config - } - with open(self.output_file, "w") as f: - json.dump(data, f, indent=2) - - self.dump_html(tasks_to_save, all_cases) - - def dump_html(self, tasks_to_save: List[Tuple[int, str]], all_cases: Dict[str, Any]): - html_file = Path(str(self.output_file) + ".html") - - cases = all_cases - completed = {tid: c for tid, c in cases.items() if c.get("status") == "ok"} - n_correct = sum(1 for c in completed.values() if c.get("correct", False)) - n_incorrect = len(completed) - n_correct - n_pending = len(tasks_to_save) - len(completed) - accuracy = n_correct / len(completed) * 100 if completed else 0.0 - ci_lower, ci_upper = wilson_interval(n_correct, len(completed)) if completed else (0.0, 1.0) - - sampling_parts = [] - for k, v in self.sampling_config.items(): - if v is not None: - sampling_parts.append(f"{k}={v}") - sampling_str = ", ".join(sampling_parts) if sampling_parts else "default" - - rows = [] - for i, task_id in tasks_to_save: - case = cases.get(task_id, {}) - status = case.get("status", "pending") - expected = case.get("expected", "") - answer = case.get("answer") or "" if status == "ok" else "" - is_correct = case.get("correct", False) if status == "ok" else False - response = case.get("response", "") or "" - prompt = case.get("prompt", "") or "" - grader_log = case.get("grader_log", {}) - - if status == "ok": - status_class = "correct" if is_correct else "incorrect" - status_text = "✓" if is_correct else "✗" - elif status == "pending": - status_class = "pending" - status_text = "–" - else: - status_class = "error" - status_text = "!" - - tokens = case.get("tokens") - tokens_str = str(tokens) if tokens is not None else "" - tps_gen = case.get("tps_gen") - tps_str = f"{tps_gen:.1f}" if tps_gen is not None else "" - t_gen_ms = case.get("t_gen_ms") - t_gen_str = f"{t_gen_ms/1000:.1f}" if t_gen_ms is not None else "" - reasoning_content = case.get("reasoning_content", "") or "" - server_name = case.get("server_name", "") or "" - - escaped_response = self._escape_html(response) - escaped_prompt = self._escape_html(prompt) - escaped_reasoning = self._escape_html(reasoning_content) - grader_log_str = self._escape_html(json.dumps(grader_log, indent=2)) - escaped_server = self._escape_html(server_name) - - answer_class = status_class if status == "ok" else "" - rows.append(f""" - {task_id} - {status_text} - {self._escape_html(expected)} - {self._escape_html(answer)} - {tokens_str} - {tps_str} - {t_gen_str} - {escaped_server} - - - -
- Prompt
{escaped_prompt}
- Response
{escaped_response}
- {f'Reasoning
{escaped_reasoning}
' if escaped_reasoning else ''} - Grader
{grader_log_str}
-
- - """) - - rows_html = "\n".join(rows) - - # ---- per-problem summary table ---- - problem_groups: Dict[int, List[Dict[str, Any]]] = {} - for _tid, _case in cases.items(): - if _case.get("status") != "ok": - continue - _pidx = _case.get("problem_idx") - if _pidx is None: - _p_parts = _tid.rsplit("_", 2) - _pidx = int(_p_parts[-1]) if len(_p_parts) >= 3 else 0 - problem_groups.setdefault(_pidx, []).append(_case) - - summary_rows_html = "" - if problem_groups: - def _stat(v, fmt=".1f", avg_fmt=None): - if not v: - return ("–", "–", "–") - af = fmt if avg_fmt is None else avg_fmt - return (f"{min(v):{fmt}}", f"{sum(v)/len(v):{af}}", f"{max(v):{fmt}}") - - summary_data = [] - for pidx, g in problem_groups.items(): - runs = len(g) - n_ok = sum(1 for c in g if c.get("correct", False)) - toks = [c["tokens"] for c in g if c.get("tokens") is not None] - tps = [c["tps_gen"] for c in g if c.get("tps_gen") is not None] - tg = [c["t_gen_ms"] / 1000 for c in g if c.get("t_gen_ms") is not None] - summary_data.append(( - pidx, runs, n_ok, - _stat(toks, "d", ".0f"), - _stat(tps), - _stat(tg), - )) - - summary_data.sort(key=lambda r: r[0]) # sort by problem index ascending - - summary_rows_html = "\n".join( - f""" - {p:03d} - {r} - {n}/{r} - {tk[0]}{tk[1]}{tk[2]} - {tp[0]}{tp[1]}{tp[2]} - {tg[0]}{tg[1]}{tg[2]} - """ - for p, r, n, tk, tp, tg in summary_data - ) - - html_content = f""" - - - -{self.dataset_type.upper()} Eval - - - -
-
Dataset
{self.dataset_type.upper()}
-
Model
{self.model_name or 'N/A'}
-
Accuracy
{accuracy:.1f}% [{ci_lower*100:.1f}%, {ci_upper*100:.1f}%]
-
Correct
{n_correct} / {len(completed)}
-
Pending
{n_pending}
-
Time
{self.total_time:.1f}s
-
Sampling
{sampling_str}
-
-
- - -
-
- - - - - - - - - - - - - - - {rows_html} - -
IDGoldAnswerTokensT/sGen sServer
-
-
- - - - - - - - - - - - - - - - - - - - - {summary_rows_html} - -
ProblemRunsCorrectTokensT/sGen s
minavgmaxminavgmaxminavgmax
-
- - -""" - - with open(html_file, "w") as f: - f.write(html_content) - - def _escape_html(self, s: str) -> str: - return (s.replace("&", "&") - .replace("<", "<") - .replace(">", ">") - .replace('"', """) - .replace("'", "'")) - - @classmethod - def load(cls, path: Path) -> "EvalState": - with open(path, "r") as f: - data = json.load(f) - - eval_state = cls( - dataset_type=data["id"], - sampling_config=data["sampling_config"], - output_file=path, - model_name=data.get("model_name") - ) - eval_state.load_dataset() - - eval_state.tasks = [] - eval_state.all_tasks = [] - for task_id in data.get("tasks", []): - parts = task_id.rsplit("_", 2) - if len(parts) >= 3: - idx = int(parts[-1]) - else: - idx = 0 - eval_state.tasks.append((idx, task_id)) - eval_state.all_tasks.append((idx, task_id)) - - eval_state.task_states = data.get("task_states", {}) - - cases = eval_state.task_states.get("cases", {}) - eval_state.total = eval_state.task_states.get("total", 0) - eval_state.correct = eval_state.task_states.get("correct", 0) - eval_state.total_time = eval_state.task_states.get("total_time", 0.0) - - if eval_state.total == 0: - eval_state.total = len(cases) - eval_state.correct = sum(1 for c in cases.values() if c.get("correct", False)) - - return eval_state - - def is_complete(self) -> bool: - if not self.all_tasks: - return False - cases = self.task_states.get("cases", {}) - completed = {tid for tid in self.task_states.get("cases", {}).keys() if cases.get(tid, {}).get("status") == "ok"} - return len(completed) == len(self.all_tasks) - - def get_pending_tasks(self) -> List[Tuple[int, str]]: - cases = self.task_states.get("cases", {}) - pending = [] - for i, task_id in self.all_tasks: - status = cases.get(task_id, {}).get("status", "pending") - if status != "ok": - pending.append((i, task_id)) - return pending - - def print_all_tasks(self): - cases = self.task_states.get("cases", {}) - tasks_to_show = self.all_tasks if self.all_tasks else self.tasks - print() - print("Tasks:") - print(" Task ID Dataset Prompt (first 40 chars) Expected Answer Tokens T/s Gen s Status") - for i, task_id in tasks_to_show: - question, prompt, expected = self.get_case(i) - case = cases.get(task_id, {}) - status = case.get("status", "pending") - answer = case.get("answer") or "N/A" if status == "ok" else "N/A" - tokens = case.get("tokens") - tokens_str = str(tokens) if tokens is not None else "N/A" - tps_gen = case.get("tps_gen") - tps_str = f"{tps_gen:.1f}" if tps_gen is not None else "N/A" - t_gen_ms = case.get("t_gen_ms") - t_gen_str = f"{t_gen_ms/1000:.1f}" if t_gen_ms is not None else "N/A" - server_name = case.get("server_name", "") or "" - is_correct = case.get("correct", False) if status == "ok" else False - symbol = "✓ " if is_correct else ("✗ " if status == "ok" else "") - first_line = question.split('\n')[0] - question_trunc = first_line[:43] - if len(first_line) > 43: - question_trunc += "..." - else: - question_trunc = question_trunc.ljust(43) + "..." - print(f" {task_id:<20} {self.dataset_type.upper()} {question_trunc:<40} {expected:<10} {answer:<10} {tokens_str:<6} {tps_str:<6} {t_gen_str:<8} {symbol}{status} {server_name}") - print() - - def print_existing_summary(self): - cases = self.task_states.get("cases", {}) - completed_cases = {tid: c for tid, c in cases.items() if c.get("status") == "ok"} - correct = sum(1 for c in completed_cases.values() if c.get("correct", False)) - total = len(completed_cases) - if total == 0: - print(f"{'='*60}") - print(f"Results: 0/0 correct (0.0%)") - print(f"{'='*60}") - else: - ci_lower, ci_upper = self.accuracy_ci() - print(f"{'='*60}") - print(f"Results: {correct}/{total} correct ({correct/total*100:.1f}%) [{ci_lower*100:.1f}%, {ci_upper*100:.1f}%]") - print(f"{'='*60}") - - def accuracy_ci(self) -> Tuple[float, float]: - """Compute Wilson score confidence interval from completed cases.""" - cases = self.task_states.get("cases", {}) - completed = {tid: c for tid, c in cases.items() if c.get("status") == "ok"} - correct = sum(1 for c in completed.values() if c.get("correct", False)) - total = len(completed) - return wilson_interval(correct, total) - -def normalize_number(s: str) -> Optional[int]: - match = re.match(r"\d+", s) # match digits from the start - if not match: - return None - return int(match.group(0)) - -class AimeDataset(BaseDataset): - def __init__(self, split: str = "train"): - self.split = split - self.questions = [] - self._load_dataset() - - def _load_dataset(self): - print(f"Loading AIME dataset (split: {self.split})...") - from datasets import load_dataset - - cache_path = cache_dir / "AI-MO___aimo-validation-aime" / "default" / "0.0.0" - if cache_path.exists(): - print(f"Using cached dataset from {cache_path}") - ds = load_dataset("AI-MO/aimo-validation-aime", split=self.split, cache_dir=str(cache_path)) - else: - ds = load_dataset("AI-MO/aimo-validation-aime", split=self.split) - - self.questions = [] - for row in ds: - question = dict(row) - question["dataset_type"] = "aime" - self.questions.append(question) - - print(f"AIME dataset loaded: {len(self.questions)} questions") - - def get_question(self, index: int) -> Dict: - """Get question by index""" - return self.questions[index] - - def get_question_text(self, question: Dict) -> str: - """Get question string""" - return question["problem"] if "problem" in question else question["question"] - - def get_answer(self, question: Dict) -> str: - answer = question["answer"] - if isinstance(answer, str): - normalized = normalize_number(answer) - return str(normalized) if normalized is not None else answer - return str(answer) - - def get_prompt(self, question: Dict) -> str: - """Get formatted prompt for the question""" - return TEMPLATE_REGISTRY[question["dataset_type"]].format( - question=self.get_question_text(question), - ) - -class Aime2025Dataset(BaseDataset): - def __init__(self): - self.questions = [] - self._load_dataset() - - def _load_dataset(self): - print(f"Loading AIME2025 dataset...") - from datasets import load_dataset - - config_name = "AIME2025-I" - cache_path = cache_dir / "opencompass___AIME2025" / "default" / "0.0.0" - if cache_path.exists(): - print(f"Using cached dataset from {cache_path}") - ds = load_dataset("opencompass/AIME2025", config_name, split="test", cache_dir=str(cache_path)) - else: - ds = load_dataset("opencompass/AIME2025", config_name, split="test") - - self.questions = [] - for row in ds: - question = dict(row) - question["dataset_type"] = "aime2025" - self.questions.append(question) - - print(f"AIME2025 dataset loaded: {len(self.questions)} questions") - - print(f"Loading AIME2025 dataset (part 2)...") - config_name_2 = "AIME2025-II" - cache_path_2 = cache_dir / "opencompass___AIME2025" / "default" / "0.0.0" - if cache_path_2.exists(): - print(f"Using cached dataset from {cache_path_2}") - ds_2 = load_dataset("opencompass/AIME2025", config_name_2, split="test", cache_dir=str(cache_path_2)) - else: - ds_2 = load_dataset("opencompass/AIME2025", config_name_2, split="test") - - for row in ds_2: - question = dict(row) - question["dataset_type"] = "aime2025" - self.questions.append(question) - - print(f"AIME2025 dataset loaded: {len(self.questions)} questions (total)") - - def get_question(self, index: int) -> Dict: - """Get question by index""" - return self.questions[index] - - def get_question_text(self, question: Dict) -> str: - """Get question string""" - return question["question"] - - def get_answer(self, question: Dict) -> str: - answer = question["answer"] - if isinstance(answer, str): - normalized = normalize_number(answer) - return str(normalized) if normalized is not None else answer - return str(answer) - - def get_prompt(self, question: Dict) -> str: - """Get formatted prompt for the question""" - return TEMPLATE_REGISTRY["aime2025"].format( - question=self.get_question_text(question), - ) - -class Aime2026Dataset(BaseDataset): - def __init__(self): - self.questions = [] - self._load_dataset() - - def _load_dataset(self): - print(f"Loading AIME2026 dataset...") - from datasets import load_dataset - - cache_path = cache_dir / "MathArena___aime_2026" / "default" / "0.0.0" - if cache_path.exists(): - print(f"Using cached dataset from {cache_path}") - ds = load_dataset("MathArena/aime_2026", "default", split="train", cache_dir=str(cache_path)) - else: - ds = load_dataset("MathArena/aime_2026", "default", split="train") - - self.questions = [] - for row in ds: - question = dict(row) - question["dataset_type"] = "aime2026" - self.questions.append(question) - - print(f"AIME2026 dataset loaded: {len(self.questions)} questions") - - def get_question(self, index: int) -> Dict: - """Get question by index""" - return self.questions[index] - - def get_question_text(self, question: Dict) -> str: - """Get question string""" - return question["problem"] - - def get_answer(self, question: Dict) -> str: - return str(question["answer"]) - - def get_prompt(self, question: Dict) -> str: - """Get formatted prompt for the question""" - return TEMPLATE_REGISTRY["aime2026"].format( - question=self.get_question_text(question), - ) - -class Gsm8kDataset(BaseDataset): - def __init__(self, split: str = "test"): - self.split = split - self.questions = [] - self._load_dataset() - - def _load_dataset(self): - print(f"Loading GSM8K dataset (split: {self.split})...") - from datasets import load_dataset - - cache_path = cache_dir / "openai___gsm8k" / "default" / "0.0.0" - if cache_path.exists(): - print(f"Using cached dataset from {cache_path}") - ds = load_dataset("openai/gsm8k", "main", split=self.split, cache_dir=str(cache_path)) - else: - ds = load_dataset("openai/gsm8k", "main", split=self.split) - - self.questions = [] - for row in ds: - question = dict(row) - question["dataset_type"] = "gsm8k" - - # Extract numeric answer from the answer field (already has #### prefix) - gold = question["answer"] - # Split by #### and take the last part - parts = gold.split("####") - if len(parts) > 1: - gold = parts[-1].strip() - # Extract the first number from the remaining text - normalized = normalize_number(gold) - question["gold"] = str(normalized) if normalized is not None else gold - - self.questions.append(question) - - print(f"GSM8K dataset loaded: {len(self.questions)} questions") - - def get_question(self, index: int) -> Dict: - """Get question by index""" - return self.questions[index] - - def get_question_text(self, question: Dict) -> str: - """Get question string""" - return question["problem"] if "problem" in question else question["question"] - - def get_answer(self, question: Dict) -> str: - # GSM8K has pre-extracted gold field, AIME uses answer field - if "gold" in question: - return question["gold"] - answer = question["answer"] - if isinstance(answer, str): - normalized = normalize_number(answer) - return str(normalized) if normalized is not None else answer - return str(answer) - - def get_prompt(self, question: Dict) -> str: - """Get formatted prompt for the question""" - return TEMPLATE_REGISTRY[question["dataset_type"]].format( - question=self.get_question_text(question), - ) - -class GpqaDataset(BaseDataset): - def __init__(self, variant: str = "diamond", seed: int = 1234): - self.variant = variant - self.seed = seed - self.questions = [] - self._load_dataset() - - def _load_dataset(self): - print(f"Loading GPQA dataset (variant: {self.variant})...") - import pandas as pd - - url = f"https://openaipublic.blob.core.windows.net/simple-evals/gpqa_{self.variant}.csv" - df = pd.read_csv(url) - - rng = random.Random(self.seed) - - self.questions = [] - for _, row in df.iterrows(): - question = row.to_dict() - question["dataset_type"] = "gpqa" - - # Shuffle the answer options - correct_answer = question["Correct Answer"] - incorrect_answers = [ - question["Incorrect Answer 1"], - question["Incorrect Answer 2"], - question["Incorrect Answer 3"] - ] - - # Create list of (answer, is_correct) tuples - options = [(ans, ans == correct_answer) for ans in incorrect_answers] - options.append((correct_answer, True)) - - # Shuffle the options - rng.shuffle(options) - - # Extract shuffled answers and determine correct letter - shuffled_answers = [ans for ans, _ in options] - correct_letter = chr(ord('A') + options.index((correct_answer, True))) - - # Store shuffled answers and correct letter - question["shuffled_answers"] = shuffled_answers - question["correct_letter"] = correct_letter - - self.questions.append(question) - - print(f"GPQA dataset loaded: {len(self.questions)} questions") - - def get_question(self, index: int) -> Dict: - """Get question by index""" - return self.questions[index] - - def get_question_text(self, question: Dict) -> str: - """Get question string""" - return question["Question"] - - def get_answer(self, question: Dict) -> str: - # GPQA returns the correct letter (A, B, C, or D) - return question["correct_letter"] - - def get_prompt(self, question: Dict) -> str: - """Get formatted prompt for the question""" - return TEMPLATE_REGISTRY["gpqa"].format( - Question=self.get_question_text(question), - A=question["shuffled_answers"][0], - B=question["shuffled_answers"][1], - C=question["shuffled_answers"][2], - D=question["shuffled_answers"][3] - ) - -class Grader: - def __init__( - self, - grader_type: str = "llm", - grader_script: Optional[str] = None, - grader_model_name: Optional[str] = None, - grader_server_url: str = "", - dataset_type: str = "aime" - ): - self.grader_type = grader_type - self.grader_script = grader_script - self.grader_model_name = grader_model_name - self.grader_server_url = grader_server_url - self.dataset_type = dataset_type - self.pattern = self._get_pattern() - - def _get_pattern(self) -> Optional[str]: - if self.grader_type == "regex": - return GRADER_PATTERNS.get(self.dataset_type) # Use dataset_type as key - return None - - def _extract_answer_regex(self, pred: str) -> Optional[str]: - """Extract answer using regex pattern""" - if not self.pattern: - return None - - # For AIME datasets, prioritize boxed answers - if self.dataset_type in ["aime", "aime2025"]: - boxed_pattern = r'\\boxed{([^}]+)}' - boxed_matches = re.findall(boxed_pattern, pred, re.IGNORECASE) - if boxed_matches: - # Return the last boxed answer found (most likely the final answer) - return boxed_matches[-1].strip() - - # For other datasets, search for numbers from the end of the text - # This prioritizes numbers that appear later in the response - matches = re.findall(self.pattern, pred, re.IGNORECASE) - if not matches: - return None - - # Process matches from end to start - for match in reversed(matches): - if isinstance(match, tuple): - match = match[0] if match[0] else match[1] - answer = match.strip() - if answer: - return answer - return None - - def _grade_regex(self, gold: str, pred: str) -> Tuple[bool, Optional[str]]: - """Grade using regex pattern matching""" - answer = self._extract_answer_regex(pred) - if answer is None: - return False, None - is_correct = answer.strip() == gold.strip() - return is_correct, answer - - def _grade_cli(self, gold: str, pred: str) -> Tuple[bool, Optional[str]]: - """Grade using external CLI script""" - if not self.grader_script: - raise ValueError("CLI grader requires --grader-script") - - script_path = Path(self.grader_script) - if not script_path.exists(): - raise FileNotFoundError(f"Grader script not found: {self.grader_script}") - - try: - result = subprocess.run( - [str(script_path), "--answer", pred, "--expected", gold], - capture_output=True, - text=True, - timeout=30 - ) - is_correct = result.returncode == 0 - answer = pred if is_correct else None - return is_correct, answer - except subprocess.TimeoutExpired: - return False, None - except Exception as e: - return False, None - - def _grade_llm(self, gold: str, pred: str, problem: str) -> Tuple[bool, Optional[str]]: - """Grade using LLM-based extraction with few-shot examples""" - sample_answers = SAMPLE_ANSWERS.get(self.dataset_type, []) - sample_examples = "\n".join([ - f"Example {i+1}: {ans}" for i, ans in enumerate(sample_answers) - ]) - - system_prompt = f"""You are an answer extraction system. Your task is to extract the answer from the model's response. - -Here are some examples of extracted answers to demonstrate what you are supposed to output: - -{sample_examples} - -When extracting the answer, provide only the extracted answer itself, nothing else. If there is no clear answer that can be extracted from the response, reply with 'no answer'.""" - - user_prompt = f"""Extract the answer from the following response: - -"{pred}" - -Please provide only the extracted answer, nothing else. If there is no clear answer that can be extracted from the response, reply with 'no answer'.""" - - url = f"{self.grader_server_url}/v1/chat/completions" - headers = {"Content-Type": "application/json"} - data = { - "model": self.grader_model_name, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt} - ], - "temperature": 0, - } - #print(json.dumps(data, indent=2)) - - try: - response = requests.post(url, headers=headers, json=data) - response.raise_for_status() - answer = response.json()["choices"][0]["message"]["content"].strip() - is_correct = answer.strip().lower() == gold.strip().lower() - return is_correct, answer - except Exception as e: - return False, None - - def _truncate_response(self, response: str, max_lines: int = 6) -> str: - """Keep only last N lines of response""" - lines = response.split('\n') - return '\n'.join(lines[-max_lines:]) if len(lines) > max_lines else response - - def grade(self, gold: str, pred: str, problem: str = "") -> Tuple[bool, Optional[str]]: - """Grade the response""" - if self.grader_type == "regex": - return self._grade_regex(gold, pred) - elif self.grader_type == "cli": - return self._grade_cli(gold, pred) - elif self.grader_type == "llm": - return self._grade_llm(gold, pred, problem) - else: - raise ValueError(f"Unknown grader type: {self.grader_type}") - -class Processor: - def __init__( - self, - server_configs: List[ServerConfig], - grader: Grader, - model_name: Optional[str] = None, - n_predict: int = -1 - ): - self.server_configs = server_configs - self.grader = grader - self.model_name = model_name - self.n_predict = n_predict - - @staticmethod - def _check_server(server_config: ServerConfig) -> List[str]: - url = f"{server_config.url}/v1/models" - try: - response = requests.get(url) - response.raise_for_status() - models = [m["id"] for m in response.json().get("data", [])] - return models - except Exception as e: - print(f"Error: Cannot reach server {server_config.name} ({server_config.url}): {e}", file=sys.stderr) - sys.exit(1) - - def _make_request( - self, server_config: ServerConfig, eval_state: EvalState, prompt: str - ) -> Tuple[Dict[str, Any], int, Optional[float], Optional[float], str]: - url = f"{server_config.url}/v1/chat/completions" - headers = {"Content-Type": "application/json"} - data = { - "model": self.model_name if self.model_name else "llama", - "messages": [{"role": "user", "content": prompt}], - "n_predict": self.n_predict - } - if eval_state.sampling_config.get("temperature") is not None: - data["temperature"] = eval_state.sampling_config["temperature"] - if eval_state.sampling_config.get("top_k") is not None: - data["top_k"] = eval_state.sampling_config["top_k"] - if eval_state.sampling_config.get("top_p") is not None: - data["top_p"] = eval_state.sampling_config["top_p"] - if eval_state.sampling_config.get("min_p") is not None: - data["min_p"] = eval_state.sampling_config["min_p"] - - response = requests.post(url, headers=headers, json=data) - response.raise_for_status() - result = response.json() - tokens = result.get("usage", {}).get("completion_tokens", 0) - timings = result.get("timings", {}) - tps_gen = timings.get("predicted_per_second") if timings else None - t_gen_ms = timings.get("predicted_ms") if timings else None - finish_reason = result.get("choices", [{}])[0].get("finish_reason", "stop") - return result, tokens, tps_gen, t_gen_ms, finish_reason - - def _process_single_case( - self, server_config: ServerConfig, eval_state: EvalState, i: int, task_id: str - ) -> TaskState: - question_text, prompt, expected = eval_state.get_case(i) - - # Extract chunk_idx from task_id: "{dataset_type}_{chunk_idx:03d}_{index:03d}" - _parts = task_id.rsplit("_", 2) - chunk_idx = int(_parts[-2]) if len(_parts) >= 3 else 0 - problem_idx = i - - task_state = TaskState( - task_id=task_id, - prompt=prompt, - expected=expected, - question_text=question_text, - server_name=server_config.name, - chunk_idx=chunk_idx, - problem_idx=problem_idx, - ) - - try: - response, tokens, tps_gen, t_gen_ms, finish_reason = self._make_request(server_config, eval_state, prompt) - result = response["choices"][0]["message"]["content"] - reasoning_content = response["choices"][0].get("message", {}).get("reasoning_content") - task_state.response = result - task_state.tokens = tokens - task_state.tps_gen = tps_gen - task_state.t_gen_ms = t_gen_ms - task_state.reasoning_content = reasoning_content - - if finish_reason != "stop": - task_state.status = f"error: finish_reason={finish_reason}" - eval_state.add_result( - task_id, prompt, expected, result, None, - {"finish_reason": finish_reason}, False, task_state.status, - tokens, tps_gen, t_gen_ms, reasoning_content, server_config.name, - chunk_idx, problem_idx, - ) - eval_state.dump() - return task_state - - result_truncated = self.grader._truncate_response(result, max_lines=10) - is_correct, answer = self.grader.grade(expected, result_truncated, prompt) - - grader_log = { - "pred": result_truncated, - "grader_type": self.grader.grader_type - } - if self.grader.grader_type == "regex" and self.grader.pattern: - grader_log["pattern"] = self.grader.pattern - - task_state.correct = is_correct - task_state.answer = answer - task_state.grader_log = grader_log - task_state.status = "ok" - - eval_state.add_result( - task_id, prompt, expected, result, answer, - grader_log, is_correct, "ok", - tokens, tps_gen, t_gen_ms, reasoning_content, server_config.name, - chunk_idx, problem_idx, - ) - - eval_state.dump() - - except Exception as e: - task_state.status = f"error: {str(e)}" - - return task_state - - @staticmethod - def _worker( - server_config: ServerConfig, - processor: "Processor", - eval_state: EvalState, - task_queue: Queue, - results_queue: Queue, - ): - """Worker that pulls tasks from a shared queue and sends them to its server.""" - while True: - task = task_queue.get() - if task is None: # sentinel - task_queue.task_done() - break - try: - i, task_id = task - result = processor._process_single_case(server_config, eval_state, i, task_id) - results_queue.put(result) - finally: - task_queue.task_done() - - def evaluate(self, eval_state: EvalState, verbose: bool = False, resume: bool = False): - total_tasks = len(eval_state.tasks) - eval_state.total = len(eval_state.all_tasks) if eval_state.all_tasks else total_tasks - eval_state.processed = 0 - start_time = time.time() - - # Check servers and list models - server_models = [self._check_server(sc) for sc in self.server_configs] - - # Print server info - print(f"\nProcessing {len(eval_state.tasks)} {eval_state.dataset_type.upper()} tasks ...") - print(f"Servers ({len(self.server_configs)}):") - for i, sc in enumerate(self.server_configs): - models_str = ", ".join(server_models[i]) if server_models[i] else "(none)" - print(f" {i+1}. {sc.name} — {sc.url} ({sc.threads} threads) [{models_str}]") - print(f"Model: {self.model_name}") - print(f"Grader: {self.grader.grader_type}") - print(f"Sampling: temp={eval_state.sampling_config.get('temperature', 'skip')}, top-k={eval_state.sampling_config.get('top_k', 'skip')}, top-p={eval_state.sampling_config.get('top_p', 'skip')}, min-p={eval_state.sampling_config.get('min_p', 'skip')}") - print() - - # Shared task queue: all workers compete for tasks - task_queue: Queue = Queue() - for i, task_id in eval_state.tasks: - task_queue.put((i, task_id)) - - # Results queue: workers push completed TaskStates here - results_queue: Queue = Queue() - - # Total worker threads across all servers - total_threads = sum(sc.threads for sc in self.server_configs) - - # Add one sentinel per worker so every worker exits cleanly - for _ in range(total_threads): - task_queue.put(None) - - # Launch workers: one ThreadPoolExecutor per server - executors: List[ThreadPoolExecutor] = [] - worker_futures: List[Any] = [] - for server_config in self.server_configs: - executor = ThreadPoolExecutor(max_workers=server_config.threads) - executors.append(executor) - for _ in range(server_config.threads): - future = executor.submit( - self._worker, server_config, self, eval_state, - task_queue, results_queue - ) - worker_futures.append(future) - - # Drain results as they complete - n_correct = 0 - session_time = 0.0 - completed_count = 0 - - while completed_count < total_tasks: - task_state = results_queue.get() - eval_state.processed += 1 - completed_count += 1 - if task_state.correct: - n_correct += 1 - elapsed = time.time() - start_time - eval_state.total_time += elapsed - session_time += elapsed - start_time = time.time() - eval_state.print_progress(task_state, total_tasks, n_correct) - - if verbose: - print(f"\nCase {eval_state.processed}: {task_state.correct}") - print(f" Expected: {task_state.expected}") - if task_state.response: - print(f" Response: {task_state.response}") - if task_state.answer: - print(f" Answer: {task_state.answer}") - print(f" Status: {task_state.status}") - - # Wait for all workers to finish and shut down executors - for future in worker_futures: - future.result() - for executor in executors: - executor.shutdown(wait=True) - - print(f"\nSession time: {session_time:.1f}s | Total accumulated time: {eval_state.total_time:.1f}s") - eval_state.print_summary() - eval_state.dump() - -def main(): - parser = argparse.ArgumentParser( - description="Simplified evaluation tool for llama.cpp" - ) - parser.add_argument( - "--server", - type=str, - default="http://localhost:8033", - help="Comma-separated llama-server URLs (default: http://localhost:8033)" - ) - parser.add_argument( - "--server-name", - type=str, - default="", - help="Comma-separated display names for servers (default: use URLs)" - ) - parser.add_argument( - "--dataset", - type=str, - default="aime", - choices=["aime", "aime2025", "aime2026", "gsm8k", "gpqa"], - help="Dataset type (default: aime)" - ) - parser.add_argument( - "--n_cases", - type=int, - default=None, - help="Number of cases to evaluate (default: all)" - ) - parser.add_argument( - "--seed", - type=int, - default=1234, - help="Random seed for shuffling (default: 1234)" - ) - parser.add_argument( - "--n_predict", - type=int, - default=-1, - help="Max tokens to predict per prompt (default: -1, infinite)" - ) - parser.add_argument( - "--temperature", - type=float, - default=None, - help="Sampling temperature (default: not passed)" - ) - parser.add_argument( - "--top-k", - type=int, - default=None, - help="Top K sampling (default: not passed)" - ) - parser.add_argument( - "--top-p", - type=float, - default=None, - help="Top P sampling (default: not passed)" - ) - parser.add_argument( - "--min-p", - type=float, - default=None, - help="Min P sampling (default: not passed)" - ) - parser.add_argument( - "--threads", - type=str, - default="32", - help="Comma-separated thread counts per server (default: 32)" - ) - parser.add_argument( - "--model", - type=str, - default=None, - help="Model name to append as query parameter (e.g., gpt-oss-20b-hf)" - ) - parser.add_argument( - "--verbose", - action="store_true", - help="Show detailed output for each case" - ) - parser.add_argument( - "--output", - type=Path, - default=Path("llama-eval-state.json"), - help="Output file for eval state (default: llama-eval-state.json)" - ) - parser.add_argument( - "--grader-type", - type=str, - default="llm", - choices=["regex", "cli", "llm"], - help="Grader type: regex, cli, or llm (default: llm)" - ) - parser.add_argument( - "--grader-script", - type=str, - default=None, - help="CLI grader script path (required for --grader-type cli)" - ) - parser.add_argument( - "--grader-server", - type=str, - default="", - help="Server URL for LLM grader (default: same as main server)" - ) - parser.add_argument( - "--grader-model", - type=str, - default="", - help="Model name for LLM grader (default: same as main model)" - ) - parser.add_argument( - "--resume", - action="store_true", - help="Resume from existing eval state" - ) - - args = parser.parse_args() - - # Parse server URLs and thread counts - server_urls = [u.strip() for u in args.server.split(",") if u.strip()] - thread_counts = [int(t.strip()) for t in args.threads.split(",") if t.strip()] - - if len(server_urls) != len(thread_counts): - print(f"Error: --server ({len(server_urls)} URLs) and --threads ({len(thread_counts)} values) must have the same count") - sys.exit(1) - - # Parse server names (optional, defaults to URLs) - if args.server_name: - server_names = [n.strip() for n in args.server_name.split(",") if n.strip()] - if len(server_names) != len(server_urls): - print(f"Error: --server-name ({len(server_names)} names) and --server ({len(server_urls)} URLs) must have the same count") - sys.exit(1) - else: - server_names = server_urls # fallback to URLs - - server_configs = [ - ServerConfig(url=url, threads=threads, name=name) - for url, threads, name in zip(server_urls, thread_counts, server_names) - ] - - if args.dataset == "gpqa" and args.grader_type != "llm": - print("Error: GPQA dataset requires --grader-type llm") - parser.print_help() - sys.exit(1) - - if args.output.exists(): - print(f"Loading existing eval state from {args.output}") - eval_state = EvalState.load(args.output) - - # Verify model matches - if eval_state.model_name is not None and args.model != eval_state.model_name: - print(f"Error: Model mismatch. State has '{eval_state.model_name}', but --model is '{args.model}'") - sys.exit(1) - - eval_state.print_all_tasks() - eval_state.print_existing_summary() - - if eval_state.is_complete(): - return - - print() - - if not args.resume: - print(f"Evaluation incomplete. Run with --resume to continue.") - return - - pending_tasks = eval_state.get_pending_tasks() - print(f"Resuming from {len(pending_tasks)} pending tasks") - - existing_cases = eval_state.task_states.get("cases", {}) - - eval_state.tasks = pending_tasks - eval_state.task_states["cases"] = existing_cases - - grader_server_url = args.grader_server if args.grader_server else server_configs[0].url - grader_model_name = args.grader_model if args.grader_model else args.model - if args.grader_type == "llm" and not grader_model_name: - print("Error: --grader-type llm requires --grader-model or --model") - sys.exit(1) - grader = Grader( - grader_type=args.grader_type, - grader_script=args.grader_script, - grader_model_name=grader_model_name, - grader_server_url=grader_server_url, - dataset_type=eval_state.dataset_type - ) - resume = True - else: - if args.resume: - print("Error: No existing eval state found to resume") - sys.exit(1) - - grader_server_url = args.grader_server if args.grader_server else server_configs[0].url - grader_model_name = args.grader_model if args.grader_model else args.model - if args.grader_type == "llm" and not grader_model_name: - print("Error: --grader-type llm requires --grader-model or --model") - sys.exit(1) - - grader = Grader( - grader_type=args.grader_type, - grader_script=args.grader_script, - grader_model_name=grader_model_name, - grader_server_url=grader_server_url, - dataset_type=args.dataset - ) - - if args.grader_type == "llm" and not args.grader_server: - print("Warning: Using same server for LLM grader (no --grader-server specified)") - - sampling_config = {} - if args.temperature is not None: - sampling_config["temperature"] = args.temperature - if args.top_k is not None: - sampling_config["top_k"] = args.top_k - if args.top_p is not None: - sampling_config["top_p"] = args.top_p - if args.min_p is not None: - sampling_config["min_p"] = args.min_p - - eval_state = EvalState( - dataset_type=args.dataset, - sampling_config=sampling_config, - output_file=args.output, - model_name=args.model - ) - eval_state.load_dataset(seed=args.seed) - eval_state.setup_tasks(n_cases=args.n_cases, seed=args.seed) - eval_state.dump() - resume = False - - eval_state.print_all_tasks() - - processor = Processor( - server_configs=server_configs, - grader=grader, - model_name=args.model, - n_predict=args.n_predict - ) - - processor.evaluate(eval_state, verbose=args.verbose, resume=resume) - print(f"\nEval state dumped to {args.output}") - -if __name__ == "__main__": - main() diff --git a/examples/llama-eval/llama-server-simulator.py b/examples/llama-eval/llama-server-simulator.py deleted file mode 100644 index e64ba8933..000000000 --- a/examples/llama-eval/llama-server-simulator.py +++ /dev/null @@ -1,376 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json -import random -import re -import time -import sys -import os -import threading -from http.server import HTTPServer, BaseHTTPRequestHandler -from typing import Dict, List, Optional -from dataclasses import dataclass -from pathlib import Path - -import datasets - -# Set cache directory for HuggingFace datasets -cache_dir = Path.home() / ".cache" / "huggingface" / "datasets" -cache_dir.mkdir(parents=True, exist_ok=True) -os.environ["HF_DATASETS_CACHE"] = str(cache_dir) - -def dice(s1: str, s2: str) -> float: - """Calculate Dice coefficient between two strings based on bigram overlap.""" - if not s1 and not s2: - return 1.0 - - def _bigrams(s: str): - return [s[i : i + 2] for i in range(len(s) - 1)] - - bigrams1 = _bigrams(s1) - bigrams2 = _bigrams(s2) - - if not bigrams1 and not bigrams2: - return 1.0 - - from collections import Counter - - freq1 = Counter(bigrams1) - freq2 = Counter(bigrams2) - - intersection = sum(min(freq1[bg], freq2[bg]) for bg in freq1) - dice_coeff = 2 * intersection / (len(bigrams1) + len(bigrams2)) - return dice_coeff - -def debug_log(message: str): - """Log debug messages to both stdout and a file""" - print(message, file=sys.stderr) - with open("/tmp/simulator-debug.log", "a") as f: - f.write(message + "\n") - -simulator: Optional["Simulator"] = None - -@dataclass -class EvalState: - id: str - tasks: List[str] - task_states: Dict[str, Dict] - sampling_config: Dict - -def normalize_number(s: str) -> Optional[int]: - match = re.match(r"\d+", s) # match digits from the start - if not match: - return None - return int(match.group(0)) - -class AimeDataset: - def __init__(self, split: str = "train", dataset_type: str = "aime"): - self.split = split - self.dataset_type = dataset_type - self.questions: List[Dict] = [] - self._load_dataset() - - def _get_question_text(self, question: Dict) -> str: - """Get question text, handling different dataset field names.""" - return question.get("problem", question.get("question", "")) - - def _load_dataset(self): - if self.dataset_type == "aime": - print(f"Loading AIME dataset (split: {self.split})...") - cache_path = Path.home() / ".cache" / "huggingface" / "datasets" / "AI-MO___aimo-validation-aime" / "default" / "0.0.0" - if cache_path.exists(): - print(f"Using cached dataset from {cache_path}") - ds = datasets.load_dataset("AI-MO/aimo-validation-aime", split=self.split, cache_dir=str(cache_path)) - else: - ds = datasets.load_dataset("AI-MO/aimo-validation-aime", split=self.split) - elif self.dataset_type == "aime2025": - print(f"Loading AIME2025 dataset...") - ds_list = [] - for config_name in ["AIME2025-I", "AIME2025-II"]: - cache_path = Path.home() / ".cache" / "huggingface" / "datasets" / "opencompass___AIME2025" / "default" / "0.0.0" - if cache_path.exists(): - print(f"Using cached dataset from {cache_path}") - ds = datasets.load_dataset("opencompass/AIME2025", config_name, split="test", cache_dir=str(cache_path)) - else: - ds = datasets.load_dataset("opencompass/AIME2025", config_name, split="test") - ds_list.extend(ds) - ds = ds_list - else: - raise ValueError(f"Unknown dataset type: {self.dataset_type}") - - self.questions = list(ds) - print(f"{self.dataset_type} dataset loaded: {len(self.questions)} questions") - - def find_question(self, request_text: str) -> Optional[Dict]: - # Strip common template prefixes to get the actual question text - # Templates include things like "Solve the following math problem step by step..." - # The actual question usually follows a blank line or after the template instruction - cleaned = request_text - # Split on double newline and take the part that looks like the problem - parts = cleaned.split('\n\n') - if len(parts) > 1: - # Find the part that's longest (likely the actual problem text) - problem_parts = [p for p in parts if len(p.strip()) > 100] - if problem_parts: - cleaned = max(problem_parts, key=lambda x: len(x)) - - best_match = None - best_distance = -1 - best_index = -1 - - for i, question in enumerate(self.questions): - question_text = self._get_question_text(question) - request_lower = cleaned.lower() - question_lower = question_text.lower() - - # Check if question text is contained in the cleaned request - if question_lower in request_lower or request_lower in question_lower: - debug_log(f"DEBUG: Found substring match at index {i}") - return question - - # Exact match - if question_lower == request_lower: - debug_log(f"DEBUG: Found exact match at index {i}") - return question - - # Remove LaTeX formatting for more flexible matching - question_no_latex = re.sub(r'\$[^$]+\$', '', question_text) - if question_no_latex.lower() == request_lower: - debug_log(f"DEBUG: Found match (no LaTeX) at index {i}") - return question - - # Calculate Dice coefficient for partial matches - # Only consider if request is at least 50% of question length - if len(request_lower) >= len(question_lower) * 0.5: - distance = dice(question_lower, request_lower) - - if distance > best_distance: - best_distance = distance - best_match = question - best_index = i - - if best_match and best_distance > 0.3: # Threshold for partial match - debug_log(f"DEBUG: Found best partial match at index {best_index} with distance {best_distance:.3f}") - return best_match - - debug_log(f"DEBUG: No matching question found for cleaned: {cleaned[:100]}...") - return None - - def get_answer(self, question: Dict) -> str: - answer = question["answer"] - if isinstance(answer, str): - normalized = normalize_number(answer) - return str(normalized) if normalized is not None else answer - return str(answer) - -class Simulator: - def __init__( - self, - port: int = 8033, - host: str = "localhost", - success_rate: float = 0.8, - dataset_split: str = "train", - dataset_type: str = "aime" - ): - self.port = port - self.host = host - self.success_rate = success_rate - self.dataset = AimeDataset(dataset_split, dataset_type) - self.eval_state = EvalState( - id=dataset_type, - tasks=[dataset_type], - task_states={}, - sampling_config={"temperature": 0, "max_tokens": 2048} - ) - - def _generate_response( - self, - question: Dict, - should_be_correct: bool - ) -> Dict: - expected_answer = self.dataset.get_answer(question) - - if should_be_correct: - response_text = expected_answer - else: - response_text = self._generate_wrong_answer(question) - - comp_tokens = random.randint(10000, 60000) - tps_gen = random.uniform(90.0, 110.0) - t_gen_ms = comp_tokens / tps_gen * 1000 - - return { - "id": f"chatcmpl-{int(time.time())}", - "object": "chat.completion", - "created": int(time.time()), - "model": "llama", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": response_text - }, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 100, - "completion_tokens": comp_tokens, - "total_tokens": 100 + comp_tokens - }, - "timings": { - "predicted_ms": t_gen_ms, - "predicted_per_second": tps_gen - } - } - - def _generate_wrong_answer(self, question: Dict) -> str: - expected_answer = self.dataset.get_answer(question) - - if expected_answer.isdigit(): - wrong_answer = str(int(expected_answer) + 1) - else: - wrong_answer = expected_answer + " (wrong)" - - return wrong_answer - - def _process_request(self, request_data: Dict) -> Dict: - messages = request_data.get("messages", []) - if not messages: - return {"error": "No messages in request"} - - request_text = messages[0].get("content", "") - debug_log(f"DEBUG: Received request with content: {request_text[:150]}...") - - question = self.dataset.find_question(request_text) - if not question: - debug_log(f"DEBUG: find_question returned None") - return {"error": "No matching question found"} - - should_be_correct = random.random() < self.success_rate - - response = self._generate_response(question, should_be_correct) - - task_id = "aime" - self.eval_state.task_states[task_id] = { - "correct": should_be_correct, - "expected": self.dataset.get_answer(question), - "predicted": response["choices"][0]["message"]["content"] - } - - return response - -class RequestHandler(BaseHTTPRequestHandler): - def do_GET(self): - if self.path == "/v1/models": - self._send_json({"data": [{"id": "llama", "object": "model"}]}, 200) - return - self._send_json({"error": "Not found"}, 404) - - def do_POST(self): - if self.path != "/v1/chat/completions": - self._send_json({"error": "Not found"}, 404) - return - - try: - content_length = int(self.headers.get("Content-Length", 0)) - body = self.rfile.read(content_length) - request_data = json.loads(body) if body else None - - if not request_data: - self._send_json({"error": "Invalid JSON"}, 400) - return - - if simulator is None: - self._send_json({"error": "Simulator not initialized"}, 500) - return - - response = simulator._process_request(request_data) - self._send_json(response, 200) - - except json.JSONDecodeError: - self._send_json({"error": "Invalid JSON"}, 400) - except Exception as e: - print(f"Error processing request: {e}") - self._send_json({"error": str(e)}, 500) - - def _send_json(self, data: dict, status: int = 200): - body = json.dumps(data).encode("utf-8") - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format, *args): - # Suppress default request logging - pass - - -def main(): - parser = argparse.ArgumentParser( - description="llama-server simulator for testing eval scripts" - ) - parser.add_argument( - "--port", - type=int, - default=8033, - help="Server port (default: 8033)" - ) - parser.add_argument( - "--host", - type=str, - default="localhost", - help="Server host (default: localhost)" - ) - parser.add_argument( - "--success-rate", - type=float, - default=0.8, - help="Success rate 0-1 (default: 0.8)" - ) - parser.add_argument( - "--dataset", - type=str, - default="aime", - choices=["aime", "aime2025"], - help="Dataset type (default: aime)" - ) - parser.add_argument( - "--dataset-split", - type=str, - default="train", - help="AIME dataset split to use (default: train)" - ) - - args = parser.parse_args() - - global simulator - simulator = Simulator( - port=args.port, - host=args.host, - success_rate=args.success_rate, - dataset_split=args.dataset_split, - dataset_type=args.dataset - ) - - server = HTTPServer((args.host, args.port), RequestHandler) - server_thread = threading.Thread(target=server.serve_forever, daemon=True) - server_thread.start() - - print("\n=== llama-server-simulator ===") - print(f"Server running on http://{args.host}:{args.port}") - print(f"Success rate: {args.success_rate}") - print(f"{args.dataset} dataset loaded: {len(simulator.dataset.questions)} questions") - print("\nPress Ctrl+C to stop\n") - - try: - server_thread.join() - except KeyboardInterrupt: - print("\nShutting down...") - server.shutdown() - -if __name__ == "__main__": - main() diff --git a/examples/llama-eval/test-simulator.sh b/examples/llama-eval/test-simulator.sh deleted file mode 100644 index f3ddf3e95..000000000 --- a/examples/llama-eval/test-simulator.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -set -e - -# Get the directory where this script is located -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -echo "=== llama-server-simulator Test Script ===" -echo "" - -PORT=8033 -SUCCESS_RATE=0.8 -TEST_PORT=8034 - -echo "Starting simulator on port $PORT with success rate $SUCCESS_RATE..." -source "$SCRIPT_DIR/venv/bin/activate" -python3 "$SCRIPT_DIR/llama-server-simulator.py" --port $PORT --success-rate $SUCCESS_RATE > /tmp/simulator-test.log 2>&1 & -SIMULATOR_PID=$! - -echo "Waiting for simulator to start..." -sleep 5 - -# Helper function to make a request and extract the answer -make_request() { - local question="$1" - curl -s -X POST http://localhost:$PORT/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d "{ - \"model\": \"llama\", - \"messages\": [ - {\"role\": \"user\", \"content\": \"$question\"} - ], - \"temperature\": 0, - \"max_tokens\": 2048 - }" | python3 -c "import sys, json; data = json.load(sys.stdin); print(data.get('choices', [{}])[0].get('message', {}).get('content', data.get('error', 'No response')))" -} - -# Test question (repeated in multiple tests) -TEST_QUESTION="Quadratic polynomials P(x) and Q(x) have leading coefficients 2 and -2, respectively. The graphs of both polynomials pass through the two points (16,54) and (20,53). Find P(0) + Q(0)." - -echo "" -echo "=== Test 1: Correct Answer ===" -echo "Sending request with known question..." -answer=$(make_request "$TEST_QUESTION") -echo "Answer: $answer" -echo "Expected: 116" -echo "Correct: $([ "$answer" == "116" ] && echo "Yes" || echo "No")" - -echo "" -echo "=== Test 2: Wrong Answer ===" -echo "Sending request with known question (success rate 0.0)..." -answer=$(make_request "$TEST_QUESTION") -echo "Answer: $answer" -echo "Expected: 116" -echo "Correct: $([ "$answer" == "116" ] && echo "Yes" || echo "No")" - -echo "" -echo "=== Test 3: No Matching Question ===" -echo "Sending request with non-matching text..." -response=$(make_request "What is the capital of France?") -echo "Response: $response" -echo "Expected: No matching question found" -echo "Correct: $([ "$response" == "No matching question found" ] && echo "Yes" || echo "No")" - -echo "" -echo "=== Test 4: Success Rate Verification ===" -echo "Sending 10 requests to test success rate..." -correct_count=0 -for i in {1..10}; do - answer=$(make_request "$TEST_QUESTION") - if [ "$answer" == "116" ]; then - correct_count=$((correct_count + 1)) - fi - echo " Request $i: Answer = $answer" -done -echo "Correct answers: $correct_count/10" -echo "Expected: ~8/10 (80% success rate)" -echo "Success rate: $(echo "scale=1; $correct_count * 10" | bc)%" - -echo "" -echo "=== Test Complete ===" -echo "Stopping simulator..." -kill $SIMULATOR_PID 2>/dev/null -wait $SIMULATOR_PID 2>/dev/null || true - -echo "Simulator stopped." diff --git a/examples/test-cmake/.gitignore b/examples/test-cmake/.gitignore deleted file mode 100644 index b630ddb7d..000000000 --- a/examples/test-cmake/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -llama-build-install -install -build -build-subdir diff --git a/examples/test-cmake/CMakeLists.txt b/examples/test-cmake/CMakeLists.txt deleted file mode 100644 index 6ceb3359e..000000000 --- a/examples/test-cmake/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -cmake_minimum_required(VERSION 3.14) -project(llama-simple) - -set(CMAKE_CXX_STANDARD 17) - -option(LLAMA_TEST_USE_SUBDIR "Use add_subdirectory instead of find_package" OFF) - -if(LLAMA_TEST_USE_SUBDIR) - add_subdirectory(../../ llama.cpp) -else() - find_package(llama 0.1.0 REQUIRED) -endif() - -add_executable(test-cmake test-cmake.cpp) -target_link_libraries(test-cmake PRIVATE llama) - -if(DEFINED LLAMA_BUILD_NUMBER) - target_compile_definitions(test-cmake PRIVATE - LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER} - LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}" - ) -endif() diff --git a/examples/test-cmake/README.md b/examples/test-cmake/README.md deleted file mode 100644 index 03895abfb..000000000 --- a/examples/test-cmake/README.md +++ /dev/null @@ -1,45 +0,0 @@ -## cmake-test - -This is just for manually testing/developing of a llama.cpp installation to -enable troubleshooting issues and exploration. The idea is that this can be used -after making changes to llama.cpp installation cmake configuration and then -verify it locally. - -### find_package -The following will configure, build, and install llama.cpp, and the build a -project that uses find_package to use the installation. - -Configuring/build/install: -```console -./build-install.sh -``` -The above command will create a directory named `install` in the current directory -which will have the following files in its lib directory: -```console -$ ls install/lib/ -cmake libggml.so libllama-common.so.0 libllama.so.0.1.0 llama.cpp -libggml-base.so libggml.so.0 libllama-common.so.0.1.0 libmtmd.so pkgconfig -libggml-base.so.0 libggml.so.0.19.0 libllama.so libmtmd.so.0 -libggml-base.so.0.19.0 libllama-common.so libllama.so.0 libmtmd.so.0.1.0 -``` - -Build/run this project using the installation created above: -```console -$ ./build.sh --- Configuring done (0.0s) --- Generating done (0.0s) --- Build files have been written to: /path/to/llama.cpp/examples/test-cmake/build -[100%] Built target test-cmake -[test-cmake] Using llama.cpp version 0.1.0-dev-b10335 -[test-cmake] Initializing backend... -load_backend: loaded CPU backend from /path/to/llama.cpp/examples/test-cmake/install/lib/llama.cpp/libggml-cpu-alderlake.so -[test-cmake] Backend initialized. -``` - -### add_subdirectory -The following will use add_subdirectory to include llama.cpp in a cmake project -and is intended to simulate projects that build llama.cpp in this way. - -```console -$ USE_SUBDIR=ON ./build.sh -``` diff --git a/examples/test-cmake/build-install.sh b/examples/test-cmake/build-install.sh deleted file mode 100755 index 77a6713d6..000000000 --- a/examples/test-cmake/build-install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e - -rm -rf llama-build-install install - -cmake --fresh -S ../../. -B llama-build-install -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=ON \ - -DGGML_BACKEND_DL=ON \ - -DGGML_CPU_ALL_VARIANTS=ON \ - -DLLAMA_TESTS_INSTALL=OFF \ - -DCMAKE_INSTALL_PREFIX="${PWD}/install" \ - -DGGML_BACKEND_DIR="${PWD}/install/lib/llama.cpp" \ - -DGGML_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \ - -DLLAMA_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \ - -DLLAMA_TOOLS_INSTALL=OFF - -cmake --build llama-build-install --parallel 12 -cmake --install llama-build-install diff --git a/examples/test-cmake/build.sh b/examples/test-cmake/build.sh deleted file mode 100755 index 869a64160..000000000 --- a/examples/test-cmake/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -if [ "${USE_SUBDIR:-OFF}" = "ON" ]; then - BUILD_DIR="build-subdir" - CMAKE_ARGS="-DLLAMA_TEST_USE_SUBDIR=ON -DLLAMA_BUILD_COMMON=ON -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON-DLLAMA_BUILD_TESTS=ON" - LIB_PATH="${PWD}/${BUILD_DIR}/bin" -else - BUILD_DIR="build" - CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PWD}/install" - LIB_PATH="${PWD}/install/lib/llama.cpp" -fi - -cmake --fresh -S . -B "${BUILD_DIR}" ${CMAKE_ARGS} -cmake --build "${BUILD_DIR}" -j 8 - -LD_LIBRARY_PATH="${LIB_PATH}:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "./${BUILD_DIR}/test-cmake" diff --git a/examples/test-cmake/test-cmake.cpp b/examples/test-cmake/test-cmake.cpp deleted file mode 100644 index fea27c7e8..000000000 --- a/examples/test-cmake/test-cmake.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include "llama.h" -#include - -int main(void) { -#ifdef LLAMA_BUILD_NUMBER - printf("[test-cmake] llama.cpp version: %s, build: %d (%s)\n", - llama_version(), LLAMA_BUILD_NUMBER, LLAMA_BUILD_COMMIT); -#else - printf("[test-cmake] llama.cpp version: %s\n", llama_version()); -#endif - printf("[test-cmake] ggml version: %s, commit: %s\n", ggml_version(), ggml_commit()); - printf("[test-cmake] Initializing backend...\n"); - llama_backend_init(); - printf("[test-cmake] Backend initialized.\n"); - llama_backend_free(); - return 0; -}