mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-10-22 06:09:59 +00:00
1242be66f6
* server: add realtime transcription API * drop transcription_sessions * proxy ws * add docs * server: let the API key middleware guard the websocket httplib runs the pre-routing handler before the websocket upgrade, so the API key middleware answers 401 at handshake like any other route. This covers the router proxy, whose children run keyless, and accepts the same key forms as HTTP (Bearer, raw key, X-Api-Key). * server: keep realtime slots out of the prompt loop A realtime slot waiting for audio is GENERATING with nothing to decode, so it fell through the prompt loop and took slot_batched. A request on a higher slot then failed can_batch_with on the task type and starved for the whole session. Realtime slots have no prompt: the loop sets them up once and always leaves them. * server: start the realtime session once its slot is live The audio stream was created in the prompt loop, which skips a slot that cannot batch with the one generating, and session.created went out before the task was posted: early appends failed and the reader quit, leaving the socket open and the slot held until the HTTP timeout. The slot now sets up its stream at launch and signals it with the begin result, session.created waits for it, and a reader that stops closes the socket so the session ends with it. * server: name the committed item in input_audio_buffer.committed Per the Realtime spec, item_id is the item made from the committed audio and previous_item_id the one committed before it, null for the first. committed now comes before the transcript that completes that item, and both go out with the item switch under one lock, so a delta cannot land between them. * server: use the GA transcription session shape The session object follows the GA Realtime schema: type transcription, with the input format, transcription model, turn detection and noise reduction under audio.input. The rate stays the model sample rate, which is what the client must send for now. --------- Co-authored-by: Pascal <admin@serveurperso.com>
117 lines
3.8 KiB
C++
117 lines
3.8 KiB
C++
#pragma once
|
|
|
|
#include <atomic>
|
|
#include <functional>
|
|
#include <map>
|
|
#include <memory>
|
|
#include <string>
|
|
#include <thread>
|
|
#include <vector>
|
|
#include <cstdint>
|
|
#include <unordered_map>
|
|
|
|
struct common_params;
|
|
|
|
// generator-like API for HTTP response generation
|
|
// this object response with one of the 2 modes:
|
|
// 1) normal response: `data` contains the full response body
|
|
// 2) streaming response: each call to next(output) generates the next chunk
|
|
// when next(output) returns false, no more data after the current chunk
|
|
// note: some chunks can be empty, in which case no data is sent for that chunk
|
|
struct server_http_res {
|
|
std::string content_type = "application/json; charset=utf-8";
|
|
int status = 200;
|
|
std::string data;
|
|
std::map<std::string, std::string> headers;
|
|
|
|
std::function<bool(std::string &)> next = nullptr;
|
|
bool is_stream() const {
|
|
return next != nullptr;
|
|
}
|
|
|
|
// fired before req and res are destroyed
|
|
virtual void on_complete() {}
|
|
|
|
virtual ~server_http_res() = default;
|
|
};
|
|
|
|
// unique pointer, used by set_chunked_content_provider
|
|
// httplib requires the stream provider to be stored in heap
|
|
using server_http_res_ptr = std::unique_ptr<server_http_res>;
|
|
using raw_buffer = std::vector<uint8_t>;
|
|
|
|
struct uploaded_file {
|
|
raw_buffer data;
|
|
std::string filename;
|
|
std::string content_type;
|
|
};
|
|
|
|
struct server_http_req {
|
|
std::map<std::string, std::string> params; // path_params + query_params
|
|
std::map<std::string, std::string> headers; // used by MCP proxy
|
|
std::string path;
|
|
std::string query_string; // query parameters string (e.g. "action=save")
|
|
std::string body;
|
|
std::map<std::string, uploaded_file> files; // used for file uploads (form data)
|
|
const std::function<bool()> & should_stop;
|
|
|
|
std::string get_param(const std::string & key, const std::string & def = "") const {
|
|
auto it = params.find(key);
|
|
if (it != params.end()) {
|
|
return it->second;
|
|
}
|
|
return def;
|
|
}
|
|
};
|
|
|
|
struct server_http_context {
|
|
class Impl;
|
|
std::unique_ptr<Impl> pimpl;
|
|
|
|
std::atomic<bool> is_ready = false;
|
|
|
|
// note: the handler should never throw exceptions
|
|
using handler_t = std::function<server_http_res_ptr(const server_http_req & req)>;
|
|
mutable std::unordered_map<std::string, handler_t> handlers;
|
|
|
|
// websocket connection, see ws()
|
|
struct ws_conn {
|
|
// blocking read, returns 1 for a text message, 2 for a binary message, 0 when the connection is closed
|
|
virtual int read(std::string & msg) = 0;
|
|
// thread-safe, can be called while another thread is in read()
|
|
virtual bool send(const std::string & msg) = 0;
|
|
virtual bool send_binary(const std::string & msg) = 0;
|
|
virtual bool is_open() const = 0;
|
|
virtual void close() = 0;
|
|
virtual ~ws_conn() = default;
|
|
};
|
|
// the handler runs on its own HTTP thread for the whole lifetime of the connection
|
|
using ws_handler_t = std::function<void(const server_http_req & req, ws_conn & conn)>;
|
|
|
|
std::string path_prefix;
|
|
int port = 9931;
|
|
bool is_ssl = false;
|
|
|
|
server_http_context();
|
|
~server_http_context();
|
|
|
|
bool init(const common_params & params);
|
|
bool start();
|
|
void stop() const;
|
|
void join();
|
|
|
|
void get(const std::string & path, const handler_t & handler) const;
|
|
void post(const std::string & path, const handler_t & handler) const;
|
|
void del(const std::string & path, const handler_t & handler) const;
|
|
void ws(const std::string & path, const ws_handler_t & handler) const;
|
|
|
|
// Register the Google Cloud Platform (Vertex AI) compat (AIP_PREDICT_ROUTE env var, or /predict)
|
|
// Must be called AFTER all other API routes are registered
|
|
void register_gcp_compat() const;
|
|
|
|
// for debugging
|
|
std::vector<std::string> listening_addresses;
|
|
|
|
private:
|
|
bool init_listener(const common_params & params);
|
|
};
|