LLM_service.h

Go to the documentation of this file.

 
 
#pragma once
 
#include <thread>
#include <condition_variable>
 
#include "LLM.h"
#include "completion_processor.h"
 
#define LLAMALIB_INF(...) LOG_TMPL(GGML_LOG_LEVEL_INFO, -1, __VA_ARGS__)
 
struct common_params;  
struct server_context; 
struct server_http_context; 
struct server_routes;
struct server_http_req;
struct server_http_res;
 
using server_http_res_ptr = std::unique_ptr<server_http_res>;
using handler_t = std::function<server_http_res_ptr(const server_http_req & req)>;
 
class UNDREAMAI_API LLMService : public LLMProvider
{
public:
    LLMService();
 
    LLMService(const std::string &model_path, int num_slots = 1, int num_threads = -1, int num_GPU_layers = 0, bool flash_attention = false, int context_size = 4096, int batch_size = 2048, bool embedding_only = false, const std::vector<std::string> &lora_paths = {});
 
    ~LLMService();
 
    static LLMService *from_params(const json &params_json);
 
    static LLMService *from_command(const std::string &command);
 
    static LLMService *from_command(int argc, char **argv);
 
    static std::vector<char *> jsonToArguments(const json &params_json);
 
    void init(int argc, char **argv);
 
    void init(const std::string &params_string);
 
    void init(const char *params_string);
 
    std::string get_command() { return command; }
 
    std::string encapsulate_route(const json &body, handler_t route_handler);
 
    //=================================== LLM METHODS START ===================================//
 
    void enable_reasoning(bool reasoning) override;
 
    std::string tokenize_json(const json &data) override;
 
    std::string detokenize_json(const json &data) override;
 
    std::string embeddings_json(const json &data) override;
 
    std::string apply_template_json(const json &data) override;
 
    std::string completion_json(const json &data, CharArrayFn callback = nullptr, bool callbackWithJSON = true) override;
 
    std::string slot_json(const json &data) override;
 
    std::string lora_weight_json(const json &data) override;
 
    std::string lora_list_json() override;
 
    void cancel(int id_slot) override;
 
    void start() override;
 
    bool started() override;
 
    void stop() override;
 
    void start_server(const std::string &host = "0.0.0.0", int port = -1, const std::string &API_key = "") override;
 
    void stop_server() override;
 
    void join_service() override;
 
    void join_server() override;
 
    void set_SSL(const std::string &SSL_cert, const std::string &SSL_key) override;
 
    int embedding_size() override;
 
    int get_next_available_slot() override;
 
    void debug(int debug_level) override;
 
    void logging_callback(CharArrayFn callback) override;
 
    std::string debug_implementation() override { return "standalone"; }
    //=================================== LLM METHODS END ===================================//
 
private:
    std::string command = "";             
    common_params *params;                
    bool llama_backend_has_init;          
    server_context *ctx_server = nullptr; 
    server_http_context* ctx_http = nullptr;
    server_routes* routes = nullptr;
 
    std::mutex start_stop_mutex;                
    std::thread service_thread;                 
    std::condition_variable service_stopped_cv; 
    bool service_stopped = false;               
    std::thread server_thread;                  
    std::condition_variable server_stopped_cv;  
    bool server_stopped = false;                
 
    int next_available_slot = 0;
 
    std::vector<std::string> splitArguments(const std::string &inputString);
 
    const std::string detect_chat_template();
 
    server_http_req escape_reasoning(server_http_req prompt);
};
 
 
extern "C"
{
    UNDREAMAI_API void LLMService_Registry(LLMProviderRegistry *existing_instance);
 
    UNDREAMAI_API LLMService *LLMService_Construct(const char *model_path, int num_slots = 1, int num_threads = -1, int num_GPU_layers = 0, bool flash_attention = false, int context_size = 4096, int batch_size = 2048, bool embedding_only = false, int lora_count = 0, const char **lora_paths = nullptr);
 
    UNDREAMAI_API LLMService *LLMService_From_Command(const char *params_string);
 
    UNDREAMAI_API const char *LLMService_Command(LLMService *llm_service);
 
    UNDREAMAI_API void LLMService_InjectErrorState(ErrorState *error_state);
}