LLM_client.h

Go to the documentation of this file.

 
 
#pragma once
 
#include <vector>
#include <thread>
#include <chrono>
#include <iostream>
#include <sstream>
#include "LLM.h"
#include "completion_processor.h"
 
#if TARGET_OS_IOS || TARGET_OS_VISION
    #include "ios_http_transport.h"
#else
    // increase max payload length to allow use of larger context size
    #define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 1048576
    // disable Nagle's algorithm
    #define CPPHTTPLIB_TCP_NODELAY true
    #include "httplib.h"
#endif
 
class UNDREAMAI_API LLMClient : public LLMLocal
{
public:
    LLMClient(LLMProvider *llm);
 
    LLMClient(const std::string &url, const int port, const std::string &API_key = "", const int max_retries = 5);
 
    ~LLMClient();
 
 
    bool is_server_alive();
 
    void set_SSL(const char *SSL_cert);
 
    bool is_remote() const { return !url.empty() && port > -1; }
 
    //=================================== LLM METHODS START ===================================//
 
    std::string tokenize_json(const json &data) override;
 
    std::string detokenize_json(const json &data) override;
 
    std::string embeddings_json(const json &data) override;
 
    std::string completion_json(const json &data, CharArrayFn callback = nullptr, bool callbackWithJSON = true) override;
 
    std::string apply_template_json(const json &data) override;
 
    std::string slot_json(const json &data) override;
 
    void cancel(int id_slot) override;
 
    int get_next_available_slot() override;
 
    int get_slot_context_size() override;
    //=================================== LLM METHODS END ===================================//
 
private:
    // Local LLM members
    LLMProvider *llm = nullptr; 
#if TARGET_OS_IOS || TARGET_OS_VISION
    IOSHttpTransport *transport = nullptr;
#else
    httplib::Client *client = nullptr;
    httplib::SSLClient *sslClient = nullptr;
#endif
    bool use_ssl = false;
 
    // Remote LLM members
    std::string url = "";      
    int port = -1;             
    std::string API_key = "";  
    std::string SSL_cert = ""; 
    int max_retries = 5;
    std::vector<bool*> active_requests;
 
 
    std::string post_request(const std::string &path, const json &payload, CharArrayFn callback = nullptr, bool callbackWithJSON = true);
};
 
 
extern "C"
{
    UNDREAMAI_API bool LLMClient_Is_Server_Alive(LLMClient *llm);
 
    UNDREAMAI_API void LLMClient_Set_SSL(LLMClient *llm, const char *SSL_cert);
 
    UNDREAMAI_API LLMClient *LLMClient_Construct(LLMProvider *llm);
 
    UNDREAMAI_API LLMClient *LLMClient_Construct_Remote(const char *url, const int port, const char *API_key = "");
}