|
async void | Awake () |
| The Unity Awake function that starts the LLM server. The server can be started asynchronously if the asynchronousStartup option is set.
|
|
async Task | WaitUntilReady () |
|
void | SetModel (string path) |
| Allows to set the model used by the LLM. The model provided is copied to the Assets/StreamingAssets folder that allows it to also work in the build. Models supported are in .gguf format.
|
|
void | SetLora (string path, float weight=1) |
| Allows to set a LORA model to use in the LLM. The model provided is copied to the Assets/StreamingAssets folder that allows it to also work in the build. Models supported are in .gguf format.
|
|
void | AddLora (string path, float weight=1) |
| Allows to add a LORA model to use in the LLM. The model provided is copied to the Assets/StreamingAssets folder that allows it to also work in the build. Models supported are in .gguf format.
|
|
void | RemoveLora (string path) |
| Allows to remove a LORA model from the LLM. Models supported are in .gguf format.
|
|
void | RemoveLoras () |
| Allows to remove all LORA models from the LLM.
|
|
void | SetLoraWeight (string path, float weight) |
| Allows to change the weight (scale) of a LORA model in the LLM.
|
|
void | SetLoraWeights (Dictionary< string, float > loraToWeight) |
| Allows to change the weights (scale) of the LORA models in the LLM.
|
|
void | UpdateLoras () |
|
void | SetTemplate (string templateName, bool setDirty=true) |
| Set the chat template for the LLM.
|
|
void | SetSSLCert (string path) |
| Use a SSL certificate for the LLM server.
|
|
void | SetSSLKey (string path) |
| Use a SSL key for the LLM server.
|
|
string | GetTemplate () |
| Returns the chat template of the LLM.
|
|
int | Register (LLMCharacter llmCharacter) |
| Registers a local LLMCharacter object. This allows to bind the LLMCharacter "client" to a specific slot of the LLM.
|
|
void | Update () |
| The Unity Update function. It is used to retrieve the LLM replies.
|
|
async Task< string > | Tokenize (string json) |
| Tokenises the provided query.
|
|
async Task< string > | Detokenize (string json) |
| Detokenises the provided query.
|
|
async Task< string > | Embeddings (string json) |
| Computes the embeddings of the provided query.
|
|
void | ApplyLoras () |
| Sets the lora scale, only works after the LLM service has started.
|
|
async Task< List< LoraWeightResult > > | ListLoras () |
| Gets a list of the lora adapters.
|
|
async Task< string > | Slot (string json) |
| Allows to save / restore the state of a slot.
|
|
async Task< string > | Completion (string json, Callback< string > streamCallback=null) |
| Allows to use the chat and completion functionality of the LLM.
|
|
async Task | SetBasePrompt (string base_prompt) |
|
void | CancelRequest (int id_slot) |
| Allows to cancel the requests in a specific slot of the LLM.
|
|
void | Destroy () |
| Stops and destroys the LLM.
|
|
void | OnDestroy () |
| The Unity OnDestroy function called when the onbject is destroyed. The function StopProcess is called to stop the LLM server.
|
|
|
bool | advancedOptions = false |
| toggle to show/hide advanced options in the GameObject
|
|
bool | remote = false |
| toggle to enable remote server functionality
|
|
int | port = 13333 |
| port to use for the LLM server
|
|
int | numThreads = -1 |
| number of threads to use (-1 = all)
|
|
int | numGPULayers = 0 |
| number of model layers to offload to the GPU (0 = GPU not used). Use a large number i.e. >30 to utilise the GPU as much as possible. If the user's GPU is not supported, the LLM will fall back to the CPU
|
|
bool | debug = false |
| select to log the output of the LLM in the Unity Editor.
|
|
int | parallelPrompts = -1 |
| number of prompts that can happen in parallel (-1 = number of LLMCharacter objects)
|
|
bool | dontDestroyOnLoad = true |
| select to not destroy the LLM GameObject when loading a new Scene.
|
|
int | contextSize = 8192 |
| Size of the prompt context (0 = context size of the model). This is the number of tokens the model can take as input when generating responses.
|
|
int | batchSize = 512 |
| Batch size for prompt processing.
|
|
string | basePrompt = "" |
| a base prompt to use as a base for all LLMCharacter objects
|
|
string | model = "" |
| the LLM model to use. Models with .gguf format are allowed.
|
|
string | chatTemplate = ChatTemplate.DefaultTemplate |
| Chat template used for the model.
|
|
string | lora = "" |
| the paths of the LORA models being used (relative to the Assets/StreamingAssets folder). Models with .gguf format are allowed.
|
|
string | loraWeights = "" |
| the weights of the LORA models being used.
|
|
bool | flashAttention = false |
| enable use of flash attention
|
|
string | APIKey |
| API key to use for the server (optional)
|
|
string | SSLCertPath = "" |
|
string | SSLKeyPath = "" |
|
Class implementing the LLM server.
Definition at line 18 of file LLM.cs.