105 public static string LlamaLibReleaseURL = $
"https://github.com/undreamai/LlamaLib/releases/download/{LlamaLibVersion}";
111 public static string LlamaLibURL = $
"{LlamaLibReleaseURL}/{libraryName}.zip";
113 public static string LLMUnityStore = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
"LLMUnity");
119 public static string cacheZipHashPath = cacheZipPath +
".sha256";
125 public static readonly Dictionary<string, (string, string, string)[]>
modelOptions =
new Dictionary<
string, (
string,
string,
string)[]>()
127 {
"Large models (more than 10B)",
new(string, string, string)[]
129 (
"Gemma 3 12B",
"https://huggingface.co/lmstudio-community/gemma-3-12b-it-GGUF/resolve/main/gemma-3-12b-it-Q4_K_M.gguf",
"https://ai.google.dev/gemma/terms"),
130 (
"Phi 4 14B",
"https://huggingface.co/bartowski/phi-4-GGUF/resolve/main/phi-4-Q4_K_M.gguf",
null),
131 (
"Qwen 3 14B",
"https://huggingface.co/unsloth/Qwen3-14B-GGUF/resolve/main/Qwen3-14B-Q4_K_M.gguf",
null),
132 (
"DeepSeek R1 Distill Qwen 14B",
"https://huggingface.co/lmstudio-community/DeepSeek-R1-Distill-Qwen-14B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-14B-Q4_K_M.gguf",
null),
134 {
"Medium models (up to 10B)",
new(string, string, string)[]
136 (
"Qwen 3.5 9B",
"https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/main/Qwen3.5-9B-Q4_K_M.gguf",
null),
137 (
"Llama 3.1 8B",
"https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf",
"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B/blob/main/LICENSE"),
138 (
"DeepSeek R1 Distill Llama 8B",
"https://huggingface.co/lmstudio-community/DeepSeek-R1-Distill-Llama-8B-GGUF/resolve/main/DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf",
null),
139 (
"DeepSeek R1 Distill Qwen 7B",
"https://huggingface.co/lmstudio-community/DeepSeek-R1-Distill-Qwen-7B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M.gguf",
null),
140 (
"Gemma 2 9B it",
"https://huggingface.co/bartowski/gemma-2-9b-it-GGUF/resolve/main/gemma-2-9b-it-Q4_K_M.gguf",
"https://ai.google.dev/gemma/terms"),
141 (
"Mistral 7B Instruct v0.2",
"https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf",
null),
142 (
"OpenHermes 2.5 7B",
"https://huggingface.co/TheBloke/OpenHermes-2.5-Mistral-7B-GGUF/resolve/main/openhermes-2.5-mistral-7b.Q4_K_M.gguf",
null),
144 {
"Small models (up to 5B)",
new(string, string, string)[]
146 (
"Qwen 3.5 4B",
"https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/Qwen3.5-4B-Q4_K_M.gguf",
null),
147 (
"Llama 3.2 3B",
"https://huggingface.co/hugging-quants/Llama-3.2-3B-Instruct-Q4_K_M-GGUF/resolve/main/llama-3.2-3b-instruct-q4_k_m.gguf",
"https://huggingface.co/meta-llama/Llama-3.2-1B/blob/main/LICENSE.txt"),
148 (
"Gemma 3 4B",
"https://huggingface.co/lmstudio-community/gemma-3-4b-it-GGUF/resolve/main/gemma-3-4b-it-Q4_K_M.gguf",
"https://ai.google.dev/gemma/terms"),
149 (
"Phi 4 4B",
"https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF/resolve/main/microsoft_Phi-4-mini-instruct-Q4_K_M.gguf",
null),
151 {
"Tiny models (up to 2B)",
new(string, string, string)[]
153 (
"Qwen 3.5 2B",
"https://huggingface.co/unsloth/Qwen3.5-2B-GGUF/resolve/main/Qwen3.5-2B-Q4_K_M.gguf",
null),
154 (
"Qwen 3.5 0.8B",
"https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q4_K_M.gguf",
null),
155 (
"Llama 3.2 1B",
"https://huggingface.co/hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF/resolve/main/llama-3.2-1b-instruct-q4_k_m.gguf",
"https://huggingface.co/meta-llama/Llama-3.2-1B/blob/main/LICENSE.txt"),
156 (
"Gemma 3 1B",
"https://huggingface.co/lmstudio-community/gemma-3-1b-it-GGUF/resolve/main/gemma-3-1b-it-Q4_K_M.gguf",
"https://ai.google.dev/gemma/terms"),
157 (
"DeepSeek R1 Distill Qwen 1.5B",
"https://huggingface.co/lmstudio-community/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q4_K_M.gguf",
null),
159 {
"RAG models",
new(string, string, string)[]
161 (
"All MiniLM L12 v2",
"https://huggingface.co/leliuga/all-MiniLM-L12-v2-GGUF/resolve/main/all-MiniLM-L12-v2.Q4_K_M.gguf",
null),
162 (
"BGE large en v1.5",
"https://huggingface.co/CompendiumLabs/bge-large-en-v1.5-gguf/resolve/main/bge-large-en-v1.5-q4_k_m.gguf",
null),
163 (
"BGE base en v1.5",
"https://huggingface.co/CompendiumLabs/bge-base-en-v1.5-gguf/resolve/main/bge-base-en-v1.5-q4_k_m.gguf",
null),
164 (
"BGE small en v1.5",
"https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-q4_k_m.gguf",
null),
169 [
LLMUnity]
public static DebugModeType DebugMode = DebugModeType.All;
170 static string DebugModeKey =
"DebugMode";
171 public static bool CUBLAS =
false;
172 static string CUBLASKey =
"CUBLAS";
173 static List<Action<string>> errorCallbacks =
new List<Action<string>>();
174 static readonly
object lockObject =
new object();
175 static Dictionary<string, Task> androidExtractTasks =
new Dictionary<string, Task>();
177 public enum DebugModeType
186 public static void Log(
string message)
188 if ((
int)DebugMode > (
int)DebugModeType.All)
return;
192 public static void LogWarning(
string message)
194 if ((
int)DebugMode > (
int)DebugModeType.Warning)
return;
195 Debug.LogWarning(message);
198 public static void LogError(
string message,
bool throwException =
false)
200 if ((
int)DebugMode <= (
int)DebugModeType.Error || throwException)
202 Debug.LogError(message);
203 foreach (Action<string> errorCallback
in errorCallbacks) errorCallback(message);
205 if (throwException)
throw new LLMUnityException(message);
208 static void LoadPlayerPrefs()
210 DebugMode = (DebugModeType)PlayerPrefs.GetInt(DebugModeKey, (
int)DebugModeType.All);
211 CUBLAS = PlayerPrefs.GetInt(CUBLASKey, 0) == 1;
214 public static void SetDebugMode(DebugModeType newDebugMode)
216 if (DebugMode == newDebugMode)
return;
217 DebugMode = newDebugMode;
218 PlayerPrefs.SetInt(DebugModeKey, (
int)DebugMode);
223 public static void SetCUBLAS(
bool value)
225 if (CUBLAS == value)
return;
227 PlayerPrefs.SetInt(CUBLASKey, value ? 1 : 0);
233 public static string GetAssetPath(
string relPath =
"")
235 string assetsDir = Application.platform == RuntimePlatform.Android ? Application.persistentDataPath : Application.streamingAssetsPath;
236 return Path.Combine(assetsDir, relPath).Replace(
'\\',
'/');
239 public static string GetDownloadAssetPath(
string relPath =
"")
241 string assetsDir = Application.streamingAssetsPath;
243 bool isVisionOS =
false;
244#if UNITY_2022_3_OR_NEWER
245 isVisionOS = Application.platform == RuntimePlatform.VisionOS;
247 if (Application.platform == RuntimePlatform.Android || Application.platform == RuntimePlatform.IPhonePlayer || isVisionOS)
249 assetsDir = Application.persistentDataPath;
251 return Path.Combine(assetsDir, relPath).Replace(
'\\',
'/');
254 static void InitializeOnLoadCommon()
256#if UNITY_EDITOR || !((UNITY_ANDROID || UNITY_IOS || UNITY_VISIONOS))
257 LlamaLib.baseLibraryPath = Path.Combine(
libraryPath, LlamaLib.GetPlatform(),
"native");
262 [InitializeOnLoadMethod]
263 static async Task InitializeOnLoad()
266 LlamaLib.libraryExclusion =
new List<string>(){CUBLAS ?
"tinyblas" :
"cublas"};
267 InitializeOnLoadCommon();
268 await DownloadLibrary();
272 [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.BeforeSceneLoad)]
273 static void InitializeOnLoad()
275 InitializeOnLoadCommon();
280 static Dictionary<string, ResumingWebClient> downloadClients =
new Dictionary<string, ResumingWebClient>();
282 public static void CancelDownload(
string savePath)
284 if (!downloadClients.ContainsKey(savePath))
return;
285 downloadClients[savePath].CancelDownloadAsync();
286 downloadClients.Remove(savePath);
289 public static async Task DownloadFile(
290 string fileUrl,
string savePath,
bool overwrite =
false,
291 Action<string> callback =
null, Action<float> progressCallback =
null,
bool debug =
true
294 if (File.Exists(savePath) && !overwrite)
296 if(debug) Log($
"File already exists at: {savePath}");
300 if(debug) Log($
"Downloading {fileUrl} to {savePath}...");
301 string tmpPath = Path.Combine(Application.temporaryCachePath, Path.GetFileName(savePath));
303 ResumingWebClient client =
new ResumingWebClient();
304 downloadClients[savePath] = client;
305 if (File.Exists(tmpPath) && overwrite) File.Delete(tmpPath);
306 await client.DownloadFileTaskAsyncResume(
new Uri(fileUrl), tmpPath, !overwrite, progressCallback);
307 downloadClients.Remove(savePath);
309 AssetDatabase.StartAssetEditing();
311 Directory.CreateDirectory(Path.GetDirectoryName(savePath));
312 if (File.Exists(savePath)) File.Delete(savePath);
313 File.Move(tmpPath, savePath);
315 AssetDatabase.StopAssetEditing();
317 if(debug) Log($
"Download complete!");
320 progressCallback?.Invoke(1f);
321 callback?.Invoke(savePath);
324 public static async Task AndroidExtractFile(
string assetName,
bool overwrite =
false,
bool log =
true,
int chunkSize = 1024 * 1024)
329 if (!androidExtractTasks.TryGetValue(assetName, out extractionTask))
331#if !UNITY_EDITOR && UNITY_ANDROID
332 extractionTask = AndroidExtractFileOnce(assetName, overwrite, log, chunkSize);
334 extractionTask = Task.CompletedTask;
336 androidExtractTasks[assetName] = extractionTask;
339 await extractionTask;
342 public static async Task AndroidExtractFileOnce(
string assetName,
bool overwrite =
false,
bool log =
true,
int chunkSize = 1024 * 1024)
344 string source =
"jar:file://" + Application.dataPath +
"!/assets/" + assetName;
345 string target = GetAssetPath(assetName);
346 if (!overwrite && File.Exists(target))
348 if (log) Log($
"File {target} already exists");
352 Log($
"Extracting {source} to {target}");
355 UnityWebRequest www = UnityWebRequest.Get(source);
357 var operation = www.SendWebRequest();
359 while (!operation.isDone) await Task.Delay(1);
360 if (www.result != UnityWebRequest.Result.Success)
362 LogError(
"Failed to load file from StreamingAssets: " + www.error);
366 byte[] buffer =
new byte[chunkSize];
367 using (Stream responseStream =
new MemoryStream(www.downloadHandler.data))
368 using (FileStream fileStream =
new FileStream(target, FileMode.Create, FileAccess.Write))
371 while ((bytesRead = await responseStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
373 await fileStream.WriteAsync(buffer, 0, bytesRead);
379 public static async Task AndroidExtractAsset(
string path,
bool overwrite =
false)
381 if (Application.platform != RuntimePlatform.Android)
return;
382 await AndroidExtractFile(Path.GetFileName(path), overwrite);
385 public static string GetFullPath(
string path)
387 return Path.GetFullPath(path).Replace(
'\\',
'/');
390 public static bool IsSubPath(
string childPath,
string parentPath)
392 return GetFullPath(childPath).StartsWith(GetFullPath(parentPath), StringComparison.OrdinalIgnoreCase);
395 public static string RelativePath(
string fullPath,
string basePath)
398 string fullParentPath = GetFullPath(basePath).TrimEnd(
'/');
399 string fullChildPath = GetFullPath(fullPath);
401 string relativePath = fullChildPath;
402 if (fullChildPath.StartsWith(fullParentPath, StringComparison.OrdinalIgnoreCase))
404 relativePath = fullChildPath.Substring(fullParentPath.Length);
405 while (relativePath.StartsWith(
"/")) relativePath = relativePath.Substring(1);
410 public static string SearchDirectory(
string directory,
string targetFileName)
412 string[] files = Directory.GetFiles(directory, targetFileName);
413 if (files.Length > 0)
return files[0];
414 string[] subdirectories = Directory.GetDirectories(directory);
415 foreach (var subdirectory
in subdirectories)
417 string result = SearchDirectory(subdirectory, targetFileName);
418 if (result !=
null)
return result;
425 [HideInInspector]
public static float libraryProgress = 1;
427 public static void CreateEmptyFile(
string path)
429 File.Create(path).Dispose();
432 static void ExtractInsideDirectory(
string zipPath,
string extractPath,
string prefix =
"",
bool overwrite =
true)
434 using (ZipArchive archive = ZipFile.OpenRead(zipPath))
436 foreach (ZipArchiveEntry entry
in archive.Entries)
438 if (
string.IsNullOrEmpty(entry.Name))
441 string destinationPath;
442 if (!String.IsNullOrEmpty(prefix))
444 string normalizedPath = entry.FullName.Replace(
'\\',
'/');
445 if (!normalizedPath.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
447 destinationPath = Path.Combine(extractPath, normalizedPath.Substring(prefix.Length));
451 destinationPath = Path.Combine(extractPath, entry.FullName);
454 Directory.CreateDirectory(Path.GetDirectoryName(destinationPath));
455 entry.ExtractToFile(destinationPath, overwrite);
460 static async Task DownloadAndExtractInsideDirectory()
462 string setupDir = Path.Combine(
libraryPath,
"setup");
463 Directory.CreateDirectory(setupDir);
465 string setupFile = Path.Combine(setupDir, Path.GetFileName(
LlamaLibURL) +
".complete");
466 if (File.Exists(setupFile))
return;
469 foreach (
string existingZipPath
in Directory.GetFiles(
cacheDownloadPath,
"*.zip"))
471 if (existingZipPath != cacheZipPath)
473 File.Delete(existingZipPath);
478 string cacheZipNewHashPath = cacheZipHashPath +
".new";
479 string hash = File.Exists(cacheZipHashPath)? File.ReadAllText(cacheZipHashPath).Trim() :
"";
480 bool same_hash =
false;
483 new ResumingWebClient().GetURLFileSize(hashurl);
484 await DownloadFile(hashurl, cacheZipNewHashPath, debug: false);
485 same_hash = File.ReadAllText(cacheZipNewHashPath).Trim() == hash;
488 if (!File.Exists(cacheZipPath) || !same_hash) await DownloadFile(
LlamaLibURL, cacheZipPath,
true,
null, SetLibraryProgress);
490 AssetDatabase.StartAssetEditing();
491 ExtractInsideDirectory(cacheZipPath,
libraryPath, $
"{libraryName}/runtimes/");
492 CreateEmptyFile(setupFile);
493 AssetDatabase.StopAssetEditing();
495 if (File.Exists(cacheZipNewHashPath))
497 if (File.Exists(cacheZipHashPath)) File.Delete(cacheZipHashPath);
498 File.Move(cacheZipNewHashPath, cacheZipHashPath);
502 static void DeleteEarlierVersions()
504 List<string> assetPathSubDirs =
new List<string>();
505 foreach (
string dir
in new string[] { GetAssetPath(), Path.Combine(Application.dataPath,
"Plugins",
"Android") })
507 if (Directory.Exists(dir)) assetPathSubDirs.AddRange(Directory.GetDirectories(dir));
510 List<Regex> versionRegexes =
new List<Regex> {
new Regex(
"undreamai-(.+)-llamacpp"),
new Regex(
"LlamaLib-(.+)") };
511 foreach (
string assetPathSubDir
in assetPathSubDirs)
513 foreach (Regex regex
in versionRegexes)
515 Match match = regex.Match(Path.GetFileName(assetPathSubDir));
518 string version = match.Groups[1].Value;
521 Debug.Log($
"Deleting other LLMUnity version folder: {assetPathSubDir}");
522 Directory.Delete(assetPathSubDir,
true);
523 if (File.Exists(assetPathSubDir +
".meta")) File.Delete(assetPathSubDir +
".meta");
530 static async Task DownloadLibrary()
532 if (libraryProgress < 1)
return;
535 DeleteEarlierVersions();
542 for (
int i=1; i<=3; i++)
544 if (i > 1) Log(
"Downloading LlamaLib failed, try #" + i);
548 await DownloadAndExtractInsideDirectory();
560 public static async Task RedownloadLibrary()
562 if (File.Exists(cacheZipPath)) File.Delete(cacheZipPath);
563 if (File.Exists(cacheZipHashPath)) File.Delete(cacheZipHashPath);
565 await DownloadLibrary();
568 private static void SetLibraryProgress(
float progress)
570 libraryProgress = Math.Min(0.99f, progress);
573 public static string AddAsset(
string assetPath)
575 if (!File.Exists(assetPath))
577 LogError($
"{assetPath} does not exist!");
580 string assetDir = GetAssetPath();
581 if (IsSubPath(assetPath, assetDir))
return RelativePath(assetPath, assetDir);
583 string filename = Path.GetFileName(assetPath);
584 string fullPath = GetAssetPath(filename);
585 AssetDatabase.StartAssetEditing();
586 foreach (
string path
in new string[] { fullPath, fullPath +
".meta" })
588 if (File.Exists(path)) File.Delete(path);
590 File.Copy(assetPath, fullPath);
591 AssetDatabase.StopAssetEditing();
601 errorCallbacks.Add(callback);
607 errorCallbacks.Remove(callback);
613 errorCallbacks.Clear();
616 public static int GetMaxFreqKHz(
int cpuId)
618 string[] paths =
new string[]
620 $
"/sys/devices/system/cpu/cpufreq/stats/cpu{cpuId}/time_in_state",
621 $
"/sys/devices/system/cpu/cpu{cpuId}/cpufreq/stats/time_in_state",
622 $
"/sys/devices/system/cpu/cpu{cpuId}/cpufreq/cpuinfo_max_freq"
625 foreach (var path
in paths)
627 if (!File.Exists(path))
continue;
630 using (StreamReader sr =
new StreamReader(path))
633 while ((line = sr.ReadLine()) !=
null)
635 string[] parts = line.Split(
' ');
636 if (parts.Length > 0 &&
int.TryParse(parts[0], out
int freqKHz))
638 if (freqKHz > maxFreqKHz)
640 maxFreqKHz = freqKHz;
645 if (maxFreqKHz != 0)
return maxFreqKHz;
650 public static bool IsSmtCpu(
int cpuId)
652 string[] paths =
new string[]
654 $
"/sys/devices/system/cpu/cpu{cpuId}/topology/core_cpus_list",
655 $
"/sys/devices/system/cpu/cpu{cpuId}/topology/thread_siblings_list"
658 foreach (var path
in paths)
660 if (!File.Exists(path))
continue;
661 using (StreamReader sr =
new StreamReader(path))
664 while ((line = sr.ReadLine()) !=
null)
666 if (line.Contains(
",") || line.Contains(
"-"))
682 int maxFreqKHzMin =
int.MaxValue;
683 int maxFreqKHzMax = 0;
684 List<int> cpuMaxFreqKHz =
new List<int>();
685 List<bool> cpuIsSmtCpu =
new List<bool>();
689 string cpuPath =
"/sys/devices/system/cpu/";
691 if (Directory.Exists(cpuPath))
693 foreach (
string cpuDir
in Directory.GetDirectories(cpuPath))
695 string dirName = Path.GetFileName(cpuDir);
696 if (!dirName.StartsWith(
"cpu"))
continue;
697 if (!
int.TryParse(dirName.Substring(3), out coreIndex))
continue;
699 int maxFreqKHz = GetMaxFreqKHz(coreIndex);
700 cpuMaxFreqKHz.Add(maxFreqKHz);
701 if (maxFreqKHz > maxFreqKHzMax) maxFreqKHzMax = maxFreqKHz;
702 if (maxFreqKHz < maxFreqKHzMin) maxFreqKHzMin = maxFreqKHz;
703 cpuIsSmtCpu.Add(IsSmtCpu(coreIndex));
713 int numCores = SystemInfo.processorCount;
714 int maxFreqKHzMedium = (maxFreqKHzMin + maxFreqKHzMax) / 2;
715 if (maxFreqKHzMedium == maxFreqKHzMax) numBigCores = numCores;
718 for (
int i = 0; i < cpuMaxFreqKHz.Count; i++)
720 if (cpuIsSmtCpu[i] || cpuMaxFreqKHz[i] >= maxFreqKHzMedium) numBigCores++;
724 if (numBigCores == 0) numBigCores = SystemInfo.processorCount / 2;
725 else numBigCores = Math.Min(numBigCores, SystemInfo.processorCount);
736 List<int> capacities =
new List<int>();
737 int minCapacity =
int.MaxValue;
740 string cpuPath =
"/sys/devices/system/cpu/";
742 if (Directory.Exists(cpuPath))
744 foreach (
string cpuDir
in Directory.GetDirectories(cpuPath))
746 string dirName = Path.GetFileName(cpuDir);
747 if (!dirName.StartsWith(
"cpu"))
continue;
748 if (!
int.TryParse(dirName.Substring(3), out coreIndex))
continue;
750 string capacityPath = Path.Combine(cpuDir,
"cpu_capacity");
751 if (!File.Exists(capacityPath))
break;
753 int capacity =
int.Parse(File.ReadAllText(capacityPath).Trim());
754 capacities.Add(capacity);
755 if (minCapacity > capacity) minCapacity = capacity;
765 foreach (
int capacity
in capacities)
767 if (capacity >= 2 * minCapacity) numBigCores++;
770 if (numBigCores == 0 || numBigCores > SystemInfo.processorCount) numBigCores = SystemInfo.processorCount;