using System.Text.Json; using AuthorBuddy.Web.Models; namespace AuthorBuddy.Web.Services; public class LlamaCppBackend : ILLMBackend { private readonly HttpClient _http; private readonly OllamaSettings _settings; public string Name => "llama.cpp"; public bool SupportsModelManagement => false; public LlamaCppBackend(OllamaSettings settings) { _settings = settings; _http = new HttpClient { Timeout = TimeSpan.FromSeconds(300) }; } private string Url => _settings.LlamaCppUrl; public async Task TestConnectionAsync() { try { var response = await _http.GetAsync($"{Url}/health"); return response.IsSuccessStatusCode; } catch { return false; } } public async Task> GetModelsAsync() { try { var response = await _http.GetAsync($"{Url}/v1/models"); if (!response.IsSuccessStatusCode) return new List { new LLM_Model("llama.cpp") }; var content = await response.Content.ReadAsStringAsync(); using var doc = JsonDocument.Parse(content); if (!doc.RootElement.TryGetProperty("data", out var data)) return new List { new LLM_Model("llama.cpp") }; var list = new List(); foreach (var m in data.EnumerateArray()) { var id = m.TryGetProperty("id", out var idProp) ? idProp.GetString() ?? "llama.cpp" : "llama.cpp"; list.Add(new LLM_Model(id)); } return list; } catch { return new List { new LLM_Model("llama.cpp") }; } } public async Task ChatAsync(string model, string prompt, string? systemPrompt = null, double? temperature = null) { try { var messages = new List(); if (!string.IsNullOrEmpty(systemPrompt)) messages.Add(new { role = "system", content = systemPrompt }); messages.Add(new { role = "user", content = prompt }); var request = new Dictionary { ["model"] = model, ["messages"] = messages, ["stream"] = false }; if (temperature.HasValue) request["temperature"] = Math.Clamp(temperature.Value, 0.0, 2.0); var response = await _http.PostAsync( $"{Url}/v1/chat/completions", new StringContent(JsonSerializer.Serialize(request), System.Text.Encoding.UTF8, "application/json") ); if (!response.IsSuccessStatusCode) return "Fehler bei der Verarbeitung."; var content = await response.Content.ReadAsStringAsync(); using var doc = JsonDocument.Parse(content); var root = doc.RootElement; if (root.TryGetProperty("choices", out var choices) && choices.GetArrayLength() > 0) { var first = choices[0]; if (first.TryGetProperty("message", out var msg) && msg.TryGetProperty("content", out var text)) return text.GetString() ?? ""; } return "Fehler bei der Verarbeitung."; } catch (Exception ex) { return $"Error: {ex.Message}"; } } public async Task ChatStreamAsync(string model, string prompt, Func onToken, string? systemPrompt = null, double? temperature = null) { var messages = new List(); if (!string.IsNullOrEmpty(systemPrompt)) messages.Add(new { role = "system", content = systemPrompt }); messages.Add(new { role = "user", content = prompt }); var request = new Dictionary { ["model"] = model, ["messages"] = messages, ["stream"] = true }; if (temperature.HasValue) request["temperature"] = Math.Clamp(temperature.Value, 0.0, 2.0); using var response = await _http.PostAsync( $"{Url}/v1/chat/completions", new StringContent(JsonSerializer.Serialize(request), System.Text.Encoding.UTF8, "application/json") ); if (!response.IsSuccessStatusCode) return; using var stream = await response.Content.ReadAsStreamAsync(); using var reader = new StreamReader(stream); string? line; while ((line = await reader.ReadLineAsync()) != null) { if (string.IsNullOrWhiteSpace(line)) continue; if (!line.StartsWith("data: ")) continue; var json = line[6..]; if (json == "[DONE]") break; try { using var doc = JsonDocument.Parse(json); var root = doc.RootElement; if (root.TryGetProperty("choices", out var choices) && choices.GetArrayLength() > 0) { var delta = choices[0].TryGetProperty("delta", out var d) ? d : default; if (delta.ValueKind != JsonValueKind.Undefined && delta.TryGetProperty("content", out var content)) { var text = content.GetString(); if (!string.IsNullOrEmpty(text)) await onToken(text); } } } catch { } } } public async Task GetEmbeddingAsync(string text, string? model = null) { try { var request = new { content = text }; var response = await _http.PostAsync( $"{Url}/embedding", new StringContent(JsonSerializer.Serialize(request), System.Text.Encoding.UTF8, "application/json") ); if (!response.IsSuccessStatusCode) return Array.Empty(); var content = await response.Content.ReadAsStringAsync(); using var doc = JsonDocument.Parse(content); if (!doc.RootElement.TryGetProperty("embedding", out var emb)) return Array.Empty(); var list = new List(); foreach (var v in emb.EnumerateArray()) list.Add(v.GetSingle()); return list.ToArray(); } catch { return Array.Empty(); } } public Task PullModelAsync(string name, IProgress? progress = null) => Task.FromResult(false); public Task DeleteModelAsync(string name) => Task.FromResult(false); public async Task GetContextSizeAsync(string model) { try { var response = await _http.GetAsync($"{Url}/slots"); if (!response.IsSuccessStatusCode) return 2048; var content = await response.Content.ReadAsStringAsync(); using var doc = JsonDocument.Parse(content); if (doc.RootElement.GetArrayLength() > 0) { var first = doc.RootElement[0]; if (first.TryGetProperty("n_ctx", out var ctx)) return ctx.GetInt32(); } return 2048; } catch { return 2048; } } public async Task MeasureLatencyAsync(string model) { try { var sw = System.Diagnostics.Stopwatch.StartNew(); var response = await _http.GetAsync($"{Url}/health"); sw.Stop(); return response.IsSuccessStatusCode ? sw.ElapsedMilliseconds : -1; } catch { return -1; } } public async Task ModelExistsAsync(string model) { try { var models = await GetModelsAsync(); return models.Any(); } catch { return false; } } public async Task<(bool Success, int ContextWindow)> GetModelContextWindowAsync(string model) { var ctx = await GetContextSizeAsync(model); return (ctx.HasValue, ctx ?? 2048); } public bool TestEmbedding(string model) { try { var response = _http.PostAsync( $"{Url}/embedding", new StringContent(JsonSerializer.Serialize(new { content = "test" }), System.Text.Encoding.UTF8, "application/json") ).GetAwaiter().GetResult(); if (!response.IsSuccessStatusCode) return false; var content = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); using var doc = JsonDocument.Parse(content); if (doc.RootElement.TryGetProperty("embedding", out var emb)) return emb.EnumerateArray().Any(); return false; } catch { return false; } } }