254 lines
9.4 KiB
C#
254 lines
9.4 KiB
C#
|
using System.Diagnostics;
|
||
|
|
using System.Text.Json;
|
||
|
|
using System.Text.RegularExpressions;
|
||
|
|
using AuthorBuddy.Web.Models;
|
||
|
|
|
||
|
|
namespace AuthorBuddy.Web.Services;
|
||
|
|
|
||
|
|
public class OllamaBackend : ILLMBackend
|
||
|
|
{
|
||
|
|
private readonly HttpClient _http;
|
||
|
|
private readonly OllamaSettings _settings;
|
||
|
|
|
||
|
|
public string Name => "Ollama";
|
||
|
|
public bool SupportsModelManagement => true;
|
||
|
|
|
||
|
|
public OllamaBackend(OllamaSettings settings)
|
||
|
|
{
|
||
|
|
_settings = settings;
|
||
|
|
_http = new HttpClient { Timeout = TimeSpan.FromSeconds(300) };
|
||
|
|
}
|
||
|
|
|
||
|
|
private string Url => _settings.OllamaUrl;
|
||
|
|
|
||
|
|
public async Task<bool> TestConnectionAsync()
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var sw = Stopwatch.StartNew();
|
||
|
|
var response = await _http.GetAsync($"{Url}/api/tags");
|
||
|
|
sw.Stop();
|
||
|
|
return response.IsSuccessStatusCode;
|
||
|
|
}
|
||
|
|
catch { return false; }
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task<IEnumerable<LLM_Model>> GetModelsAsync()
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var response = await _http.GetAsync($"{Url}/api/tags");
|
||
|
|
if (!response.IsSuccessStatusCode) return Enumerable.Empty<LLM_Model>();
|
||
|
|
var content = await response.Content.ReadAsStringAsync();
|
||
|
|
using var doc = JsonDocument.Parse(content);
|
||
|
|
if (!doc.RootElement.TryGetProperty("models", out var models))
|
||
|
|
return Enumerable.Empty<LLM_Model>();
|
||
|
|
var list = new List<LLM_Model>();
|
||
|
|
foreach (var m in models.EnumerateArray())
|
||
|
|
list.Add(new LLM_Model(m));
|
||
|
|
return list;
|
||
|
|
}
|
||
|
|
catch { return Enumerable.Empty<LLM_Model>(); }
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task<string> ChatAsync(string model, string prompt, string? systemPrompt = null, double? temperature = null)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var request = new Dictionary<string, object?>
|
||
|
|
{
|
||
|
|
["model"] = model, ["prompt"] = prompt, ["stream"] = false
|
||
|
|
};
|
||
|
|
if (!string.IsNullOrEmpty(systemPrompt)) request["system"] = systemPrompt;
|
||
|
|
if (temperature.HasValue) request["temperature"] = Math.Clamp(temperature.Value, 0.0, 2.0);
|
||
|
|
|
||
|
|
var response = await _http.PostAsJsonAsync($"{Url}/api/generate", request);
|
||
|
|
if (!response.IsSuccessStatusCode) return "Fehler bei der Verarbeitung.";
|
||
|
|
var content = await response.Content.ReadAsStringAsync();
|
||
|
|
using var doc = JsonDocument.Parse(content);
|
||
|
|
if (doc.RootElement.TryGetProperty("response", out var resp))
|
||
|
|
return resp.GetString() ?? "";
|
||
|
|
return "Fehler bei der Verarbeitung.";
|
||
|
|
}
|
||
|
|
catch (Exception ex) { return $"Error: {ex.Message}"; }
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task ChatStreamAsync(string model, string prompt, Func<string, Task> onToken, string? systemPrompt = null, double? temperature = null)
|
||
|
|
{
|
||
|
|
var request = new Dictionary<string, object?>
|
||
|
|
{
|
||
|
|
["model"] = model, ["prompt"] = prompt, ["stream"] = true
|
||
|
|
};
|
||
|
|
if (!string.IsNullOrEmpty(systemPrompt)) request["system"] = systemPrompt;
|
||
|
|
if (temperature.HasValue) request["temperature"] = Math.Clamp(temperature.Value, 0.0, 2.0);
|
||
|
|
|
||
|
|
using var response = await _http.PostAsync(
|
||
|
|
$"{Url}/api/generate",
|
||
|
|
new StringContent(JsonSerializer.Serialize(request), System.Text.Encoding.UTF8, "application/json")
|
||
|
|
);
|
||
|
|
if (!response.IsSuccessStatusCode) return;
|
||
|
|
|
||
|
|
using var stream = await response.Content.ReadAsStreamAsync();
|
||
|
|
using var reader = new StreamReader(stream);
|
||
|
|
string? line;
|
||
|
|
while ((line = await reader.ReadLineAsync()) != null)
|
||
|
|
{
|
||
|
|
var text = ExtractResponseText(line);
|
||
|
|
if (!string.IsNullOrEmpty(text))
|
||
|
|
await onToken(text);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private string ExtractResponseText(string jsonLine)
|
||
|
|
{
|
||
|
|
if (string.IsNullOrWhiteSpace(jsonLine)) return string.Empty;
|
||
|
|
try
|
||
|
|
{
|
||
|
|
using var doc = JsonDocument.Parse(jsonLine);
|
||
|
|
if (doc.RootElement.TryGetProperty("response", out var resp))
|
||
|
|
return resp.GetString() ?? string.Empty;
|
||
|
|
}
|
||
|
|
catch { }
|
||
|
|
return string.Empty;
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task<float[]> GetEmbeddingAsync(string text, string? model = null)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var m = model ?? _settings.EmbedModel;
|
||
|
|
var request = new { model = m, prompt = text };
|
||
|
|
var response = await _http.PostAsJsonAsync($"{Url}/api/embeddings", request);
|
||
|
|
if (!response.IsSuccessStatusCode) return Array.Empty<float>();
|
||
|
|
var content = await response.Content.ReadAsStringAsync();
|
||
|
|
using var doc = JsonDocument.Parse(content);
|
||
|
|
if (!doc.RootElement.TryGetProperty("embedding", out var emb)) return Array.Empty<float>();
|
||
|
|
var list = new List<float>();
|
||
|
|
foreach (var v in emb.EnumerateArray()) list.Add(v.GetSingle());
|
||
|
|
return list.ToArray();
|
||
|
|
}
|
||
|
|
catch { return Array.Empty<float>(); }
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task<bool> PullModelAsync(string name, IProgress<string>? progress = null)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var request = new { name, stream = true };
|
||
|
|
var response = await _http.PostAsync(
|
||
|
|
$"{Url}/api/pull",
|
||
|
|
new StringContent(JsonSerializer.Serialize(request), System.Text.Encoding.UTF8, "application/json")
|
||
|
|
);
|
||
|
|
if (!response.IsSuccessStatusCode) return false;
|
||
|
|
|
||
|
|
using var stream = await response.Content.ReadAsStreamAsync();
|
||
|
|
using var reader = new StreamReader(stream);
|
||
|
|
string? line;
|
||
|
|
while ((line = await reader.ReadLineAsync()) != null)
|
||
|
|
{
|
||
|
|
if (string.IsNullOrWhiteSpace(line)) continue;
|
||
|
|
try
|
||
|
|
{
|
||
|
|
using var doc = JsonDocument.Parse(line);
|
||
|
|
var root = doc.RootElement;
|
||
|
|
if (root.TryGetProperty("status", out var status))
|
||
|
|
{
|
||
|
|
var st = status.GetString() ?? "";
|
||
|
|
var msg = st;
|
||
|
|
if (root.TryGetProperty("completed", out var c) && root.TryGetProperty("total", out var t))
|
||
|
|
{
|
||
|
|
var comp = c.GetInt64(); var tot = t.GetInt64();
|
||
|
|
if (tot > 0) msg = $"{st}: {comp}/{tot} ({(int)(comp * 100 / tot)}%)";
|
||
|
|
}
|
||
|
|
progress?.Report(msg);
|
||
|
|
if (st == "success") return true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch { }
|
||
|
|
}
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
catch { return false; }
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task<bool> DeleteModelAsync(string name)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var req = new HttpRequestMessage(HttpMethod.Delete, $"{Url}/api/delete")
|
||
|
|
{
|
||
|
|
Content = new StringContent(JsonSerializer.Serialize(new { name }), System.Text.Encoding.UTF8, "application/json")
|
||
|
|
};
|
||
|
|
var response = await _http.SendAsync(req);
|
||
|
|
return response.IsSuccessStatusCode;
|
||
|
|
}
|
||
|
|
catch { return false; }
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task<int?> GetContextSizeAsync(string model)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var response = await _http.PostAsJsonAsync($"{Url}/api/show", new { model });
|
||
|
|
if (!response.IsSuccessStatusCode) return null;
|
||
|
|
var content = await response.Content.ReadAsStringAsync();
|
||
|
|
using var doc = JsonDocument.Parse(content);
|
||
|
|
if (doc.RootElement.TryGetProperty("modelfile", out var mf))
|
||
|
|
{
|
||
|
|
var modelfile = mf.GetString() ?? "";
|
||
|
|
var match = Regex.Match(modelfile, @"num_ctx\s+(\d+)");
|
||
|
|
if (match.Success && int.TryParse(match.Groups[1].Value, out var ctx))
|
||
|
|
return ctx;
|
||
|
|
}
|
||
|
|
return 2048;
|
||
|
|
}
|
||
|
|
catch { return null; }
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task<long> MeasureLatencyAsync(string model)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var sw = Stopwatch.StartNew();
|
||
|
|
var request = new { model, prompt = "Hallo", stream = false };
|
||
|
|
var response = await _http.PostAsJsonAsync($"{Url}/api/generate", request);
|
||
|
|
sw.Stop();
|
||
|
|
return response.IsSuccessStatusCode ? sw.ElapsedMilliseconds : -1;
|
||
|
|
}
|
||
|
|
catch { return -1; }
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task<bool> ModelExistsAsync(string model)
|
||
|
|
{
|
||
|
|
if (string.IsNullOrWhiteSpace(model)) return false;
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var models = await GetModelsAsync();
|
||
|
|
return models.Any(m => m.Name == model || m.Name.StartsWith(model.Split(':')[0]));
|
||
|
|
}
|
||
|
|
catch { return false; }
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task<(bool Success, int ContextWindow)> GetModelContextWindowAsync(string model)
|
||
|
|
{
|
||
|
|
var ctx = await GetContextSizeAsync(model);
|
||
|
|
return (ctx.HasValue, ctx ?? 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
public bool TestEmbedding(string model)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
var request = new { model, prompt = "test" };
|
||
|
|
var response = _http.PostAsJsonAsync($"{Url}/api/embeddings", request).GetAwaiter().GetResult();
|
||
|
|
if (!response.IsSuccessStatusCode) return false;
|
||
|
|
var content = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
|
||
|
|
using var doc = JsonDocument.Parse(content);
|
||
|
|
if (doc.RootElement.TryGetProperty("embedding", out var emb))
|
||
|
|
return emb.EnumerateArray().Any();
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
catch { return false; }
|
||
|
|
}
|
||
|
|
}
|