AuthorBuddy/Services/RagService.cs

using AuthorBuddy.Web.Data;
using LiteDB;
using System.Security.Cryptography;
using System.Text;

namespace AuthorBuddy.Web.Services;

public class RagService : IRagService
{
    private readonly ILiteDatabase _db;
    private readonly IOllamaService _ollama;

    public RagService(ILiteDatabase db, IOllamaService ollama)
    {
        _db = db;
        _ollama = ollama;
    }

    public async Task IndexProjectAsync(int projectId, string rootPath, IProgress<string> progress)
    {
        var files = Directory.GetFiles(rootPath, "*.md", SearchOption.AllDirectories)
                             .Where(f => f.Contains("00_Drafts") || f.Contains("01_Fakten"));

        var metaCol = _db.GetCollection<DocumentMetaEntity>("documentMeta");
        var vecCol = _db.GetCollection<DocumentVector>("vectors");

        foreach (var file in files)
        {
            progress?.Report($"Indexiere: {Path.GetFileName(file)}...");

            string content = await File.ReadAllTextAsync(file);
            string hash = ComputeHash(content);

            var meta = metaCol.FindOne(m => m.ProjectId == projectId && m.FilePath == file);

            if (meta != null && meta.Hash == hash)
            {
                var existingVec = vecCol.FindOne(v => v.DocumentMetaId == meta.Id);
                if (existingVec != null)
                {
                    progress?.Report($"  ↳ unverändert, übersprungen");
                    continue;
                }
            }

            var embedding = await _ollama.GetEmbeddingAsync(content);
            if (embedding.Length == 0)
            {
                progress?.Report($"  ↳ Fehler beim Embedding, übersprungen");
                continue;
            }

            if (meta == null)
            {
                meta = new DocumentMetaEntity
                {
                    ProjectId = projectId,
                    FilePath = file,
                    LastModified = File.GetLastWriteTimeUtc(file),
                    Hash = hash,
                    FileType = ".md"
                };
                metaCol.Insert(meta);
            }
            else
            {
                meta.LastModified = File.GetLastWriteTimeUtc(file);
                meta.Hash = hash;
                metaCol.Update(meta);
            }

            var vec = vecCol.FindOne(v => v.DocumentMetaId == meta.Id);
            if (vec != null)
            {
                vec.Text = content;
                vec.Hash = hash;
                vec.Embedding = embedding;
                vecCol.Update(vec);
            }
            else
            {
                vecCol.Insert(new DocumentVector
                {
                    DocumentMetaId = meta.Id,
                    ProjectId = projectId,
                    Text = content,
                    Hash = hash,
                    Embedding = embedding
                });
            }

            progress?.Report($"  ↳ ✓ {embedding.Length} Dimensionen");
        }

        progress?.Report("Indexierung abgeschlossen.");
    }

    public async Task<List<RagSearchResult>> SearchAsync(int projectId, string query, int topN = 5)
    {
        var queryVec = await _ollama.GetEmbeddingAsync(query);
        if (queryVec.Length == 0)
            return new List<RagSearchResult>();

        var vecCol = _db.GetCollection<DocumentVector>("vectors");
        var metaCol = _db.GetCollection<DocumentMetaEntity>("documentMeta");

        var vectors = vecCol.Find(v => v.ProjectId == projectId).ToList();
        var results = new List<RagSearchResult>();

        foreach (var vec in vectors)
        {
            var sim = CosineSimilarity(queryVec, vec.Embedding);
            if (sim > 0)
            {
                var meta = metaCol.FindById(vec.DocumentMetaId);
                results.Add(new RagSearchResult
                {
                    Text = vec.Text.Length > 500 ? vec.Text[..500] + "..." : vec.Text,
                    FilePath = meta?.FilePath ?? "",
                    Similarity = sim
                });
            }
        }

        return results.OrderByDescending(r => r.Similarity).Take(topN).ToList();
    }

    private static string ComputeHash(string content)
    {
        var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(content));
        return Convert.ToHexStringLower(bytes);
    }

    private static double CosineSimilarity(float[] a, float[] b)
    {
        double dotProduct = 0.0, l2A = 0.0, l2B = 0.0;
        for (int i = 0; i < a.Length; i++)
        {
            dotProduct += a[i] * b[i];
            l2A += Math.Pow(a[i], 2);
            l2B += Math.Pow(b[i], 2);
        }
        return dotProduct / (Math.Sqrt(l2A) * Math.Sqrt(l2B));
    }
}