101 lines
3.7 KiB
C#
101 lines
3.7 KiB
C#
using AuthorBuddy.Web.Data;
|
|
using Microsoft.Data.Sqlite;
|
|
using Microsoft.EntityFrameworkCore;
|
|
|
|
namespace AuthorBuddy.Web.Services;
|
|
|
|
public class FileScannerService : IFileScannerService
|
|
{
|
|
private readonly IOllamaService _ollama;
|
|
private readonly IDbContextFactory<AppDbContext> _contextFactory;
|
|
private readonly string _connectionString;
|
|
|
|
public FileScannerService(IOllamaService ollama, IDbContextFactory<AppDbContext> contextFactory)
|
|
{
|
|
_ollama = ollama;
|
|
_contextFactory = contextFactory;
|
|
_connectionString = Models.Constants.SQLiteConnection;
|
|
}
|
|
|
|
public async Task ScanProjectFolderAsync(int projectId, string rootPath, IProgress<string> progress)
|
|
{
|
|
var files = Directory.GetFiles(rootPath, "*.md", SearchOption.AllDirectories)
|
|
.Where(f => f.Contains("00_Drafts") || f.Contains("01_Fakten"));
|
|
|
|
foreach (var file in files)
|
|
{
|
|
var fileInfo = new FileInfo(file);
|
|
bool needsUpdate = await CheckIfUpdateRequired(projectId, file, fileInfo.LastWriteTimeUtc);
|
|
|
|
if (needsUpdate)
|
|
{
|
|
progress?.Report($"Verarbeite: {fileInfo.Name}...");
|
|
await ProcessAndVectorizeFile(projectId, file, fileInfo.LastWriteTimeUtc);
|
|
}
|
|
}
|
|
progress?.Report("Scan abgeschlossen.");
|
|
}
|
|
|
|
private async Task<bool> CheckIfUpdateRequired(int projectId, string file, DateTime lastWriteTimeUtc)
|
|
{
|
|
await using var context = await _contextFactory.CreateDbContextAsync();
|
|
|
|
var meta = await context.DocumentMeta
|
|
.Where(m => m.ProjectId == projectId && m.FilePath == file)
|
|
.Select(m => m.LastModified)
|
|
.FirstOrDefaultAsync();
|
|
|
|
if (meta == null) return true;
|
|
return meta.Value < lastWriteTimeUtc;
|
|
}
|
|
|
|
private async Task<bool> ProcessAndVectorizeFile(int projectId, string path, DateTime lastModified)
|
|
{
|
|
string content = await File.ReadAllTextAsync(path);
|
|
|
|
var vector = await _ollama.GetEmbeddingAsync(content);
|
|
|
|
using var connection = new SqliteConnection(_connectionString);
|
|
connection.Open();
|
|
using var transaction = connection.BeginTransaction();
|
|
|
|
try
|
|
{
|
|
var cmdMeta = connection.CreateCommand();
|
|
cmdMeta.CommandText = @"
|
|
INSERT INTO DocumentMeta (project_id, file_path, last_modified)
|
|
VALUES (@pid, @path, @mod)
|
|
ON CONFLICT(file_path) DO UPDATE SET last_modified = @mod
|
|
RETURNING id;";
|
|
cmdMeta.Parameters.AddWithValue("@pid", projectId);
|
|
cmdMeta.Parameters.AddWithValue("@path", path);
|
|
cmdMeta.Parameters.AddWithValue("@mod", lastModified);
|
|
|
|
var docId = await cmdMeta.ExecuteScalarAsync();
|
|
|
|
byte[] blob = ConvertVectorToBytes(vector);
|
|
var cmdVec = connection.CreateCommand();
|
|
cmdVec.CommandText = "INSERT INTO vec_documents(rowid, embedding) VALUES (@id, @vec) " +
|
|
"ON CONFLICT(rowid) DO UPDATE SET embedding = @vec;";
|
|
cmdVec.Parameters.AddWithValue("@id", docId);
|
|
cmdVec.Parameters.AddWithValue("@vec", blob);
|
|
await cmdVec.ExecuteNonQueryAsync();
|
|
|
|
transaction.Commit();
|
|
return true;
|
|
}
|
|
catch
|
|
{
|
|
transaction.Rollback();
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private static byte[] ConvertVectorToBytes(IList<double> vector)
|
|
{
|
|
float[] floatVector = vector.Select(d => (float)d).ToArray();
|
|
byte[] bytes = new byte[floatVector.Length * sizeof(float)];
|
|
Buffer.BlockCopy(floatVector, 0, bytes, 0, bytes.Length);
|
|
return bytes;
|
|
}
|
|
}
|