AuthorBuddy/Services/DuckDuckGoService.cs

using AuthorBuddy.Web.Models;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Web;

namespace AuthorBuddy.Web.Services;

public class DuckDuckGoService : IDuckDuckGoService
{
    private readonly HttpClient _http;
    private readonly HttpClient _htmlHttp;

    public DuckDuckGoService()
    {
        _http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
        _http.DefaultRequestHeaders.UserAgent.ParseAdd("AuthorBuddy/1.0 (fact-checker)");

        _htmlHttp = new HttpClient { Timeout = TimeSpan.FromSeconds(20) };
        _htmlHttp.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
        _htmlHttp.DefaultRequestHeaders.Accept.ParseAdd("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        _htmlHttp.DefaultRequestHeaders.Add("Accept-Language", "de,en;q=0.8");
    }

    public async Task<List<DuckDuckGoResult>> SearchWebAsync(string query, int maxResults = 5)
    {
        var results = new List<DuckDuckGoResult>();
        try
        {
            var url = "https://html.duckduckgo.com/html/?q=" + HttpUtility.UrlEncode(query);
            var html = await _htmlHttp.GetStringAsync(url);

            if (html.Contains("anomaly", StringComparison.OrdinalIgnoreCase) ||
                html.Contains("challenge-form", StringComparison.OrdinalIgnoreCase))
            {
                return results;
            }

            var blocks = Regex.Split(html, "<div[^>]*class=\"result[^\"]*\"", RegexOptions.IgnoreCase);
            foreach (var block in blocks.Skip(1))
            {
                var link = Regex.Match(block, "<a[^>]*class=\"result__a\"[^>]*href=\"([^\"]+)\"[^>]*>(.*?)</a>", RegexOptions.Singleline);
                if (!link.Success) continue;

                var realUrl = DecodeDdgUrl(System.Net.WebUtility.HtmlDecode(link.Groups[1].Value));
                if (string.IsNullOrWhiteSpace(realUrl)) continue;

                var title = StripHtml(link.Groups[2].Value);
                var snippet = string.Empty;
                var snip = Regex.Match(block, "<a[^>]*class=\"result__snippet\"[^>]*>(.*?)</a>", RegexOptions.Singleline);
                if (snip.Success) snippet = StripHtml(snip.Groups[1].Value);

                results.Add(new DuckDuckGoResult { Heading = title, AbstractText = snippet, AbstractUrl = realUrl });
                if (results.Count >= maxResults) break;
            }
        }
        catch
        {
            // Keine Ergebnisse bei Fehler
        }

        return results;
    }

    public async Task<List<DuckDuckGoResult>> SearchAsync(string query, int maxResults = 3)
    {
        var results = new List<DuckDuckGoResult>();
        try
        {
            var url = "https://api.duckduckgo.com/" +
                      $"?q={HttpUtility.UrlEncode(query)}" +
                      "&format=json" +
                      "&no_html=1" +
                      "&skip_disambig=1" +
                      "&t=AuthorBuddy";

            var response = await _http.GetStringAsync(url);
            using var doc = JsonDocument.Parse(response);
            var root = doc.RootElement;

            var result = new DuckDuckGoResult
            {
                Heading = GetString(root, "Heading"),
                AbstractText = GetString(root, "AbstractText"),
                AbstractUrl = GetString(root, "AbstractURL"),
                Answer = GetString(root, "Answer")
            };

            if (root.TryGetProperty("RelatedTopics", out var related) && related.ValueKind == JsonValueKind.Array)
            {
                foreach (var topic in related.EnumerateArray())
                {
                    if (topic.TryGetProperty("Text", out var text))
                    {
                        var t = text.GetString();
                        if (!string.IsNullOrWhiteSpace(t))
                            result.RelatedTexts.Add(t);
                        if (topic.TryGetProperty("FirstURL", out var firstUrl))
                        {
                            var u = firstUrl.GetString();
                            if (!string.IsNullOrWhiteSpace(u))
                                result.RelatedUrls.Add(u);
                        }
                    }
                    else if (topic.TryGetProperty("Topics", out var subTopics) && subTopics.ValueKind == JsonValueKind.Array)
                    {
                        foreach (var sub in subTopics.EnumerateArray())
                        {
                            if (sub.TryGetProperty("Text", out var subText))
                            {
                                var t = subText.GetString();
                                if (!string.IsNullOrWhiteSpace(t))
                                    result.RelatedTexts.Add(t);
                                if (sub.TryGetProperty("FirstURL", out var firstUrl))
                                {
                                    var u = firstUrl.GetString();
                                    if (!string.IsNullOrWhiteSpace(u))
                                        result.RelatedUrls.Add(u);
                                }
                            }
                        }
                    }
                }
            }

            if (!string.IsNullOrWhiteSpace(result.AbstractText) ||
                !string.IsNullOrWhiteSpace(result.Answer) ||
                result.RelatedTexts.Count > 0)
            {
                results.Add(result);
            }
        }
        catch
        {
            // Kein Ergebnis bei Fehler
        }

        return results.Take(maxResults).ToList();
    }

    private static string GetString(JsonElement element, string property)
    {
        if (element.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String)
            return value.GetString() ?? string.Empty;
        return string.Empty;
    }

    private static string DecodeDdgUrl(string rawUrl)
    {
        if (string.IsNullOrWhiteSpace(rawUrl)) return rawUrl;

        var uddg = Regex.Match(rawUrl, "[?&]uddg=([^&]+)");
        if (uddg.Success)
            return HttpUtility.UrlDecode(uddg.Groups[1].Value);

        if (rawUrl.StartsWith("//"))
            return "https:" + rawUrl;

        return rawUrl;
    }

    private static string StripHtml(string html)
    {
        var text = Regex.Replace(html, "<[^>]+>", string.Empty);
        return System.Net.WebUtility.HtmlDecode(text).Trim();
    }
}