Wp2ghost/CSharpGhostImporter/PostFetcher.cs

using System.Text.Json;

namespace wp2ghost
{
    // =============================================================================
    // PostFetcher — Posts von WordPress abrufen, verarbeiten
    // =============================================================================
    static class PostFetcher
    {
        private static readonly HttpClient httpClient = new();
        private static HashSet<int> ProcessedPosts = new();
        private static List<GhostPost> GhostPosts = new();
        private static string ProcessedPostsPath = "";
        //private static string MarkdownDir = "";
        private static string GhostPostsDir = "";
        private static string BaseDir = "";

        public static int ProcessedCount => ProcessedPosts.Count;
        public static int PostCount => GhostPosts.Count;
        public static IReadOnlyList<GhostPost> Posts => GhostPosts;

        public static void Init(string baseDir)
        {
            BaseDir = baseDir;
            ProcessedPostsPath = Path.Combine(baseDir, "processed_posts.txt");
            //MarkdownDir = Path.Combine(baseDir, "markdown");
            GhostPostsDir = Path.Combine(baseDir, "ghostPosts");

            ////if (!Directory.Exists(MarkdownDir))
            ////    Directory.CreateDirectory(MarkdownDir);
            if (!Directory.Exists(GhostPostsDir))
                Directory.CreateDirectory(GhostPostsDir);

            LoadProcessedPosts();
        }

        public static void ClearProcessedPosts()
        {
            ProcessedPosts.Clear();
            File.WriteAllText(ProcessedPostsPath, "wp_id,wp_slug,ghost_id" + Environment.NewLine);
            Console.WriteLine("Abgefragte Posts wurden zurückgesetzt.");
        }

        public static void ShowInfo()
        {
            if (ProcessedPosts.Count > 0)
            {
                Console.WriteLine($"Letzte abgefragte Objekte: {string.Join(", ", ProcessedPosts.OrderByDescending(x => x).Take(10))}");
            }
        }

        public static async Task FetchFromWordPressAsync(string wpBaseUrl, int maxPosts)
        {
            Console.WriteLine();
            Console.WriteLine("Starte Abruf von WordPress...");

            string wpHost = "";
            try { wpHost = new Uri(wpBaseUrl).Host.ToLowerInvariant(); } catch { }

            try
            {
                var allPostIds = new List<int>();
                var postTypes = new[] { "post", "page" };

                foreach (var postType in postTypes)
                {
                    Console.WriteLine($"Hole {postType}-IDs von der WordPress-API...");
                    string idsUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/posts-ids?post_type={postType}";
                    var idsResponse = await httpClient.GetStringAsync(idsUrl);
                    var idsObj = JsonSerializer.Deserialize<IdsResponse>(idsResponse);
                    if (idsObj?.Ids != null)
                        allPostIds.AddRange(idsObj.Ids);
                }

                if (allPostIds.Count == 0)
                {
                    Console.WriteLine("Keine Beiträge oder Seiten gefunden.");
                    return;
                }

                ProcessedPosts.Clear();
                LoadProcessedPosts();

                var pendingIds = allPostIds.Where(id => !ProcessedPosts.Contains(id)).ToList();
                Console.WriteLine($"Insgesamt gefunden: {allPostIds.Count}, Bereits abgefragt: {allPostIds.Count - pendingIds.Count}, Noch zu verarbeiten: {pendingIds.Count}");

                if (pendingIds.Count == 0)
                {
                    Console.WriteLine("Alle Beiträge und Seiten wurden bereits abgefragt. Nichts zu tun. (Nutze Option 5 zum Zurücksetzen)");
                    return;
                }

                if (maxPosts > 0)
                    pendingIds = pendingIds.Take(maxPosts).ToList();

                Console.WriteLine($"Es werden {pendingIds.Count} Beiträge/Seiten verarbeitet.");

                int internalId = 1;
                if (GhostPosts.Count > 0)
                {
                    var lastId = GhostPosts.Select(gp => int.TryParse(gp.Id, out int id) ? id : 0).Max();
                    internalId = lastId + 1;
                }
                int processed = 0;

                foreach (int wpId in pendingIds)
                {
                    Console.WriteLine($"[{++processed}/{pendingIds.Count}] -> Beitrag/Seite ID: {wpId}...");
                    string detailUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/post-details/{wpId}";

                    //string postUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/wp/v2/posts/{wpId}?_embed";
         
                    try
                    {
                        var detailJson = await httpClient.GetStringAsync(detailUrl);
                        var wpPost = JsonSerializer.Deserialize<WpPostResponse>(detailJson);

                        //var postJson = await httpClient.GetStringAsync(postUrl);

                        if (wpPost == null)
                        {
                            Console.WriteLine($"   Übersprungen (null).");
                            MarkProcessed(wpId, null);
                            continue;
                        }

                        string htmlContent = wpPost.Content ?? "";
                        var parsedDate = DateTime.MinValue;
                        var postDate = DateTime.TryParse(wpPost.Date, out parsedDate) ? parsedDate : new DateTime(2026, 9, 1);

                        // Featured Image: falls von der WP API nicht gesetzt, ersten Bild-URL aus HTML nehmen
                        if (string.IsNullOrEmpty(wpPost.FeaturedImage))
                        {
                            var firstImgMatch = System.Text.RegularExpressions.Regex.Match(htmlContent, @"<img\s+[^>]*src=[""']([^""']+)[""'][^>]*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
                            if (firstImgMatch.Success)
                            {
                                string firstImgUrl = firstImgMatch.Groups[1].Value;
                                if (IsInternalUrl(firstImgUrl, wpHost))
                                {
                                    wpPost.FeaturedImage = firstImgUrl;
                                    Console.WriteLine($"   [Featured Image] Aus HTML extrahiert (da API keine lieferte): {Path.GetFileName(new Uri(firstImgUrl).AbsolutePath)}");
                                }
                            }
                        }

                        // Alle internen Bild-URLs aus dem HTML extrahieren (keine externen Links)
                        var htmlImageUrls = ExtractImageUrls(htmlContent, wpHost);

                        // Bevorzugte Media-Dateien ermitteln
                        var preferredMedia = GetPreferredMediaFiles(wpPost.MediaFiles, htmlImageUrls, wpHost);

                        // Prüfe: Referenzierte Bilder im HTML, die NICHT in MediaFiles sind
                        if (htmlImageUrls.Count > 0 && wpPost.MediaFiles != null)
                        {
                            var mediaFileNames = wpPost.MediaFiles.Select(m => GetFileNameFromUrl(m).ToLowerInvariant()).ToHashSet();
                            foreach (var imgUrl in htmlImageUrls)
                            {
                                string imgFileName = GetFileNameFromUrl(imgUrl).ToLowerInvariant();
                                if (!mediaFileNames.Contains(imgFileName))
                                {
                                    Console.WriteLine($"   [Warnung] Bild im HTML nicht in MediaFiles: {Path.GetFileName(new Uri(imgUrl).AbsolutePath)}");
                                }
                            }
                        }

                        if (preferredMedia.Count > 0)
                        {
                            foreach (string mediaUrl in preferredMedia)
                            {
                                var downloaded = await ImageManager.DownloadAsync(mediaUrl, postDate);
                                if (downloaded.HasValue)
                                {
                                    string ghostMediaPath = $"__GHOST_URL__/content/images/{downloaded.Value.RelativePath}";
                                    htmlContent = htmlContent.Replace(mediaUrl, ghostMediaPath);
                                }
                            }
                        }

                        // WordPress HTML zu Ghost-kompatibles HTML konvertieren (Galerien, Figures, etc.)
                        htmlContent = HtmlGhostConverter.Convert(htmlContent);

                        // Lokale Bildpfade für JSON-Datei sammeln
                        var imageFiles = new List<string>();
                        if (preferredMedia.Count > 0)
                        {
                            foreach (string mediaUrl in preferredMedia)
                            {
                                if (ImageManager.TryGetRelativePath(mediaUrl, out string relPath))
                                    imageFiles.Add(relPath);
                            }
                        }

                        // Featured Image verarbeiten
                        string featureImageGhostPath = "";
                        if (!string.IsNullOrEmpty(wpPost.FeaturedImage))
                        {
                            if (ImageManager.TryGetRelativePath(wpPost.FeaturedImage, out string featRel))
                            {
                                featureImageGhostPath = $"__GHOST_URL__/content/images/{featRel}";
                            }
                            else
                            {
                                var featDownload = await ImageManager.DownloadAsync(wpPost.FeaturedImage, postDate);
                                if (featDownload.HasValue)
                                {
                                    featureImageGhostPath = $"__GHOST_URL__/content/images/{featDownload.Value.RelativePath}";
                                    if (!imageFiles.Contains(featDownload.Value.RelativePath))
                                        imageFiles.Add(featDownload.Value.RelativePath);
                                }
                            }
                        }

                        var ghostPost = new GhostPost
                        {
                            Id = internalId.ToString(),
                            Title = wpPost.Title,
                            Slug = wpPost.Slug,
                            Html = htmlContent,
                            Status = "published",
                            CreatedAt = postDate.ToString("o"),
                            PublishedAt = postDate.ToString("o")
                        };

                        GhostPosts.Add(ghostPost);

                        // JSON-Datei in ghostPosts/ speichern
                        var ghostPostFile = new GhostPostFile
                        {
                            WpId = wpId,
                            WpSlug = wpPost.Slug,
                            GhostId = ghostPost.Id,
                            Title = wpPost.Title,
                            Slug = wpPost.Slug,
                            Excerpt = wpPost.Excerpt,
                            Status = "published",
                            CreatedAt = postDate.ToString("o"),
                            PublishedAt = postDate.ToString("o"),
                            FeatureImage = featureImageGhostPath,
                            Categories = wpPost.Categories ?? new(),
                            Tags = wpPost.Tags ?? new(),
                            Keywords = wpPost.Keywords ?? "",
                            Comments = wpPost.Comments ?? new(),
                            ImageFiles = imageFiles,
                            Html = htmlContent
                        };
                        string jsonPath = Path.Combine(GhostPostsDir, $"{ghostPostFile.WpId}.json");
                        File.WriteAllText(jsonPath, JsonSerializer.Serialize(ghostPostFile, new JsonSerializerOptions { WriteIndented = true }));

                        // Cache sofort speichern (nicht erst beim Beenden)
                        ImageManager.SaveCache();

                        MarkProcessed(wpId, wpPost.Slug);
                        internalId++;
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine($"   Fehler beim Abrufen von ID {wpId}: {ex.Message}");
                        MarkProcessed(wpId, null);
                    }
                }

                Console.WriteLine();
                Console.WriteLine("=========================================");
                Console.WriteLine($"  Abruf abgeschlossen! {GhostPosts.Count} Beiträge geladen.");
                Console.WriteLine($"  Abgefragte Posts insgesamt: {ProcessedPosts.Count}");
                Console.WriteLine("  Nutze Option 2 um die Ghost ZIP zu erstellen.");
                Console.WriteLine("=========================================");
            }
            catch (Exception ex)
            {
                Console.WriteLine($"\nFehler beim Abruf: {ex.Message}");
                Console.WriteLine($"Stack Trace: {ex.StackTrace}");
            }
        }

        //public static void SaveAllMarkdownFiles()
        //{
        //    foreach (var post in GhostPosts)
        //    {
        //        if (string.IsNullOrEmpty(post.Slug)) continue;
        //        string mdPath = Path.Combine(MarkdownDir, $"{post.Slug}.md");
        //        if (!string.IsNullOrEmpty(post.Markdown))
        //            File.WriteAllText(mdPath, post.Markdown);
        //    }
        //    Console.WriteLine($"Markdown-Dateien gespeichert: {GhostPosts.Count} Dateien unter {MarkdownDir}");
        //}

        private static List<string> ExtractImageUrls(string html, string wpHost)
        {
            var urls = new List<string>();
            var imageExts = new[] { ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".tiff", ".tif" };

            // 1) <img src="..."> extrahieren
            var imgMatches = System.Text.RegularExpressions.Regex.Matches(html, @"<img\s+[^>]*src=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            foreach (System.Text.RegularExpressions.Match m in imgMatches)
            {
                string url = m.Groups[1].Value;
                if (!string.IsNullOrEmpty(url) && IsInternalUrl(url, wpHost) && !urls.Contains(url))
                    urls.Add(url);
            }

            // 2) data-orig-file (Vollbild) extrahieren
            var origFileMatches = System.Text.RegularExpressions.Regex.Matches(html, @"data-orig-file=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            foreach (System.Text.RegularExpressions.Match m in origFileMatches)
            {
                string url = m.Groups[1].Value;
                if (!string.IsNullOrEmpty(url) && IsInternalUrl(url, wpHost) && !urls.Contains(url))
                    urls.Add(url);
            }

            // 3) data-large-file extrahieren
            var largeFileMatches = System.Text.RegularExpressions.Regex.Matches(html, @"data-large-file=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            foreach (System.Text.RegularExpressions.Match m in largeFileMatches)
            {
                string url = m.Groups[1].Value;
                if (!string.IsNullOrEmpty(url) && IsInternalUrl(url, wpHost) && !urls.Contains(url))
                    urls.Add(url);
            }

            // 4) srcset (größte Variante pro img) extrahieren
            var srcsetMatches = System.Text.RegularExpressions.Regex.Matches(html, @"srcset=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            foreach (System.Text.RegularExpressions.Match m in srcsetMatches)
            {
                string srcset = m.Groups[1].Value;
                string widestUrl = GetWidestFromSrcset(srcset);
                if (!string.IsNullOrEmpty(widestUrl) && IsInternalUrl(widestUrl, wpHost) && !urls.Contains(widestUrl))
                    urls.Add(widestUrl);
            }

            // 5) <a href="..."> mit Bildendung (Figure-Links auf Originalbild)
            var aMatches = System.Text.RegularExpressions.Regex.Matches(html, @"<a\s+[^>]*href=[""']([^""']+)[""'][^>]*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            foreach (System.Text.RegularExpressions.Match m in aMatches)
            {
                string url = m.Groups[1].Value;
                if (string.IsNullOrEmpty(url)) continue;
                try
                {
                    string ext = Path.GetExtension(new Uri(url).AbsolutePath).ToLowerInvariant();
                    if (imageExts.Contains(ext) && IsInternalUrl(url, wpHost) && !urls.Contains(url))
                        urls.Add(url);
                }
                catch { }
            }

            return urls;
        }

        private static string GetWidestFromSrcset(string srcset)
        {
            string bestUrl = "";
            int bestWidth = 0;
            foreach (string part in srcset.Split(',', StringSplitOptions.RemoveEmptyEntries))
            {
                string[] tokens = part.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries);
                if (tokens.Length < 2) continue;
                if (int.TryParse(tokens[1].Replace("w", ""), out int w) && w > bestWidth)
                {
                    bestWidth = w;
                    bestUrl = tokens[0];
                }
            }
            return bestUrl;
        }

        private static bool IsInternalUrl(string url, string wpHost)
        {
            if (string.IsNullOrEmpty(wpHost)) return true;
            try
            {
                var uri = new Uri(url);
                return uri.Host.ToLowerInvariant().EndsWith(wpHost) || wpHost.EndsWith(uri.Host.ToLowerInvariant());
            }
            catch
            {
                return url.StartsWith("/") || url.StartsWith("wp-content");
            }
        }

        private static string NormalizeImageUrl(string url)
        {
            try
            {
                var uri = new Uri(url);
                string path = uri.AbsolutePath;
                // Entferne -scaled UND Dimensionspostfixe für den Vergleich
                path = System.Text.RegularExpressions.Regex.Replace(path, @"-scaled(?=\.\w+$)", "");
                path = System.Text.RegularExpressions.Regex.Replace(path, @"-\d+x\d+(?=\.\w+$)", "");
                return path;
            }
            catch
            {
                return url;
            }
        }

        private static string GetFileNameFromUrl(string url)
        {
            try
            {
                return Path.GetFileName(new Uri(url).AbsolutePath);
            }
            catch
            {
                return url;
            }
        }

        private static List<string> GetPreferredMediaFiles(List<string>? mediaFiles, List<string> htmlImageUrls, string wpHost)
        {
            var result = new List<string>();
            var imageExts = new[] { ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".tiff", ".tif" };

            // Alle internen Bild-URLs zusammenführen: MediaFiles + HTML-referenzierte
            var allUrls = new List<string>();
            if (mediaFiles != null)
                allUrls.AddRange(mediaFiles.Where(u => IsInternalUrl(u, wpHost)));
            foreach (var u in htmlImageUrls)
            {
                if (!allUrls.Contains(u))
                    allUrls.Add(u);
            }

            if (allUrls.Count == 0)
                return result;

            // Dateinamen aus den HTML-Links extrahieren
            var htmlFileNames = htmlImageUrls.Select(u => GetFileNameFromUrl(u).ToLowerInvariant()).ToHashSet();

            // Gruppiere nach normalisiertem Dateinamen (ohne -scaled und Dimensionspostfixe)
            var grouped = new Dictionary<string, List<string>>();
            foreach (var url in allUrls)
            {
                string normalized = NormalizeImageUrl(url);
                if (!grouped.ContainsKey(normalized))
                    grouped[normalized] = new List<string>();
                grouped[normalized].Add(url);
            }

            foreach (var group in grouped)
            {
                // Nur Hauptbilder berücksichtigen (keine WordPress-Sizes wie -300x200, keine -scaled)
                var mainImages = group.Value.Where(u => ImageManager.IsMainImage(u)).ToList();
                if (mainImages.Count == 0)
                {
                    Console.WriteLine($"   [Bild] Übersprungen (kein Hauptbild): {Path.GetFileName(new Uri(@group.Value[0]).AbsolutePath)}");
                    continue;
                }

                if (mainImages.Count == 1)
                {
                    result.Add(mainImages[0]);
                }
                else
                {
                    // Prüfe ob eine der Varianten als <a href> im HTML referenziert wird
                    string? linked = mainImages.FirstOrDefault(u => htmlFileNames.Contains(GetFileNameFromUrl(u).ToLowerInvariant()));

                    if (linked != null)
                    {
                        result.Add(linked);
                        var skipped = mainImages.Where(u => u != linked).ToList();
                        foreach (var s in skipped)
                            Console.WriteLine($"   [Bild] Übersprungen (nicht verlinkt): {Path.GetFileName(new Uri(s).AbsolutePath)}");
                    }
                    else
                    {
                        // Keine Verlinkung: Original (ohne -scaled) bevorzugen
                        var preferred = mainImages.FirstOrDefault(u => !u.Contains("-scaled")) ?? mainImages[0];
                        result.Add(preferred);
                        var skipped = mainImages.Where(u => u != preferred).ToList();
                        foreach (var s in skipped)
                            Console.WriteLine($"   [Bild] Übersprungen (scaled): {Path.GetFileName(new Uri(s).AbsolutePath)}");
                    }
                }
            }

            return result;
        }

        private static void MarkProcessed(int wpId, string? wpSlug = null)
        {
            ProcessedPosts.Add(wpId);
            SaveProcessedPosts(wpId, wpSlug);
        }

        private static void LoadProcessedPosts()
        {
            if (!File.Exists(ProcessedPostsPath)) return;
            try
            {
                var lines = File.ReadAllLines(ProcessedPostsPath);
                foreach (var line in lines)
                {
                    if (string.IsNullOrWhiteSpace(line) || line.StartsWith("wp_id")) continue;
                    var parts = line.Split(',');
                    if (parts.Length >= 1 && int.TryParse(parts[0].Trim(), out int postId))
                        ProcessedPosts.Add(postId);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Fehler beim Laden der abgefragten Posts: {ex.Message}");
            }
        }

        private static void SaveProcessedPosts(int wpId, string? wpSlug)
        {
            string entry = $"{wpId},{wpSlug ?? ""}";
            using var writer = File.AppendText(ProcessedPostsPath);
            writer.WriteLine(entry);
        }
    }
}