Wp2ghost/CSharpGhostImporter/PostFetcher.cs

using System.Text.Json;

namespace wp2ghost
{
    // =============================================================================
    // PostFetcher — Posts von WordPress abrufen, verarbeiten
    // =============================================================================
    static class PostFetcher
    {
        private static readonly HttpClient httpClient = new();
        private static HashSet<int> ProcessedPosts = new();
        private static List<GhostPost> GhostPosts = new();
        private static string ProcessedPostsPath = "";
        //private static string MarkdownDir = "";
        private static string GhostPostsDir = "";
        private static string BaseDir = "";

        public static int ProcessedCount => ProcessedPosts.Count;
        public static int PostCount => GhostPosts.Count;
        public static IReadOnlyList<GhostPost> Posts => GhostPosts;

        public static void Init(string baseDir)
        {
            BaseDir = baseDir;
            ProcessedPostsPath = Path.Combine(baseDir, "processed_posts.txt");
            //MarkdownDir = Path.Combine(baseDir, "markdown");
            GhostPostsDir = Path.Combine(baseDir, "ghostPosts");

            ////if (!Directory.Exists(MarkdownDir))
            ////    Directory.CreateDirectory(MarkdownDir);
            if (!Directory.Exists(GhostPostsDir))
                Directory.CreateDirectory(GhostPostsDir);

            LoadProcessedPosts();
        }

        public static void ClearProcessedPosts()
        {
            ProcessedPosts.Clear();
            File.WriteAllText(ProcessedPostsPath, "wp_id,wp_slug,ghost_id" + Environment.NewLine);
            Console.WriteLine("Abgefragte Posts wurden zurückgesetzt.");
        }

        public static void ShowInfo()
        {
            if (ProcessedPosts.Count > 0)
            {
                Console.WriteLine($"Letzte abgefragte Objekte: {string.Join(", ", ProcessedPosts.OrderByDescending(x => x).Take(10))}");
            }
        }

        public static async Task FetchFromWordPressAsync(string wpBaseUrl, int maxPosts)
        {
            Console.WriteLine();
            Console.WriteLine("Starte Abruf von WordPress...");

            string wpHost = "";
            try { wpHost = new Uri(wpBaseUrl).Host.ToLowerInvariant(); } catch { }

            try
            {
                var allPostIds = new List<int>();
                var postTypes = new[] { "post", "page" };

                foreach (var postType in postTypes)
                {
                    Console.WriteLine($"Hole {postType}-IDs von der WordPress-API...");
                    string idsUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/posts-ids?post_type={postType}";
                    var idsResponse = await httpClient.GetStringAsync(idsUrl);
                    var idsObj = JsonSerializer.Deserialize<IdsResponse>(idsResponse);
                    if (idsObj?.Ids != null)
                        allPostIds.AddRange(idsObj.Ids);
                }

                if (allPostIds.Count == 0)
                {
                    Console.WriteLine("Keine Beiträge oder Seiten gefunden.");
                    return;
                }

                ProcessedPosts.Clear();
                LoadProcessedPosts();

                var pendingIds = allPostIds.Where(id => !ProcessedPosts.Contains(id)).ToList();
                Console.WriteLine($"Insgesamt gefunden: {allPostIds.Count}, Bereits abgefragt: {allPostIds.Count - pendingIds.Count}, Noch zu verarbeiten: {pendingIds.Count}");

                if (pendingIds.Count == 0)
                {
                    Console.WriteLine("Alle Beiträge und Seiten wurden bereits abgefragt. Nichts zu tun. (Nutze Option 5 zum Zurücksetzen)");
                    return;
                }

                if (maxPosts > 0)
                    pendingIds = pendingIds.Take(maxPosts).ToList();

                Console.WriteLine($"Es werden {pendingIds.Count} Beiträge/Seiten verarbeitet.");

                int internalId = 1;
                if (GhostPosts.Count > 0)
                {
                    var lastId = GhostPosts.Select(gp => int.TryParse(gp.Id, out int id) ? id : 0).Max();
                    internalId = lastId + 1;
                }
                int processed = 0;

                foreach (int wpId in pendingIds)
                {
                    Console.WriteLine($"[{++processed}/{pendingIds.Count}] -> Beitrag/Seite ID: {wpId}...");
                    string detailUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/post-details/{wpId}";

                    //string postUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/wp/v2/posts/{wpId}?_embed";
         
                    try
                    {
                        var detailJson = await httpClient.GetStringAsync(detailUrl);
                        var wpPost = JsonSerializer.Deserialize<WpPostResponse>(detailJson);

                        //var postJson = await httpClient.GetStringAsync(postUrl);

                        if (wpPost == null)
                        {
                            Console.WriteLine($"   Übersprungen (null).");
                            MarkProcessed(wpId, null);
                            continue;
                        }

                        string htmlContent = wpPost.Content ?? "";
                        var parsedDate = DateTime.MinValue;
                        var postDate = DateTime.TryParse(wpPost.Date, out parsedDate) ? parsedDate : new DateTime(2026, 9, 1);

                        // Featured Image: falls von der WP API nicht gesetzt, ersten Bild-URL aus HTML nehmen
                        if (string.IsNullOrEmpty(wpPost.FeaturedImage))
                        {
                            var firstImgMatch = System.Text.RegularExpressions.Regex.Match(htmlContent, @"<img\s+[^>]*src=[""']([^""']+)[""'][^>]*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
                            if (firstImgMatch.Success)
                            {
                                string firstImgUrl = firstImgMatch.Groups[1].Value;
                                if (IsInternalUrl(firstImgUrl, wpHost))
                                {
                                    wpPost.FeaturedImage = firstImgUrl;
                                    Console.WriteLine($"   [Featured Image] Aus HTML extrahiert (da API keine lieferte): {Path.GetFileName(new Uri(firstImgUrl).AbsolutePath)}");
                                }
                            }
                        }

                        // Alle internen Bild-URLs im HTML (src, srcset, data-*, href) — keine externen Links
                        var htmlImageUrls = ExtractImageUrls(htmlContent, wpHost);

                        // Varianten desselben Bildes (Original, -scaled, -1024x683, ?w=...) zusammenfassen:
                        // pro Bild wird genau EINE Datei geladen, auf die alle Varianten umgeschrieben werden.
                        var imageGroups = new Dictionary<string, List<string>>();
                        var mediaUrls = (wpPost.MediaFiles ?? new()).Where(u => IsInternalUrl(u, wpHost)).ToList();
                        if (!string.IsNullOrEmpty(wpPost.FeaturedImage))
                            mediaUrls.Add(wpPost.FeaturedImage);
                        foreach (string url in htmlImageUrls.Concat(mediaUrls))
                        {
                            string key = ImageManager.GetImageKey(url);
                            if (!imageGroups.TryGetValue(key, out var variants))
                                imageGroups[key] = variants = new List<string>();
                            variants.Add(url);
                        }

                        var imageFiles = new List<string>();
                        var ghostPathByKey = new Dictionary<string, string>();
                        foreach (var group in imageGroups)
                        {
                            var downloaded = await ImageManager.DownloadBestAsync(group.Value, postDate);
                            if (!downloaded.HasValue) continue;
                            ghostPathByKey[group.Key] = $"__GHOST_URL__/content/images/{downloaded.Value.RelativePath}";
                            if (!imageFiles.Contains(downloaded.Value.RelativePath))
                                imageFiles.Add(downloaded.Value.RelativePath);
                        }

                        // Längste URLs zuerst ersetzen, damit "bild.jpg" nicht in "bild.jpg?w=1024" hineinersetzt
                        foreach (string url in htmlImageUrls.OrderByDescending(u => u.Length))
                        {
                            if (ghostPathByKey.TryGetValue(ImageManager.GetImageKey(url), out string? ghostMediaPath))
                                htmlContent = htmlContent.Replace(url, ghostMediaPath);
                        }

                        // WordPress HTML zu Ghost-kompatibles HTML konvertieren (Galerien, Figures, etc.)
                        htmlContent = HtmlGhostConverter.Convert(htmlContent);

                        int remainingWpImages = ExtractImageUrls(htmlContent, wpHost).Count;
                        if (remainingWpImages > 0)
                            Console.WriteLine($"   [Warnung] {remainingWpImages} Bild-URL(s) zeigen weiterhin auf WordPress");

                        // Featured Image (wurde oben mit heruntergeladen)
                        string featureImageGhostPath = "";
                        if (!string.IsNullOrEmpty(wpPost.FeaturedImage)
                            && ghostPathByKey.TryGetValue(ImageManager.GetImageKey(wpPost.FeaturedImage), out string? featPath))
                        {
                            featureImageGhostPath = featPath;
                        }

                        var ghostPost = new GhostPost
                        {
                            Id = internalId.ToString(),
                            Title = wpPost.Title,
                            Slug = wpPost.Slug,
                            Html = htmlContent,
                            FeatureImage = string.IsNullOrEmpty(featureImageGhostPath) ? null : featureImageGhostPath,
                            Status = "published",
                            CreatedAt = postDate.ToString("o"),
                            PublishedAt = postDate.ToString("o")
                        };

                        GhostPosts.Add(ghostPost);

                        // JSON-Datei in ghostPosts/ speichern
                        var ghostPostFile = new GhostPostFile
                        {
                            WpId = wpId,
                            WpSlug = wpPost.Slug,
                            GhostId = ghostPost.Id,
                            Title = wpPost.Title,
                            Slug = wpPost.Slug,
                            Excerpt = wpPost.Excerpt,
                            Status = "published",
                            CreatedAt = postDate.ToString("o"),
                            PublishedAt = postDate.ToString("o"),
                            FeatureImage = featureImageGhostPath,
                            Categories = wpPost.Categories ?? new(),
                            Tags = wpPost.Tags ?? new(),
                            Keywords = wpPost.Keywords ?? "",
                            Comments = wpPost.Comments ?? new(),
                            ImageFiles = imageFiles,
                            Html = htmlContent
                        };
                        string jsonPath = Path.Combine(GhostPostsDir, $"{ghostPostFile.WpId}.json");
                        File.WriteAllText(jsonPath, JsonSerializer.Serialize(ghostPostFile, new JsonSerializerOptions { WriteIndented = true }));

                        // Cache sofort speichern (nicht erst beim Beenden)
                        ImageManager.SaveCache();

                        MarkProcessed(wpId, wpPost.Slug);
                        internalId++;
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine($"   Fehler beim Abrufen von ID {wpId}: {ex.Message}");
                        MarkProcessed(wpId, null);
                    }
                }

                Console.WriteLine();
                Console.WriteLine("=========================================");
                Console.WriteLine($"  Abruf abgeschlossen! {GhostPosts.Count} Beiträge geladen.");
                Console.WriteLine($"  Abgefragte Posts insgesamt: {ProcessedPosts.Count}");
                Console.WriteLine("  Nutze Option 2 um die Ghost ZIP zu erstellen.");
                Console.WriteLine("=========================================");
            }
            catch (Exception ex)
            {
                Console.WriteLine($"\nFehler beim Abruf: {ex.Message}");
                Console.WriteLine($"Stack Trace: {ex.StackTrace}");
            }
        }

        //public static void SaveAllMarkdownFiles()
        //{
        //    foreach (var post in GhostPosts)
        //    {
        //        if (string.IsNullOrEmpty(post.Slug)) continue;
        //        string mdPath = Path.Combine(MarkdownDir, $"{post.Slug}.md");
        //        if (!string.IsNullOrEmpty(post.Markdown))
        //            File.WriteAllText(mdPath, post.Markdown);
        //    }
        //    Console.WriteLine($"Markdown-Dateien gespeichert: {GhostPosts.Count} Dateien unter {MarkdownDir}");
        //}

        // Bild-URL irgendwo im HTML: src, srcset, data-orig-file, data-large-file, href ...
        // inkl. Query (?w=1024), damit beim Ersetzen nichts davon stehen bleibt.
        private static readonly System.Text.RegularExpressions.Regex ImageUrlRegex = new(
            @"https?://[^\s""'<>()]+?\.(?:jpe?g|png|gif|webp|svg|bmp|tiff?)(?:\?[^\s""'<>]*)?(?=[\s""'<>,)&\\]|$)",
            System.Text.RegularExpressions.RegexOptions.IgnoreCase | System.Text.RegularExpressions.RegexOptions.Compiled);

        private static List<string> ExtractImageUrls(string html, string wpHost)
        {
            var urls = new List<string>();
            foreach (System.Text.RegularExpressions.Match m in ImageUrlRegex.Matches(html))
            {
                string url = m.Value;
                if (IsInternalUrl(url, wpHost) && !urls.Contains(url))
                    urls.Add(url);
            }
            return urls;
        }

        private static bool IsInternalUrl(string url, string wpHost)
        {
            if (string.IsNullOrEmpty(wpHost)) return true;
            try
            {
                var uri = new Uri(url);
                return uri.Host.ToLowerInvariant().EndsWith(wpHost) || wpHost.EndsWith(uri.Host.ToLowerInvariant());
            }
            catch
            {
                return url.StartsWith("/") || url.StartsWith("wp-content");
            }
        }

        private static void MarkProcessed(int wpId, string? wpSlug = null)
        {
            ProcessedPosts.Add(wpId);
            SaveProcessedPosts(wpId, wpSlug);
        }

        private static void LoadProcessedPosts()
        {
            if (!File.Exists(ProcessedPostsPath)) return;
            try
            {
                var lines = File.ReadAllLines(ProcessedPostsPath);
                foreach (var line in lines)
                {
                    if (string.IsNullOrWhiteSpace(line) || line.StartsWith("wp_id")) continue;
                    var parts = line.Split(',');
                    if (parts.Length >= 1 && int.TryParse(parts[0].Trim(), out int postId))
                        ProcessedPosts.Add(postId);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Fehler beim Laden der abgefragten Posts: {ex.Message}");
            }
        }

        private static void SaveProcessedPosts(int wpId, string? wpSlug)
        {
            string entry = $"{wpId},{wpSlug ?? ""}";
            using var writer = File.AppendText(ProcessedPostsPath);
            writer.WriteLine(entry);
        }
    }
}