using System.Text.Json; namespace wp2ghost { // ============================================================================= // PostFetcher — Posts von WordPress abrufen, verarbeiten // ============================================================================= static class PostFetcher { private static readonly HttpClient httpClient = new(); private static HashSet ProcessedPosts = new(); private static List GhostPosts = new(); private static string ProcessedPostsPath = ""; //private static string MarkdownDir = ""; private static string GhostPostsDir = ""; private static string BaseDir = ""; public static int ProcessedCount => ProcessedPosts.Count; public static int PostCount => GhostPosts.Count; public static IReadOnlyList Posts => GhostPosts; public static void Init(string baseDir) { BaseDir = baseDir; ProcessedPostsPath = Path.Combine(baseDir, "processed_posts.txt"); //MarkdownDir = Path.Combine(baseDir, "markdown"); GhostPostsDir = Path.Combine(baseDir, "ghostPosts"); ////if (!Directory.Exists(MarkdownDir)) //// Directory.CreateDirectory(MarkdownDir); if (!Directory.Exists(GhostPostsDir)) Directory.CreateDirectory(GhostPostsDir); LoadProcessedPosts(); } public static void ClearProcessedPosts() { ProcessedPosts.Clear(); File.WriteAllText(ProcessedPostsPath, "wp_id,wp_slug,ghost_id" + Environment.NewLine); Console.WriteLine("Abgefragte Posts wurden zurückgesetzt."); } public static void ShowInfo() { if (ProcessedPosts.Count > 0) { Console.WriteLine($"Letzte abgefragte Objekte: {string.Join(", ", ProcessedPosts.OrderByDescending(x => x).Take(10))}"); } } public static async Task FetchFromWordPressAsync(string wpBaseUrl, int maxPosts) { Console.WriteLine(); Console.WriteLine("Starte Abruf von WordPress..."); string wpHost = ""; try { wpHost = new Uri(wpBaseUrl).Host.ToLowerInvariant(); } catch { } try { var allPostIds = new List(); var postTypes = new[] { "post", "page" }; foreach (var postType in postTypes) { Console.WriteLine($"Hole {postType}-IDs von der WordPress-API..."); string idsUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/posts-ids?post_type={postType}"; var idsResponse = await httpClient.GetStringAsync(idsUrl); var idsObj = JsonSerializer.Deserialize(idsResponse); if (idsObj?.Ids != null) allPostIds.AddRange(idsObj.Ids); } if (allPostIds.Count == 0) { Console.WriteLine("Keine Beiträge oder Seiten gefunden."); return; } ProcessedPosts.Clear(); LoadProcessedPosts(); var pendingIds = allPostIds.Where(id => !ProcessedPosts.Contains(id)).ToList(); Console.WriteLine($"Insgesamt gefunden: {allPostIds.Count}, Bereits abgefragt: {allPostIds.Count - pendingIds.Count}, Noch zu verarbeiten: {pendingIds.Count}"); if (pendingIds.Count == 0) { Console.WriteLine("Alle Beiträge und Seiten wurden bereits abgefragt. Nichts zu tun. (Nutze Option 5 zum Zurücksetzen)"); return; } if (maxPosts > 0) pendingIds = pendingIds.Take(maxPosts).ToList(); Console.WriteLine($"Es werden {pendingIds.Count} Beiträge/Seiten verarbeitet."); int internalId = 1; if (GhostPosts.Count > 0) { var lastId = GhostPosts.Select(gp => int.TryParse(gp.Id, out int id) ? id : 0).Max(); internalId = lastId + 1; } int processed = 0; foreach (int wpId in pendingIds) { Console.WriteLine($"[{++processed}/{pendingIds.Count}] -> Beitrag/Seite ID: {wpId}..."); string detailUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/post-details/{wpId}"; //string postUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/wp/v2/posts/{wpId}?_embed"; try { var detailJson = await httpClient.GetStringAsync(detailUrl); var wpPost = JsonSerializer.Deserialize(detailJson); //var postJson = await httpClient.GetStringAsync(postUrl); if (wpPost == null) { Console.WriteLine($" Übersprungen (null)."); MarkProcessed(wpId, null); continue; } string htmlContent = wpPost.Content ?? ""; var parsedDate = DateTime.MinValue; var postDate = DateTime.TryParse(wpPost.Date, out parsedDate) ? parsedDate : new DateTime(2026, 9, 1); // Featured Image: falls von der WP API nicht gesetzt, ersten Bild-URL aus HTML nehmen if (string.IsNullOrEmpty(wpPost.FeaturedImage)) { var firstImgMatch = System.Text.RegularExpressions.Regex.Match(htmlContent, @"]*src=[""']([^""']+)[""'][^>]*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase); if (firstImgMatch.Success) { string firstImgUrl = firstImgMatch.Groups[1].Value; if (IsInternalUrl(firstImgUrl, wpHost)) { wpPost.FeaturedImage = firstImgUrl; Console.WriteLine($" [Featured Image] Aus HTML extrahiert (da API keine lieferte): {Path.GetFileName(new Uri(firstImgUrl).AbsolutePath)}"); } } } // Alle internen Bild-URLs im HTML (src, srcset, data-*, href) — keine externen Links var htmlImageUrls = ExtractImageUrls(htmlContent, wpHost); // Varianten desselben Bildes (Original, -scaled, -1024x683, ?w=...) zusammenfassen: // pro Bild wird genau EINE Datei geladen, auf die alle Varianten umgeschrieben werden. var imageGroups = new Dictionary>(); var mediaUrls = (wpPost.MediaFiles ?? new()).Where(u => IsInternalUrl(u, wpHost)).ToList(); if (!string.IsNullOrEmpty(wpPost.FeaturedImage)) mediaUrls.Add(wpPost.FeaturedImage); foreach (string url in htmlImageUrls.Concat(mediaUrls)) { string key = ImageManager.GetImageKey(url); if (!imageGroups.TryGetValue(key, out var variants)) imageGroups[key] = variants = new List(); variants.Add(url); } var imageFiles = new List(); var ghostPathByKey = new Dictionary(); foreach (var group in imageGroups) { var downloaded = await ImageManager.DownloadBestAsync(group.Value, postDate); if (!downloaded.HasValue) continue; ghostPathByKey[group.Key] = $"__GHOST_URL__/content/images/{downloaded.Value.RelativePath}"; if (!imageFiles.Contains(downloaded.Value.RelativePath)) imageFiles.Add(downloaded.Value.RelativePath); } // Längste URLs zuerst ersetzen, damit "bild.jpg" nicht in "bild.jpg?w=1024" hineinersetzt foreach (string url in htmlImageUrls.OrderByDescending(u => u.Length)) { if (ghostPathByKey.TryGetValue(ImageManager.GetImageKey(url), out string? ghostMediaPath)) htmlContent = htmlContent.Replace(url, ghostMediaPath); } // WordPress HTML zu Ghost-kompatibles HTML konvertieren (Galerien, Figures, etc.) htmlContent = HtmlGhostConverter.Convert(htmlContent); int remainingWpImages = ExtractImageUrls(htmlContent, wpHost).Count; if (remainingWpImages > 0) Console.WriteLine($" [Warnung] {remainingWpImages} Bild-URL(s) zeigen weiterhin auf WordPress"); // Featured Image (wurde oben mit heruntergeladen) string featureImageGhostPath = ""; if (!string.IsNullOrEmpty(wpPost.FeaturedImage) && ghostPathByKey.TryGetValue(ImageManager.GetImageKey(wpPost.FeaturedImage), out string? featPath)) { featureImageGhostPath = featPath; } var ghostPost = new GhostPost { Id = internalId.ToString(), Title = wpPost.Title, Slug = wpPost.Slug, Html = htmlContent, FeatureImage = string.IsNullOrEmpty(featureImageGhostPath) ? null : featureImageGhostPath, Status = "published", CreatedAt = postDate.ToString("o"), PublishedAt = postDate.ToString("o") }; GhostPosts.Add(ghostPost); // JSON-Datei in ghostPosts/ speichern var ghostPostFile = new GhostPostFile { WpId = wpId, WpSlug = wpPost.Slug, GhostId = ghostPost.Id, Title = wpPost.Title, Slug = wpPost.Slug, Excerpt = wpPost.Excerpt, Status = "published", CreatedAt = postDate.ToString("o"), PublishedAt = postDate.ToString("o"), FeatureImage = featureImageGhostPath, Categories = wpPost.Categories ?? new(), Tags = wpPost.Tags ?? new(), Keywords = wpPost.Keywords ?? "", Comments = wpPost.Comments ?? new(), ImageFiles = imageFiles, Html = htmlContent }; string jsonPath = Path.Combine(GhostPostsDir, $"{ghostPostFile.WpId}.json"); File.WriteAllText(jsonPath, JsonSerializer.Serialize(ghostPostFile, new JsonSerializerOptions { WriteIndented = true })); // Cache sofort speichern (nicht erst beim Beenden) ImageManager.SaveCache(); MarkProcessed(wpId, wpPost.Slug); internalId++; } catch (Exception ex) { Console.WriteLine($" Fehler beim Abrufen von ID {wpId}: {ex.Message}"); MarkProcessed(wpId, null); } } Console.WriteLine(); Console.WriteLine("========================================="); Console.WriteLine($" Abruf abgeschlossen! {GhostPosts.Count} Beiträge geladen."); Console.WriteLine($" Abgefragte Posts insgesamt: {ProcessedPosts.Count}"); Console.WriteLine(" Nutze Option 2 um die Ghost ZIP zu erstellen."); Console.WriteLine("========================================="); } catch (Exception ex) { Console.WriteLine($"\nFehler beim Abruf: {ex.Message}"); Console.WriteLine($"Stack Trace: {ex.StackTrace}"); } } //public static void SaveAllMarkdownFiles() //{ // foreach (var post in GhostPosts) // { // if (string.IsNullOrEmpty(post.Slug)) continue; // string mdPath = Path.Combine(MarkdownDir, $"{post.Slug}.md"); // if (!string.IsNullOrEmpty(post.Markdown)) // File.WriteAllText(mdPath, post.Markdown); // } // Console.WriteLine($"Markdown-Dateien gespeichert: {GhostPosts.Count} Dateien unter {MarkdownDir}"); //} // Bild-URL irgendwo im HTML: src, srcset, data-orig-file, data-large-file, href ... // inkl. Query (?w=1024), damit beim Ersetzen nichts davon stehen bleibt. private static readonly System.Text.RegularExpressions.Regex ImageUrlRegex = new( @"https?://[^\s""'<>()]+?\.(?:jpe?g|png|gif|webp|svg|bmp|tiff?)(?:\?[^\s""'<>]*)?(?=[\s""'<>,)&\\]|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase | System.Text.RegularExpressions.RegexOptions.Compiled); private static List ExtractImageUrls(string html, string wpHost) { var urls = new List(); foreach (System.Text.RegularExpressions.Match m in ImageUrlRegex.Matches(html)) { string url = m.Value; if (IsInternalUrl(url, wpHost) && !urls.Contains(url)) urls.Add(url); } return urls; } private static bool IsInternalUrl(string url, string wpHost) { if (string.IsNullOrEmpty(wpHost)) return true; try { var uri = new Uri(url); return uri.Host.ToLowerInvariant().EndsWith(wpHost) || wpHost.EndsWith(uri.Host.ToLowerInvariant()); } catch { return url.StartsWith("/") || url.StartsWith("wp-content"); } } private static void MarkProcessed(int wpId, string? wpSlug = null) { ProcessedPosts.Add(wpId); SaveProcessedPosts(wpId, wpSlug); } private static void LoadProcessedPosts() { if (!File.Exists(ProcessedPostsPath)) return; try { var lines = File.ReadAllLines(ProcessedPostsPath); foreach (var line in lines) { if (string.IsNullOrWhiteSpace(line) || line.StartsWith("wp_id")) continue; var parts = line.Split(','); if (parts.Length >= 1 && int.TryParse(parts[0].Trim(), out int postId)) ProcessedPosts.Add(postId); } } catch (Exception ex) { Console.WriteLine($"Fehler beim Laden der abgefragten Posts: {ex.Message}"); } } private static void SaveProcessedPosts(int wpId, string? wpSlug) { string entry = $"{wpId},{wpSlug ?? ""}"; using var writer = File.AppendText(ProcessedPostsPath); writer.WriteLine(entry); } } }