using System.Text.Json; namespace wp2ghost { // ============================================================================= // PostFetcher — Posts von WordPress abrufen, verarbeiten // ============================================================================= static class PostFetcher { private static readonly HttpClient httpClient = new(); private static HashSet ProcessedPosts = new(); private static List GhostPosts = new(); private static string ProcessedPostsPath = ""; //private static string MarkdownDir = ""; private static string GhostPostsDir = ""; private static string BaseDir = ""; public static int ProcessedCount => ProcessedPosts.Count; public static int PostCount => GhostPosts.Count; public static IReadOnlyList Posts => GhostPosts; public static void Init(string baseDir) { BaseDir = baseDir; ProcessedPostsPath = Path.Combine(baseDir, "processed_posts.txt"); //MarkdownDir = Path.Combine(baseDir, "markdown"); GhostPostsDir = Path.Combine(baseDir, "ghostPosts"); ////if (!Directory.Exists(MarkdownDir)) //// Directory.CreateDirectory(MarkdownDir); if (!Directory.Exists(GhostPostsDir)) Directory.CreateDirectory(GhostPostsDir); LoadProcessedPosts(); } public static void ClearProcessedPosts() { ProcessedPosts.Clear(); File.WriteAllText(ProcessedPostsPath, "wp_id,wp_slug,ghost_id" + Environment.NewLine); Console.WriteLine("Abgefragte Posts wurden zurückgesetzt."); } public static void ShowInfo() { if (ProcessedPosts.Count > 0) { Console.WriteLine($"Letzte abgefragte Objekte: {string.Join(", ", ProcessedPosts.OrderByDescending(x => x).Take(10))}"); } } public static async Task FetchFromWordPressAsync(string wpBaseUrl, int maxPosts) { Console.WriteLine(); Console.WriteLine("Starte Abruf von WordPress..."); string wpHost = ""; try { wpHost = new Uri(wpBaseUrl).Host.ToLowerInvariant(); } catch { } try { var allPostIds = new List(); var postTypes = new[] { "post", "page" }; foreach (var postType in postTypes) { Console.WriteLine($"Hole {postType}-IDs von der WordPress-API..."); string idsUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/posts-ids?post_type={postType}"; var idsResponse = await httpClient.GetStringAsync(idsUrl); var idsObj = JsonSerializer.Deserialize(idsResponse); if (idsObj?.Ids != null) allPostIds.AddRange(idsObj.Ids); } if (allPostIds.Count == 0) { Console.WriteLine("Keine Beiträge oder Seiten gefunden."); return; } ProcessedPosts.Clear(); LoadProcessedPosts(); var pendingIds = allPostIds.Where(id => !ProcessedPosts.Contains(id)).ToList(); Console.WriteLine($"Insgesamt gefunden: {allPostIds.Count}, Bereits abgefragt: {allPostIds.Count - pendingIds.Count}, Noch zu verarbeiten: {pendingIds.Count}"); if (pendingIds.Count == 0) { Console.WriteLine("Alle Beiträge und Seiten wurden bereits abgefragt. Nichts zu tun. (Nutze Option 5 zum Zurücksetzen)"); return; } if (maxPosts > 0) pendingIds = pendingIds.Take(maxPosts).ToList(); Console.WriteLine($"Es werden {pendingIds.Count} Beiträge/Seiten verarbeitet."); int internalId = 1; if (GhostPosts.Count > 0) { var lastId = GhostPosts.Select(gp => int.TryParse(gp.Id, out int id) ? id : 0).Max(); internalId = lastId + 1; } int processed = 0; foreach (int wpId in pendingIds) { Console.WriteLine($"[{++processed}/{pendingIds.Count}] -> Beitrag/Seite ID: {wpId}..."); string detailUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/post-details/{wpId}"; //string postUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/wp/v2/posts/{wpId}?_embed"; try { var detailJson = await httpClient.GetStringAsync(detailUrl); var wpPost = JsonSerializer.Deserialize(detailJson); //var postJson = await httpClient.GetStringAsync(postUrl); if (wpPost == null) { Console.WriteLine($" Übersprungen (null)."); MarkProcessed(wpId, null); continue; } string htmlContent = wpPost.Content ?? ""; var parsedDate = DateTime.MinValue; var postDate = DateTime.TryParse(wpPost.Date, out parsedDate) ? parsedDate : new DateTime(2026, 9, 1); // Featured Image: falls von der WP API nicht gesetzt, ersten Bild-URL aus HTML nehmen if (string.IsNullOrEmpty(wpPost.FeaturedImage)) { var firstImgMatch = System.Text.RegularExpressions.Regex.Match(htmlContent, @"]*src=[""']([^""']+)[""'][^>]*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase); if (firstImgMatch.Success) { string firstImgUrl = firstImgMatch.Groups[1].Value; if (IsInternalUrl(firstImgUrl, wpHost)) { wpPost.FeaturedImage = firstImgUrl; Console.WriteLine($" [Featured Image] Aus HTML extrahiert (da API keine lieferte): {Path.GetFileName(new Uri(firstImgUrl).AbsolutePath)}"); } } } // Alle internen Bild-URLs aus dem HTML extrahieren (keine externen Links) var htmlImageUrls = ExtractImageUrls(htmlContent, wpHost); // Bevorzugte Media-Dateien ermitteln var preferredMedia = GetPreferredMediaFiles(wpPost.MediaFiles, htmlImageUrls, wpHost); // Prüfe: Referenzierte Bilder im HTML, die NICHT in MediaFiles sind if (htmlImageUrls.Count > 0 && wpPost.MediaFiles != null) { var mediaFileNames = wpPost.MediaFiles.Select(m => GetFileNameFromUrl(m).ToLowerInvariant()).ToHashSet(); foreach (var imgUrl in htmlImageUrls) { string imgFileName = GetFileNameFromUrl(imgUrl).ToLowerInvariant(); if (!mediaFileNames.Contains(imgFileName)) { Console.WriteLine($" [Warnung] Bild im HTML nicht in MediaFiles: {Path.GetFileName(new Uri(imgUrl).AbsolutePath)}"); } } } if (preferredMedia.Count > 0) { foreach (string mediaUrl in preferredMedia) { var downloaded = await ImageManager.DownloadAsync(mediaUrl, postDate); if (downloaded.HasValue) { string ghostMediaPath = $"__GHOST_URL__/content/images/{downloaded.Value.RelativePath}"; htmlContent = htmlContent.Replace(mediaUrl, ghostMediaPath); } } } // WordPress HTML zu Ghost-kompatibles HTML konvertieren (Galerien, Figures, etc.) htmlContent = HtmlGhostConverter.Convert(htmlContent); // Lokale Bildpfade für JSON-Datei sammeln var imageFiles = new List(); if (preferredMedia.Count > 0) { foreach (string mediaUrl in preferredMedia) { if (ImageManager.TryGetRelativePath(mediaUrl, out string relPath)) imageFiles.Add(relPath); } } // Featured Image verarbeiten string featureImageGhostPath = ""; if (!string.IsNullOrEmpty(wpPost.FeaturedImage)) { if (ImageManager.TryGetRelativePath(wpPost.FeaturedImage, out string featRel)) { featureImageGhostPath = $"__GHOST_URL__/content/images/{featRel}"; } else { var featDownload = await ImageManager.DownloadAsync(wpPost.FeaturedImage, postDate); if (featDownload.HasValue) { featureImageGhostPath = $"__GHOST_URL__/content/images/{featDownload.Value.RelativePath}"; if (!imageFiles.Contains(featDownload.Value.RelativePath)) imageFiles.Add(featDownload.Value.RelativePath); } } } var ghostPost = new GhostPost { Id = internalId.ToString(), Title = wpPost.Title, Slug = wpPost.Slug, Html = htmlContent, Status = "published", CreatedAt = postDate.ToString("o"), PublishedAt = postDate.ToString("o") }; GhostPosts.Add(ghostPost); // JSON-Datei in ghostPosts/ speichern var ghostPostFile = new GhostPostFile { WpId = wpId, WpSlug = wpPost.Slug, GhostId = ghostPost.Id, Title = wpPost.Title, Slug = wpPost.Slug, Excerpt = wpPost.Excerpt, Status = "published", CreatedAt = postDate.ToString("o"), PublishedAt = postDate.ToString("o"), FeatureImage = featureImageGhostPath, Categories = wpPost.Categories ?? new(), Tags = wpPost.Tags ?? new(), Keywords = wpPost.Keywords ?? "", Comments = wpPost.Comments ?? new(), ImageFiles = imageFiles, Html = htmlContent }; string jsonPath = Path.Combine(GhostPostsDir, $"{ghostPostFile.WpId}.json"); File.WriteAllText(jsonPath, JsonSerializer.Serialize(ghostPostFile, new JsonSerializerOptions { WriteIndented = true })); // Cache sofort speichern (nicht erst beim Beenden) ImageManager.SaveCache(); MarkProcessed(wpId, wpPost.Slug); internalId++; } catch (Exception ex) { Console.WriteLine($" Fehler beim Abrufen von ID {wpId}: {ex.Message}"); MarkProcessed(wpId, null); } } Console.WriteLine(); Console.WriteLine("========================================="); Console.WriteLine($" Abruf abgeschlossen! {GhostPosts.Count} Beiträge geladen."); Console.WriteLine($" Abgefragte Posts insgesamt: {ProcessedPosts.Count}"); Console.WriteLine(" Nutze Option 2 um die Ghost ZIP zu erstellen."); Console.WriteLine("========================================="); } catch (Exception ex) { Console.WriteLine($"\nFehler beim Abruf: {ex.Message}"); Console.WriteLine($"Stack Trace: {ex.StackTrace}"); } } //public static void SaveAllMarkdownFiles() //{ // foreach (var post in GhostPosts) // { // if (string.IsNullOrEmpty(post.Slug)) continue; // string mdPath = Path.Combine(MarkdownDir, $"{post.Slug}.md"); // if (!string.IsNullOrEmpty(post.Markdown)) // File.WriteAllText(mdPath, post.Markdown); // } // Console.WriteLine($"Markdown-Dateien gespeichert: {GhostPosts.Count} Dateien unter {MarkdownDir}"); //} private static List ExtractImageUrls(string html, string wpHost) { var urls = new List(); var imageExts = new[] { ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".tiff", ".tif" }; // 1) extrahieren var imgMatches = System.Text.RegularExpressions.Regex.Matches(html, @"]*src=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase); foreach (System.Text.RegularExpressions.Match m in imgMatches) { string url = m.Groups[1].Value; if (!string.IsNullOrEmpty(url) && IsInternalUrl(url, wpHost) && !urls.Contains(url)) urls.Add(url); } // 2) data-orig-file (Vollbild) extrahieren var origFileMatches = System.Text.RegularExpressions.Regex.Matches(html, @"data-orig-file=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase); foreach (System.Text.RegularExpressions.Match m in origFileMatches) { string url = m.Groups[1].Value; if (!string.IsNullOrEmpty(url) && IsInternalUrl(url, wpHost) && !urls.Contains(url)) urls.Add(url); } // 3) data-large-file extrahieren var largeFileMatches = System.Text.RegularExpressions.Regex.Matches(html, @"data-large-file=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase); foreach (System.Text.RegularExpressions.Match m in largeFileMatches) { string url = m.Groups[1].Value; if (!string.IsNullOrEmpty(url) && IsInternalUrl(url, wpHost) && !urls.Contains(url)) urls.Add(url); } // 4) srcset (größte Variante pro img) extrahieren var srcsetMatches = System.Text.RegularExpressions.Regex.Matches(html, @"srcset=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase); foreach (System.Text.RegularExpressions.Match m in srcsetMatches) { string srcset = m.Groups[1].Value; string widestUrl = GetWidestFromSrcset(srcset); if (!string.IsNullOrEmpty(widestUrl) && IsInternalUrl(widestUrl, wpHost) && !urls.Contains(widestUrl)) urls.Add(widestUrl); } // 5) mit Bildendung (Figure-Links auf Originalbild) var aMatches = System.Text.RegularExpressions.Regex.Matches(html, @"]*href=[""']([^""']+)[""'][^>]*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase); foreach (System.Text.RegularExpressions.Match m in aMatches) { string url = m.Groups[1].Value; if (string.IsNullOrEmpty(url)) continue; try { string ext = Path.GetExtension(new Uri(url).AbsolutePath).ToLowerInvariant(); if (imageExts.Contains(ext) && IsInternalUrl(url, wpHost) && !urls.Contains(url)) urls.Add(url); } catch { } } return urls; } private static string GetWidestFromSrcset(string srcset) { string bestUrl = ""; int bestWidth = 0; foreach (string part in srcset.Split(',', StringSplitOptions.RemoveEmptyEntries)) { string[] tokens = part.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries); if (tokens.Length < 2) continue; if (int.TryParse(tokens[1].Replace("w", ""), out int w) && w > bestWidth) { bestWidth = w; bestUrl = tokens[0]; } } return bestUrl; } private static bool IsInternalUrl(string url, string wpHost) { if (string.IsNullOrEmpty(wpHost)) return true; try { var uri = new Uri(url); return uri.Host.ToLowerInvariant().EndsWith(wpHost) || wpHost.EndsWith(uri.Host.ToLowerInvariant()); } catch { return url.StartsWith("/") || url.StartsWith("wp-content"); } } private static string NormalizeImageUrl(string url) { try { var uri = new Uri(url); string path = uri.AbsolutePath; // Entferne -scaled UND Dimensionspostfixe für den Vergleich path = System.Text.RegularExpressions.Regex.Replace(path, @"-scaled(?=\.\w+$)", ""); path = System.Text.RegularExpressions.Regex.Replace(path, @"-\d+x\d+(?=\.\w+$)", ""); return path; } catch { return url; } } private static string GetFileNameFromUrl(string url) { try { return Path.GetFileName(new Uri(url).AbsolutePath); } catch { return url; } } private static List GetPreferredMediaFiles(List? mediaFiles, List htmlImageUrls, string wpHost) { var result = new List(); var imageExts = new[] { ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".tiff", ".tif" }; // Alle internen Bild-URLs zusammenführen: MediaFiles + HTML-referenzierte var allUrls = new List(); if (mediaFiles != null) allUrls.AddRange(mediaFiles.Where(u => IsInternalUrl(u, wpHost))); foreach (var u in htmlImageUrls) { if (!allUrls.Contains(u)) allUrls.Add(u); } if (allUrls.Count == 0) return result; // Dateinamen aus den HTML-Links extrahieren var htmlFileNames = htmlImageUrls.Select(u => GetFileNameFromUrl(u).ToLowerInvariant()).ToHashSet(); // Gruppiere nach normalisiertem Dateinamen (ohne -scaled und Dimensionspostfixe) var grouped = new Dictionary>(); foreach (var url in allUrls) { string normalized = NormalizeImageUrl(url); if (!grouped.ContainsKey(normalized)) grouped[normalized] = new List(); grouped[normalized].Add(url); } foreach (var group in grouped) { // Nur Hauptbilder berücksichtigen (keine WordPress-Sizes wie -300x200, keine -scaled) var mainImages = group.Value.Where(u => ImageManager.IsMainImage(u)).ToList(); if (mainImages.Count == 0) { Console.WriteLine($" [Bild] Übersprungen (kein Hauptbild): {Path.GetFileName(new Uri(@group.Value[0]).AbsolutePath)}"); continue; } if (mainImages.Count == 1) { result.Add(mainImages[0]); } else { // Prüfe ob eine der Varianten als im HTML referenziert wird string? linked = mainImages.FirstOrDefault(u => htmlFileNames.Contains(GetFileNameFromUrl(u).ToLowerInvariant())); if (linked != null) { result.Add(linked); var skipped = mainImages.Where(u => u != linked).ToList(); foreach (var s in skipped) Console.WriteLine($" [Bild] Übersprungen (nicht verlinkt): {Path.GetFileName(new Uri(s).AbsolutePath)}"); } else { // Keine Verlinkung: Original (ohne -scaled) bevorzugen var preferred = mainImages.FirstOrDefault(u => !u.Contains("-scaled")) ?? mainImages[0]; result.Add(preferred); var skipped = mainImages.Where(u => u != preferred).ToList(); foreach (var s in skipped) Console.WriteLine($" [Bild] Übersprungen (scaled): {Path.GetFileName(new Uri(s).AbsolutePath)}"); } } } return result; } private static void MarkProcessed(int wpId, string? wpSlug = null) { ProcessedPosts.Add(wpId); SaveProcessedPosts(wpId, wpSlug); } private static void LoadProcessedPosts() { if (!File.Exists(ProcessedPostsPath)) return; try { var lines = File.ReadAllLines(ProcessedPostsPath); foreach (var line in lines) { if (string.IsNullOrWhiteSpace(line) || line.StartsWith("wp_id")) continue; var parts = line.Split(','); if (parts.Length >= 1 && int.TryParse(parts[0].Trim(), out int postId)) ProcessedPosts.Add(postId); } } catch (Exception ex) { Console.WriteLine($"Fehler beim Laden der abgefragten Posts: {ex.Message}"); } } private static void SaveProcessedPosts(int wpId, string? wpSlug) { string entry = $"{wpId},{wpSlug ?? ""}"; using var writer = File.AppendText(ProcessedPostsPath); writer.WriteLine(entry); } } }