487 lines
No EOL
22 KiB
C#
487 lines
No EOL
22 KiB
C#
using System.Text.Json;
|
|
|
|
namespace wp2ghost
|
|
{
|
|
// =============================================================================
|
|
// PostFetcher — Posts von WordPress abrufen, verarbeiten
|
|
// =============================================================================
|
|
static class PostFetcher
|
|
{
|
|
private static readonly HttpClient httpClient = new();
|
|
private static HashSet<int> ProcessedPosts = new();
|
|
private static List<GhostPost> GhostPosts = new();
|
|
private static string ProcessedPostsPath = "";
|
|
//private static string MarkdownDir = "";
|
|
private static string GhostPostsDir = "";
|
|
private static string BaseDir = "";
|
|
|
|
public static int ProcessedCount => ProcessedPosts.Count;
|
|
public static int PostCount => GhostPosts.Count;
|
|
public static IReadOnlyList<GhostPost> Posts => GhostPosts;
|
|
|
|
public static void Init(string baseDir)
|
|
{
|
|
BaseDir = baseDir;
|
|
ProcessedPostsPath = Path.Combine(baseDir, "processed_posts.txt");
|
|
//MarkdownDir = Path.Combine(baseDir, "markdown");
|
|
GhostPostsDir = Path.Combine(baseDir, "ghostPosts");
|
|
|
|
////if (!Directory.Exists(MarkdownDir))
|
|
//// Directory.CreateDirectory(MarkdownDir);
|
|
if (!Directory.Exists(GhostPostsDir))
|
|
Directory.CreateDirectory(GhostPostsDir);
|
|
|
|
LoadProcessedPosts();
|
|
}
|
|
|
|
public static void ClearProcessedPosts()
|
|
{
|
|
ProcessedPosts.Clear();
|
|
File.WriteAllText(ProcessedPostsPath, "wp_id,wp_slug,ghost_id" + Environment.NewLine);
|
|
Console.WriteLine("Abgefragte Posts wurden zurückgesetzt.");
|
|
}
|
|
|
|
public static void ShowInfo()
|
|
{
|
|
if (ProcessedPosts.Count > 0)
|
|
{
|
|
Console.WriteLine($"Letzte abgefragte Objekte: {string.Join(", ", ProcessedPosts.OrderByDescending(x => x).Take(10))}");
|
|
}
|
|
}
|
|
|
|
public static async Task FetchFromWordPressAsync(string wpBaseUrl, int maxPosts)
|
|
{
|
|
Console.WriteLine();
|
|
Console.WriteLine("Starte Abruf von WordPress...");
|
|
|
|
string wpHost = "";
|
|
try { wpHost = new Uri(wpBaseUrl).Host.ToLowerInvariant(); } catch { }
|
|
|
|
try
|
|
{
|
|
var allPostIds = new List<int>();
|
|
var postTypes = new[] { "post", "page" };
|
|
|
|
foreach (var postType in postTypes)
|
|
{
|
|
Console.WriteLine($"Hole {postType}-IDs von der WordPress-API...");
|
|
string idsUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/posts-ids?post_type={postType}";
|
|
var idsResponse = await httpClient.GetStringAsync(idsUrl);
|
|
var idsObj = JsonSerializer.Deserialize<IdsResponse>(idsResponse);
|
|
if (idsObj?.Ids != null)
|
|
allPostIds.AddRange(idsObj.Ids);
|
|
}
|
|
|
|
if (allPostIds.Count == 0)
|
|
{
|
|
Console.WriteLine("Keine Beiträge oder Seiten gefunden.");
|
|
return;
|
|
}
|
|
|
|
var pendingIds = allPostIds.Where(id => !ProcessedPosts.Contains(id)).ToList();
|
|
Console.WriteLine($"Insgesamt gefunden: {allPostIds.Count}, Bereits abgefragt: {allPostIds.Count - pendingIds.Count}, Noch zu verarbeiten: {pendingIds.Count}");
|
|
|
|
if (pendingIds.Count == 0)
|
|
{
|
|
Console.WriteLine("Alle Beiträge und Seiten wurden bereits abgefragt. Nichts zu tun. (Nutze Option 5 zum Zurücksetzen)");
|
|
return;
|
|
}
|
|
|
|
if (maxPosts > 0)
|
|
pendingIds = pendingIds.Take(maxPosts).ToList();
|
|
|
|
Console.WriteLine($"Es werden {pendingIds.Count} Beiträge/Seiten verarbeitet.");
|
|
|
|
int internalId = 1;
|
|
if (GhostPosts.Count > 0)
|
|
{
|
|
var lastId = GhostPosts.Select(gp => int.TryParse(gp.Id, out int id) ? id : 0).Max();
|
|
internalId = lastId + 1;
|
|
}
|
|
int processed = 0;
|
|
|
|
foreach (int wpId in pendingIds)
|
|
{
|
|
Console.WriteLine($"[{++processed}/{pendingIds.Count}] -> Beitrag/Seite ID: {wpId}...");
|
|
string detailUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/post-details/{wpId}";
|
|
|
|
try
|
|
{
|
|
var detailJson = await httpClient.GetStringAsync(detailUrl);
|
|
var wpPost = JsonSerializer.Deserialize<WpPostResponse>(detailJson);
|
|
|
|
if (wpPost == null)
|
|
{
|
|
Console.WriteLine($" Übersprungen (null).");
|
|
MarkProcessed(wpId, null);
|
|
continue;
|
|
}
|
|
|
|
string htmlContent = wpPost.Content ?? "";
|
|
var parsedDate = DateTime.MinValue;
|
|
var postDate = DateTime.TryParse(wpPost.Date, out parsedDate) ? parsedDate : new DateTime(2026, 9, 1);
|
|
|
|
// Alle internen Bild-URLs aus dem HTML extrahieren (keine externen Links)
|
|
var htmlImageUrls = ExtractImageUrls(htmlContent, wpHost);
|
|
|
|
// Bevorzugte Media-Dateien ermitteln
|
|
var preferredMedia = GetPreferredMediaFiles(wpPost.MediaFiles, htmlImageUrls, wpHost);
|
|
|
|
// Prüfe: Referenzierte Bilder im HTML, die NICHT in MediaFiles sind
|
|
if (htmlImageUrls.Count > 0 && wpPost.MediaFiles != null)
|
|
{
|
|
var mediaFileNames = wpPost.MediaFiles.Select(m => GetFileNameFromUrl(m).ToLowerInvariant()).ToHashSet();
|
|
foreach (var imgUrl in htmlImageUrls)
|
|
{
|
|
string imgFileName = GetFileNameFromUrl(imgUrl).ToLowerInvariant();
|
|
if (!mediaFileNames.Contains(imgFileName))
|
|
{
|
|
Console.WriteLine($" [Warnung] Bild im HTML nicht in MediaFiles: {Path.GetFileName(new Uri(imgUrl).AbsolutePath)}");
|
|
}
|
|
}
|
|
}
|
|
|
|
if (preferredMedia.Count > 0)
|
|
{
|
|
foreach (string mediaUrl in preferredMedia)
|
|
{
|
|
var downloaded = await ImageManager.DownloadAsync(mediaUrl, postDate);
|
|
if (downloaded.HasValue)
|
|
{
|
|
string ghostMediaPath = $"__GHOST_URL__/content/images/{downloaded.Value.RelativePath}";
|
|
htmlContent = htmlContent.Replace(mediaUrl, ghostMediaPath);
|
|
}
|
|
}
|
|
}
|
|
|
|
// WordPress HTML zu Ghost-kompatibles HTML konvertieren (Galerien, Figures, etc.)
|
|
htmlContent = HtmlGhostConverter.Convert(htmlContent);
|
|
|
|
// Lokale Bildpfade für JSON-Datei sammeln
|
|
var imageFiles = new List<string>();
|
|
if (preferredMedia.Count > 0)
|
|
{
|
|
foreach (string mediaUrl in preferredMedia)
|
|
{
|
|
if (ImageManager.TryGetRelativePath(mediaUrl, out string relPath))
|
|
imageFiles.Add(relPath);
|
|
}
|
|
}
|
|
|
|
// Featured Image verarbeiten
|
|
string featureImageGhostPath = "";
|
|
if (!string.IsNullOrEmpty(wpPost.FeaturedImage))
|
|
{
|
|
if (ImageManager.TryGetRelativePath(wpPost.FeaturedImage, out string featRel))
|
|
{
|
|
featureImageGhostPath = $"__GHOST_URL__/content/images/{featRel}";
|
|
}
|
|
else
|
|
{
|
|
var featDownload = await ImageManager.DownloadAsync(wpPost.FeaturedImage, postDate);
|
|
if (featDownload.HasValue)
|
|
{
|
|
featureImageGhostPath = $"__GHOST_URL__/content/images/{featDownload.Value.RelativePath}";
|
|
if (!imageFiles.Contains(featDownload.Value.RelativePath))
|
|
imageFiles.Add(featDownload.Value.RelativePath);
|
|
}
|
|
}
|
|
}
|
|
|
|
var ghostPost = new GhostPost
|
|
{
|
|
Id = internalId.ToString(),
|
|
Title = wpPost.Title,
|
|
Slug = wpPost.Slug,
|
|
Html = htmlContent,
|
|
Status = "published",
|
|
CreatedAt = postDate.ToString("o"),
|
|
PublishedAt = postDate.ToString("o")
|
|
};
|
|
|
|
GhostPosts.Add(ghostPost);
|
|
|
|
// JSON-Datei in ghostPosts/ speichern
|
|
var postFile = new GhostPostFile
|
|
{
|
|
WpId = wpId,
|
|
WpSlug = wpPost.Slug,
|
|
GhostId = ghostPost.Id,
|
|
Title = wpPost.Title,
|
|
Slug = wpPost.Slug,
|
|
Excerpt = wpPost.Excerpt,
|
|
Status = "published",
|
|
CreatedAt = postDate.ToString("o"),
|
|
PublishedAt = postDate.ToString("o"),
|
|
FeatureImage = featureImageGhostPath,
|
|
Categories = wpPost.Categories ?? new(),
|
|
Keywords = wpPost.Keywords ?? "",
|
|
Comments = wpPost.Comments ?? new(),
|
|
ImageFiles = imageFiles,
|
|
Html = htmlContent
|
|
};
|
|
string jsonPath = Path.Combine(GhostPostsDir, $"{postFile.WpId}.json");
|
|
File.WriteAllText(jsonPath, JsonSerializer.Serialize(postFile, new JsonSerializerOptions { WriteIndented = true }));
|
|
|
|
MarkProcessed(wpId, wpPost.Slug);
|
|
internalId++;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($" Fehler beim Abrufen von ID {wpId}: {ex.Message}");
|
|
MarkProcessed(wpId, null);
|
|
}
|
|
}
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine("=========================================");
|
|
Console.WriteLine($" Abruf abgeschlossen! {GhostPosts.Count} Beiträge geladen.");
|
|
Console.WriteLine($" Abgefragte Posts insgesamt: {ProcessedPosts.Count}");
|
|
Console.WriteLine(" Nutze Option 2 um die Ghost ZIP zu erstellen.");
|
|
Console.WriteLine("=========================================");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"\nFehler beim Abruf: {ex.Message}");
|
|
Console.WriteLine($"Stack Trace: {ex.StackTrace}");
|
|
}
|
|
}
|
|
|
|
//public static void SaveAllMarkdownFiles()
|
|
//{
|
|
// foreach (var post in GhostPosts)
|
|
// {
|
|
// if (string.IsNullOrEmpty(post.Slug)) continue;
|
|
// string mdPath = Path.Combine(MarkdownDir, $"{post.Slug}.md");
|
|
// if (!string.IsNullOrEmpty(post.Markdown))
|
|
// File.WriteAllText(mdPath, post.Markdown);
|
|
// }
|
|
// Console.WriteLine($"Markdown-Dateien gespeichert: {GhostPosts.Count} Dateien unter {MarkdownDir}");
|
|
//}
|
|
|
|
private static List<string> ExtractImageUrls(string html, string wpHost)
|
|
{
|
|
var urls = new List<string>();
|
|
var imageExts = new[] { ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".tiff", ".tif" };
|
|
|
|
// 1) <img src="..."> extrahieren
|
|
var imgMatches = System.Text.RegularExpressions.Regex.Matches(html, @"<img\s+[^>]*src=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
|
foreach (System.Text.RegularExpressions.Match m in imgMatches)
|
|
{
|
|
string url = m.Groups[1].Value;
|
|
if (!string.IsNullOrEmpty(url) && IsInternalUrl(url, wpHost) && !urls.Contains(url))
|
|
urls.Add(url);
|
|
}
|
|
|
|
// 2) data-orig-file (Vollbild) extrahieren
|
|
var origFileMatches = System.Text.RegularExpressions.Regex.Matches(html, @"data-orig-file=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
|
foreach (System.Text.RegularExpressions.Match m in origFileMatches)
|
|
{
|
|
string url = m.Groups[1].Value;
|
|
if (!string.IsNullOrEmpty(url) && IsInternalUrl(url, wpHost) && !urls.Contains(url))
|
|
urls.Add(url);
|
|
}
|
|
|
|
// 3) data-large-file extrahieren
|
|
var largeFileMatches = System.Text.RegularExpressions.Regex.Matches(html, @"data-large-file=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
|
foreach (System.Text.RegularExpressions.Match m in largeFileMatches)
|
|
{
|
|
string url = m.Groups[1].Value;
|
|
if (!string.IsNullOrEmpty(url) && IsInternalUrl(url, wpHost) && !urls.Contains(url))
|
|
urls.Add(url);
|
|
}
|
|
|
|
// 4) srcset (größte Variante pro img) extrahieren
|
|
var srcsetMatches = System.Text.RegularExpressions.Regex.Matches(html, @"srcset=[""']([^""']+)[""']", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
|
foreach (System.Text.RegularExpressions.Match m in srcsetMatches)
|
|
{
|
|
string srcset = m.Groups[1].Value;
|
|
string widestUrl = GetWidestFromSrcset(srcset);
|
|
if (!string.IsNullOrEmpty(widestUrl) && IsInternalUrl(widestUrl, wpHost) && !urls.Contains(widestUrl))
|
|
urls.Add(widestUrl);
|
|
}
|
|
|
|
// 5) <a href="..."> mit Bildendung (Figure-Links auf Originalbild)
|
|
var aMatches = System.Text.RegularExpressions.Regex.Matches(html, @"<a\s+[^>]*href=[""']([^""']+)[""'][^>]*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
|
foreach (System.Text.RegularExpressions.Match m in aMatches)
|
|
{
|
|
string url = m.Groups[1].Value;
|
|
if (string.IsNullOrEmpty(url)) continue;
|
|
try
|
|
{
|
|
string ext = Path.GetExtension(new Uri(url).AbsolutePath).ToLowerInvariant();
|
|
if (imageExts.Contains(ext) && IsInternalUrl(url, wpHost) && !urls.Contains(url))
|
|
urls.Add(url);
|
|
}
|
|
catch { }
|
|
}
|
|
|
|
return urls;
|
|
}
|
|
|
|
private static string GetWidestFromSrcset(string srcset)
|
|
{
|
|
string bestUrl = "";
|
|
int bestWidth = 0;
|
|
foreach (string part in srcset.Split(',', StringSplitOptions.RemoveEmptyEntries))
|
|
{
|
|
string[] tokens = part.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
|
if (tokens.Length < 2) continue;
|
|
if (int.TryParse(tokens[1].Replace("w", ""), out int w) && w > bestWidth)
|
|
{
|
|
bestWidth = w;
|
|
bestUrl = tokens[0];
|
|
}
|
|
}
|
|
return bestUrl;
|
|
}
|
|
|
|
private static bool IsInternalUrl(string url, string wpHost)
|
|
{
|
|
if (string.IsNullOrEmpty(wpHost)) return true;
|
|
try
|
|
{
|
|
var uri = new Uri(url);
|
|
return uri.Host.ToLowerInvariant().EndsWith(wpHost) || wpHost.EndsWith(uri.Host.ToLowerInvariant());
|
|
}
|
|
catch
|
|
{
|
|
return url.StartsWith("/") || url.StartsWith("wp-content");
|
|
}
|
|
}
|
|
|
|
private static string NormalizeImageUrl(string url)
|
|
{
|
|
try
|
|
{
|
|
var uri = new Uri(url);
|
|
string path = uri.AbsolutePath;
|
|
// Entferne -scaled UND Dimensionspostfixe für den Vergleich
|
|
path = System.Text.RegularExpressions.Regex.Replace(path, @"-scaled(?=\.\w+$)", "");
|
|
path = System.Text.RegularExpressions.Regex.Replace(path, @"-\d+x\d+(?=\.\w+$)", "");
|
|
return path;
|
|
}
|
|
catch
|
|
{
|
|
return url;
|
|
}
|
|
}
|
|
|
|
private static string GetFileNameFromUrl(string url)
|
|
{
|
|
try
|
|
{
|
|
return Path.GetFileName(new Uri(url).AbsolutePath);
|
|
}
|
|
catch
|
|
{
|
|
return url;
|
|
}
|
|
}
|
|
|
|
private static List<string> GetPreferredMediaFiles(List<string>? mediaFiles, List<string> htmlImageUrls, string wpHost)
|
|
{
|
|
var result = new List<string>();
|
|
var imageExts = new[] { ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".tiff", ".tif" };
|
|
|
|
// Alle internen Bild-URLs zusammenführen: MediaFiles + HTML-referenzierte
|
|
var allUrls = new List<string>();
|
|
if (mediaFiles != null)
|
|
allUrls.AddRange(mediaFiles.Where(u => IsInternalUrl(u, wpHost)));
|
|
foreach (var u in htmlImageUrls)
|
|
{
|
|
if (!allUrls.Contains(u))
|
|
allUrls.Add(u);
|
|
}
|
|
|
|
if (allUrls.Count == 0)
|
|
return result;
|
|
|
|
// Dateinamen aus den HTML-Links extrahieren
|
|
var htmlFileNames = htmlImageUrls.Select(u => GetFileNameFromUrl(u).ToLowerInvariant()).ToHashSet();
|
|
|
|
// Gruppiere nach normalisiertem Dateinamen (ohne -scaled und Dimensionspostfixe)
|
|
var grouped = new Dictionary<string, List<string>>();
|
|
foreach (var url in allUrls)
|
|
{
|
|
string normalized = NormalizeImageUrl(url);
|
|
if (!grouped.ContainsKey(normalized))
|
|
grouped[normalized] = new List<string>();
|
|
grouped[normalized].Add(url);
|
|
}
|
|
|
|
foreach (var group in grouped)
|
|
{
|
|
// Nur Hauptbilder berücksichtigen (keine WordPress-Sizes wie -300x200, keine -scaled)
|
|
var mainImages = group.Value.Where(u => ImageManager.IsMainImage(u)).ToList();
|
|
if (mainImages.Count == 0)
|
|
{
|
|
Console.WriteLine($" [Bild] Übersprungen (kein Hauptbild): {Path.GetFileName(new Uri(@group.Value[0]).AbsolutePath)}");
|
|
continue;
|
|
}
|
|
|
|
if (mainImages.Count == 1)
|
|
{
|
|
result.Add(mainImages[0]);
|
|
}
|
|
else
|
|
{
|
|
// Prüfe ob eine der Varianten als <a href> im HTML referenziert wird
|
|
string? linked = mainImages.FirstOrDefault(u => htmlFileNames.Contains(GetFileNameFromUrl(u).ToLowerInvariant()));
|
|
|
|
if (linked != null)
|
|
{
|
|
result.Add(linked);
|
|
var skipped = mainImages.Where(u => u != linked).ToList();
|
|
foreach (var s in skipped)
|
|
Console.WriteLine($" [Bild] Übersprungen (nicht verlinkt): {Path.GetFileName(new Uri(s).AbsolutePath)}");
|
|
}
|
|
else
|
|
{
|
|
// Keine Verlinkung: Original (ohne -scaled) bevorzugen
|
|
var preferred = mainImages.FirstOrDefault(u => !u.Contains("-scaled")) ?? mainImages[0];
|
|
result.Add(preferred);
|
|
var skipped = mainImages.Where(u => u != preferred).ToList();
|
|
foreach (var s in skipped)
|
|
Console.WriteLine($" [Bild] Übersprungen (scaled): {Path.GetFileName(new Uri(s).AbsolutePath)}");
|
|
}
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private static void MarkProcessed(int wpId, string? wpSlug = null)
|
|
{
|
|
ProcessedPosts.Add(wpId);
|
|
SaveProcessedPosts(wpId, wpSlug);
|
|
}
|
|
|
|
private static void LoadProcessedPosts()
|
|
{
|
|
if (!File.Exists(ProcessedPostsPath)) return;
|
|
try
|
|
{
|
|
var lines = File.ReadAllLines(ProcessedPostsPath);
|
|
foreach (var line in lines)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(line) || line.StartsWith("wp_id")) continue;
|
|
var parts = line.Split(',');
|
|
if (parts.Length >= 1 && int.TryParse(parts[0].Trim(), out int postId))
|
|
ProcessedPosts.Add(postId);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Fehler beim Laden der abgefragten Posts: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
private static void SaveProcessedPosts(int wpId, string? wpSlug)
|
|
{
|
|
string entry = $"{wpId},{wpSlug ?? ""}";
|
|
using var writer = File.AppendText(ProcessedPostsPath);
|
|
writer.WriteLine(entry);
|
|
}
|
|
}
|
|
} |