449 lines
No EOL
22 KiB
C#
449 lines
No EOL
22 KiB
C#
using System.Text.Json;
|
|
|
|
namespace wp2ghost
|
|
{
|
|
// =============================================================================
|
|
// PostFetcher — Posts von WordPress abrufen, verarbeiten
|
|
// =============================================================================
|
|
static class PostFetcher
|
|
{
|
|
private static readonly HttpClient httpClient = new();
|
|
private static HashSet<int> ProcessedPosts = new();
|
|
private static List<GhostPost> GhostPosts = new();
|
|
private static string ProcessedPostsPath = "";
|
|
//private static string MarkdownDir = "";
|
|
private static string GhostPostsDir = "";
|
|
private static string BaseDir = "";
|
|
|
|
public static int ProcessedCount => ProcessedPosts.Count;
|
|
public static int PostCount => GhostPosts.Count;
|
|
public static IReadOnlyList<GhostPost> Posts => GhostPosts;
|
|
|
|
public static void Init(string baseDir)
|
|
{
|
|
BaseDir = baseDir;
|
|
ProcessedPostsPath = Path.Combine(baseDir, "processed_posts.txt");
|
|
//MarkdownDir = Path.Combine(baseDir, "markdown");
|
|
GhostPostsDir = Path.Combine(baseDir, "ghostPosts");
|
|
|
|
////if (!Directory.Exists(MarkdownDir))
|
|
//// Directory.CreateDirectory(MarkdownDir);
|
|
if (!Directory.Exists(GhostPostsDir))
|
|
Directory.CreateDirectory(GhostPostsDir);
|
|
|
|
LoadProcessedPosts();
|
|
}
|
|
|
|
public static void ClearProcessedPosts()
|
|
{
|
|
ProcessedPosts.Clear();
|
|
File.WriteAllText(ProcessedPostsPath, "wp_id,wp_slug,ghost_id" + Environment.NewLine);
|
|
if (Directory.Exists(GhostPostsDir))
|
|
{
|
|
foreach (var file in Directory.GetFiles(GhostPostsDir, "*.json"))
|
|
{
|
|
try { File.Delete(file); } catch { }
|
|
}
|
|
}
|
|
Console.WriteLine("Abgefragte Posts wurden zurückgesetzt.");
|
|
}
|
|
|
|
public static void ShowInfo()
|
|
{
|
|
if (ProcessedPosts.Count > 0)
|
|
{
|
|
Console.WriteLine($"Letzte abgefragte Objekte: {string.Join(", ", ProcessedPosts.OrderByDescending(x => x).Take(10))}");
|
|
}
|
|
}
|
|
|
|
public static async Task FetchFromWordPressAsync(string wpBaseUrl, int maxPosts)
|
|
{
|
|
Console.WriteLine();
|
|
Console.WriteLine("Starte Abruf von WordPress...");
|
|
|
|
string wpHost = "";
|
|
try { wpHost = new Uri(wpBaseUrl).Host.ToLowerInvariant(); } catch { }
|
|
|
|
try
|
|
{
|
|
var allPostIds = new List<int>();
|
|
var postTypes = new[] { "post", "page" };
|
|
|
|
foreach (var postType in postTypes)
|
|
{
|
|
Console.WriteLine($"Hole {postType}-IDs von der WordPress-API...");
|
|
string idsUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/posts-ids?post_type={postType}";
|
|
var idsResponse = await httpClient.GetStringAsync(idsUrl);
|
|
var idsObj = JsonSerializer.Deserialize<IdsResponse>(idsResponse);
|
|
if (idsObj?.Ids != null)
|
|
allPostIds.AddRange(idsObj.Ids);
|
|
}
|
|
|
|
if (allPostIds.Count == 0)
|
|
{
|
|
Console.WriteLine("Keine Beiträge oder Seiten gefunden.");
|
|
return;
|
|
}
|
|
|
|
ProcessedPosts.Clear();
|
|
LoadProcessedPosts();
|
|
|
|
var pendingIds = allPostIds.Where(id => !ProcessedPosts.Contains(id)).ToList();
|
|
Console.WriteLine($"Insgesamt gefunden: {allPostIds.Count}, Bereits abgefragt: {allPostIds.Count - pendingIds.Count}, Noch zu verarbeiten: {pendingIds.Count}");
|
|
|
|
if (pendingIds.Count == 0)
|
|
{
|
|
Console.WriteLine("Alle Beiträge und Seiten wurden bereits abgefragt. Nichts zu tun. (Nutze Option 5 zum Zurücksetzen)");
|
|
return;
|
|
}
|
|
|
|
if (maxPosts > 0)
|
|
pendingIds = pendingIds.Take(maxPosts).ToList();
|
|
|
|
Console.WriteLine($"Es werden {pendingIds.Count} Beiträge/Seiten verarbeitet.");
|
|
|
|
int internalId = 1;
|
|
if (GhostPosts.Count > 0)
|
|
{
|
|
var lastId = GhostPosts.Select(gp => int.TryParse(gp.Id, out int id) ? id : 0).Max();
|
|
internalId = lastId + 1;
|
|
}
|
|
int processed = 0;
|
|
|
|
foreach (int wpId in pendingIds)
|
|
{
|
|
Console.WriteLine($"[{++processed}/{pendingIds.Count}] -> Beitrag/Seite ID: {wpId}...");
|
|
string detailUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/custom/v1/post-details/{wpId}";
|
|
|
|
//string postUrl = $"{wpBaseUrl.TrimEnd('/')}/wp-json/wp/v2/posts/{wpId}?_embed";
|
|
|
|
try
|
|
{
|
|
var detailJson = await httpClient.GetStringAsync(detailUrl);
|
|
var wpPost = JsonSerializer.Deserialize<WpPostResponse>(detailJson);
|
|
|
|
//var postJson = await httpClient.GetStringAsync(postUrl);
|
|
|
|
if (wpPost == null)
|
|
{
|
|
Console.WriteLine($" Übersprungen (null).");
|
|
MarkProcessed(wpId, null);
|
|
continue;
|
|
}
|
|
|
|
string htmlContent = wpPost.Content ?? "";
|
|
htmlContent = await VideoLinkCleaner.RemoveDeadVideosAsync(htmlContent);
|
|
// Veröffentlichungsdatum des WordPress-Beitrags: für Ghost in UTC, lokal für Dateinamen
|
|
string? postDateUtc = DateHelper.FromGmt(wpPost.DateGmt) ?? DateHelper.ToGhostUtc(wpPost.Date);
|
|
if (postDateUtc == null)
|
|
Console.WriteLine(" [Warnung] WordPress liefert kein lesbares Datum für diesen Beitrag");
|
|
var postDate = DateTime.TryParse(wpPost.Date, out var parsedDate) ? parsedDate : DateTime.Now;
|
|
|
|
// Auf WordPress gehostete Videos herunterladen und als Ghost-Video-Karte einbinden
|
|
var mediaFiles = new List<string>();
|
|
htmlContent = await MigrateVideosAsync(htmlContent, wpHost, postDate, mediaFiles);
|
|
|
|
// Featured Image: falls von der WP API nicht gesetzt, ersten Bild-URL aus HTML nehmen
|
|
if (string.IsNullOrEmpty(wpPost.FeaturedImage))
|
|
{
|
|
var firstImgMatch = System.Text.RegularExpressions.Regex.Match(htmlContent, @"<img\s+[^>]*src=[""']([^""']+)[""'][^>]*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
|
if (firstImgMatch.Success)
|
|
{
|
|
string firstImgUrl = firstImgMatch.Groups[1].Value;
|
|
if (IsInternalUrl(firstImgUrl, wpHost))
|
|
{
|
|
wpPost.FeaturedImage = firstImgUrl;
|
|
Console.WriteLine($" [Featured Image] Aus HTML extrahiert (da API keine lieferte): {Path.GetFileName(new Uri(firstImgUrl).AbsolutePath)}");
|
|
}
|
|
}
|
|
}
|
|
|
|
// Alle internen Bild-URLs im HTML (src, srcset, data-*, href) — keine externen Links
|
|
var htmlImageUrls = ExtractImageUrls(htmlContent, wpHost);
|
|
|
|
// Varianten desselben Bildes (Original, -scaled, -1024x683, ?w=...) zusammenfassen:
|
|
// pro Bild wird genau EINE Datei geladen, auf die alle Varianten umgeschrieben werden.
|
|
var imageGroups = new Dictionary<string, List<string>>();
|
|
var mediaUrls = (wpPost.MediaFiles ?? new()).Where(u => IsInternalUrl(u, wpHost)).ToList();
|
|
if (!string.IsNullOrEmpty(wpPost.FeaturedImage))
|
|
mediaUrls.Add(wpPost.FeaturedImage);
|
|
foreach (string url in htmlImageUrls.Concat(mediaUrls))
|
|
{
|
|
string key = ImageManager.GetImageKey(url);
|
|
if (!imageGroups.TryGetValue(key, out var variants))
|
|
imageGroups[key] = variants = new List<string>();
|
|
variants.Add(url);
|
|
}
|
|
|
|
// Im Beitrag verwendet = kommt im HTML vor oder ist das Featured Image
|
|
var usedKeys = htmlImageUrls.Select(ImageManager.GetImageKey).ToHashSet();
|
|
if (!string.IsNullOrEmpty(wpPost.FeaturedImage))
|
|
usedKeys.Add(ImageManager.GetImageKey(wpPost.FeaturedImage));
|
|
|
|
var imageFiles = new List<string>();
|
|
var ghostPathByKey = new Dictionary<string, string>();
|
|
foreach (var group in imageGroups)
|
|
{
|
|
// Nur angehängte, nicht verwendete Bilder: in den Extra-Ordner, nicht nach Ghost
|
|
if (!usedKeys.Contains(group.Key))
|
|
{
|
|
await ImageManager.DownloadUnusedAsync(group.Value, wpId);
|
|
continue;
|
|
}
|
|
|
|
var downloaded = await ImageManager.DownloadBestAsync(group.Value, postDate);
|
|
if (!downloaded.HasValue) continue;
|
|
ghostPathByKey[group.Key] = $"__GHOST_URL__/content/images/{downloaded.Value.RelativePath}";
|
|
if (!imageFiles.Contains(downloaded.Value.RelativePath))
|
|
imageFiles.Add(downloaded.Value.RelativePath);
|
|
}
|
|
|
|
// Längste URLs zuerst ersetzen, damit "bild.jpg" nicht in "bild.jpg?w=1024" hineinersetzt
|
|
foreach (string url in htmlImageUrls.OrderByDescending(u => u.Length))
|
|
{
|
|
if (ghostPathByKey.TryGetValue(ImageManager.GetImageKey(url), out string? ghostMediaPath))
|
|
htmlContent = htmlContent.Replace(url, ghostMediaPath);
|
|
}
|
|
|
|
// WordPress HTML zu Ghost-kompatibles HTML konvertieren (Galerien, Figures, etc.)
|
|
htmlContent = HtmlGhostConverter.Convert(htmlContent, ImageManager.TryGetImageSize);
|
|
|
|
int remainingWpImages = ExtractImageUrls(htmlContent, wpHost).Count;
|
|
if (remainingWpImages > 0)
|
|
Console.WriteLine($" [Warnung] {remainingWpImages} Bild-URL(s) zeigen weiterhin auf WordPress");
|
|
|
|
// WordPress-Kommentare als Block ans Beitragsende hängen
|
|
int realComments = CommentBlockBuilder.CountRealComments(wpPost.Comments);
|
|
if (realComments > 0)
|
|
{
|
|
htmlContent = CommentBlockBuilder.Append(htmlContent, wpPost.Comments);
|
|
Console.WriteLine($" [Kommentare] {realComments} Kommentar(e) angehängt");
|
|
}
|
|
|
|
// Featured Image (wurde oben mit heruntergeladen)
|
|
string featureImageGhostPath = "";
|
|
if (!string.IsNullOrEmpty(wpPost.FeaturedImage)
|
|
&& ghostPathByKey.TryGetValue(ImageManager.GetImageKey(wpPost.FeaturedImage), out string? featPath))
|
|
{
|
|
featureImageGhostPath = featPath;
|
|
}
|
|
|
|
var ghostPost = new GhostPost
|
|
{
|
|
Id = internalId.ToString(),
|
|
Title = wpPost.Title,
|
|
Slug = SlugHelper.Clean(wpPost.Slug, $"post-{wpId}"),
|
|
Html = htmlContent,
|
|
FeatureImage = string.IsNullOrEmpty(featureImageGhostPath) ? null : featureImageGhostPath,
|
|
Status = "published",
|
|
CreatedAt = postDateUtc,
|
|
PublishedAt = postDateUtc
|
|
};
|
|
|
|
GhostPosts.Add(ghostPost);
|
|
|
|
// JSON-Datei in ghostPosts/ speichern
|
|
var ghostPostFile = new GhostPostFile
|
|
{
|
|
WpId = wpId,
|
|
WpSlug = wpPost.Slug,
|
|
GhostId = ghostPost.Id,
|
|
Title = wpPost.Title,
|
|
Slug = SlugHelper.Clean(wpPost.Slug, $"post-{wpId}"),
|
|
Excerpt = wpPost.Excerpt,
|
|
Status = "published",
|
|
CreatedAt = postDateUtc,
|
|
PublishedAt = postDateUtc,
|
|
FeatureImage = featureImageGhostPath,
|
|
Categories = wpPost.Categories ?? new(),
|
|
Tags = wpPost.Tags ?? new(),
|
|
Keywords = wpPost.Keywords ?? "",
|
|
Comments = wpPost.Comments ?? new(),
|
|
ImageFiles = imageFiles,
|
|
MediaFiles = mediaFiles,
|
|
Html = htmlContent
|
|
};
|
|
string jsonPath = Path.Combine(GhostPostsDir, $"{ghostPostFile.WpId}.json");
|
|
File.WriteAllText(jsonPath, JsonSerializer.Serialize(ghostPostFile, new JsonSerializerOptions { WriteIndented = true }));
|
|
|
|
// Cache sofort speichern (nicht erst beim Beenden)
|
|
ImageManager.SaveCache();
|
|
|
|
InternalLinkManager.Register(wpId, wpPost.Slug, ghostPostFile.Slug ?? "");
|
|
MarkProcessed(wpId, wpPost.Slug);
|
|
internalId++;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($" Fehler beim Abrufen von ID {wpId}: {ex.Message}");
|
|
MarkProcessed(wpId, null);
|
|
}
|
|
}
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine("=========================================");
|
|
Console.WriteLine($" Abruf abgeschlossen! {GhostPosts.Count} Beiträge geladen.");
|
|
Console.WriteLine($" Abgefragte Posts insgesamt: {ProcessedPosts.Count}");
|
|
Console.WriteLine(" Nutze Option 2 um die Ghost ZIP zu erstellen.");
|
|
Console.WriteLine("=========================================");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"\nFehler beim Abruf: {ex.Message}");
|
|
Console.WriteLine($"Stack Trace: {ex.StackTrace}");
|
|
}
|
|
}
|
|
|
|
//public static void SaveAllMarkdownFiles()
|
|
//{
|
|
// foreach (var post in GhostPosts)
|
|
// {
|
|
// if (string.IsNullOrEmpty(post.Slug)) continue;
|
|
// string mdPath = Path.Combine(MarkdownDir, $"{post.Slug}.md");
|
|
// if (!string.IsNullOrEmpty(post.Markdown))
|
|
// File.WriteAllText(mdPath, post.Markdown);
|
|
// }
|
|
// Console.WriteLine($"Markdown-Dateien gespeichert: {GhostPosts.Count} Dateien unter {MarkdownDir}");
|
|
//}
|
|
|
|
// Bild-URL irgendwo im HTML: src, srcset, data-orig-file, data-large-file, href ...
|
|
// inkl. Query (?w=1024), damit beim Ersetzen nichts davon stehen bleibt.
|
|
private static readonly System.Text.RegularExpressions.Regex ImageUrlRegex = new(
|
|
@"https?://[^\s""'<>()]+?\.(?:jpe?g|png|gif|webp|svg|bmp|tiff?)(?:\?[^\s""'<>]*)?(?=[\s""'<>,)&\\]|$)",
|
|
System.Text.RegularExpressions.RegexOptions.IgnoreCase | System.Text.RegularExpressions.RegexOptions.Compiled);
|
|
|
|
private static List<string> ExtractImageUrls(string html, string wpHost)
|
|
{
|
|
var urls = new List<string>();
|
|
foreach (System.Text.RegularExpressions.Match m in ImageUrlRegex.Matches(html))
|
|
{
|
|
string url = m.Value;
|
|
if (IsInternalUrl(url, wpHost) && !urls.Contains(url))
|
|
urls.Add(url);
|
|
}
|
|
return urls;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Ersetzt jedes auf WordPress gehostete <video> durch eine Ghost-Video-Karte, deren Quelle
|
|
/// auf die heruntergeladene Datei zeigt (__GHOST_URL__/content/media/...).
|
|
/// </summary>
|
|
private static async Task<string> MigrateVideosAsync(string html, string wpHost, DateTime postDate, List<string> mediaFiles)
|
|
{
|
|
if (!html.Contains("<video", StringComparison.OrdinalIgnoreCase)) return html;
|
|
|
|
var doc = new HtmlAgilityPack.HtmlDocument();
|
|
doc.LoadHtml(html);
|
|
var videos = doc.DocumentNode.SelectNodes("//video");
|
|
if (videos == null) return html;
|
|
|
|
bool changed = false;
|
|
foreach (var video in videos.ToList())
|
|
{
|
|
string url = video.GetAttributeValue("src", "");
|
|
if (url.Length == 0)
|
|
url = video.SelectSingleNode("./source[@src]")?.GetAttributeValue("src", "") ?? "";
|
|
url = HtmlAgilityPack.HtmlEntity.DeEntitize(url).Trim();
|
|
if (!url.StartsWith("http", StringComparison.OrdinalIgnoreCase) || !IsInternalUrl(url, wpHost)) continue;
|
|
|
|
var downloaded = await MediaManager.DownloadAsync(url, postDate);
|
|
if (!downloaded.HasValue) continue;
|
|
if (!mediaFiles.Contains(downloaded.Value.RelativePath))
|
|
mediaFiles.Add(downloaded.Value.RelativePath);
|
|
|
|
// Die umschließende Figure (wp-block-video) wird mit ersetzt, ihre Unterschrift übernommen
|
|
var figure = video.Ancestors("figure").FirstOrDefault();
|
|
bool figureOnlyHoldsVideo = figure != null && figure.SelectSingleNode(".//img") == null && figure.SelectNodes(".//video")?.Count == 1;
|
|
string caption = figureOnlyHoldsVideo ? figure!.SelectSingleNode(".//figcaption")?.InnerHtml.Trim() ?? "" : "";
|
|
|
|
var card = doc.CreateElement("figure");
|
|
card.SetAttributeValue("class", "kg-card kg-video-card kg-width-regular" + (caption.Length > 0 ? " kg-card-hascaption" : ""));
|
|
var container = doc.CreateElement("div");
|
|
container.SetAttributeValue("class", "kg-video-container");
|
|
var newVideo = doc.CreateElement("video");
|
|
newVideo.SetAttributeValue("src", MediaManager.GhostMediaPrefix + downloaded.Value.RelativePath);
|
|
newVideo.SetAttributeValue("playsinline", "");
|
|
newVideo.SetAttributeValue("preload", "metadata");
|
|
container.AppendChild(newVideo);
|
|
|
|
// Ghost braucht Bildgröße und Dauer für die Darstellung der Karte
|
|
var info = MediaManager.ProbeMp4(downloaded.Value.LocalPath);
|
|
if (info.HasValue)
|
|
{
|
|
newVideo.SetAttributeValue("width", info.Value.Width.ToString());
|
|
newVideo.SetAttributeValue("height", info.Value.Height.ToString());
|
|
int seconds = (int)Math.Round(info.Value.DurationSeconds);
|
|
var duration = doc.CreateElement("div");
|
|
duration.SetAttributeValue("class", "kg-video-duration");
|
|
duration.InnerHtml = $"{seconds / 60}:{seconds % 60:00}";
|
|
container.AppendChild(duration);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($" [Warnung] Bildgröße/Dauer von {Path.GetFileName(downloaded.Value.LocalPath)} nicht lesbar");
|
|
}
|
|
card.AppendChild(container);
|
|
|
|
if (caption.Length > 0)
|
|
{
|
|
var figcaption = doc.CreateElement("figcaption");
|
|
figcaption.InnerHtml = caption;
|
|
card.AppendChild(figcaption);
|
|
}
|
|
|
|
var target = figureOnlyHoldsVideo ? figure! : video;
|
|
target.ParentNode!.ReplaceChild(card, target);
|
|
changed = true;
|
|
}
|
|
|
|
return changed ? doc.DocumentNode.OuterHtml : html;
|
|
}
|
|
|
|
private static bool IsInternalUrl(string url, string wpHost)
|
|
{
|
|
if (string.IsNullOrEmpty(wpHost)) return true;
|
|
try
|
|
{
|
|
var uri = new Uri(url);
|
|
return uri.Host.ToLowerInvariant().EndsWith(wpHost) || wpHost.EndsWith(uri.Host.ToLowerInvariant());
|
|
}
|
|
catch
|
|
{
|
|
return url.StartsWith("/") || url.StartsWith("wp-content");
|
|
}
|
|
}
|
|
|
|
private static void MarkProcessed(int wpId, string? wpSlug = null)
|
|
{
|
|
ProcessedPosts.Add(wpId);
|
|
SaveProcessedPosts(wpId, wpSlug);
|
|
}
|
|
|
|
private static void LoadProcessedPosts()
|
|
{
|
|
if (!File.Exists(ProcessedPostsPath)) return;
|
|
try
|
|
{
|
|
var lines = File.ReadAllLines(ProcessedPostsPath);
|
|
foreach (var line in lines)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(line) || line.StartsWith("wp_id")) continue;
|
|
var parts = line.Split(',');
|
|
if (parts.Length >= 1 && int.TryParse(parts[0].Trim(), out int postId))
|
|
ProcessedPosts.Add(postId);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Fehler beim Laden der abgefragten Posts: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
private static void SaveProcessedPosts(int wpId, string? wpSlug)
|
|
{
|
|
string entry = $"{wpId},{wpSlug ?? ""}";
|
|
using var writer = File.AppendText(ProcessedPostsPath);
|
|
writer.WriteLine(entry);
|
|
}
|
|
}
|
|
} |