262 lines
12 KiB
C#
262 lines
12 KiB
C#
|
using System.Net;
|
||
|
|
using System.Net.Sockets;
|
||
|
|
using System.Text.RegularExpressions;
|
||
|
|
using HtmlAgilityPack;
|
||
|
|
|
||
|
|
namespace wp2ghost
|
||
|
|
{
|
||
|
|
// =============================================================================
|
||
|
|
// VideoLinkCleaner — tote Verweise auf Videos (Links, <video>, Einbettungen) entfernen
|
||
|
|
// =============================================================================
|
||
|
|
static class VideoLinkCleaner
|
||
|
|
{
|
||
|
|
private static readonly HttpClient httpClient = CreateClient();
|
||
|
|
// Ergebnis pro URL merken: dieselben Videos kommen in mehreren Beiträgen vor
|
||
|
|
private static readonly Dictionary<string, bool> DeadCache = new();
|
||
|
|
|
||
|
|
private static readonly string[] VideoExtensions = { ".mp4", ".mov", ".m4v", ".webm", ".avi", ".mkv", ".3gp", ".wmv" };
|
||
|
|
private static readonly string[] VideoHosts = { "youtube.com", "youtube-nocookie.com", "youtu.be", "vimeo.com", "videopress.com", "video.wordpress.com", "videos.files.wordpress.com" };
|
||
|
|
|
||
|
|
private static readonly Regex VimeoRegex = new(@"vimeo\.com/(?:video/)?(\d+)(?:/([0-9a-f]+))?", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||
|
|
private static readonly Regex VimeoHashRegex = new(@"[?&]h=([0-9a-f]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||
|
|
private static readonly Regex YouTubeRegex = new(@"(?:youtube(?:-nocookie)?\.com/(?:embed/|shorts/|watch\?(?:[^""'\s]*&)?v=)|youtu\.be/)([\w-]{11})", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||
|
|
|
||
|
|
private static HttpClient CreateClient()
|
||
|
|
{
|
||
|
|
// HTTP/2 ist nötig: über HTTP/1.1 beantworten die wordpress.com-Videoserver
|
||
|
|
// jede Anfrage dieses Clients mit 403 — auch für funktionierende Videos.
|
||
|
|
var client = new HttpClient
|
||
|
|
{
|
||
|
|
Timeout = TimeSpan.FromSeconds(10),
|
||
|
|
DefaultRequestVersion = HttpVersion.Version20,
|
||
|
|
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrLower
|
||
|
|
};
|
||
|
|
client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36");
|
||
|
|
return client;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// Entfernt Links, <video>-Elemente und Einbettungen, deren Video nicht mehr existiert.
|
||
|
|
/// Nicht eindeutig prüfbare Fälle (Timeout, Serverfehler) bleiben unverändert.
|
||
|
|
/// </summary>
|
||
|
|
public static async Task<string> RemoveDeadVideosAsync(string html)
|
||
|
|
{
|
||
|
|
if (string.IsNullOrWhiteSpace(html)) return html;
|
||
|
|
|
||
|
|
var doc = new HtmlDocument();
|
||
|
|
doc.LoadHtml(html);
|
||
|
|
|
||
|
|
int changes = ConvertPrestoPlayers(doc);
|
||
|
|
if (changes > 0)
|
||
|
|
Console.WriteLine($" [Video] {changes} Presto-Player-Block/Blöcke in normale Video-Einbettung umgewandelt");
|
||
|
|
|
||
|
|
var nodes = doc.DocumentNode.SelectNodes("//a[@href] | //video | //iframe[@src] | //embed[@src]");
|
||
|
|
if (nodes == null) return changes > 0 ? doc.DocumentNode.OuterHtml : html;
|
||
|
|
|
||
|
|
foreach (var node in nodes.ToList())
|
||
|
|
{
|
||
|
|
string url = GetUrl(node);
|
||
|
|
|
||
|
|
// <video> mit lokaler Quelle (file://, blob:, content://) hat nie funktioniert
|
||
|
|
if (node.Name == "video" && Uri.TryCreate(url, UriKind.Absolute, out var videoUri)
|
||
|
|
&& videoUri.Scheme != Uri.UriSchemeHttp && videoUri.Scheme != Uri.UriSchemeHttps)
|
||
|
|
{
|
||
|
|
Console.WriteLine($" [Video] Totes Video entfernt (lokale Quelle): {url}");
|
||
|
|
RemoveNode(node);
|
||
|
|
changes++;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!IsVideoUrl(node, url)) continue;
|
||
|
|
if (!await IsDeadAsync(url, isVideoFile: node.Name == "video" || HasVideoExtension(url))) continue;
|
||
|
|
|
||
|
|
Console.WriteLine($" [Video] Totes Video entfernt ({DescribeNode(node)}): {url}");
|
||
|
|
RemoveNode(node);
|
||
|
|
changes++;
|
||
|
|
}
|
||
|
|
|
||
|
|
return changes > 0 ? doc.DocumentNode.OuterHtml : html;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// Presto-Player-Blöcke (<presto-player src="https://vimeo.com/ID/HASH">) funktionieren nur mit
|
||
|
|
/// dem WordPress-Plugin. Sie werden durch eine normale Einbettung bzw. das enthaltene
|
||
|
|
/// <video> ersetzt, damit das Video in Ghost abspielbar bleibt.
|
||
|
|
/// </summary>
|
||
|
|
private static int ConvertPrestoPlayers(HtmlDocument doc)
|
||
|
|
{
|
||
|
|
var players = doc.DocumentNode.SelectNodes("//presto-player");
|
||
|
|
if (players == null) return 0;
|
||
|
|
|
||
|
|
int converted = 0;
|
||
|
|
foreach (var player in players.ToList())
|
||
|
|
{
|
||
|
|
string src = HtmlEntity.DeEntitize(player.GetAttributeValue("src", "")).Trim();
|
||
|
|
var vimeo = VimeoRegex.Match(src);
|
||
|
|
var youtube = YouTubeRegex.Match(src);
|
||
|
|
|
||
|
|
HtmlNode? replacement = null;
|
||
|
|
if (vimeo.Success)
|
||
|
|
{
|
||
|
|
// Der Hash gehört bei nicht gelisteten Videos zwingend dazu
|
||
|
|
string hash = vimeo.Groups[2].Success ? "?h=" + vimeo.Groups[2].Value : "";
|
||
|
|
replacement = CreateEmbedIframe(doc, $"https://player.vimeo.com/video/{vimeo.Groups[1].Value}{hash}");
|
||
|
|
}
|
||
|
|
else if (youtube.Success)
|
||
|
|
{
|
||
|
|
replacement = CreateEmbedIframe(doc, $"https://www.youtube.com/embed/{youtube.Groups[1].Value}");
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
// Selbst gehostet: das enthaltene <video> übernehmen
|
||
|
|
replacement = player.SelectSingleNode(".//video");
|
||
|
|
replacement?.Remove();
|
||
|
|
}
|
||
|
|
|
||
|
|
if (replacement == null || player.ParentNode == null) continue;
|
||
|
|
player.ParentNode.ReplaceChild(replacement, player);
|
||
|
|
converted++;
|
||
|
|
}
|
||
|
|
return converted;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static HtmlNode CreateEmbedIframe(HtmlDocument doc, string src)
|
||
|
|
{
|
||
|
|
var iframe = doc.CreateElement("iframe");
|
||
|
|
iframe.SetAttributeValue("src", src);
|
||
|
|
iframe.SetAttributeValue("width", "640");
|
||
|
|
iframe.SetAttributeValue("height", "360");
|
||
|
|
iframe.SetAttributeValue("frameborder", "0");
|
||
|
|
iframe.SetAttributeValue("allow", "autoplay; fullscreen; picture-in-picture");
|
||
|
|
iframe.SetAttributeValue("allowfullscreen", "");
|
||
|
|
return iframe;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static string GetUrl(HtmlNode node)
|
||
|
|
{
|
||
|
|
string raw = node.Name switch
|
||
|
|
{
|
||
|
|
"a" => node.GetAttributeValue("href", ""),
|
||
|
|
"video" => node.GetAttributeValue("src", "") is { Length: > 0 } src
|
||
|
|
? src
|
||
|
|
: node.SelectSingleNode("./source[@src]")?.GetAttributeValue("src", "") ?? "",
|
||
|
|
_ => node.GetAttributeValue("src", "")
|
||
|
|
};
|
||
|
|
return HtmlEntity.DeEntitize(raw).Trim();
|
||
|
|
}
|
||
|
|
|
||
|
|
private static string DescribeNode(HtmlNode node) => node.Name switch
|
||
|
|
{
|
||
|
|
"a" => "Link",
|
||
|
|
"video" => "Video",
|
||
|
|
_ => "Einbettung"
|
||
|
|
};
|
||
|
|
|
||
|
|
private static bool IsVideoUrl(HtmlNode node, string url)
|
||
|
|
{
|
||
|
|
// Nur absolute URLs sind prüfbar
|
||
|
|
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps))
|
||
|
|
return false;
|
||
|
|
if (node.Name == "video") return true;
|
||
|
|
|
||
|
|
string host = uri.Host.ToLowerInvariant();
|
||
|
|
if (VideoHosts.Any(h => host == h || host.EndsWith("." + h))) return true;
|
||
|
|
return HasVideoExtension(url);
|
||
|
|
}
|
||
|
|
|
||
|
|
private static bool HasVideoExtension(string url) =>
|
||
|
|
Uri.TryCreate(url, UriKind.Absolute, out var uri) && VideoExtensions.Contains(Path.GetExtension(uri.AbsolutePath).ToLowerInvariant());
|
||
|
|
|
||
|
|
private static async Task<bool> IsDeadAsync(string url, bool isVideoFile)
|
||
|
|
{
|
||
|
|
if (DeadCache.TryGetValue(url, out bool known)) return known;
|
||
|
|
|
||
|
|
bool dead;
|
||
|
|
var vimeo = VimeoRegex.Match(url);
|
||
|
|
var youtube = YouTubeRegex.Match(url);
|
||
|
|
if (vimeo.Success)
|
||
|
|
{
|
||
|
|
// Die Player-Seite antwortet Skripten mit 401/403 — maßgeblich ist die oEmbed-Auskunft.
|
||
|
|
// Nicht gelistete Videos brauchen dafür ihren Hash (?h=...).
|
||
|
|
string hash = vimeo.Groups[2].Success ? vimeo.Groups[2].Value : VimeoHashRegex.Match(url).Groups[1].Value;
|
||
|
|
string target = $"https://vimeo.com/{vimeo.Groups[1].Value}" + (hash.Length > 0 ? "/" + hash : "");
|
||
|
|
var status = await GetStatusAsync("https://vimeo.com/api/oembed.json?url=" + Uri.EscapeDataString(target));
|
||
|
|
dead = false;
|
||
|
|
if (status == HttpStatusCode.NotFound)
|
||
|
|
{
|
||
|
|
// oEmbed meldet 404 auch für private Videos; die Player-Seite nur für gelöschte
|
||
|
|
var playerStatus = await GetStatusAsync($"https://player.vimeo.com/video/{vimeo.Groups[1].Value}");
|
||
|
|
dead = playerStatus == HttpStatusCode.NotFound;
|
||
|
|
if (!dead)
|
||
|
|
Console.WriteLine($" [Warnung] Vimeo-Video ist privat und für Leser nicht abspielbar (bleibt erhalten): {url}");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else if (youtube.Success)
|
||
|
|
{
|
||
|
|
// Die Watch-/Embed-Seite liefert auch für gelöschte Videos 200 — oEmbed meldet 404
|
||
|
|
string target = "https://www.youtube.com/watch?v=" + youtube.Groups[1].Value;
|
||
|
|
var status = await GetStatusAsync("https://www.youtube.com/oembed?format=json&url=" + Uri.EscapeDataString(target));
|
||
|
|
dead = status == HttpStatusCode.NotFound || status == HttpStatusCode.BadRequest;
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
var status = await GetStatusAsync(url);
|
||
|
|
// 403 gilt nur bei einer Videodatei als tot — Seiten/Einbettungen sperren damit oft bloß Skripte aus
|
||
|
|
dead = status == HttpStatusCode.NotFound || status == HttpStatusCode.Gone || status == HostNotFound
|
||
|
|
|| (isVideoFile && status == HttpStatusCode.Forbidden);
|
||
|
|
}
|
||
|
|
|
||
|
|
DeadCache[url] = dead;
|
||
|
|
return dead;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Kein echter HTTP-Status: der Hostname existiert nicht mehr
|
||
|
|
private const HttpStatusCode HostNotFound = (HttpStatusCode)(-1);
|
||
|
|
|
||
|
|
/// <summary>HTTP-Status der URL; null, wenn er nicht ermittelt werden konnte (Timeout o.ä.).</summary>
|
||
|
|
private static async Task<HttpStatusCode?> GetStatusAsync(string url)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
using var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||
|
|
request.Headers.Range = new System.Net.Http.Headers.RangeHeaderValue(0, 0); // Video nicht herunterladen
|
||
|
|
using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
|
||
|
|
return response.StatusCode;
|
||
|
|
}
|
||
|
|
catch (HttpRequestException ex) when (ex.InnerException is SocketException { SocketErrorCode: SocketError.HostNotFound })
|
||
|
|
{
|
||
|
|
return HostNotFound;
|
||
|
|
}
|
||
|
|
catch
|
||
|
|
{
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private static void RemoveNode(HtmlNode node)
|
||
|
|
{
|
||
|
|
if (node.Name == "a")
|
||
|
|
{
|
||
|
|
// Linktext bleibt stehen — außer er ist selbst nur die URL
|
||
|
|
string text = HtmlEntity.DeEntitize(node.InnerText).Trim();
|
||
|
|
bool keepContent = text.Length > 0 && !text.StartsWith("http", StringComparison.OrdinalIgnoreCase) && node.ParentNode != null;
|
||
|
|
if (keepContent)
|
||
|
|
{
|
||
|
|
foreach (var child in node.ChildNodes.ToList())
|
||
|
|
node.ParentNode!.InsertBefore(child, node);
|
||
|
|
}
|
||
|
|
node.Remove();
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Umschließende Figure (wp-block-video / wp-block-embed) samt Unterschrift mit entfernen,
|
||
|
|
// wenn sie nichts anderes enthält
|
||
|
|
var figure = node.Ancestors("figure").FirstOrDefault();
|
||
|
|
bool figureOnlyHoldsThis = figure != null
|
||
|
|
&& figure.SelectSingleNode(".//img") == null
|
||
|
|
&& (figure.SelectNodes(".//video | .//iframe | .//embed")?.Count ?? 0) == 1;
|
||
|
|
(figureOnlyHoldsThis ? figure! : node).Remove();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|