diff --git a/src/N_m3u8DL-RE.Common/Entity/MediaSegment.cs b/src/N_m3u8DL-RE.Common/Entity/MediaSegment.cs index ac37e10..e108efa 100644 --- a/src/N_m3u8DL-RE.Common/Entity/MediaSegment.cs +++ b/src/N_m3u8DL-RE.Common/Entity/MediaSegment.cs @@ -19,5 +19,23 @@ namespace N_m3u8DL_RE.Common.Entity public EncryptInfo EncryptInfo { get; set; } = new EncryptInfo(); public string Url { get; set; } + + public override bool Equals(object? obj) + { + return obj is MediaSegment segment && + Index == segment.Index && + Duration == segment.Duration && + Title == segment.Title && + StartRange == segment.StartRange && + StopRange == segment.StopRange && + ExpectLength == segment.ExpectLength && + EqualityComparer.Default.Equals(EncryptInfo, segment.EncryptInfo) && + Url == segment.Url; + } + + public override int GetHashCode() + { + return HashCode.Combine(Index, Duration, Title, StartRange, StopRange, ExpectLength, EncryptInfo, Url); + } } } diff --git a/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs b/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs index e7ea974..934c4a2 100644 --- a/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs +++ b/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs @@ -23,6 +23,7 @@ namespace N_m3u8DL_RE.Common.Entity public string? Resolution { get; set; } public double? FrameRate { get; set; } public string? Channels { get; set; } + public string? Extension { get; set; } //外部轨道GroupId (后续寻找对应轨道信息) @@ -34,6 +35,40 @@ namespace N_m3u8DL_RE.Common.Entity public Playlist? Playlist { get; set; } + public string ToShortString() + { + var prefixStr = ""; + var returnStr = ""; + var encStr = string.Empty; + + if (MediaType == Enum.MediaType.AUDIO) + { + prefixStr = $"[deepskyblue3]Aud[/] {encStr}"; + var d = $"{GroupId} | {(Bandwidth != null ? (Bandwidth / 1000) + " Kbps" : "")} | {Name} | {Codecs} | {Language} | {(Channels != null ? Channels + "CH" : "")}"; + returnStr = d.EscapeMarkup(); + } + else if (MediaType == Enum.MediaType.SUBTITLES) + { + prefixStr = $"[deepskyblue3_1]Sub[/] {encStr}"; + var d = $"{GroupId} | {Language} | {Name} | {Codecs}"; + returnStr = d.EscapeMarkup(); + } + else + { + prefixStr = $"[aqua]Vid[/] {encStr}"; + var d = $"{Resolution} | {Bandwidth / 1000} Kbps | {GroupId} | {FrameRate} | {Codecs}"; + returnStr = d.EscapeMarkup(); + } + + returnStr = prefixStr + returnStr.Trim().Trim('|').Trim(); + while (returnStr.Contains("| |")) + { + returnStr = returnStr.Replace("| |", "|"); + } + + return returnStr.TrimEnd().TrimEnd('|').TrimEnd(); + } + public override string ToString() { var prefixStr = ""; diff --git a/src/N_m3u8DL-RE.Common/Entity/SubCue.cs b/src/N_m3u8DL-RE.Common/Entity/SubCue.cs index 6d944aa..2e22911 100644 --- a/src/N_m3u8DL-RE.Common/Entity/SubCue.cs +++ b/src/N_m3u8DL-RE.Common/Entity/SubCue.cs @@ -10,7 +10,21 @@ namespace N_m3u8DL_RE.Common.Entity { public TimeSpan StartTime { get; set; } public TimeSpan EndTime { get; set; } - public string Payload { get; set; } - public string Settings { get; set; } + public required string Payload { get; set; } + public required string Settings { get; set; } + + public override bool Equals(object? obj) + { + return obj is SubCue cue && + StartTime.Equals(cue.StartTime) && + EndTime.Equals(cue.EndTime) && + Payload == cue.Payload && + Settings == cue.Settings; + } + + public override int GetHashCode() + { + return HashCode.Combine(StartTime, EndTime, Payload, Settings); + } } } diff --git a/src/N_m3u8DL-RE.Common/Entity/WebSub.cs b/src/N_m3u8DL-RE.Common/Entity/WebSub.cs deleted file mode 100644 index 584b1cc..0000000 --- a/src/N_m3u8DL-RE.Common/Entity/WebSub.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace N_m3u8DL_RE.Common.Entity -{ - public class WebSub - { - public List Cues { get; set; } = new List(); - public long MpegtsTimestamp { get; set; } = 0L; - - /// - /// 从字节数组解析WEBVTT - /// - /// - /// - public static WebSub Parse(byte[] textBytes) - { - return Parse(Encoding.UTF8.GetString(textBytes)); - } - - /// - /// 从字节数组解析WEBVTT - /// - /// - /// - /// - public static WebSub Parse(byte[] textBytes, Encoding encoding) - { - return Parse(encoding.GetString(textBytes)); - } - - /// - /// 从字符串解析WEBVTT - /// - /// - /// - public static WebSub Parse(string text) - { - if (!text.Trim().StartsWith("WEBVTT")) - throw new Exception("Bad vtt!"); - - - var webSub = new WebSub(); - var needPayload = false; - var timeLine = ""; - var regex1 = new Regex("X-TIMESTAMP-MAP.*"); - - if (regex1.IsMatch(text)) - { - var timestamp = Regex.Match(regex1.Match(text).Value, "MPEGTS:(\\d+)").Groups[1].Value; - webSub.MpegtsTimestamp = Convert.ToInt64(timestamp); - } - - foreach (var line in text.Split('\n')) - { - if (string.IsNullOrEmpty(line)) continue; - - if (!needPayload && line.Contains(" --> ")) - { - needPayload = true; - timeLine = line.Trim(); - continue; - } - - if (needPayload) - { - var payload = line.Trim(); - var arr = Regex.Split(timeLine.Replace("-->", ""), "\\s").Where(s => !string.IsNullOrEmpty(s)).ToList(); - var startTime = ConvertToTS(arr[0]); - var endTime = ConvertToTS(arr[1]); - var style = arr.Count > 2 ? string.Join(" ", arr.Skip(2)) : ""; - webSub.Cues.Add(new SubCue() - { - StartTime = startTime, - EndTime = endTime, - Payload = string.Join("", payload.Where(c => c != 8203)), //Remove Zero Width Space! - Settings = style - }) ; - needPayload = false; - } - } - - return webSub; - } - - private static TimeSpan ConvertToTS(string str) - { - var ms = Convert.ToInt32(str.Split('.').Last()); - var o = str.Split('.').First(); - var t = o.Split(':').Reverse().ToList(); - var time = 0L + ms; - for (int i = 0; i < t.Count(); i++) - { - time += (int)Math.Pow(60, i) * Convert.ToInt32(t[i]) * 1000; - } - return TimeSpan.FromMilliseconds(time); - } - - public override string ToString() - { - StringBuilder sb = new StringBuilder(); - foreach (var c in this.Cues) - { - sb.AppendLine(c.StartTime.ToString(@"hh\:mm\:ss\.fff") + " --> " + c.EndTime.ToString(@"hh\:mm\:ss\.fff") + " " + c.Settings); - sb.AppendLine(c.Payload); - sb.AppendLine(); - } - return sb.ToString(); - } - } -} diff --git a/src/N_m3u8DL-RE.Common/Entity/WebVttSub.cs b/src/N_m3u8DL-RE.Common/Entity/WebVttSub.cs new file mode 100644 index 0000000..26a9d31 --- /dev/null +++ b/src/N_m3u8DL-RE.Common/Entity/WebVttSub.cs @@ -0,0 +1,166 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Common.Entity +{ + public partial class WebVttSub + { + [RegexGenerator("X-TIMESTAMP-MAP.*")] + private static partial Regex TSMapRegex(); + [RegexGenerator("MPEGTS:(\\d+)")] + private static partial Regex TSValueRegex(); + [RegexGenerator("\\s")] + private static partial Regex SplitRegex(); + + public List Cues { get; set; } = new List(); + public long MpegtsTimestamp { get; set; } = 0L; + + /// + /// 从字节数组解析WEBVTT + /// + /// + /// + public static WebVttSub Parse(byte[] textBytes) + { + return Parse(Encoding.UTF8.GetString(textBytes)); + } + + /// + /// 从字节数组解析WEBVTT + /// + /// + /// + /// + public static WebVttSub Parse(byte[] textBytes, Encoding encoding) + { + return Parse(encoding.GetString(textBytes)); + } + + /// + /// 从字符串解析WEBVTT + /// + /// + /// + public static WebVttSub Parse(string text) + { + if (!text.Trim().StartsWith("WEBVTT")) + throw new Exception("Bad vtt!"); + + + var webSub = new WebVttSub(); + var needPayload = false; + var timeLine = ""; + var regex1 = TSMapRegex(); + + if (regex1.IsMatch(text)) + { + var timestamp = TSValueRegex().Match(regex1.Match(text).Value).Groups[1].Value; + webSub.MpegtsTimestamp = Convert.ToInt64(timestamp); + } + + var payloads = new List(); + foreach (var line in text.Split('\n')) + { + if (line.Contains(" --> ")) + { + needPayload = true; + timeLine = line.Trim(); + continue; + } + + if (needPayload) + { + if (string.IsNullOrEmpty(line.Trim())) + { + var payload = string.Join(Environment.NewLine, payloads); + var arr = SplitRegex().Split(timeLine.Replace("-->", "")).Where(s => !string.IsNullOrEmpty(s)).ToList(); + var startTime = ConvertToTS(arr[0]); + var endTime = ConvertToTS(arr[1]); + var style = arr.Count > 2 ? string.Join(" ", arr.Skip(2)) : ""; + webSub.Cues.Add(new SubCue() + { + StartTime = startTime, + EndTime = endTime, + Payload = string.Join("", payload.Where(c => c != 8203)), //Remove Zero Width Space! + Settings = style + }); + payloads.Clear(); + needPayload = false; + } + else + { + payloads.Add(line.Trim()); + } + } + } + + return webSub; + } + + /// + /// 从另一个字幕中获取所有Cue，并加载此字幕中，且自动修正偏移 + /// + /// + /// + public WebVttSub AddCuesFromOne(WebVttSub webSub) + { + FixTimestamp(webSub, this.MpegtsTimestamp); + foreach (var item in webSub.Cues) + { + if (!this.Cues.Contains(item)) this.Cues.Add(item); + } + return this; + } + + public static void FixTimestamp(WebVttSub sub, long baseTimestamp) + { + if (baseTimestamp == 0 || sub.MpegtsTimestamp == 0) + { + return; + } + + //The MPEG2 transport stream clocks (PCR, PTS, DTS) all have units of 1/90000 second + var seconds = (sub.MpegtsTimestamp - baseTimestamp) / 90000; + for (int i = 0; i < sub.Cues.Count; i++) + { + sub.Cues[i].StartTime += TimeSpan.FromSeconds(seconds); + sub.Cues[i].EndTime += TimeSpan.FromSeconds(seconds); + } + } + + private static TimeSpan ConvertToTS(string str) + { + var ms = Convert.ToInt32(str.Split('.').Last()); + var o = str.Split('.').First(); + var t = o.Split(':').Reverse().ToList(); + var time = 0L + ms; + for (int i = 0; i < t.Count(); i++) + { + time += (int)Math.Pow(60, i) * Convert.ToInt32(t[i]) * 1000; + } + return TimeSpan.FromMilliseconds(time); + } + + public override string ToString() + { + StringBuilder sb = new StringBuilder(); + foreach (var c in this.Cues) + { + sb.AppendLine(c.StartTime.ToString(@"hh\:mm\:ss\.fff") + " --> " + c.EndTime.ToString(@"hh\:mm\:ss\.fff") + " " + c.Settings); + sb.AppendLine(c.Payload); + sb.AppendLine(); + } + sb.AppendLine(); + return sb.ToString(); + } + + public string ToStringWithHeader() + { + return "WEBVTT" + Environment.NewLine + Environment.NewLine + ToString(); + } + } +} diff --git a/src/N_m3u8DL-RE.Common/JsonContext/JsonContext.cs b/src/N_m3u8DL-RE.Common/JsonContext/JsonContext.cs new file mode 100644 index 0000000..01b89c1 --- /dev/null +++ b/src/N_m3u8DL-RE.Common/JsonContext/JsonContext.cs @@ -0,0 +1,20 @@ +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Common.Enum; +using System.Text.Json.Serialization; + +namespace N_m3u8DL_RE.Common +{ + [JsonSourceGenerationOptions( + WriteIndented = true, + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + GenerationMode = JsonSourceGenerationMode.Metadata)] + [JsonSerializable(typeof(MediaType))] + [JsonSerializable(typeof(EncryptMethod))] + [JsonSerializable(typeof(ExtractorType))] + [JsonSerializable(typeof(Choise))] + [JsonSerializable(typeof(StreamSpec))] + [JsonSerializable(typeof(IOrderedEnumerable))] + [JsonSerializable(typeof(List))] + [JsonSerializable(typeof(Dictionary))] + internal partial class JsonContext : JsonSerializerContext { } +} diff --git a/src/N_m3u8DL-RE.Common/Log/Logger.cs b/src/N_m3u8DL-RE.Common/Log/Logger.cs index 03aa7a0..9e4a647 100644 --- a/src/N_m3u8DL-RE.Common/Log/Logger.cs +++ b/src/N_m3u8DL-RE.Common/Log/Logger.cs @@ -8,8 +8,11 @@ using System.Threading.Tasks; namespace N_m3u8DL_RE.Common.Log { - public class Logger + public partial class Logger { + [RegexGenerator("{}")] + private static partial Regex VarsRepRegex(); + /// /// 日志级别，默认为INFO /// @@ -46,7 +49,7 @@ namespace N_m3u8DL_RE.Common.Log { for (int i = 0; i < ps.Length; i++) { - data = new Regex("{}").Replace(data, $"{ps[i]}", 1); + data = VarsRepRegex().Replace(data, $"{ps[i]}", 1); } return data; } diff --git a/src/N_m3u8DL-RE.Common/N_m3u8DL-RE.Common.csproj b/src/N_m3u8DL-RE.Common/N_m3u8DL-RE.Common.csproj index 4c184bd..6360901 100644 --- a/src/N_m3u8DL-RE.Common/N_m3u8DL-RE.Common.csproj +++ b/src/N_m3u8DL-RE.Common/N_m3u8DL-RE.Common.csproj @@ -1,9 +1,11 @@  - net6.0 + library + net7.0 N_m3u8DL_RE.Common enable + preview enable diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs b/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs index 2e461b3..d7e568b 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs @@ -69,6 +69,15 @@ namespace N_m3u8DL_RE.Common.Resource { } } + /// + /// 查找类似 二进制合并中... 的本地化字符串。 + /// + public static string binaryMerge { + get { + return ResourceManager.GetString("binaryMerge", resourceCulture); + } + } + /// /// 查找类似 验证最后一个分片有效性 的本地化字符串。 /// @@ -87,6 +96,42 @@ namespace N_m3u8DL_RE.Common.Resource { } } + /// + /// 查找类似 正在提取TTML(raw)字幕... 的本地化字符串。 + /// + public static string fixingTTML { + get { + return ResourceManager.GetString("fixingTTML", resourceCulture); + } + } + + /// + /// 查找类似 正在提取TTML(mp4)字幕... 的本地化字符串。 + /// + public static string fixingTTMLmp4 { + get { + return ResourceManager.GetString("fixingTTMLmp4", resourceCulture); + } + } + + /// + /// 查找类似 正在提取VTT(raw)字幕... 的本地化字符串。 + /// + public static string fixingVTT { + get { + return ResourceManager.GetString("fixingVTT", resourceCulture); + } + } + + /// + /// 查找类似 正在提取VTT(mp4)字幕... 的本地化字符串。 + /// + public static string fixingVTTmp4 { + get { + return ResourceManager.GetString("fixingVTTmp4", resourceCulture); + } + } + /// /// 查找类似 找不到支持的Processor 的本地化字符串。 /// @@ -186,6 +231,15 @@ namespace N_m3u8DL_RE.Common.Resource { } } + /// + /// 查找类似 分片数量校验不通过, 共{}个,已下载{}. 的本地化字符串。 + /// + public static string segmentCountCheckNotPass { + get { + return ResourceManager.GetString("segmentCountCheckNotPass", resourceCulture); + } + } + /// /// 查找类似 已选择的流: 的本地化字符串。 /// @@ -195,6 +249,15 @@ namespace N_m3u8DL_RE.Common.Resource { } } + /// + /// 查找类似 开始下载... 的本地化字符串。 + /// + public static string startDownloading { + get { + return ResourceManager.GetString("startDownloading", resourceCulture); + } + } + /// /// 查找类似 已解析, 共计 {} 条媒体流, 基本流 {} 条, 可选音频流 {} 条, 可选字幕流 {} 条 的本地化字符串。 /// diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx b/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx index b323cd8..68cb5fb 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx @@ -131,7 +131,7 @@ Live stream found - Selected Streams: + Selected streams: Writing meta.json @@ -148,4 +148,25 @@ No Processor matched + + Start downloading... + + + Segment count check not pass, total: {}, downloaded: {}. + + + Extracting VTT(raw) subtitle... + + + Extracting VTT(mp4) subtitle... + + + Binary merging... + + + Extracting TTML(mp4) subtitle... + + + Extracting TTML(raw) subtitle... + \ No newline at end of file diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.resx b/src/N_m3u8DL-RE.Common/Resource/ResString.resx index 28e8636..9b4e647 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.resx +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.resx @@ -148,4 +148,25 @@ 找不到支持的Processor + + 开始下载... + + + 分片数量校验不通过, 共{}个,已下载{}. + + + 正在提取VTT(raw)字幕... + + + 正在提取VTT(mp4)字幕... + + + 二进制合并中... + + + 正在提取TTML(mp4)字幕... + + + 正在提取TTML(raw)字幕... + \ No newline at end of file diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx b/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx index 39c8c4b..2cda1c5 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx @@ -148,4 +148,25 @@ 找不到支持的Processor + + 開始下載... + + + 分片數量校驗不通過, 共{}個,已下載{}. + + + 正在提取VTT(raw)字幕... + + + 正在提取VTT(mp4)字幕... + + + 二進製合併中... + + + 正在提取TTML(mp4)字幕... + + + 正在提取TTML(raw)字幕... + \ No newline at end of file diff --git a/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs b/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs index 5210240..24b453b 100644 --- a/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs +++ b/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs @@ -1,4 +1,5 @@ -using N_m3u8DL_RE.Common.JsonConverter; +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Common.JsonConverter; using System; using System.Collections.Generic; using System.Linq; @@ -11,16 +12,30 @@ namespace N_m3u8DL_RE.Common.Util { public class GlobalUtil { + private static readonly JsonSerializerOptions Options = new JsonSerializerOptions + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = { new JsonStringEnumConverter(), new BytesBase64Converter() } + }; + private static readonly JsonContext Context = new JsonContext(Options); + public static string ConvertToJson(object o) { - var options = new JsonSerializerOptions + if (o is StreamSpec s) { - Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - WriteIndented = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - Converters = { new JsonStringEnumConverter(), new BytesBase64Converter() } - }; - return JsonSerializer.Serialize(o, options); + return JsonSerializer.Serialize(s, Context.StreamSpec); + } + else if (o is IOrderedEnumerable ss) + { + return JsonSerializer.Serialize(ss, Context.IOrderedEnumerableStreamSpec); + } + else if (o is List sList) + { + return JsonSerializer.Serialize(sList, Context.ListStreamSpec); + } + return JsonSerializer.Serialize(o, Options); } //此函数用于格式化输出时长 diff --git a/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor2.cs b/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor2.cs index 96ccf4f..4455d20 100644 --- a/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor2.cs +++ b/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor2.cs @@ -126,7 +126,7 @@ namespace N_m3u8DL_RE.Parser.Extractor if (mimeType == null) { - mimeType = representation.Attribute("contentType")?.Value ?? representation.Attribute("mimeType")?.Value ?? ""; + mimeType = representation.Attribute("contentType")?.Value ?? adaptationSet.Attribute("mimeType")?.Value ?? ""; } var bandwidth = representation.Attribute("bandwidth"); StreamSpec streamSpec = new(); @@ -139,17 +139,34 @@ namespace N_m3u8DL_RE.Parser.Extractor streamSpec.FrameRate = frameRate ?? GetFrameRate(representation); streamSpec.Resolution = representation.Attribute("width")?.Value != null ? $"{representation.Attribute("width")?.Value}x{representation.Attribute("height")?.Value}" : null; streamSpec.Url = MpdUrl; - streamSpec.MediaType = mimeType.Split("/")[0] switch + streamSpec.MediaType = mimeType.Split('/')[0] switch { "text" => MediaType.SUBTITLES, "audio" => MediaType.AUDIO, _ => null }; + //推测后缀名 + var mType = representation.Attribute("mimeType")?.Value ?? adaptationSet.Attribute("mimeType")?.Value; + if (mType != null) + { + var mTypeSplit = mType.Split('/'); + streamSpec.Extension = mTypeSplit.Length == 2 ? mTypeSplit[1] : null; + } //优化字幕场景识别 if (streamSpec.Codecs == "stpp" || streamSpec.Codecs == "wvtt") { streamSpec.MediaType = MediaType.SUBTITLES; } + //优化字幕场景识别 + var role = representation.Elements().Where(e => e.Name.LocalName == "Role").FirstOrDefault() ?? adaptationSet.Elements().Where(e => e.Name.LocalName == "Role").FirstOrDefault(); + if (role != null) + { + var v = role.Attribute("value")?.Value; + if (v == "subtitle") + streamSpec.MediaType = MediaType.SUBTITLES; + if (mType != null && mType.Contains("ttml")) + streamSpec.Extension = "ttml"; + } streamSpec.Playlist.IsLive = isLive; //设置刷新间隔 timeShiftBufferDepth / 2 if (timeShiftBufferDepth != null) @@ -188,7 +205,7 @@ namespace N_m3u8DL_RE.Parser.Extractor } else { - var initUrl = ParserUtil.CombineURL(segBaseUrl, initialization.Attribute("sourceURL")?.Value); + var initUrl = ParserUtil.CombineURL(segBaseUrl, initialization.Attribute("sourceURL")?.Value!); var initRange = initialization.Attribute("range")?.Value; streamSpec.Playlist.MediaInit = new MediaSegment(); streamSpec.Playlist.MediaInit.Url = PreProcessUrl(initUrl); @@ -211,7 +228,7 @@ namespace N_m3u8DL_RE.Parser.Extractor var initialization = segmentList.Elements().Where(e => e.Name.LocalName == "Initialization").FirstOrDefault(); if (initialization != null) { - var initUrl = ParserUtil.CombineURL(segBaseUrl, initialization.Attribute("sourceURL")?.Value); + var initUrl = ParserUtil.CombineURL(segBaseUrl, initialization.Attribute("sourceURL")?.Value!); var initRange = initialization.Attribute("range")?.Value; streamSpec.Playlist.MediaInit = new MediaSegment(); streamSpec.Playlist.MediaInit.Url = PreProcessUrl(initUrl); @@ -227,7 +244,7 @@ namespace N_m3u8DL_RE.Parser.Extractor for (int segmentIndex = 0; segmentIndex < segmentURLs.Count(); segmentIndex++) { var segmentURL = segmentURLs.ElementAt(segmentIndex); - var mediaUrl = ParserUtil.CombineURL(segBaseUrl, segmentURL.Attribute("media")?.Value); + var mediaUrl = ParserUtil.CombineURL(segBaseUrl, segmentURL.Attribute("media")?.Value!); var mediaRange = segmentURL.Attribute("range")?.Value; MediaSegment mediaSegment = new(); mediaSegment.Duration = Convert.ToDouble(duration); @@ -253,20 +270,23 @@ namespace N_m3u8DL_RE.Parser.Extractor if (segmentTemplateElements.Any() || segmentTemplateElementsOuter.Any()) { //优先使用最近的元素 - var segmentTemplate = segmentTemplateElements.FirstOrDefault() ?? segmentTemplateElementsOuter.FirstOrDefault(); - var segmentTemplateOuter = segmentTemplateElementsOuter.FirstOrDefault() ?? segmentTemplateElements.FirstOrDefault(); + var segmentTemplate = (segmentTemplateElements.FirstOrDefault() ?? segmentTemplateElementsOuter.FirstOrDefault())!; + var segmentTemplateOuter = (segmentTemplateElementsOuter.FirstOrDefault() ?? segmentTemplateElements.FirstOrDefault())!; var varDic = new Dictionary(); varDic[DASHTags.TemplateRepresentationID] = streamSpec.GroupId; varDic[DASHTags.TemplateBandwidth] = bandwidth?.Value; //timesacle var timescaleStr = segmentTemplate.Attribute("timescale")?.Value ?? segmentTemplateOuter.Attribute("timescale")?.Value ?? "1"; var durationStr = segmentTemplate.Attribute("duration")?.Value ?? segmentTemplateOuter.Attribute("duration")?.Value; - var startNumberStr = segmentTemplate.Attribute("startNumber")?.Value ?? segmentTemplateOuter.Attribute("startNumber")?.Value ?? "0"; + var startNumberStr = segmentTemplate.Attribute("startNumber")?.Value ?? segmentTemplateOuter.Attribute("startNumber")?.Value ?? "1"; //处理init url var initialization = segmentTemplate.Attribute("initialization")?.Value ?? segmentTemplateOuter.Attribute("initialization")?.Value; - var initUrl = ParserUtil.ReplaceVars(ParserUtil.CombineURL(segBaseUrl, initialization), varDic); - streamSpec.Playlist.MediaInit = new MediaSegment(); - streamSpec.Playlist.MediaInit.Url = PreProcessUrl(initUrl); + if (initialization != null) + { + var initUrl = ParserUtil.ReplaceVars(ParserUtil.CombineURL(segBaseUrl, initialization), varDic); + streamSpec.Playlist.MediaInit = new MediaSegment(); + streamSpec.Playlist.MediaInit.Url = PreProcessUrl(initUrl); + } //处理分片 var media = segmentTemplate.Attribute("media")?.Value ?? segmentTemplateOuter.Attribute("media")?.Value; var segmentTimeline = segmentTemplate.Elements().Where(e => e.Name.LocalName == "SegmentTimeline").FirstOrDefault(); @@ -290,7 +310,7 @@ namespace N_m3u8DL_RE.Parser.Extractor var _repeatCount = Convert.ToInt64(_repeatCountStr); varDic[DASHTags.TemplateTime] = currentTime; varDic[DASHTags.TemplateNumber] = segNumber++; - var mediaUrl = ParserUtil.ReplaceVars(ParserUtil.CombineURL(segBaseUrl, media), varDic); + var mediaUrl = ParserUtil.ReplaceVars(ParserUtil.CombineURL(segBaseUrl, media!), varDic); MediaSegment mediaSegment = new(); mediaSegment.Url = PreProcessUrl(mediaUrl); mediaSegment.Duration = _duration / (double)timescale; @@ -307,7 +327,7 @@ namespace N_m3u8DL_RE.Parser.Extractor MediaSegment _mediaSegment = new(); varDic[DASHTags.TemplateTime] = currentTime; varDic[DASHTags.TemplateNumber] = segNumber++; - var _mediaUrl = ParserUtil.ReplaceVars(ParserUtil.CombineURL(segBaseUrl, media), varDic); + var _mediaUrl = ParserUtil.ReplaceVars(ParserUtil.CombineURL(segBaseUrl, media!), varDic); _mediaSegment.Url = PreProcessUrl(_mediaUrl); _mediaSegment.Index = segIndex++; _mediaSegment.Duration = _duration / (double)timescale; @@ -327,9 +347,9 @@ namespace N_m3u8DL_RE.Parser.Extractor if (totalNumber == 0 && isLive) { var now = publishTime == null ? DateTime.Now : DateTime.Parse(publishTime); - var availableTime = DateTime.Parse(availabilityStartTime); + var availableTime = DateTime.Parse(availabilityStartTime!); var ts = now - availableTime; - var updateTs = XmlConvert.ToTimeSpan(timeShiftBufferDepth); + var updateTs = XmlConvert.ToTimeSpan(timeShiftBufferDepth!); //(当前时间到发布时间的时间差 - 最小刷新间隔) / 分片时长 startNumber += (long)((ts.TotalSeconds - updateTs.TotalSeconds) * timescale / duration); totalNumber = (long)(updateTs.TotalSeconds * timescale / duration); @@ -337,7 +357,7 @@ namespace N_m3u8DL_RE.Parser.Extractor for (long index = startNumber, segIndex = 0; index < startNumber + totalNumber; index++, segIndex++) { varDic[DASHTags.TemplateNumber] = index; - var mediaUrl = ParserUtil.ReplaceVars(ParserUtil.CombineURL(segBaseUrl, media), varDic); + var mediaUrl = ParserUtil.ReplaceVars(ParserUtil.CombineURL(segBaseUrl, media!), varDic); MediaSegment mediaSegment = new(); mediaSegment.Url = PreProcessUrl(mediaUrl); mediaSegment.Index = isLive ? index : segIndex; //直播直接用startNumber @@ -389,7 +409,7 @@ namespace N_m3u8DL_RE.Parser.Extractor var startIndex = streamList[_index].Playlist?.MediaParts.Last().MediaSegments.Last().Index + 1; foreach (var item in streamSpec.Playlist.MediaParts[0].MediaSegments) { - item.Index = item.Index + startIndex.Value; + item.Index = item.Index + startIndex!.Value; } streamList[_index].Playlist?.MediaParts.Add(new MediaPart() { @@ -400,6 +420,11 @@ namespace N_m3u8DL_RE.Parser.Extractor } else { + //分片默认后缀m4s + if (streamSpec.Extension == null || streamSpec.Playlist.MediaParts.Sum(x => x.MediaSegments.Count) > 1) + { + streamSpec.Extension = "m4s"; + } streamList.Add(streamSpec); //将segBaseUrl恢复 （重要） segBaseUrl = this.BaseUrl; diff --git a/src/N_m3u8DL-RE.Parser/Extractor/HLSExtractor.cs b/src/N_m3u8DL-RE.Parser/Extractor/HLSExtractor.cs index 338ecf0..50c23cf 100644 --- a/src/N_m3u8DL-RE.Parser/Extractor/HLSExtractor.cs +++ b/src/N_m3u8DL-RE.Parser/Extractor/HLSExtractor.cs @@ -87,7 +87,7 @@ namespace N_m3u8DL_RE.Parser.Extractor List streams = new List(); using StringReader sr = new StringReader(M3u8Content); - string line; + string? line; bool expectPlaylist = false; StreamSpec streamSpec = new(); @@ -205,7 +205,7 @@ namespace N_m3u8DL_RE.Parser.Extractor bool hasAd = false; using StringReader sr = new StringReader(M3u8Content); - string line; + string? line; bool expectSegment = false; bool isEndlist = false; long segIndex = 0; @@ -449,7 +449,7 @@ namespace N_m3u8DL_RE.Parser.Extractor return playlist; } - private byte[] ParseKey(string method, string uriText) + private byte[]? ParseKey(string method, string uriText) { foreach (var p in ParserConfig.KeyProcessors) { @@ -476,12 +476,14 @@ namespace N_m3u8DL_RE.Parser.Extractor } else { + var playlist = await ParseListAsync(); return new List() { new StreamSpec() { Url = ParserConfig.Url, - Playlist = await ParseListAsync() + Playlist = playlist, + Extension = playlist.MediaInit != null ? "mp4" : "ts" } }; } @@ -508,8 +510,9 @@ namespace N_m3u8DL_RE.Parser.Extractor for (int i = 0; i < lists.Count; i++) { //重新加载m3u8 - await LoadM3u8FromUrlAsync(lists[i].Url); + await LoadM3u8FromUrlAsync(lists[i].Url!); lists[i].Playlist = await ParseListAsync(); + lists[i].Extension = lists[i].Playlist!.MediaInit != null ? "mp4" : "ts"; } } } diff --git a/src/N_m3u8DL-RE.Parser/Mp4/BinaryReader2.cs b/src/N_m3u8DL-RE.Parser/Mp4/BinaryReader2.cs new file mode 100644 index 0000000..51ef5e5 --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Mp4/BinaryReader2.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Mp4SubtitleParser +{ + //make BinaryReader in Big Endian + class BinaryReader2 : BinaryReader + { + public BinaryReader2(System.IO.Stream stream) : base(stream) { } + + public bool HasMoreData() + { + return BaseStream.Position < BaseStream.Length; + } + + public long GetLength() + { + return BaseStream.Length; + } + + public long GetPosition() + { + return BaseStream.Position; + } + + public override int ReadInt32() + { + var data = base.ReadBytes(4); + if (BitConverter.IsLittleEndian) + Array.Reverse(data); + return BitConverter.ToInt32(data, 0); + } + + public override short ReadInt16() + { + var data = base.ReadBytes(2); + if (BitConverter.IsLittleEndian) + Array.Reverse(data); + return BitConverter.ToInt16(data, 0); + } + + public override long ReadInt64() + { + var data = base.ReadBytes(8); + if (BitConverter.IsLittleEndian) + Array.Reverse(data); + return BitConverter.ToInt64(data, 0); + } + + public override uint ReadUInt32() + { + var data = base.ReadBytes(4); + if (BitConverter.IsLittleEndian) + Array.Reverse(data); + return BitConverter.ToUInt32(data, 0); + } + + } +} diff --git a/src/N_m3u8DL-RE.Parser/Mp4/MP4Parser.cs b/src/N_m3u8DL-RE.Parser/Mp4/MP4Parser.cs new file mode 100644 index 0000000..fd1ef78 --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Mp4/MP4Parser.cs @@ -0,0 +1,345 @@ +using System.Text; + +/** + * Translated from shaka-player project + * https://github.com/nilaoda/Mp4SubtitleParser + * https://github.com/shaka-project/shaka-player + */ +namespace Mp4SubtitleParser +{ + class ParsedBox + { + public MP4Parser Parser { get; set; } + public bool PartialOkay { get; set; } + public long Start { get; set; } + public uint Version { get; set; } = 1000; + public uint Flags { get; set; } = 1000; + public BinaryReader2 Reader { get; set; } + public bool Has64BitSize { get; set; } + } + + class TFHD + { + public uint TrackId { get; set; } + public uint DefaultSampleDuration { get; set; } + public uint DefaultSampleSize { get; set; } + } + + class TRUN + { + public uint SampleCount { get; set; } + public List SampleData { get; set; } = new List(); + } + + class Sample + { + public uint SampleDuration { get; set; } + public uint SampleSize { get; set; } + public uint SampleCompositionTimeOffset { get; set; } + } + + enum BoxType + { + BASIC_BOX = 0, + FULL_BOX = 1 + }; + + class MP4Parser + { + public bool Done { get; set; } = false; + public Dictionary Headers { get; set; } = new Dictionary(); + public Dictionary BoxDefinitions { get; set; } = new Dictionary(); + + public delegate void BoxHandler(ParsedBox box); + public delegate void DataHandler(byte[] data); + + public static BoxHandler AllData(DataHandler handler) + { + return (box) => + { + var all = box.Reader.GetLength() - box.Reader.GetPosition(); + handler(box.Reader.ReadBytes((int)all)); + }; + } + + public static void Children(ParsedBox box) + { + var headerSize = HeaderSize(box); + while (box.Reader.HasMoreData() && !box.Parser.Done) + { + box.Parser.ParseNext(box.Start + headerSize, box.Reader, box.PartialOkay); + } + } + + public static void SampleDescription(ParsedBox box) + { + var headerSize = HeaderSize(box); + var count = box.Reader.ReadUInt32(); + for (int i = 0; i < count; i++) + { + box.Parser.ParseNext(box.Start + headerSize, box.Reader, box.PartialOkay); + if (box.Parser.Done) + { + break; + } + } + } + + public void Parse(byte[] data, bool partialOkay = false, bool stopOnPartial = false) + { + var reader = new BinaryReader2(new MemoryStream(data)); + this.Done = false; + while (reader.HasMoreData() && !this.Done) + { + this.ParseNext(0, reader, partialOkay, stopOnPartial); + } + } + + private void ParseNext(long absStart, BinaryReader2 reader, bool partialOkay, bool stopOnPartial = false) + { + var start = reader.GetPosition(); + + // size(4 bytes) + type(4 bytes) = 8 bytes + if (stopOnPartial && start + 8 > reader.GetLength()) + { + this.Done = true; + return; + } + + long size = reader.ReadUInt32(); + long type = reader.ReadUInt32(); + var name = TypeToString(type); + var has64BitSize = false; + + //Console.WriteLine($"Parsing MP4 box: {name}"); + + switch (size) + { + case 0: + size = reader.GetLength() - start; + break; + case 1: + if (stopOnPartial && reader.GetPosition() + 8 > reader.GetLength()) + { + this.Done = true; + return; + } + size = (long)reader.ReadUInt64(); + has64BitSize = true; + break; + } + + BoxHandler boxDefinition = null; + this.BoxDefinitions.TryGetValue(type, out boxDefinition); + + if (boxDefinition != null) + { + uint version = 1000; + uint flags = 1000; + + if (this.Headers[type] == (int)BoxType.FULL_BOX) + { + if (stopOnPartial && reader.GetPosition() + 4 > reader.GetLength()) + { + this.Done = true; + return; + } + var versionAndFlags = reader.ReadUInt32(); + version = versionAndFlags >> 24; + flags = versionAndFlags & 0xFFFFFF; + } + var end = start + size; + if (partialOkay && end > reader.GetLength()) + { + // For partial reads, truncate the payload if we must. + end = reader.GetLength(); + } + + if (stopOnPartial && end > reader.GetLength()) + { + this.Done = true; + return; + } + + int payloadSize = (int)(end - reader.GetPosition()); + var payload = (payloadSize > 0) ? reader.ReadBytes(payloadSize) : new byte[0]; + var box = new ParsedBox() + { + Parser = this, + PartialOkay = partialOkay || false, + Version = version, + Flags = flags, + Reader = new BinaryReader2(new MemoryStream(payload)), + Start = start + absStart, + Has64BitSize = has64BitSize, + }; + + boxDefinition(box); + } + else + { + // Move the read head to be at the end of the box. + // If the box is longer than the remaining parts of the file, e.g. the + // mp4 is improperly formatted, or this was a partial range request that + // ended in the middle of a box, just skip to the end. + var skipLength = Math.Min( + start + size - reader.GetPosition(), + reader.GetLength() - reader.GetPosition()); + reader.ReadBytes((int)skipLength); + } + } + + + private static int HeaderSize(ParsedBox box) + { + return /* basic header */ 8 + + /* additional 64-bit size field */ (box.Has64BitSize ? 8 : 0) + + /* version and flags for a "full" box */ (box.Flags != 0 ? 4 : 0); + } + + public static string TypeToString(long type) + { + return Encoding.UTF8.GetString(new byte[] + { + (byte)((type >> 24) & 0xff), + (byte)((type >> 16) & 0xff), + (byte)((type >> 8) & 0xff), + (byte)(type & 0xff) + }); + } + + private static int TypeFromString(string name) + { + if (name.Length != 4) + throw new Exception("Mp4 box names must be 4 characters long"); + var code = 0; + foreach (var chr in name) { + code = (code << 8) | chr; + } + return code; + } + + public MP4Parser Box(string type, BoxHandler handler) + { + var typeCode = TypeFromString(type); + this.Headers[typeCode] = (int)BoxType.BASIC_BOX; + this.BoxDefinitions[typeCode] = handler; + return this; + } + + public MP4Parser FullBox(string type, BoxHandler handler) + { + var typeCode = TypeFromString(type); + this.Headers[typeCode] = (int)BoxType.FULL_BOX; + this.BoxDefinitions[typeCode] = handler; + return this; + } + + public static uint ParseMDHD(BinaryReader2 reader, uint version) + { + if (version == 1) + { + reader.ReadBytes(8); // Skip "creation_time" + reader.ReadBytes(8); // Skip "modification_time" + } + else + { + reader.ReadBytes(4); // Skip "creation_time" + reader.ReadBytes(4); // Skip "modification_time" + } + + return reader.ReadUInt32(); + } + + public static ulong ParseTFDT(BinaryReader2 reader, uint version) + { + return version == 1 ? reader.ReadUInt64() : reader.ReadUInt32(); + } + + public static TFHD ParseTFHD(BinaryReader2 reader, uint flags) + { + var trackId = reader.ReadUInt32(); + uint defaultSampleDuration = 0; + uint defaultSampleSize = 0; + + // Skip "base_data_offset" if present. + if ((flags & 0x000001) != 0) + { + reader.ReadBytes(8); + } + + // Skip "sample_description_index" if present. + if ((flags & 0x000002) != 0) + { + reader.ReadBytes(4); + } + + // Read "default_sample_duration" if present. + if ((flags & 0x000008) != 0) + { + defaultSampleDuration = reader.ReadUInt32(); + } + + // Read "default_sample_size" if present. + if ((flags & 0x000010) != 0) + { + defaultSampleSize = reader.ReadUInt32(); + } + + return new TFHD() { TrackId = trackId, DefaultSampleDuration = defaultSampleDuration, DefaultSampleSize = defaultSampleSize }; + } + + public static TRUN ParseTRUN(BinaryReader2 reader, uint version, uint flags) + { + var trun = new TRUN(); + trun.SampleCount = reader.ReadUInt32(); + + // Skip "data_offset" if present. + if ((flags & 0x000001) != 0) + { + reader.ReadBytes(4); + } + + // Skip "first_sample_flags" if present. + if ((flags & 0x000004) != 0) + { + reader.ReadBytes(4); + } + + for (int i = 0; i < trun.SampleCount; i++) + { + var sample = new Sample(); + + // Read "sample duration" if present. + if ((flags & 0x000100) != 0) + { + sample.SampleDuration = reader.ReadUInt32(); + } + + // Read "sample_size" if present. + if ((flags & 0x000200) != 0) + { + sample.SampleSize = reader.ReadUInt32(); + } + + // Skip "sample_flags" if present. + if ((flags & 0x000400) != 0) + { + reader.ReadBytes(4); + } + + // Read "sample_time_offset" if present. + if ((flags & 0x000800) != 0) + { + sample.SampleCompositionTimeOffset = version == 0 ? + reader.ReadUInt32() : + (uint)reader.ReadInt32(); + } + + trun.SampleData.Add(sample); + } + + return trun; + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/Mp4/MP4TtmlUtil.cs b/src/N_m3u8DL-RE.Parser/Mp4/MP4TtmlUtil.cs new file mode 100644 index 0000000..ea0a9f7 --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Mp4/MP4TtmlUtil.cs @@ -0,0 +1,290 @@ +using N_m3u8DL_RE.Common.Entity; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; + +namespace Mp4SubtitleParser +{ + class SubEntity + { + public string Begin { get; set; } + public string End { get; set; } + public string Region { get; set; } + public List Contents { get; set; } = new List(); + public List ContentStrings { get; set; } = new List(); + + public override bool Equals(object? obj) + { + return obj is SubEntity entity && + Begin == entity.Begin && + End == entity.End && + Region == entity.Region && + ContentStrings.SequenceEqual(entity.ContentStrings); + } + + public override int GetHashCode() + { + return HashCode.Combine(Begin, End, Region, ContentStrings); + } + } + + public partial class MP4TtmlUtil + { + [RegexGenerator(">(.+?)<\\/p>")] + private static partial Regex LabelFixRegex(); + + public static bool CheckInit(byte[] data) + { + bool sawSTPP = false; + + //parse init + new MP4Parser() + .Box("moov", MP4Parser.Children) + .Box("trak", MP4Parser.Children) + .Box("mdia", MP4Parser.Children) + .Box("minf", MP4Parser.Children) + .Box("stbl", MP4Parser.Children) + .FullBox("stsd", MP4Parser.SampleDescription) + .Box("stpp", (box) => { + sawSTPP = true; + }) + .Parse(data); + + return sawSTPP; + } + + private static string ShiftTime(string xmlSrc, long segTimeMs, int index) + { + string Add(string xmlTime) + { + var dt = DateTime.ParseExact(xmlTime, "HH:mm:ss.fff", System.Globalization.CultureInfo.InvariantCulture); + var ts = TimeSpan.FromMilliseconds(dt.TimeOfDay.TotalMilliseconds + segTimeMs * index); + return string.Format("{0:00}:{1:00}:{2:00}.{3:000}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); + } + + if (!xmlSrc.Contains("")) return xmlSrc; + var xmlDoc = new XmlDocument(); + XmlNamespaceManager? nsMgr = null; + xmlDoc.LoadXml(xmlSrc); + var ttNode = xmlDoc.LastChild; + if (nsMgr == null) + { + var ns = ((XmlElement)ttNode!).GetAttribute("xmlns"); + nsMgr = new XmlNamespaceManager(xmlDoc.NameTable); + nsMgr.AddNamespace("ns", ns); + } + + var bodyNode = ttNode!.SelectSingleNode("ns:body", nsMgr); + if (bodyNode == null) + return xmlSrc; + + var _div = bodyNode.SelectSingleNode("ns:div", nsMgr); + //Parse

label + foreach (XmlElement _p in _div!.SelectNodes("ns:p", nsMgr)!) + { + var _begin = _p.GetAttribute("begin"); + var _end = _p.GetAttribute("end"); + _p.SetAttribute("begin", Add(_begin)); + _p.SetAttribute("end", Add(_end)); + //Console.WriteLine($"{_begin} {_p.GetAttribute("begin")}"); + //Console.WriteLine($"{_end} {_p.GetAttribute("begin")}"); + } + + return xmlDoc.OuterXml; + } + + private static string GetTextFromElement(XmlElement node) + { + var sb = new StringBuilder(); + foreach (XmlNode item in node.ChildNodes) + { + if (item.NodeType == XmlNodeType.Text) + { + sb.Append(item.InnerText.Trim()); + } + else if(item.NodeType == XmlNodeType.Element && item.Name == "br") + { + sb.AppendLine(); + } + } + return sb.ToString(); + } + + public static WebVttSub ExtractFromMp4s(IEnumerable items, long segTimeMs) + { + //read ttmls + List xmls = new List(); + int segIndex = 0; + foreach (var item in items) + { + var dataSeg = File.ReadAllBytes(item); + + var sawMDAT = false; + //parse media + new MP4Parser() + .Box("mdat", MP4Parser.AllData((data) => + { + sawMDAT = true; + // Join this to any previous payload, in case the mp4 has multiple + // mdats. + if (segTimeMs != 0) + { + xmls.Add(ShiftTime(Encoding.UTF8.GetString(data), segTimeMs, segIndex)); + } + else + { + xmls.Add(Encoding.UTF8.GetString(data)); + } + })) + .Parse(dataSeg,/* partialOkay= */ false); + segIndex++; + } + + return ExtractSub(xmls); + } + + public static WebVttSub ExtractFromTTMLs(IEnumerable items, long segTimeMs) + { + //read ttmls + List xmls = new List(); + int segIndex = 0; + foreach (var item in items) + { + var xml = File.ReadAllText(item); + if (segTimeMs != 0) + { + xmls.Add(ShiftTime(xml, segTimeMs, segIndex)); + } + else + { + xmls.Add(xml); + } + segIndex++; + } + + return ExtractSub(xmls); + } + + private static WebVttSub ExtractSub(List xmls) + { + //parsing + var xmlDoc = new XmlDocument(); + var finalSubs = new List(); + XmlNode? headNode = null; + XmlNamespaceManager? nsMgr = null; + var regex = LabelFixRegex(); + foreach (var item in xmls) + { + var xmlContent = item; + if (!xmlContent.Contains(" label + foreach (XmlElement _p in _div!.SelectNodes("ns:p", nsMgr)!) + { + var _begin = _p.GetAttribute("begin"); + var _end = _p.GetAttribute("end"); + var _region = _p.GetAttribute("region"); + var sub = new SubEntity + { + Begin = _begin, + End = _end, + Region = _region + }; + var _spans = _p.ChildNodes; + //Collect + foreach (XmlNode _node in _spans) + { + if (_node.NodeType == XmlNodeType.Element) + { + var _span = (XmlElement)_node; + if (string.IsNullOrEmpty(_span.InnerText)) + continue; + sub.Contents.Add(_span); + sub.ContentStrings.Add(_span.OuterXml); + } + else if (_node.NodeType == XmlNodeType.Text) + { + var _span = new XmlDocument().CreateElement("span"); + _span.InnerText = _node.Value!; + sub.Contents.Add(_span); + sub.ContentStrings.Add(_span.OuterXml); + } + } + //Check if one

has been splitted + var index = finalSubs.FindLastIndex(s => s.End == _begin && s.Region == _region && s.ContentStrings.SequenceEqual(sub.ContentStrings)); + //Skip empty lines + if (sub.ContentStrings.Count > 0) + { + //Extend

duration + if (index != -1) + finalSubs[index].End = sub.End; + else if (!finalSubs.Contains(sub)) + finalSubs.Add(sub); + } + } + } + + + var dic = new Dictionary(); + foreach (var sub in finalSubs) + { + var key = $"{sub.Begin} --> {sub.End}"; + foreach (var item in sub.Contents) + { + if (dic.ContainsKey(key)) + { + if (item.GetAttribute("tts:fontStyle") == "italic" || item.GetAttribute("tts:fontStyle") == "oblique") + dic[key] = $"{dic[key]}\r\n{GetTextFromElement(item)}"; + else + dic[key] = $"{dic[key]}\r\n{GetTextFromElement(item)}"; + } + else + { + if (item.GetAttribute("tts:fontStyle") == "italic" || item.GetAttribute("tts:fontStyle") == "oblique") + dic.Add(key, $"{GetTextFromElement(item)}"); + else + dic.Add(key, GetTextFromElement(item)); + } + } + } + + + StringBuilder vtt = new StringBuilder(); + vtt.AppendLine("WEBVTT"); + foreach (var item in dic) + { + vtt.AppendLine(item.Key); + vtt.AppendLine(item.Value); + vtt.AppendLine(); + } + + return WebVttSub.Parse(vtt.ToString()); + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/Mp4/MP4VttUtil.cs b/src/N_m3u8DL-RE.Parser/Mp4/MP4VttUtil.cs new file mode 100644 index 0000000..fdb62e3 --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Mp4/MP4VttUtil.cs @@ -0,0 +1,216 @@ +using N_m3u8DL_RE.Common.Entity; +using System.Text; + +namespace Mp4SubtitleParser +{ + public class MP4VttUtil + { + public static (bool, uint) CheckInit(byte[] data) + { + uint timescale = 0; + bool sawWVTT = false; + + //parse init + new MP4Parser() + .Box("moov", MP4Parser.Children) + .Box("trak", MP4Parser.Children) + .Box("mdia", MP4Parser.Children) + .FullBox("mdhd", (box) => + { + if (!(box.Version == 0 || box.Version == 1)) + throw new Exception("MDHD version can only be 0 or 1"); + timescale = MP4Parser.ParseMDHD(box.Reader, box.Version); + }) + .Box("minf", MP4Parser.Children) + .Box("stbl", MP4Parser.Children) + .FullBox("stsd", MP4Parser.SampleDescription) + .Box("wvtt", (box) => { + // A valid vtt init segment, though we have no actual subtitles yet. + sawWVTT = true; + }) + .Parse(data); + + return (sawWVTT, timescale); + } + + public static WebVttSub ExtractSub(IEnumerable files, uint timescale) + { + if (timescale == 0) + throw new Exception("Missing timescale for VTT content!"); + + List cues = new(); + + foreach (var item in files) + { + var dataSeg = File.ReadAllBytes(item); + + bool sawTFDT = false; + bool sawTRUN = false; + bool sawMDAT = false; + byte[]? rawPayload = null; + ulong baseTime = 0; + ulong defaultDuration = 0; + List presentations = new(); + + + //parse media + new MP4Parser() + .Box("moof", MP4Parser.Children) + .Box("traf", MP4Parser.Children) + .FullBox("tfdt", (box) => + { + sawTFDT = true; + if (!(box.Version == 0 || box.Version == 1)) + throw new Exception("TFDT version can only be 0 or 1"); + baseTime = MP4Parser.ParseTFDT(box.Reader, box.Version); + }) + .FullBox("tfhd", (box) => + { + if (box.Flags == 1000) + throw new Exception("A TFHD box should have a valid flags value"); + defaultDuration = MP4Parser.ParseTFHD(box.Reader, box.Flags).DefaultSampleDuration; + }) + .FullBox("trun", (box) => + { + sawTRUN = true; + if (box.Version == 1000) + throw new Exception("A TRUN box should have a valid version value"); + if (box.Flags == 1000) + throw new Exception("A TRUN box should have a valid flags value"); + presentations = MP4Parser.ParseTRUN(box.Reader, box.Version, box.Flags).SampleData; + }) + .Box("mdat", MP4Parser.AllData((data) => + { + if (sawMDAT) + throw new Exception("VTT cues in mp4 with multiple MDAT are not currently supported"); + sawMDAT = true; + rawPayload = data; + })) + .Parse(dataSeg,/* partialOkay= */ false); + + if (!sawMDAT && !sawTFDT && !sawTRUN) + { + throw new Exception("A required box is missing"); + } + + var currentTime = baseTime; + var reader = new BinaryReader2(new MemoryStream(rawPayload!)); + + foreach (var presentation in presentations) + { + var duration = presentation.SampleDuration == 0 ? defaultDuration : presentation.SampleDuration; + var startTime = presentation.SampleCompositionTimeOffset != 0 ? + baseTime + presentation.SampleCompositionTimeOffset : + currentTime; + currentTime = startTime + duration; + var totalSize = 0; + do + { + // Read the payload size. + var payloadSize = (int)reader.ReadUInt32(); + totalSize += payloadSize; + + // Skip the type. + var payloadType = reader.ReadUInt32(); + var payloadName = MP4Parser.TypeToString(payloadType); + + // Read the data payload. + byte[]? payload = null; + if (payloadName == "vttc") + { + if (payloadSize > 8) + { + payload = reader.ReadBytes(payloadSize - 8); + } + } + else if (payloadName == "vtte") + { + // It's a vtte, which is a vtt cue that is empty. Ignore any data that + // does exist. + reader.ReadBytes(payloadSize - 8); + } + else + { + Console.WriteLine($"Unknown box {payloadName}! Skipping!"); + reader.ReadBytes(payloadSize - 8); + } + + if (duration != 0) + { + if (payload != null) + { + if (timescale == 0) + throw new Exception("Timescale should not be zero!"); + var cue = ParseVTTC( + payload, + 0 + (double)startTime / timescale, + 0 + (double)currentTime / timescale); + //Check if same subtitle has been splitted + if (cue != null) + { + var index = cues.FindLastIndex(s => s.EndTime == cue.StartTime && s.Settings == cue.Settings && s.Payload == cue.Payload); + if (index != -1) + { + cues[index].EndTime = cue.EndTime; + } + else + { + cues.Add(cue); + } + } + } + } + else + { + throw new Exception("WVTT sample duration unknown, and no default found!"); + } + + if (!(presentation.SampleSize == 0 || totalSize <= presentation.SampleSize)) + { + throw new Exception("The samples do not fit evenly into the sample sizes given in the TRUN box!"); + } + + } while (presentation.SampleSize != 0 && (totalSize < presentation.SampleSize)); + + if (reader.HasMoreData()) + { + //throw new Exception("MDAT which contain VTT cues and non-VTT data are not currently supported!"); + } + } + } + + if (cues.Count > 0) + { + return new WebVttSub() { Cues = cues }; + } + return new WebVttSub(); + } + + private static SubCue? ParseVTTC(byte[] data, double startTime, double endTime) + { + string payload = string.Empty; + string id = string.Empty; + string settings = string.Empty; + new MP4Parser() + .Box("payl", MP4Parser.AllData((data) => + { + payload = Encoding.UTF8.GetString(data); + })) + .Box("iden", MP4Parser.AllData((data) => + { + id = Encoding.UTF8.GetString(data); + })) + .Box("sttg", MP4Parser.AllData((data) => + { + settings = Encoding.UTF8.GetString(data); + })) + .Parse(data); + + if (!string.IsNullOrEmpty(payload)) + { + return new SubCue() { StartTime = TimeSpan.FromSeconds(startTime), EndTime = TimeSpan.FromSeconds(endTime), Payload = payload, Settings = settings }; + } + return null; + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/N_m3u8DL-RE.Parser.csproj b/src/N_m3u8DL-RE.Parser/N_m3u8DL-RE.Parser.csproj index 8846f53..03732ee 100644 --- a/src/N_m3u8DL-RE.Parser/N_m3u8DL-RE.Parser.csproj +++ b/src/N_m3u8DL-RE.Parser/N_m3u8DL-RE.Parser.csproj @@ -1,9 +1,11 @@ - net6.0 + library + net7.0 N_m3u8DL_RE.Parser enable + preview enable diff --git a/src/N_m3u8DL-RE.Parser/Processor/DefaultUrlProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/DefaultUrlProcessor.cs index 1ba0d87..d0e3ca9 100644 --- a/src/N_m3u8DL-RE.Parser/Processor/DefaultUrlProcessor.cs +++ b/src/N_m3u8DL-RE.Parser/Processor/DefaultUrlProcessor.cs @@ -10,8 +10,10 @@ using System.Threading.Tasks; namespace N_m3u8DL_RE.Parser.Processor { - public class DefaultUrlProcessor : UrlProcessor + public partial class DefaultUrlProcessor : UrlProcessor { + [RegexGenerator("\\?.*")] + private static partial Regex ParaRegex(); public override bool CanProcess(ExtractorType extractorType, string oriUrl, ParserConfig paserConfig) => true; @@ -20,7 +22,7 @@ namespace N_m3u8DL_RE.Parser.Processor if (paserConfig.AppendUrlParams) { Logger.Debug("Before: " + oriUrl); - oriUrl += new Regex("\\?.*").Match(paserConfig.Url).Value; + oriUrl += ParaRegex().Match(paserConfig.Url).Value; Logger.Debug("After: " + oriUrl); } diff --git a/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSContentProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSContentProcessor.cs index fbcb25d..fea98d4 100644 --- a/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSContentProcessor.cs +++ b/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSContentProcessor.cs @@ -10,8 +10,17 @@ using System.Threading.Tasks; namespace N_m3u8DL_RE.Parser.Processor.HLS { - public class DefaultHLSContentProcessor : ContentProcessor + public partial class DefaultHLSContentProcessor : ContentProcessor { + [RegexGenerator("#EXT-X-DISCONTINUITY\\s+#EXT-X-MAP:URI=\\\"(.*?)\\\",BYTERANGE=\\\"(.*?)\\\"")] + private static partial Regex YkDVRegex(); + [RegexGenerator("#EXT-X-MAP:URI=\\\".*?BUMPER/[\\s\\S]+?#EXT-X-DISCONTINUITY")] + private static partial Regex DNSPRegex(); + [RegexGenerator("(#EXTINF.*)(\\s+)(#EXT-X-KEY.*)")] + private static partial Regex OrderFixRegex(); + [RegexGenerator("#EXT-X-MAP.*\\.apple\\.com/")] + private static partial Regex ATVRegex(); + public override bool CanProcess(ExtractorType extractorType, string rawText, ParserConfig parserConfig) => extractorType == ExtractorType.HLS; public override string Process(string m3u8Content, ParserConfig parserConfig) @@ -38,7 +47,7 @@ namespace N_m3u8DL_RE.Parser.Processor.HLS //针对优酷#EXT-X-VERSION:7杜比视界片源修正 if (m3u8Content.Contains("#EXT-X-DISCONTINUITY") && m3u8Content.Contains("#EXT-X-MAP") && m3u8Content.Contains("ott.cibntv.net") && m3u8Content.Contains("ccode=")) { - Regex ykmap = new Regex("#EXT-X-DISCONTINUITY\\s+#EXT-X-MAP:URI=\\\"(.*?)\\\",BYTERANGE=\\\"(.*?)\\\""); + Regex ykmap = YkDVRegex(); foreach (Match m in ykmap.Matches(m3u8Content)) { m3u8Content = m3u8Content.Replace(m.Value, $"#EXTINF:0.000000,\n#EXT-X-BYTERANGE:{m.Groups[2].Value}\n{m.Groups[1].Value}"); @@ -48,7 +57,7 @@ namespace N_m3u8DL_RE.Parser.Processor.HLS //针对Disney+修正 if (m3u8Content.Contains("#EXT-X-DISCONTINUITY") && m3u8Content.Contains("#EXT-X-MAP") && m3u8Url.Contains("media.dssott.com/")) { - Regex ykmap = new Regex("#EXT-X-MAP:URI=\\\".*?BUMPER/[\\s\\S]+?#EXT-X-DISCONTINUITY"); + Regex ykmap = DNSPRegex(); if (ykmap.IsMatch(m3u8Content)) { m3u8Content = m3u8Content.Replace(ykmap.Match(m3u8Content).Value, "#XXX"); @@ -56,10 +65,10 @@ namespace N_m3u8DL_RE.Parser.Processor.HLS } //针对AppleTv修正 - if (m3u8Content.Contains("#EXT-X-DISCONTINUITY") && m3u8Content.Contains("#EXT-X-MAP") && (m3u8Url.Contains(".apple.com/") || Regex.IsMatch(m3u8Content, "#EXT-X-MAP.*\\.apple\\.com/"))) + if (m3u8Content.Contains("#EXT-X-DISCONTINUITY") && m3u8Content.Contains("#EXT-X-MAP") && (m3u8Url.Contains(".apple.com/") || ATVRegex().IsMatch(m3u8Content))) { //只取加密部分即可 - Regex ykmap = new Regex("(#EXT-X-KEY:[\\s\\S]*?)(#EXT-X-DISCONTINUITY|#EXT-X-ENDLIST)"); + Regex ykmap = DNSPRegex(); if (ykmap.IsMatch(m3u8Content)) { m3u8Content = "#EXTM3U\r\n" + ykmap.Match(m3u8Content).Groups[1].Value + "\r\n#EXT-X-ENDLIST"; @@ -67,9 +76,10 @@ namespace N_m3u8DL_RE.Parser.Processor.HLS } //修复#EXT-X-KEY与#EXTINF出现次序异常问题 - if (Regex.IsMatch(m3u8Content, "(#EXTINF.*)(\\s+)(#EXT-X-KEY.*)")) + var regex = OrderFixRegex(); + if (regex.IsMatch(m3u8Content)) { - m3u8Content = Regex.Replace(m3u8Content, "(#EXTINF.*)(\\s+)(#EXT-X-KEY.*)", "$3$2$1"); + m3u8Content = regex.Replace(m3u8Content, "$3$2$1"); } return m3u8Content; diff --git a/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSKeyProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSKeyProcessor.cs index 1b5fc46..c7cee1d 100644 --- a/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSKeyProcessor.cs +++ b/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSKeyProcessor.cs @@ -16,7 +16,7 @@ namespace N_m3u8DL_RE.Parser.Processor.HLS public override bool CanProcess(ExtractorType extractorType, string method, string uriText, ParserConfig paserConfig) => extractorType == ExtractorType.HLS; - public override byte[] Process(string method, string uriText, ParserConfig parserConfig) + public override byte[]? Process(string method, string uriText, ParserConfig parserConfig) { var encryptInfo = new EncryptInfo(); diff --git a/src/N_m3u8DL-RE.Parser/Processor/KeyProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/KeyProcessor.cs index f2736d7..1d77666 100644 --- a/src/N_m3u8DL-RE.Parser/Processor/KeyProcessor.cs +++ b/src/N_m3u8DL-RE.Parser/Processor/KeyProcessor.cs @@ -12,6 +12,6 @@ namespace N_m3u8DL_RE.Parser.Processor public abstract class KeyProcessor { public abstract bool CanProcess(ExtractorType extractorType, string method, string uriText, ParserConfig parserConfig); - public abstract byte[] Process(string method, string uriText, ParserConfig parserConfig); + public abstract byte[]? Process(string method, string uriText, ParserConfig parserConfig); } } diff --git a/src/N_m3u8DL-RE.Parser/StreamExtractor.cs b/src/N_m3u8DL-RE.Parser/StreamExtractor.cs index 607a089..0bd3004 100644 --- a/src/N_m3u8DL-RE.Parser/StreamExtractor.cs +++ b/src/N_m3u8DL-RE.Parser/StreamExtractor.cs @@ -72,7 +72,7 @@ namespace N_m3u8DL_RE.Parser } else { - throw new Exception(ResString.notSupported); + throw new NotSupportedException(ResString.notSupported); } } diff --git a/src/N_m3u8DL-RE.Parser/Util/ParserUtil.cs b/src/N_m3u8DL-RE.Parser/Util/ParserUtil.cs index ef098e3..6fb4993 100644 --- a/src/N_m3u8DL-RE.Parser/Util/ParserUtil.cs +++ b/src/N_m3u8DL-RE.Parser/Util/ParserUtil.cs @@ -8,8 +8,11 @@ using System.Threading.Tasks; namespace N_m3u8DL_RE.Parser.Util { - internal class ParserUtil + internal partial class ParserUtil { + [RegexGenerator("\\$Number%([^$]+)d\\$")] + private static partial Regex VarsNumberRegex(); + ///

/// 从以下文本中获取参数 /// #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=2149280,CODECS="mp4a.40.2,avc1.64001f",RESOLUTION=1280x720,NAME="720" @@ -21,18 +24,25 @@ namespace N_m3u8DL_RE.Parser.Util { line = line.Trim(); if (key == "") - return line.Substring(line.IndexOf(':') + 1); + return line[(line.IndexOf(':') + 1)..]; - if (line.Contains(key + "=\"")) + var index = -1; + var result = string.Empty; + if ((index = line.IndexOf(key + "=\"")) > -1) { - return Regex.Match(line, key + "=\"([^\"]*)\"").Groups[1].Value; + var startIndex = index + (key + "=\"").Length; + var endIndex = startIndex + line[startIndex..].IndexOf('\"'); + result = line[startIndex..endIndex]; } - else if (line.Contains(key + "=")) + else if ((index = line.IndexOf(key + "=")) > -1) { - return Regex.Match(line, key + "=([^,]*)").Groups[1].Value; + var startIndex = index + (key + "=").Length; + var endIndex = startIndex + line[startIndex..].IndexOf(','); + if (endIndex >= startIndex) result = line[startIndex..endIndex]; + else result = line[startIndex..]; } - return string.Empty; + return result; } /// @@ -80,10 +90,10 @@ namespace N_m3u8DL_RE.Parser.Util { foreach (var item in keyValuePairs) if (text.Contains(item.Key)) - text = text.Replace(item.Key, item.Value.ToString()); + text = text.Replace(item.Key, item.Value!.ToString()); //处理特殊形式数字 如 $Number%05d$ - var regex = new Regex("\\$Number%([^$]+)d\\$"); + var regex = VarsNumberRegex(); if (regex.IsMatch(text) && keyValuePairs.ContainsKey(DASHTags.TemplateNumber)) { foreach (Match m in regex.Matches(text)) diff --git a/src/N_m3u8DL-RE/Config/DownloaderConfig.cs b/src/N_m3u8DL-RE/Config/DownloaderConfig.cs new file mode 100644 index 0000000..4f23a9d --- /dev/null +++ b/src/N_m3u8DL-RE/Config/DownloaderConfig.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Config +{ + internal class DownloaderConfig + { + /// + /// 临时文件存储目录 + /// + public string? TmpDir { get; set; } + /// + /// 文件存储目录 + /// + public string? SaveDir { get; set; } + /// + /// 文件名 + /// + public string? SaveName { get; set; } + /// + /// 线程数 + /// + public int ThreadCount { get; set; } = 8; + /// + /// 跳过合并 + /// + public bool SkipMerge { get; set; } = false; + /// + /// 二进制合并 + /// + public bool BinaryMerge { get; set; } = false; + /// + /// 完成后是否删除临时文件 + /// + public bool DelAfterDone { get; set; } = false; + /// + /// 校验有没有下完全部分片 + /// + public bool CheckSegmentsCount { get; set; } = true; + /// + /// 校验响应头的文件大小和实际大小 + /// + public bool CheckContentLength { get; set; } = true; + /// + /// 自动修复字幕 + /// + public bool AutoSubtitleFix { get; set; } = true; + /// + /// 请求头 + /// + public Dictionary Headers { get; set; } = new Dictionary() + { + ["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36" + }; + } +} diff --git a/src/N_m3u8DL-RE/Crypto/AESUtil.cs b/src/N_m3u8DL-RE/Crypto/AESUtil.cs index edb07ed..b2b270d 100644 --- a/src/N_m3u8DL-RE/Crypto/AESUtil.cs +++ b/src/N_m3u8DL-RE/Crypto/AESUtil.cs @@ -9,26 +9,19 @@ namespace N_m3u8DL_RE.Crypto { internal class AESUtil { - public static byte[] AES128Decrypt(string filePath, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC, PaddingMode padding = PaddingMode.PKCS7) + /// + /// AES-128解密，解密后原地替换文件 + /// + /// + /// + /// + /// + /// + public static void AES128Decrypt(string filePath, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC, PaddingMode padding = PaddingMode.PKCS7) { - FileStream fs = new FileStream(filePath, FileMode.Open); - //获取文件大小 - long size = fs.Length; - byte[] inBuff = new byte[size]; - fs.Read(inBuff, 0, inBuff.Length); - fs.Close(); - - Aes dcpt = Aes.Create(); - dcpt.BlockSize = 128; - dcpt.KeySize = 128; - dcpt.Key = keyByte; - dcpt.IV = ivByte; - dcpt.Mode = mode; - dcpt.Padding = padding; - - ICryptoTransform cTransform = dcpt.CreateDecryptor(); - byte[] resultArray = cTransform.TransformFinalBlock(inBuff, 0, inBuff.Length); - return resultArray; + var fileBytes = File.ReadAllBytes(filePath); + var decrypted = AES128Decrypt(fileBytes, keyByte, ivByte, mode, padding); + File.WriteAllBytes(filePath, decrypted); } public static byte[] AES128Decrypt(byte[] encryptedBuff, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC, PaddingMode padding = PaddingMode.PKCS7) diff --git a/src/N_m3u8DL-RE/Directory.Build.props b/src/N_m3u8DL-RE/Directory.Build.props index d50d822..ad71932 100644 --- a/src/N_m3u8DL-RE/Directory.Build.props +++ b/src/N_m3u8DL-RE/Directory.Build.props @@ -4,8 +4,7 @@ Speed true true - Link - link + full 7.0.0-* @@ -17,7 +16,6 @@ - diff --git a/src/N_m3u8DL-RE/DownloadManager/SimpleDownloadManager.cs b/src/N_m3u8DL-RE/DownloadManager/SimpleDownloadManager.cs new file mode 100644 index 0000000..c397fea --- /dev/null +++ b/src/N_m3u8DL-RE/DownloadManager/SimpleDownloadManager.cs @@ -0,0 +1,303 @@ +using Mp4SubtitleParser; +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Common.Log; +using N_m3u8DL_RE.Common.Resource; +using N_m3u8DL_RE.Common.Util; +using N_m3u8DL_RE.Config; +using N_m3u8DL_RE.Downloader; +using N_m3u8DL_RE.Entity; +using N_m3u8DL_RE.Util; +using Spectre.Console; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.DownloadManager +{ + internal class SimpleDownloadManager + { + IDownloader Downloader; + DownloaderConfig DownloaderConfig; + + public SimpleDownloadManager(DownloaderConfig downloaderConfig) + { + this.DownloaderConfig = downloaderConfig; + Downloader = new SimpleDownloader(DownloaderConfig); + } + + private async Task DownloadStreamAsync(StreamSpec streamSpec, ProgressTask task) + { + ConcurrentDictionary FileDic = new(); + + var segments = streamSpec.Playlist?.MediaParts.SelectMany(m => m.MediaSegments); + if (segments == null) return false; + + var dirName = $"{streamSpec.GroupId}_{streamSpec.Codecs}_{streamSpec.Language}"; + var tmpDir = DownloaderConfig.TmpDir ?? Path.Combine(Environment.CurrentDirectory, dirName); + var saveDir = DownloaderConfig.SaveDir ?? Environment.CurrentDirectory; + var saveName = DownloaderConfig.SaveName ?? dirName; + var headers = DownloaderConfig.Headers; + var output = Path.Combine(saveDir, saveName + $".{streamSpec.Extension ?? "ts"}"); + + Logger.Debug($"dirName: {dirName}; tmpDir: {tmpDir}; saveDir: {saveDir}; saveName: {saveName}; output: {output}"); + + //创建文件夹 + if (!Directory.Exists(tmpDir)) Directory.CreateDirectory(tmpDir); + if (!Directory.Exists(saveDir)) Directory.CreateDirectory(saveDir); + + var totalCount = segments.Count(); + if (streamSpec.Playlist?.MediaInit != null) + { + totalCount++; + } + + task.MaxValue = totalCount; + task.StartTask(); + + //开始下载 + Logger.InfoMarkUp(ResString.startDownloading + streamSpec.ToShortString()); + + //下载init + if (streamSpec.Playlist?.MediaInit != null) + { + totalCount++; + var path = Path.Combine(tmpDir, "_init.mp4.tmp"); + var result = await Downloader.DownloadSegmentAsync(streamSpec.Playlist.MediaInit, path, headers); + FileDic[streamSpec.Playlist.MediaInit] = result; + task.Increment(1); + //修改输出后缀 + output = Path.ChangeExtension(output, ".mp4"); + } + + //开始下载 + var pad = "0".PadLeft(segments.Count().ToString().Length, '0'); + var options = new ParallelOptions() + { + MaxDegreeOfParallelism = DownloaderConfig.ThreadCount + }; + await Parallel.ForEachAsync(segments, options, async (seg, _) => + { + var index = seg.Index; + var path = Path.Combine(tmpDir, index.ToString(pad) + $".{streamSpec.Extension ?? "clip"}.tmp"); + var result = await Downloader.DownloadSegmentAsync(seg, path, headers); + FileDic[seg] = result; + task.Increment(1); + }); + + //校验分片数量 + if (DownloaderConfig.CheckSegmentsCount && FileDic.Values.Any(s => s == null)) + { + Logger.WarnMarkUp(ResString.segmentCountCheckNotPass, totalCount, FileDic.Values.Where(s => s != null).Count()); + return false; + } + + //移除无效片段 + var badKeys = FileDic.Where(i => i.Value == null).Select(i => i.Key); + foreach (var badKey in badKeys) + { + FileDic!.Remove(badKey, out _); + } + + //校验完整性 + if (DownloaderConfig.CheckContentLength && FileDic.Values.Any(a => a!.Success == false)) + { + return false; + } + + //自动修复VTT raw字幕 + if (DownloaderConfig.AutoSubtitleFix && streamSpec.MediaType == Common.Enum.MediaType.SUBTITLES + && streamSpec.Extension != null && streamSpec.Extension.Contains("vtt")) + { + Logger.WarnMarkUp(ResString.fixingVTT); + //排序字幕并修正时间戳 + bool first = true; + var finalVtt = new WebVttSub(); + var keys = FileDic.Keys.OrderBy(k => k.Index); + foreach (var seg in keys) + { + var vttContent = File.ReadAllText(FileDic[seg]!.ActualFilePath); + var vtt = WebVttSub.Parse(vttContent); + if (first) + { + finalVtt = vtt; + first = false; + } + else + { + finalVtt.AddCuesFromOne(vtt); + } + } + //写出字幕 + var files = FileDic.Values.Select(v => v!.ActualFilePath).OrderBy(s => s).ToArray(); + foreach (var item in files) File.Delete(item); + FileDic.Clear(); + var index = 0; + var path = Path.Combine(tmpDir, index.ToString(pad) + $".fix.{streamSpec.Extension ?? "clip"}"); + var vttContentFixed = finalVtt.ToStringWithHeader(); + await File.WriteAllTextAsync(path, vttContentFixed, new UTF8Encoding(false)); + FileDic[keys.First()] = new DownloadResult() + { + ActualContentLength = vttContentFixed.Length, + ActualFilePath = path + }; + } + + //自动修复VTT mp4字幕 + if (DownloaderConfig.AutoSubtitleFix && streamSpec.MediaType == Common.Enum.MediaType.SUBTITLES + && streamSpec.Codecs != "stpp" && streamSpec.Extension != null && streamSpec.Extension.Contains("m4s")) + { + var initFile = FileDic.Values.Where(v => Path.GetFileName(v!.ActualFilePath).StartsWith("_init")).FirstOrDefault(); + var iniFileBytes = File.ReadAllBytes(initFile!.ActualFilePath); + var (sawVtt, timescale) = MP4VttUtil.CheckInit(iniFileBytes); + if (sawVtt) + { + Logger.WarnMarkUp(ResString.fixingVTTmp4); + var mp4s = FileDic.Values.Select(v => v!.ActualFilePath).Where(p => p.EndsWith(".m4s")).OrderBy(s => s).ToArray(); + var finalVtt = MP4VttUtil.ExtractSub(mp4s, timescale); + //写出字幕 + var firstKey = FileDic.Keys.First(); + var files = FileDic.Values.Select(v => v!.ActualFilePath).OrderBy(s => s).ToArray(); + foreach (var item in files) File.Delete(item); + FileDic.Clear(); + var index = 0; + var path = Path.Combine(tmpDir, index.ToString(pad) + ".fix.vtt"); + var vttContentFixed = finalVtt.ToStringWithHeader(); + await File.WriteAllTextAsync(path, vttContentFixed, new UTF8Encoding(false)); + FileDic[firstKey] = new DownloadResult() + { + ActualContentLength = vttContentFixed.Length, + ActualFilePath = path + }; + //修改输出后缀 + output = Path.ChangeExtension(output, ".vtt"); + } + } + + //自动修复TTML raw字幕 + if (DownloaderConfig.AutoSubtitleFix && streamSpec.MediaType == Common.Enum.MediaType.SUBTITLES + && streamSpec.Extension != null && streamSpec.Extension.Contains("ttml")) + { + Logger.WarnMarkUp(ResString.fixingTTML); + var mp4s = FileDic.Values.Select(v => v!.ActualFilePath).Where(p => p.EndsWith(".ttml")).OrderBy(s => s).ToArray(); + var finalVtt = MP4TtmlUtil.ExtractFromTTMLs(mp4s, 0); + //写出字幕 + var firstKey = FileDic.Keys.First(); + var files = FileDic.Values.Select(v => v!.ActualFilePath).OrderBy(s => s).ToArray(); + foreach (var item in files) File.Delete(item); + FileDic.Clear(); + var index = 0; + var path = Path.Combine(tmpDir, index.ToString(pad) + ".fix.vtt"); + var vttContentFixed = finalVtt.ToStringWithHeader(); + await File.WriteAllTextAsync(path, vttContentFixed, new UTF8Encoding(false)); + FileDic[firstKey] = new DownloadResult() + { + ActualContentLength = vttContentFixed.Length, + ActualFilePath = path + }; + //修改输出后缀 + output = Path.ChangeExtension(output, ".vtt"); + } + + //自动修复TTML mp4字幕 + if (DownloaderConfig.AutoSubtitleFix && streamSpec.MediaType == Common.Enum.MediaType.SUBTITLES + && streamSpec.Extension != null && streamSpec.Extension.Contains("m4s") + && streamSpec.Codecs != null && streamSpec.Codecs.Contains("stpp")) + { + Logger.WarnMarkUp(ResString.fixingTTMLmp4); + //sawTtml暂时不判断 + //var initFile = FileDic.Values.Where(v => Path.GetFileName(v!.ActualFilePath).StartsWith("_init")).FirstOrDefault(); + //var iniFileBytes = File.ReadAllBytes(initFile!.ActualFilePath); + //var sawTtml = MP4TtmlUtil.CheckInit(iniFileBytes); + var mp4s = FileDic.Values.Select(v => v!.ActualFilePath).Where(p => p.EndsWith(".m4s")).OrderBy(s => s).ToArray(); + var finalVtt = MP4TtmlUtil.ExtractFromMp4s(mp4s, 0); + //写出字幕 + var firstKey = FileDic.Keys.First(); + var files = FileDic.Values.Select(v => v!.ActualFilePath).OrderBy(s => s).ToArray(); + foreach (var item in files) File.Delete(item); + FileDic.Clear(); + var index = 0; + var path = Path.Combine(tmpDir, index.ToString(pad) + ".fix.vtt"); + var vttContentFixed = finalVtt.ToStringWithHeader(); + await File.WriteAllTextAsync(path, vttContentFixed, new UTF8Encoding(false)); + FileDic[firstKey] = new DownloadResult() + { + ActualContentLength = vttContentFixed.Length, + ActualFilePath = path + }; + //修改输出后缀 + output = Path.ChangeExtension(output, ".vtt"); + } + + //合并 + if (!DownloaderConfig.SkipMerge) + { + if (DownloaderConfig.BinaryMerge) + { + Logger.InfoMarkUp(ResString.binaryMerge); + var files = FileDic.Values.Select(v => v!.ActualFilePath).OrderBy(s => s).ToArray(); + DownloadUtil.CombineMultipleFilesIntoSingleFile(files, output); + } + else + { + throw new NotImplementedException(); + } + } + //删除临时文件夹 + if (DownloaderConfig.DelAfterDone) + { + var files = FileDic.Values.Select(v => v!.ActualFilePath); + foreach (var file in files) + { + File.Delete(file); + } + if (!Directory.EnumerateFiles(tmpDir).Any()) + { + Directory.Delete(tmpDir); + } + } + + return true; + } + + public async Task StartDownloadAsync(IEnumerable streamSpecs) + { + ConcurrentDictionary Results = new(); + + var progress = AnsiConsole.Progress().AutoClear(true); + + //进度条的列定义 + progress.Columns(new ProgressColumn[] + { + new TaskDescriptionColumn() { Alignment = Justify.Left }, + new ProgressBarColumn(), + new PercentageColumn(), + new RemainingTimeColumn(), + new SpinnerColumn(), + }); + + await progress.StartAsync(async ctx => + { + //创建任务 + var dic = streamSpecs.Select(item => + { + var task = ctx.AddTask(item.ToShortString(), autoStart: false); + return (item, task); + }).ToDictionary(item => item.item, item => item.task); + //遍历，顺序下载 + foreach (var kp in dic) + { + var task = kp.Value; + var result = await DownloadStreamAsync(kp.Key, task); + Results[kp.Key] = result; + } + }); + + return Results.Values.All(v => v == true); + } + } +} diff --git a/src/N_m3u8DL-RE/Downloader/IDownloader.cs b/src/N_m3u8DL-RE/Downloader/IDownloader.cs new file mode 100644 index 0000000..4282e65 --- /dev/null +++ b/src/N_m3u8DL-RE/Downloader/IDownloader.cs @@ -0,0 +1,15 @@ +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Entity; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Downloader +{ + internal interface IDownloader + { + Task DownloadSegmentAsync(MediaSegment segment, string savePath, Dictionary? headers = null); + } +} diff --git a/src/N_m3u8DL-RE/Downloader/SimpleDownloader.cs b/src/N_m3u8DL-RE/Downloader/SimpleDownloader.cs new file mode 100644 index 0000000..e24f7c5 --- /dev/null +++ b/src/N_m3u8DL-RE/Downloader/SimpleDownloader.cs @@ -0,0 +1,90 @@ +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Common.Enum; +using N_m3u8DL_RE.Common.Log; +using N_m3u8DL_RE.Config; +using N_m3u8DL_RE.Entity; +using N_m3u8DL_RE.Util; +using Spectre.Console; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Downloader +{ + /// + /// 简单下载器 + /// + internal class SimpleDownloader : IDownloader + { + DownloaderConfig DownloaderConfig; + + public SimpleDownloader(DownloaderConfig config) + { + DownloaderConfig = config; + } + + public async Task DownloadSegmentAsync(MediaSegment segment, string savePath, Dictionary? headers = null) + { + var url = segment.Url; + var dResult = await DownClipAsync(url, savePath, segment.StartRange, segment.StopRange, headers); + if (dResult != null && dResult.Success && segment.EncryptInfo != null) + { + if (segment.EncryptInfo.Method == EncryptMethod.AES_128) + { + var key = segment.EncryptInfo.Key; + var iv = segment.EncryptInfo.IV; + Crypto.AESUtil.AES128Decrypt(dResult.ActualFilePath, key!, iv!); + } + else if (segment.EncryptInfo.Method == EncryptMethod.AES_128_ECB) + { + var key = segment.EncryptInfo.Key; + var iv = segment.EncryptInfo.IV; + Crypto.AESUtil.AES128Decrypt(dResult.ActualFilePath, key!, iv!, System.Security.Cryptography.CipherMode.ECB); + } + else if (segment.EncryptInfo.Method == EncryptMethod.SAMPLE_AES_CTR) + { + throw new NotSupportedException("SAMPLE-AES-CTR"); + } + } + return dResult; + } + + private async Task DownClipAsync(string url, string path, long? fromPosition, long? toPosition, Dictionary? headers = null, int retryCount = 3) + { + retry: + try + { + var des = Path.ChangeExtension(path, null); + //已下载过跳过 + if (File.Exists(des)) + { + return new DownloadResult() { ActualContentLength = 0, ActualFilePath = des }; + } + var result = await DownloadUtil.DownloadToFileAsync(url, path, headers, fromPosition, toPosition); + //下载完成后改名 + if (result.Success || !DownloaderConfig.CheckContentLength) + { + File.Move(path, des); + result.ActualFilePath = des; + return result; + } + throw new Exception("please retry"); + } + catch (Exception ex) + { + Logger.Debug(ex.ToString()); + if (retryCount-- > 0) + { + await Task.Delay(200); + goto retry; + } + //throw new Exception("download failed", ex); + return null; + } + } + } +} diff --git a/src/N_m3u8DL-RE/Entity/DownloadResult.cs b/src/N_m3u8DL-RE/Entity/DownloadResult.cs new file mode 100644 index 0000000..e12870f --- /dev/null +++ b/src/N_m3u8DL-RE/Entity/DownloadResult.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Entity +{ + internal class DownloadResult + { + public bool Success { get => (ActualContentLength != null && RespContentLength != null) ? (RespContentLength == ActualContentLength) : (ActualContentLength == null ? false : true); } + public long? RespContentLength { get; set; } + public long? ActualContentLength { get; set; } + public required string ActualFilePath { get; set; } + } +} diff --git a/src/N_m3u8DL-RE/N_m3u8DL-RE.csproj b/src/N_m3u8DL-RE/N_m3u8DL-RE.csproj index 1b0354f..bd6cbb6 100644 --- a/src/N_m3u8DL-RE/N_m3u8DL-RE.csproj +++ b/src/N_m3u8DL-RE/N_m3u8DL-RE.csproj @@ -2,9 +2,10 @@ Exe - net6.0 + net7.0 N_m3u8DL_RE enable + preview enable @@ -17,7 +18,7 @@ - + diff --git a/src/N_m3u8DL-RE/Processor/DemoProcessor2.cs b/src/N_m3u8DL-RE/Processor/DemoProcessor2.cs index 413c830..120881c 100644 --- a/src/N_m3u8DL-RE/Processor/DemoProcessor2.cs +++ b/src/N_m3u8DL-RE/Processor/DemoProcessor2.cs @@ -20,11 +20,11 @@ namespace N_m3u8DL_RE.Processor return extractorType == ExtractorType.HLS && parserConfig.Url.Contains("playertest.longtailvideo.com"); } - public override byte[] Process(string method, string uriText, ParserConfig parserConfig) + public override byte[]? Process(string method, string uriText, ParserConfig parserConfig) { Logger.InfoMarkUp($"[white on green]My Key Processor => {uriText}[/]"); var key = new DefaultHLSKeyProcessor().Process(method, uriText, parserConfig); - Logger.InfoMarkUp("[red]" + HexUtil.BytesToHex(key, " ") + "[/]"); + Logger.InfoMarkUp("[red]" + HexUtil.BytesToHex(key!, " ") + "[/]"); return key; } } diff --git a/src/N_m3u8DL-RE/Program.cs b/src/N_m3u8DL-RE/Program.cs index a51b079..e13685c 100644 --- a/src/N_m3u8DL-RE/Program.cs +++ b/src/N_m3u8DL-RE/Program.cs @@ -11,10 +11,14 @@ using N_m3u8DL_RE.Common.Log; using System.Globalization; using System.Text; using System.Text.RegularExpressions; -using N_m3u8DL_RE.Subtitle; using System.Collections.Concurrent; using N_m3u8DL_RE.Common.Util; using N_m3u8DL_RE.Processor; +using N_m3u8DL_RE.Downloader; +using N_m3u8DL_RE.Config; +using N_m3u8DL_RE.Util; +using System.Diagnostics; +using N_m3u8DL_RE.DownloadManager; namespace N_m3u8DL_RE { @@ -34,18 +38,30 @@ namespace N_m3u8DL_RE try { - var config = new ParserConfig(); + var parserConfig = new ParserConfig(); //demo1 - config.ContentProcessors.Insert(0, new DemoProcessor()); + parserConfig.ContentProcessors.Insert(0, new DemoProcessor()); //demo2 - config.KeyProcessors.Insert(0, new DemoProcessor2()); + parserConfig.KeyProcessors.Insert(0, new DemoProcessor2()); var url = string.Empty; - //url = "http://livesim.dashif.org/livesim/mup_300/tsbd_500/testpic_2s/Manifest.mpd"; - url = "http://playertest.longtailvideo.com/adaptive/oceans_aes/oceans_aes.m3u8"; - //url = "https://bitmovin-a.akamaihd.net/content/art-of-motion_drm/mpds/11331.mpd"; - + //url = "https://cmafref.akamaized.net/cmaf/live-ull/2006350/akambr/out.mpd"; //直播 + //url = "http://playertest.longtailvideo.com/adaptive/oceans_aes/oceans_aes.m3u8"; + //url = "https://vod.sdn.wavve.com/hls/S01/S01_E461382925.1/1/5000/chunklist.m3u8"; + url = "https://bitmovin-a.akamaihd.net/content/art-of-motion_drm/mpds/11331.mpd"; + //url = "http://tv-live.ynkmit.com/tv/anning.m3u8?txSecret=7528f35fb4b62bd24d55b891899db68f&txTime=632C8680"; //直播 + //url = "https://rest-as.ott.kaltura.com/api_v3/service/assetFile/action/playManifest/partnerId/147/assetId/1304099/assetType/media/assetFileId/16136929/contextType/PLAYBACK/isAltUrl/False/ks/djJ8MTQ3fMusTFH6PCZpcrfKLQwI-pPm9ex6b6r49wioe32WH2udXeM4reyWIkSDpi7HhvhxBHAHAKiHrcnkmIJQpyAt4MuDBG0ywGQ-jOeqQFcTRQ8BGJGw6g-smSBLwSbo4CCx9M9vWNJX3GkOfhoMAY4yRU-ur3okHiVq1mUJ82XBd_iVqLuzodnc9sJEtcHH0zc5CoPiTq2xor-dq3yDURnZm3isfSN3t9uLIJEW09oE-SJ84DM5GUuFUdbnIV8bdcWUsPicUg-Top1G2D3WcWXq4EvPnwvD8jrC_vsiOpLHf5akAwtdGsJ6__cXUmT7a-QlfjdvaZ5T8UhDLnttHmsxYs2E5c0lh4uOvvJou8dD8iYxUexlPI2j4QUkBRxqOEVLSNV3Y82-5TTRqgnK_uGYXHwk7EAmDws7hbLj2-DJ1heXDcye3OJYdunJgAS-9ma5zmQQNiY_HYh6wj2N1HpCTNAtWWga6R9fC0VgBTZbidW-YwMSGzIvMQfIfWKe15X7Oc_hCs-zGfW9XeRJZrutcWKK_D_HlzpQVBF2vIF3XgaI/a.mpd"; + //url = "https://dash.akamaized.net/dash264/TestCases/2c/qualcomm/1/MultiResMPEG2.mpd"; + //url = "http://playertest.longtailvideo.com/adaptive/oceans_aes/oceans_aes.m3u8"; + //url = "https://cmaf.lln.latam.hbomaxcdn.com/videos/GYPGKMQjoDkVLBQEAAAAo/1/1b5ad5/1_single_J8sExA_1080hi.mpd"; + //url = "https://livesim.dashif.org/dash/vod/testpic_2s/multi_subs.mpd"; //ttml + mp4 + //url = "http://media.axprod.net/TestVectors/v6-Clear/Manifest_1080p.mpd"; //vtt + mp4 + url = "https://livesim.dashif.org/dash/vod/testpic_2s/xml_subs.mpd"; //ttml + if (args.Length > 0) + { + url = args[0]; + } if (string.IsNullOrEmpty(url)) { @@ -53,7 +69,7 @@ namespace N_m3u8DL_RE } //流提取器配置 - var extractor = new StreamExtractor(config); + var extractor = new StreamExtractor(parserConfig); extractor.LoadSourceFromUrl(url); //解析流信息 @@ -91,8 +107,23 @@ namespace N_m3u8DL_RE Logger.InfoMarkUp(item.ToString()); } - Logger.Info("按任意键继续"); Console.ReadKey(); + + //下载配置 + var downloadConfig = new DownloaderConfig() + { + Headers = parserConfig.Headers, + BinaryMerge = true, + DelAfterDone = true, + CheckSegmentsCount = true + }; + //开始下载 + var sdm = new SimpleDownloadManager(downloadConfig); + var result = await sdm.StartDownloadAsync(selectedStreams); + if (result) + Logger.InfoMarkUp("[white on green]成功[/]"); + else + Logger.ErrorMarkUp("[white on red]失败[/]"); } catch (Exception ex) { @@ -100,10 +131,5 @@ namespace N_m3u8DL_RE } //Console.ReadKey(); } - - static void Print(object o) - { - Console.WriteLine(GlobalUtil.ConvertToJson(o)); - } } } \ No newline at end of file diff --git a/src/N_m3u8DL-RE/Subtitle/WebVTTUtil.cs b/src/N_m3u8DL-RE/Subtitle/WebVTTUtil.cs deleted file mode 100644 index c40154c..0000000 --- a/src/N_m3u8DL-RE/Subtitle/WebVTTUtil.cs +++ /dev/null @@ -1,35 +0,0 @@ -using N_m3u8DL_RE.Common.Entity; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace N_m3u8DL_RE.Subtitle -{ - public class WebVTTUtil - { - /// - /// 修复VTT起始时间戳
- /// X-TIMESTAMP-MAP=MPEGTS:8528254208,LOCAL:00:00:00.000 - /// - /// - /// 基础时间戳 - /// - public static void FixTimestamp(WebSub sub, long baseTimestamp) - { - if (baseTimestamp == 0 || sub.MpegtsTimestamp == 0) - { - return; - } - - //The MPEG2 transport stream clocks (PCR, PTS, DTS) all have units of 1/90000 second - var seconds = (sub.MpegtsTimestamp - baseTimestamp) / 90000; - for (int i = 0; i < sub.Cues.Count; i++) - { - sub.Cues[i].StartTime += TimeSpan.FromSeconds(seconds); - sub.Cues[i].EndTime += TimeSpan.FromSeconds(seconds); - } - } - } -} diff --git a/src/N_m3u8DL-RE/Util/DownloadUtil.cs b/src/N_m3u8DL-RE/Util/DownloadUtil.cs new file mode 100644 index 0000000..5d4661a --- /dev/null +++ b/src/N_m3u8DL-RE/Util/DownloadUtil.cs @@ -0,0 +1,104 @@ +using N_m3u8DL_RE.Common.Log; +using N_m3u8DL_RE.Common.Resource; +using N_m3u8DL_RE.Entity; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Util +{ + internal class DownloadUtil + { + private static readonly HttpClient AppHttpClient = new(new HttpClientHandler + { + AllowAutoRedirect = false, + AutomaticDecompression = DecompressionMethods.All, + ServerCertificateCustomValidationCallback = (sender, cert, chain, sslPolicyErrors) => true + }) + { + Timeout = TimeSpan.FromMinutes(2) + }; + + public static async Task DownloadToFileAsync(string url, string path, Dictionary? headers = null, long? fromPosition = null, long? toPosition = null) + { + Logger.Debug(ResString.fetch + url); + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(url)); + if (fromPosition != null || toPosition != null) + request.Headers.Range = new(fromPosition, toPosition); + if (headers != null) + { + foreach (var item in headers) + { + request.Headers.TryAddWithoutValidation(item.Key, item.Value); + } + } + Logger.Debug(request.Headers.ToString()); + using var response = await AppHttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + if (response.StatusCode == HttpStatusCode.Found || response.StatusCode == HttpStatusCode.Moved) + { + HttpResponseHeaders respHeaders = response.Headers; + Logger.Debug(respHeaders.ToString()); + if (respHeaders != null && respHeaders.Location != null) + { + var redirectedUrl = respHeaders.Location.AbsoluteUri; + return await DownloadToFileAsync(redirectedUrl, path, headers); + } + } + response.EnsureSuccessStatusCode(); + var contentLength = response.Content.Headers.ContentLength; + using var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None); + using var responseStream = await response.Content.ReadAsStreamAsync(); + var buffer = new byte[16 * 1024]; + var size = 0; + while ((size = await responseStream.ReadAsync(buffer)) > 0) + { + await stream.WriteAsync(buffer, 0, size); + } + + return new DownloadResult() + { + ActualContentLength = stream.Length, + RespContentLength = contentLength, + ActualFilePath = path + }; + } + + /// + /// 输入一堆已存在的文件，合并到新文件 + /// + /// + /// + public static void CombineMultipleFilesIntoSingleFile(string[] files, string outputFilePath) + { + if (files.Length == 0) return; + if (files.Length == 1) + { + FileInfo fi = new FileInfo(files[0]); + fi.CopyTo(outputFilePath, true); + return; + } + + if (!Directory.Exists(Path.GetDirectoryName(outputFilePath))) + Directory.CreateDirectory(Path.GetDirectoryName(outputFilePath)!); + + string[] inputFilePaths = files; + using (var outputStream = File.Create(outputFilePath)) + { + foreach (var inputFilePath in inputFilePaths) + { + if (inputFilePath == "") + continue; + using (var inputStream = File.OpenRead(inputFilePath)) + { + inputStream.CopyTo(outputStream); + } + } + } + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/Util/PromptUtil.cs b/src/N_m3u8DL-RE/Util/PromptUtil.cs similarity index 98% rename from src/N_m3u8DL-RE.Parser/Util/PromptUtil.cs rename to src/N_m3u8DL-RE/Util/PromptUtil.cs index b00c58b..de45bf5 100644 --- a/src/N_m3u8DL-RE.Parser/Util/PromptUtil.cs +++ b/src/N_m3u8DL-RE/Util/PromptUtil.cs @@ -8,7 +8,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace N_m3u8DL_RE.Parser.Util +namespace N_m3u8DL_RE.Util { public class PromptUtil { diff --git a/src/N_m3u8DL-RE/rd.xml b/src/N_m3u8DL-RE/rd.xml index 398d5c5..ef7a25e 100644 --- a/src/N_m3u8DL-RE/rd.xml +++ b/src/N_m3u8DL-RE/rd.xml @@ -3,19 +3,5 @@ - - - - - - - - - - - - - - \ No newline at end of file