From d21d3924cc17b33a2d90e66061c0ce4a24212a80 Mon Sep 17 00:00:00 2001 From: nilaoda Date: Mon, 27 Jun 2022 20:16:38 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=BC=BA=E5=8F=AF=E5=AE=9A=E5=88=B6?= =?UTF-8?q?=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/N_m3u8DL-RE.Common/Config/ParserConfig.cs | 44 ------- src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs | 8 ++ src/N_m3u8DL-RE.Common/Entity/SubCue.cs | 16 +++ src/N_m3u8DL-RE.Common/Entity/WebSub.cs | 104 ++++++++++++++++ .../Resource/ResString.Designer.cs | 9 ++ .../Resource/ResString.en-US.resx | 3 + .../Resource/ResString.resx | 3 + .../Resource/ResString.zh-TW.resx | 3 + src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs | 9 ++ src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs | 37 ++++++ .../N_m3u8DL-RE.Extends.csproj | 18 +++ .../Subtitle/WebVTTUtil.cs | 35 ++++++ src/N_m3u8DL-RE.Parser/Config/ParserConfig.cs | 69 +++++++++++ .../Extractor/DASHExtractor.cs | 18 +-- .../Extractor/HLSExtractor.cs | 115 ++++-------------- .../Extractor/IExtractor.cs | 2 +- .../Processor/ContentProcessor.cs | 15 +++ .../DASH/DefaultDASHContentProcessor.cs | 33 +++++ .../Processor/DefaultUrlProcessor.cs | 28 +++++ .../HLS/DefaultHLSContentProcessor.cs | 77 ++++++++++++ .../Processor/HLS/DefaultHLSKeyProcessor.cs | 76 ++++++++++++ .../Processor/KeyProcessor.cs | 16 +++ .../Processor/UrlProcessor.cs | 15 +++ src/N_m3u8DL-RE.Parser/StreamExtractor.cs | 10 +- src/N_m3u8DL-RE.sln | 12 +- src/N_m3u8DL-RE/N_m3u8DL-RE.csproj | 1 + src/N_m3u8DL-RE/Processor/DemoProcessor.cs | 25 ++++ src/N_m3u8DL-RE/Processor/DemoProcessor2.cs | 27 ++++ src/N_m3u8DL-RE/Program.cs | 32 +++-- 29 files changed, 702 insertions(+), 158 deletions(-) delete mode 100644 src/N_m3u8DL-RE.Common/Config/ParserConfig.cs create mode 100644 src/N_m3u8DL-RE.Common/Entity/SubCue.cs create mode 100644 src/N_m3u8DL-RE.Common/Entity/WebSub.cs create mode 100644 src/N_m3u8DL-RE.Extends/N_m3u8DL-RE.Extends.csproj create mode 100644 src/N_m3u8DL-RE.Extends/Subtitle/WebVTTUtil.cs create mode 100644 src/N_m3u8DL-RE.Parser/Config/ParserConfig.cs create mode 100644 src/N_m3u8DL-RE.Parser/Processor/ContentProcessor.cs create mode 100644 src/N_m3u8DL-RE.Parser/Processor/DASH/DefaultDASHContentProcessor.cs create mode 100644 src/N_m3u8DL-RE.Parser/Processor/DefaultUrlProcessor.cs create mode 100644 src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSContentProcessor.cs create mode 100644 src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSKeyProcessor.cs create mode 100644 src/N_m3u8DL-RE.Parser/Processor/KeyProcessor.cs create mode 100644 src/N_m3u8DL-RE.Parser/Processor/UrlProcessor.cs create mode 100644 src/N_m3u8DL-RE/Processor/DemoProcessor.cs create mode 100644 src/N_m3u8DL-RE/Processor/DemoProcessor2.cs diff --git a/src/N_m3u8DL-RE.Common/Config/ParserConfig.cs b/src/N_m3u8DL-RE.Common/Config/ParserConfig.cs deleted file mode 100644 index a48dc6d..0000000 --- a/src/N_m3u8DL-RE.Common/Config/ParserConfig.cs +++ /dev/null @@ -1,44 +0,0 @@ -using N_m3u8DL_RE.Common.Enum; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace N_m3u8DL_RE.Common.Config -{ - public class ParserConfig - { - public string Url { get; set; } - public string BaseUrl { get; set; } - public Dictionary Headers { get; set; } = new Dictionary() - { - ["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36" - }; - - /// - /// 自定义的加密方式 默认AES_128_CBC - /// - public EncryptMethod CustomMethod { get; set; } = EncryptMethod.AES_128; - - /// - /// 自定义的解密KEY - /// - public byte[]? CustomeKey { get; set; } - - /// - /// 自定义的解密IV - /// - public byte[]? CustomeIV { get; set; } - - /// - /// 组装视频分段的URL时，是否要把原本URL后的参数也加上去 - /// 如 Base URL = "http://xxx.com/playlist.m3u8?hmac=xxx&token=xxx" - /// 相对路径 = clip_01.ts - /// 如果 AppendUrlParams=false，得 http://xxx.com/clip_01.ts - /// 如果 AppendUrlParams=true，得 http://xxx.com/clip_01.ts?hmac=xxx&token=xxx - /// - public bool AppendUrlParams { get; set; } = false; - - } -} diff --git a/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs b/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs index 5027b26..3de9264 100644 --- a/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs +++ b/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs @@ -1,4 +1,5 @@ using N_m3u8DL_RE.Common.Enum; +using N_m3u8DL_RE.Common.Util; using Spectre.Console; using System; using System.Collections.Generic; @@ -70,6 +71,13 @@ namespace N_m3u8DL_RE.Common.Entity returnStr = returnStr.Replace("| |", "|"); } + //计算时长 + if (Playlist != null) + { + var total = Playlist.MediaParts.Sum(x => x.MediaSegments.Sum(m => m.Duration)); + returnStr += " | " + GlobalUtil.FormatTime((int)total); + } + return returnStr; } } diff --git a/src/N_m3u8DL-RE.Common/Entity/SubCue.cs b/src/N_m3u8DL-RE.Common/Entity/SubCue.cs new file mode 100644 index 0000000..6d944aa --- /dev/null +++ b/src/N_m3u8DL-RE.Common/Entity/SubCue.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Common.Entity +{ + public class SubCue + { + public TimeSpan StartTime { get; set; } + public TimeSpan EndTime { get; set; } + public string Payload { get; set; } + public string Settings { get; set; } + } +} diff --git a/src/N_m3u8DL-RE.Common/Entity/WebSub.cs b/src/N_m3u8DL-RE.Common/Entity/WebSub.cs new file mode 100644 index 0000000..01f3994 --- /dev/null +++ b/src/N_m3u8DL-RE.Common/Entity/WebSub.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Common.Entity +{ + public class WebSub + { + public List Cues { get; set; } = new List(); + public long MpegtsTimestamp { get; set; } = 0L; + + /// + /// 从字节数组解析WEBVTT + /// + /// + /// + public static WebSub Parse(byte[] textBytes) + { + return Parse(Encoding.UTF8.GetString(textBytes)); + } + + /// + /// 从字符串解析WEBVTT + /// + /// + /// + public static WebSub Parse(string text) + { + if (!text.Trim().StartsWith("WEBVTT")) + throw new Exception("Bad vtt!"); + + + var webSub = new WebSub(); + var needPayload = false; + var timeLine = ""; + var regex1 = new Regex("X-TIMESTAMP-MAP.*"); + + if (regex1.IsMatch(text)) + { + var timestamp = Regex.Match(regex1.Match(text).Value, "MPEGTS:(\\d+)").Groups[1].Value; + webSub.MpegtsTimestamp = Convert.ToInt64(timestamp); + } + + foreach (var line in text.Split('\n')) + { + if (string.IsNullOrEmpty(line)) continue; + + if (!needPayload && line.Contains(" --> ")) + { + needPayload = true; + timeLine = line.Trim(); + continue; + } + + if (needPayload) + { + var payload = line.Trim(); + var arr = Regex.Split(timeLine.Replace("-->", ""), "\\s").Where(s => !string.IsNullOrEmpty(s)).ToList(); + var startTime = ConvertToTS(arr[0]); + var endTime = ConvertToTS(arr[1]); + var style = arr.Count > 2 ? string.Join(" ", arr.Skip(2)) : ""; + webSub.Cues.Add(new SubCue() + { + StartTime = startTime, + EndTime = endTime, + Payload = payload, + Settings = style + }); + needPayload = false; + } + } + + return webSub; + } + + private static TimeSpan ConvertToTS(string str) + { + var ms = Convert.ToInt32(str.Split('.').Last()); + var o = str.Split('.').First(); + var t = o.Split(':').Reverse().ToList(); + var time = 0L + ms; + for (int i = 0; i < t.Count(); i++) + { + time += (int)Math.Pow(60, i) * Convert.ToInt32(t[i]) * 1000; + } + return TimeSpan.FromMilliseconds(time); + } + + public override string ToString() + { + StringBuilder sb = new StringBuilder(); + foreach (var c in this.Cues) + { + sb.AppendLine(c.StartTime.ToString(@"hh\:mm\:ss\.fff") + " --> " + c.EndTime.ToString(@"hh\:mm\:ss\.fff") + " " + c.Settings); + sb.AppendLine(c.Payload); + sb.AppendLine(); + } + return sb.ToString(); + } + } +} diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs b/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs index eab35b4..2e461b3 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs @@ -87,6 +87,15 @@ namespace N_m3u8DL_RE.Common.Resource { } } + /// + /// 查找类似 找不到支持的Processor 的本地化字符串。 + /// + public static string keyProcessorNotFound { + get { + return ResourceManager.GetString("keyProcessorNotFound", resourceCulture); + } + } + /// /// 查找类似 检测到直播流 的本地化字符串。 /// diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx b/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx index 335f124..b323cd8 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx @@ -145,4 +145,7 @@ Parsing streams... + + No Processor matched + \ No newline at end of file diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.resx b/src/N_m3u8DL-RE.Common/Resource/ResString.resx index a7e8d14..28e8636 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.resx +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.resx @@ -145,4 +145,7 @@ 正在解析媒体信息... + + 找不到支持的Processor + \ No newline at end of file diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx b/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx index 8f07b5c..39c8c4b 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx @@ -145,4 +145,7 @@ 正在解析媒體信息... + + 找不到支持的Processor + \ No newline at end of file diff --git a/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs b/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs index 0d64542..5210240 100644 --- a/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs +++ b/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs @@ -22,5 +22,14 @@ namespace N_m3u8DL_RE.Common.Util }; return JsonSerializer.Serialize(o, options); } + + //此函数用于格式化输出时长 + public static string FormatTime(int time) + { + TimeSpan ts = new TimeSpan(0, 0, time); + string str = ""; + str = (ts.Hours.ToString("00") == "00" ? "" : ts.Hours.ToString("00") + "h") + ts.Minutes.ToString("00") + "m" + ts.Seconds.ToString("00") + "s"; + return str; + } } } diff --git a/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs b/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs index da64b33..6197d76 100644 --- a/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs +++ b/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs @@ -53,6 +53,43 @@ namespace N_m3u8DL_RE.Common.Util return webResponse; } + //重定向 + public static async Task Get302Async(string url, Dictionary? headers = null) + { + Logger.Debug(ResString.fetch + url); + var handler = new HttpClientHandler() + { + AllowAutoRedirect = false + }; + string redirectedUrl = url; + + using (HttpClient client = new HttpClient(handler)) + { + if (headers != null) + { + foreach (var item in headers) + { + client.DefaultRequestHeaders.TryAddWithoutValidation(item.Key, item.Value); + } + } + using (HttpResponseMessage response = await client.GetAsync(url)) + using (HttpContent content = response.Content) + { + Logger.Debug(ResString.fetch + response.Headers); + if (response.StatusCode == HttpStatusCode.Found || response.StatusCode == HttpStatusCode.Moved) + { + HttpResponseHeaders respHeaders = response.Headers; + if (respHeaders != null && respHeaders.Location != null) + { + redirectedUrl = respHeaders.Location.AbsoluteUri; + } + } + } + } + + return redirectedUrl; + } + public static async Task GetBytesAsync(string url, Dictionary? headers = null) { byte[] bytes = new byte[0]; diff --git a/src/N_m3u8DL-RE.Extends/N_m3u8DL-RE.Extends.csproj b/src/N_m3u8DL-RE.Extends/N_m3u8DL-RE.Extends.csproj new file mode 100644 index 0000000..7958b21 --- /dev/null +++ b/src/N_m3u8DL-RE.Extends/N_m3u8DL-RE.Extends.csproj @@ -0,0 +1,18 @@ + + + + net6.0 + N_m3u8DL_RE.Extends + enable + enable + + + + + + + + + + + diff --git a/src/N_m3u8DL-RE.Extends/Subtitle/WebVTTUtil.cs b/src/N_m3u8DL-RE.Extends/Subtitle/WebVTTUtil.cs new file mode 100644 index 0000000..fdb5eb9 --- /dev/null +++ b/src/N_m3u8DL-RE.Extends/Subtitle/WebVTTUtil.cs @@ -0,0 +1,35 @@ +using N_m3u8DL_RE.Common.Entity; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Extends.Subtitle +{ + public class WebVTTUtil + { + /// + /// 修复VTT起始时间戳
+ /// X-TIMESTAMP-MAP=MPEGTS:8528254208,LOCAL:00:00:00.000 + /// + /// + /// 基础时间戳 + /// + public static void FixTimestamp(WebSub sub, long baseTimestamp) + { + if (baseTimestamp == 0 || sub.MpegtsTimestamp == 0) + { + return; + } + + //The MPEG2 transport stream clocks (PCR, PTS, DTS) all have units of 1/90000 second + var seconds = (sub.MpegtsTimestamp - baseTimestamp) / 90000; + for (int i = 0; i < sub.Cues.Count; i++) + { + sub.Cues[i].StartTime += TimeSpan.FromSeconds(seconds); + sub.Cues[i].EndTime += TimeSpan.FromSeconds(seconds); + } + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/Config/ParserConfig.cs b/src/N_m3u8DL-RE.Parser/Config/ParserConfig.cs new file mode 100644 index 0000000..439ef23 --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Config/ParserConfig.cs @@ -0,0 +1,69 @@ +using N_m3u8DL_RE.Common.Enum; +using N_m3u8DL_RE.Parser.Processor; +using N_m3u8DL_RE.Parser.Processor.DASH; +using N_m3u8DL_RE.Parser.Processor.HLS; + +namespace N_m3u8DL_RE.Parser.Config +{ + public class ParserConfig + { + public string Url { get; set; } + + public string BaseUrl { get; set; } + + public Dictionary Headers { get; set; } = new Dictionary() + { + ["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36" + }; + + /// + /// HLS内容前置处理器. 调用顺序与列表顺序相同 + /// + public IList HLSContentProcessors { get; } = new List() { new DefaultHLSContentProcessor() }; + + /// + /// DASH内容前置处理器. 调用顺序与列表顺序相同 + /// + public IList DASHContentProcessors { get; } = new List() { new DefaultDASHContentProcessor() }; + + /// + /// 添加分片URL前置处理器. 调用顺序与列表顺序相同 + /// + public IList HLSUrlProcessors { get; } = new List() { new DefaultUrlProcessor() }; + + /// + /// DASH内容前置处理器. 调用顺序与列表顺序相同 + /// + public IList DASHUrlProcessors { get; } = new List() { new DefaultUrlProcessor() }; + + /// + /// HLS-KEY解析器. 调用顺序与列表顺序相同 + /// + public IList HLSKeyProcessors { get; } = new List() { new DefaultHLSKeyProcessor() }; + + + /// + /// 自定义的加密方式 + /// + public EncryptMethod? CustomMethod { get; set; } + + /// + /// 自定义的解密KEY + /// + public byte[]? CustomeKey { get; set; } + + /// + /// 自定义的解密IV + /// + public byte[]? CustomeIV { get; set; } + + /// + /// 组装视频分段的URL时，是否要把原本URL后的参数也加上去 + /// 如 Base URL = "http://xxx.com/playlist.m3u8?hmac=xxx&token=xxx" + /// 相对路径 = clip_01.ts + /// 如果 AppendUrlParams=false，得 http://xxx.com/clip_01.ts + /// 如果 AppendUrlParams=true，得 http://xxx.com/clip_01.ts?hmac=xxx&token=xxx + /// + public bool AppendUrlParams { get; set; } = false; + } +} diff --git a/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor.cs b/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor.cs index fa8742d..c1af0c5 100644 --- a/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor.cs +++ b/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor.cs @@ -1,9 +1,8 @@ -using N_m3u8DL_RE.Common.Config; +using N_m3u8DL_RE.Parser.Config; using N_m3u8DL_RE.Common.Entity; using N_m3u8DL_RE.Common.Enum; using N_m3u8DL_RE.Common.Log; using N_m3u8DL_RE.Common.Resource; -using N_m3u8DL_RE.Common.Util; using N_m3u8DL_RE.Parser.Util; using System; using System.Collections.Generic; @@ -723,9 +722,12 @@ namespace N_m3u8DL_RE.Parser.Extractor /// private string PreProcessUrl(string url) { - if (ParserConfig.AppendUrlParams) + foreach (var p in ParserConfig.DASHUrlProcessors) { - url += new Regex("\\?.*").Match(MpdUrl).Value; + if (p.CanProcess(url, ParserConfig)) + { + url = p.Process(url, ParserConfig); + } } return url; @@ -733,10 +735,12 @@ namespace N_m3u8DL_RE.Parser.Extractor private void PreProcessContent() { - //XiGua - if (this.MpdContent.Contains(" @@ -46,60 +50,13 @@ namespace N_m3u8DL_RE.Parser.Extractor throw new Exception(ResString.badM3u8); } - //央视频回放 - if (M3u8Url.Contains("tlivecloud-playback-cdn.ysp.cctv.cn") && M3u8Url.Contains("endtime=")) + foreach (var p in ParserConfig.HLSContentProcessors) { - M3u8Content += Environment.NewLine + HLSTags.ext_x_endlist; - } - - //IMOOC - if (M3u8Url.Contains("imooc.com/")) - { - //M3u8Content = DecodeImooc.DecodeM3u8(M3u8Content); - } - - //iqy - if (M3u8Content.StartsWith("{\"payload\"")) - { - // - } - - //针对优酷#EXT-X-VERSION:7杜比视界片源修正 - if (M3u8Content.Contains("#EXT-X-DISCONTINUITY") && M3u8Content.Contains("#EXT-X-MAP") && M3u8Content.Contains("ott.cibntv.net") && M3u8Content.Contains("ccode=")) - { - Regex ykmap = new Regex("#EXT-X-DISCONTINUITY\\s+#EXT-X-MAP:URI=\\\"(.*?)\\\",BYTERANGE=\\\"(.*?)\\\""); - foreach (Match m in ykmap.Matches(M3u8Content)) + if (p.CanProcess(M3u8Content, ParserConfig)) { - M3u8Content = M3u8Content.Replace(m.Value, $"#EXTINF:0.000000,\n#EXT-X-BYTERANGE:{m.Groups[2].Value}\n{m.Groups[1].Value}"); + M3u8Content = p.Process(M3u8Content, ParserConfig); } } - - //针对Disney+修正 - if (M3u8Content.Contains("#EXT-X-DISCONTINUITY") && M3u8Content.Contains("#EXT-X-MAP") && M3u8Url.Contains("media.dssott.com/")) - { - Regex ykmap = new Regex("#EXT-X-MAP:URI=\\\".*?BUMPER/[\\s\\S]+?#EXT-X-DISCONTINUITY"); - if (ykmap.IsMatch(M3u8Content)) - { - M3u8Content = M3u8Content.Replace(ykmap.Match(M3u8Content).Value, "#XXX"); - } - } - - //针对AppleTv修正 - if (M3u8Content.Contains("#EXT-X-DISCONTINUITY") && M3u8Content.Contains("#EXT-X-MAP") && (M3u8Url.Contains(".apple.com/") || Regex.IsMatch(M3u8Content, "#EXT-X-MAP.*\\.apple\\.com/"))) - { - //只取加密部分即可 - Regex ykmap = new Regex("(#EXT-X-KEY:[\\s\\S]*?)(#EXT-X-DISCONTINUITY|#EXT-X-ENDLIST)"); - if (ykmap.IsMatch(M3u8Content)) - { - M3u8Content = "#EXTM3U\r\n" + ykmap.Match(M3u8Content).Groups[1].Value + "\r\n#EXT-X-ENDLIST"; - } - } - - //修复#EXT-X-KEY与#EXTINF出现次序异常问题 - if (Regex.IsMatch(M3u8Content, "(#EXTINF.*)(\\s+)(#EXT-X-KEY.*)")) - { - M3u8Content = Regex.Replace(M3u8Content, "(#EXTINF.*)(\\s+)(#EXT-X-KEY.*)", "$3$2$1"); - } } /// @@ -107,9 +64,12 @@ namespace N_m3u8DL_RE.Parser.Extractor /// private string PreProcessUrl(string url) { - if (ParserConfig.AppendUrlParams) + foreach (var p in ParserConfig.HLSUrlProcessors) { - url += new Regex("\\?.*").Match(M3u8Url).Value; + if (p.CanProcess(url, ParserConfig)) + { + url = p.Process(url, ParserConfig); + } } return url; @@ -338,7 +298,7 @@ namespace N_m3u8DL_RE.Parser.Extractor //自定义KEY情况 判断是否需要读取IV if (line.Contains("IV=0x") && ParserConfig.CustomeKey != null && ParserConfig.CustomeIV == null) { - currentEncryptInfo.Method = ParserConfig.CustomMethod; + currentEncryptInfo.Method = ParserConfig.CustomMethod ?? EncryptMethod.AES_128; currentEncryptInfo.Key = ParserConfig.CustomeKey; currentEncryptInfo.IV = HexUtil.HexToBytes(iv); } @@ -346,25 +306,7 @@ namespace N_m3u8DL_RE.Parser.Extractor if (uri != uri_last) { //解析key - currentEncryptInfo.Key = await ParseKeyAsync(uri); - //加密方式 - if (Enum.TryParse(method.Replace("-", "_"), out EncryptMethod m)) - { - currentEncryptInfo.Method = m; - } - else - { - currentEncryptInfo.Method = EncryptMethod.UNKNOWN; - } - //没有读取到IV，自己生成 - if (string.IsNullOrEmpty(iv)) - { - currentEncryptInfo.IV = HexUtil.HexToBytes(Convert.ToString(segIndex, 16).PadLeft(32, '0')); - } - else - { - currentEncryptInfo.IV = HexUtil.HexToBytes(iv); - } + currentEncryptInfo = ParseKey(method, uri, iv, segIndex); } lastKeyLine = line; } @@ -475,22 +417,17 @@ namespace N_m3u8DL_RE.Parser.Extractor return playlist; } - private async Task ParseKeyAsync(string uri) + private EncryptInfo ParseKey(string method, string uriText, string ivText, int segIndex) { - if (uri.ToLower().StartsWith("base64:")) + foreach (var p in ParserConfig.HLSKeyProcessors) { - return Convert.FromBase64String(uri.Substring(7)); - } - else if (uri.ToLower().StartsWith("data:text/plain;base64,")) - { - return Convert.FromBase64String(uri.Substring(23)); - } - else - { - var segUrl = PreProcessUrl(ParserUtil.CombineURL(BaseUrl, uri)); - var bytes = await HTTPUtil.GetBytesAsync(segUrl, ParserConfig.Headers); - return bytes; + if (p.CanProcess(method, uriText, ivText, ParserConfig)) + { + return p.Process(method, uriText, ivText, segIndex, ParserConfig); + } } + + throw new Exception(ResString.keyProcessorNotFound); } public async Task> ExtractStreamsAsync(string rawText) diff --git a/src/N_m3u8DL-RE.Parser/Extractor/IExtractor.cs b/src/N_m3u8DL-RE.Parser/Extractor/IExtractor.cs index 9c40a7b..de74826 100644 --- a/src/N_m3u8DL-RE.Parser/Extractor/IExtractor.cs +++ b/src/N_m3u8DL-RE.Parser/Extractor/IExtractor.cs @@ -1,4 +1,4 @@ -using N_m3u8DL_RE.Common.Config; +using N_m3u8DL_RE.Parser.Config; using N_m3u8DL_RE.Common.Entity; using System; using System.Collections.Generic; diff --git a/src/N_m3u8DL-RE.Parser/Processor/ContentProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/ContentProcessor.cs new file mode 100644 index 0000000..caa061a --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Processor/ContentProcessor.cs @@ -0,0 +1,15 @@ +using N_m3u8DL_RE.Parser.Config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Parser.Processor +{ + public abstract class ContentProcessor + { + public abstract bool CanProcess(string rawText, ParserConfig parserConfig); + public abstract string Process(string rawText, ParserConfig parserConfig); + } +} diff --git a/src/N_m3u8DL-RE.Parser/Processor/DASH/DefaultDASHContentProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/DASH/DefaultDASHContentProcessor.cs new file mode 100644 index 0000000..a0a6b9c --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Processor/DASH/DefaultDASHContentProcessor.cs @@ -0,0 +1,33 @@ +using N_m3u8DL_RE.Common.Log; +using N_m3u8DL_RE.Parser.Config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Parser.Processor.DASH +{ + /// + /// 西瓜视频处理 + /// + public class DefaultDASHContentProcessor : ContentProcessor + { + public override bool CanProcess(string mpdContent, ParserConfig parserConfig) + { + if (mpdContent.Contains(" true; + + public override string Process(string oriUrl, ParserConfig paserConfig) + { + if (paserConfig.AppendUrlParams) + { + Logger.Debug("Before: " + oriUrl); + oriUrl += new Regex("\\?.*").Match(paserConfig.Url).Value; + Logger.Debug("After: " + oriUrl); + } + + return oriUrl; + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSContentProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSContentProcessor.cs new file mode 100644 index 0000000..bcbe42e --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSContentProcessor.cs @@ -0,0 +1,77 @@ +using N_m3u8DL_RE.Parser.Config; +using N_m3u8DL_RE.Parser.Constants; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Parser.Processor.HLS +{ + public class DefaultHLSContentProcessor : ContentProcessor + { + public override bool CanProcess(string rawText, ParserConfig parserConfig) => true; + + public override string Process(string m3u8Content, ParserConfig parserConfig) + { + var m3u8Url = parserConfig.Url; + //央视频回放 + if (m3u8Url.Contains("tlivecloud-playback-cdn.ysp.cctv.cn") && m3u8Url.Contains("endtime=")) + { + m3u8Content += Environment.NewLine + HLSTags.ext_x_endlist; + } + + //IMOOC + if (m3u8Url.Contains("imooc.com/")) + { + //M3u8Content = DecodeImooc.DecodeM3u8(M3u8Content); + } + + //iqy + if (m3u8Content.StartsWith("{\"payload\"")) + { + // + } + + //针对优酷#EXT-X-VERSION:7杜比视界片源修正 + if (m3u8Content.Contains("#EXT-X-DISCONTINUITY") && m3u8Content.Contains("#EXT-X-MAP") && m3u8Content.Contains("ott.cibntv.net") && m3u8Content.Contains("ccode=")) + { + Regex ykmap = new Regex("#EXT-X-DISCONTINUITY\\s+#EXT-X-MAP:URI=\\\"(.*?)\\\",BYTERANGE=\\\"(.*?)\\\""); + foreach (Match m in ykmap.Matches(m3u8Content)) + { + m3u8Content = m3u8Content.Replace(m.Value, $"#EXTINF:0.000000,\n#EXT-X-BYTERANGE:{m.Groups[2].Value}\n{m.Groups[1].Value}"); + } + } + + //针对Disney+修正 + if (m3u8Content.Contains("#EXT-X-DISCONTINUITY") && m3u8Content.Contains("#EXT-X-MAP") && m3u8Url.Contains("media.dssott.com/")) + { + Regex ykmap = new Regex("#EXT-X-MAP:URI=\\\".*?BUMPER/[\\s\\S]+?#EXT-X-DISCONTINUITY"); + if (ykmap.IsMatch(m3u8Content)) + { + m3u8Content = m3u8Content.Replace(ykmap.Match(m3u8Content).Value, "#XXX"); + } + } + + //针对AppleTv修正 + if (m3u8Content.Contains("#EXT-X-DISCONTINUITY") && m3u8Content.Contains("#EXT-X-MAP") && (m3u8Url.Contains(".apple.com/") || Regex.IsMatch(m3u8Content, "#EXT-X-MAP.*\\.apple\\.com/"))) + { + //只取加密部分即可 + Regex ykmap = new Regex("(#EXT-X-KEY:[\\s\\S]*?)(#EXT-X-DISCONTINUITY|#EXT-X-ENDLIST)"); + if (ykmap.IsMatch(m3u8Content)) + { + m3u8Content = "#EXTM3U\r\n" + ykmap.Match(m3u8Content).Groups[1].Value + "\r\n#EXT-X-ENDLIST"; + } + } + + //修复#EXT-X-KEY与#EXTINF出现次序异常问题 + if (Regex.IsMatch(m3u8Content, "(#EXTINF.*)(\\s+)(#EXT-X-KEY.*)")) + { + m3u8Content = Regex.Replace(m3u8Content, "(#EXTINF.*)(\\s+)(#EXT-X-KEY.*)", "$3$2$1"); + } + + return m3u8Content; + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSKeyProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSKeyProcessor.cs new file mode 100644 index 0000000..36e96f1 --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Processor/HLS/DefaultHLSKeyProcessor.cs @@ -0,0 +1,76 @@ +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Common.Enum; +using N_m3u8DL_RE.Common.Util; +using N_m3u8DL_RE.Parser.Config; +using N_m3u8DL_RE.Parser.Util; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Parser.Processor.HLS +{ + public class DefaultHLSKeyProcessor : KeyProcessor + { + public override bool CanProcess(string method, string uriText, string ivText, ParserConfig paserConfig) => true; + + public override EncryptInfo Process(string method, string uriText, string ivText, int segIndex, ParserConfig parserConfig) + { + var encryptInfo = new EncryptInfo(); + + + if (uriText.ToLower().StartsWith("base64:")) + { + encryptInfo.Key = Convert.FromBase64String(uriText.Substring(7)); + } + else if (uriText.ToLower().StartsWith("data:text/plain;base64,")) + { + encryptInfo.Key = Convert.FromBase64String(uriText.Substring(23)); + } + else + { + var segUrl = PreProcessUrl(ParserUtil.CombineURL(parserConfig.BaseUrl, uriText), parserConfig); + var bytes = HTTPUtil.GetBytesAsync(segUrl, parserConfig.Headers).Result; + encryptInfo.Key = bytes; + } + + //加密方式 + if (Enum.TryParse(method.Replace("-", "_"), out EncryptMethod m)) + { + encryptInfo.Method = m; + } + else + { + encryptInfo.Method = EncryptMethod.UNKNOWN; + } + //没有读取到IV，自己生成 + if (string.IsNullOrEmpty(ivText)) + { + encryptInfo.IV = HexUtil.HexToBytes(Convert.ToString(segIndex, 16).PadLeft(32, '0')); + } + else + { + encryptInfo.IV = HexUtil.HexToBytes(ivText); + } + + return encryptInfo; + } + + /// + /// 预处理URL + /// + private string PreProcessUrl(string url, ParserConfig parserConfig) + { + foreach (var p in parserConfig.HLSUrlProcessors) + { + if (p.CanProcess(url, parserConfig)) + { + url = p.Process(url, parserConfig); + } + } + + return url; + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/Processor/KeyProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/KeyProcessor.cs new file mode 100644 index 0000000..2839be0 --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Processor/KeyProcessor.cs @@ -0,0 +1,16 @@ +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Parser.Config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Parser.Processor +{ + public abstract class KeyProcessor + { + public abstract bool CanProcess(string method, string uriText, string ivText, ParserConfig parserConfig); + public abstract EncryptInfo Process(string method, string uriText, string ivText, int segIndex, ParserConfig parserConfig); + } +} diff --git a/src/N_m3u8DL-RE.Parser/Processor/UrlProcessor.cs b/src/N_m3u8DL-RE.Parser/Processor/UrlProcessor.cs new file mode 100644 index 0000000..81a4986 --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Processor/UrlProcessor.cs @@ -0,0 +1,15 @@ +using N_m3u8DL_RE.Parser.Config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Parser.Processor +{ + public abstract class UrlProcessor + { + public abstract bool CanProcess(string oriUrl, ParserConfig parserConfig); + public abstract string Process(string oriUrl, ParserConfig parserConfig); + } +} diff --git a/src/N_m3u8DL-RE.Parser/StreamExtractor.cs b/src/N_m3u8DL-RE.Parser/StreamExtractor.cs index a2360e2..9851491 100644 --- a/src/N_m3u8DL-RE.Parser/StreamExtractor.cs +++ b/src/N_m3u8DL-RE.Parser/StreamExtractor.cs @@ -1,8 +1,8 @@ -using N_m3u8DL_RE.Common.Config; +using N_m3u8DL_RE.Parser.Config; using N_m3u8DL_RE.Common.Entity; using N_m3u8DL_RE.Common.Log; using N_m3u8DL_RE.Common.Resource; -using N_m3u8DL_RE.Common.Util; +using N_m3u8DL_RE.Parser.Util; using N_m3u8DL_RE.Parser.Constants; using N_m3u8DL_RE.Parser.Extractor; using System; @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using N_m3u8DL_RE.Common.Util; namespace N_m3u8DL_RE.Parser { @@ -19,6 +20,11 @@ namespace N_m3u8DL_RE.Parser private ParserConfig parserConfig = new ParserConfig(); private string rawText; + public StreamExtractor() + { + + } + public StreamExtractor(ParserConfig parserConfig) { this.parserConfig = parserConfig; diff --git a/src/N_m3u8DL-RE.sln b/src/N_m3u8DL-RE.sln index 085c7a3..217b470 100644 --- a/src/N_m3u8DL-RE.sln +++ b/src/N_m3u8DL-RE.sln @@ -3,11 +3,13 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.3.32505.426 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "N_m3u8DL-RE", "N_m3u8DL-RE\N_m3u8DL-RE.csproj", "{E6915BF9-8306-4F62-B357-23430F0D80B5}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "N_m3u8DL-RE", "N_m3u8DL-RE\N_m3u8DL-RE.csproj", "{E6915BF9-8306-4F62-B357-23430F0D80B5}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "N_m3u8DL-RE.Common", "N_m3u8DL-RE.Common\N_m3u8DL-RE.Common.csproj", "{18D8BC30-512C-4A07-BD4C-E96DF5CF6341}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "N_m3u8DL-RE.Common", "N_m3u8DL-RE.Common\N_m3u8DL-RE.Common.csproj", "{18D8BC30-512C-4A07-BD4C-E96DF5CF6341}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "N_m3u8DL-RE.Parser", "N_m3u8DL-RE.Parser\N_m3u8DL-RE.Parser.csproj", "{0DA02925-AF3A-4598-AF01-91AE5539FCA1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "N_m3u8DL-RE.Parser", "N_m3u8DL-RE.Parser\N_m3u8DL-RE.Parser.csproj", "{0DA02925-AF3A-4598-AF01-91AE5539FCA1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "N_m3u8DL-RE.Extends", "N_m3u8DL-RE.Extends\N_m3u8DL-RE.Extends.csproj", "{99175570-6FE1-45C0-87BD-D2E1B52A35CC}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -27,6 +29,10 @@ Global {0DA02925-AF3A-4598-AF01-91AE5539FCA1}.Debug|Any CPU.Build.0 = Debug|Any CPU {0DA02925-AF3A-4598-AF01-91AE5539FCA1}.Release|Any CPU.ActiveCfg = Release|Any CPU {0DA02925-AF3A-4598-AF01-91AE5539FCA1}.Release|Any CPU.Build.0 = Release|Any CPU + {99175570-6FE1-45C0-87BD-D2E1B52A35CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {99175570-6FE1-45C0-87BD-D2E1B52A35CC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {99175570-6FE1-45C0-87BD-D2E1B52A35CC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {99175570-6FE1-45C0-87BD-D2E1B52A35CC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/N_m3u8DL-RE/N_m3u8DL-RE.csproj b/src/N_m3u8DL-RE/N_m3u8DL-RE.csproj index 19ccf75..6f69e13 100644 --- a/src/N_m3u8DL-RE/N_m3u8DL-RE.csproj +++ b/src/N_m3u8DL-RE/N_m3u8DL-RE.csproj @@ -13,6 +13,7 @@ + diff --git a/src/N_m3u8DL-RE/Processor/DemoProcessor.cs b/src/N_m3u8DL-RE/Processor/DemoProcessor.cs new file mode 100644 index 0000000..f24c6a9 --- /dev/null +++ b/src/N_m3u8DL-RE/Processor/DemoProcessor.cs @@ -0,0 +1,25 @@ +using N_m3u8DL_RE.Common.Log; +using N_m3u8DL_RE.Parser.Config; +using N_m3u8DL_RE.Parser.Processor; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Processor +{ + internal class DemoProcessor : ContentProcessor + { + public override bool CanProcess(string rawText, ParserConfig parserConfig) + { + return parserConfig.Url.Contains("bitmovin"); + } + + public override string Process(string rawText, ParserConfig parserConfig) + { + Logger.InfoMarkUp("[red]Match bitmovin![/]"); + return rawText; + } + } +} diff --git a/src/N_m3u8DL-RE/Processor/DemoProcessor2.cs b/src/N_m3u8DL-RE/Processor/DemoProcessor2.cs new file mode 100644 index 0000000..ceb4869 --- /dev/null +++ b/src/N_m3u8DL-RE/Processor/DemoProcessor2.cs @@ -0,0 +1,27 @@ +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Common.Log; +using N_m3u8DL_RE.Parser.Config; +using N_m3u8DL_RE.Parser.Processor; +using N_m3u8DL_RE.Parser.Processor.HLS; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Processor +{ + internal class DemoProcessor2 : KeyProcessor + { + public override bool CanProcess(string method, string uriText, string ivText, ParserConfig parserConfig) + { + return parserConfig.Url.Contains("playertest.longtailvideo.com"); + } + + public override EncryptInfo Process(string method, string uriText, string ivText, int segIndex, ParserConfig parserConfig) + { + Logger.InfoMarkUp("[white on green]My Key Processor![/]"); + return new DefaultHLSKeyProcessor().Process(method, uriText, ivText, segIndex, parserConfig); + } + } +} diff --git a/src/N_m3u8DL-RE/Program.cs b/src/N_m3u8DL-RE/Program.cs index 448de46..2d09737 100644 --- a/src/N_m3u8DL-RE/Program.cs +++ b/src/N_m3u8DL-RE/Program.cs @@ -1,7 +1,7 @@ -using N_m3u8DL_RE.Common.Config; +using N_m3u8DL_RE.Parser.Config; using N_m3u8DL_RE.Common.Entity; using N_m3u8DL_RE.Common.Enum; -using N_m3u8DL_RE.Common.Util; +using N_m3u8DL_RE.Parser.Util; using N_m3u8DL_RE.Parser; using Spectre.Console; using System.Text.Json; @@ -10,7 +10,11 @@ using N_m3u8DL_RE.Common.Resource; using N_m3u8DL_RE.Common.Log; using System.Globalization; using System.Text; -using N_m3u8DL_RE.Parser.Util; +using System.Text.RegularExpressions; +using N_m3u8DL_RE.Extends.Subtitle; +using System.Collections.Concurrent; +using N_m3u8DL_RE.Common.Util; +using N_m3u8DL_RE.Processor; namespace N_m3u8DL_RE { @@ -26,18 +30,25 @@ namespace N_m3u8DL_RE //设置语言 CultureInfo.DefaultThreadCurrentCulture = new CultureInfo(loc); Thread.CurrentThread.CurrentUICulture = CultureInfo.GetCultureInfo(loc); + //Logger.LogLevel = LogLevel.DEBUG; try { - //Logger.LogLevel = LogLevel.DEBUG; var config = new ParserConfig(); + //demo1 + config.DASHContentProcessors.Insert(0, new DemoProcessor()); + //demo2 + config.HLSKeyProcessors.Insert(0, new DemoProcessor2()); + var url = string.Empty; - //url = "http://playertest.longtailvideo.com/adaptive/oceans_aes/oceans_aes.m3u8"; - url = "https://bitmovin-a.akamaihd.net/content/MI201109210084_1/mpds/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.mpd"; + url = "http://playertest.longtailvideo.com/adaptive/oceans_aes/oceans_aes.m3u8"; + //url = "https://bitmovin-a.akamaihd.net/content/art-of-motion_drm/mpds/11331.mpd"; + + if (string.IsNullOrEmpty(url)) { - url = AnsiConsole.Ask("Input [green]URL[/]: "); + url = AnsiConsole.Ask("请输入 [green]URL[/]: "); } //流提取器配置 @@ -47,11 +58,6 @@ namespace N_m3u8DL_RE //解析流信息 var streams = await extractor.ExtractStreamsAsync(); - if (streams.Count == 0) - { - throw new Exception("解析失败"); - } - //全部媒体 var lists = streams.OrderByDescending(p => p.Bandwidth); //基本流 @@ -83,6 +89,8 @@ namespace N_m3u8DL_RE Logger.InfoMarkUp(item.ToString()); } + Logger.Info("按任意键继续"); + Console.ReadKey(); } catch (Exception ex) {