From eb6421ce7218a8aeec01e770cdd33def467eefbe Mon Sep 17 00:00:00 2001 From: nilaoda Date: Sun, 19 Jun 2022 23:22:34 +0800 Subject: [PATCH] =?UTF-8?q?=E5=9F=BA=E6=9C=AC=E5=AE=8C=E6=88=90MPD?= =?UTF-8?q?=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/N_m3u8DL-RE.Common/Config/ParserConfig.cs | 9 + src/N_m3u8DL-RE.Common/Entity/Playlist.cs | 2 +- src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs | 25 +- .../Resource/ResString.Designer.cs | 27 + .../Resource/ResString.en-US.resx | 9 + .../Resource/ResString.resx | 9 + .../Resource/ResString.zh-TW.resx | 9 + src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs | 2 +- .../Extractor/DASHExtractor.cs | 740 ++++++++++++++++++ .../Extractor/HLSExtractor.cs | 36 +- .../Extractor/IExtractor.cs | 2 + src/N_m3u8DL-RE.Parser/StreamExtractor.cs | 29 +- src/N_m3u8DL-RE.Parser/Util/PromptUtil.cs | 3 + src/N_m3u8DL-RE/Program.cs | 47 +- 14 files changed, 898 insertions(+), 51 deletions(-) create mode 100644 src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor.cs diff --git a/src/N_m3u8DL-RE.Common/Config/ParserConfig.cs b/src/N_m3u8DL-RE.Common/Config/ParserConfig.cs index f59d9b3..a48dc6d 100644 --- a/src/N_m3u8DL-RE.Common/Config/ParserConfig.cs +++ b/src/N_m3u8DL-RE.Common/Config/ParserConfig.cs @@ -31,5 +31,14 @@ namespace N_m3u8DL_RE.Common.Config /// public byte[]? CustomeIV { get; set; } + /// + /// 组装视频分段的URL时，是否要把原本URL后的参数也加上去 + /// 如 Base URL = "http://xxx.com/playlist.m3u8?hmac=xxx&token=xxx" + /// 相对路径 = clip_01.ts + /// 如果 AppendUrlParams=false，得 http://xxx.com/clip_01.ts + /// 如果 AppendUrlParams=true，得 http://xxx.com/clip_01.ts?hmac=xxx&token=xxx + /// + public bool AppendUrlParams { get; set; } = false; + } } diff --git a/src/N_m3u8DL-RE.Common/Entity/Playlist.cs b/src/N_m3u8DL-RE.Common/Entity/Playlist.cs index eb5880d..0265166 100644 --- a/src/N_m3u8DL-RE.Common/Entity/Playlist.cs +++ b/src/N_m3u8DL-RE.Common/Entity/Playlist.cs @@ -12,7 +12,7 @@ namespace N_m3u8DL_RE.Common.Entity //对应Url信息 public string Url { get; set; } //是否直播 - public bool IsLive { get; set; } + public bool IsLive { get; set; } = false; //INIT信息 public MediaSegment? MediaInit { get; set; } //分片信息 diff --git a/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs b/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs index ffcbe99..7198c79 100644 --- a/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs +++ b/src/N_m3u8DL-RE.Common/Entity/StreamSpec.cs @@ -35,6 +35,8 @@ namespace N_m3u8DL_RE.Common.Entity public override string ToString() { + var prefixStr = ""; + var returnStr = ""; var encStr = string.Empty; //增加加密标志 @@ -45,19 +47,30 @@ namespace N_m3u8DL_RE.Common.Entity if (MediaType == Enum.MediaType.AUDIO) { - var d = $"{GroupId} | {Name} | {Language} | {(Channels != null ? Channels + "CH" : "")} | {(Playlist != null ? Playlist.MediaParts.Sum(x => x.MediaSegments.Count) + " Segments" : "")}".Replace("| |", "|"); - return $"[deepskyblue3]Aud[/] {encStr}" + d.EscapeMarkup().Trim().Trim('|').Trim(); + prefixStr = $"[deepskyblue3]Aud[/] {encStr}"; + var d = $"{GroupId} | {(Bandwidth != null ? (Bandwidth / 1000) + " Kbps" : "")} | {Name} | {Language} | {(Channels != null ? Channels + "CH" : "")} | {(Playlist != null ? Playlist.MediaParts.Sum(x => x.MediaSegments.Count) + " Segments" : "")}"; + returnStr = d.EscapeMarkup(); } else if (MediaType == Enum.MediaType.SUBTITLES) { - var d = $"{GroupId} | {Language} | {Name} | {(Playlist != null ? Playlist.MediaParts.Sum(x => x.MediaSegments.Count) + " Segments" : "")}".Replace("| |", "|"); - return $"[deepskyblue3_1]Sub[/] {encStr}" + d.EscapeMarkup().Trim().Trim('|').Trim(); + prefixStr = $"[deepskyblue3_1]Sub[/] {encStr}"; + var d = $"{GroupId} | {Language} | {Name} | {(Playlist != null ? Playlist.MediaParts.Sum(x => x.MediaSegments.Count) + " Segments" : "")}"; + returnStr = d.EscapeMarkup(); } else { - var d = $"{Resolution} | {Bandwidth / 1000} Kbps | {FrameRate} | {Codecs} | {(Playlist != null ? Playlist.MediaParts.Sum(x => x.MediaSegments.Count) + " Segments" : "")}".Replace("| |", "|"); - return $"[aqua]Vid[/] {encStr}" + d.EscapeMarkup().Trim().Trim('|').Trim(); + prefixStr = $"[aqua]Vid[/] {encStr}"; + var d = $"{Resolution} | {Bandwidth / 1000} Kbps | {Name} | {FrameRate} | {Codecs} | {(Playlist != null ? Playlist.MediaParts.Sum(x => x.MediaSegments.Count) + " Segments" : "")}"; + returnStr = d.EscapeMarkup(); } + + returnStr = prefixStr + returnStr.Trim().Trim('|').Trim(); + while (returnStr.Contains("| |")) + { + returnStr = returnStr.Replace("| |", "|"); + } + + return returnStr; } } } diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs b/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs index 3780251..eab35b4 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.Designer.cs @@ -69,6 +69,15 @@ namespace N_m3u8DL_RE.Common.Resource { } } + /// + /// 查找类似 验证最后一个分片有效性 的本地化字符串。 + /// + public static string checkingLast { + get { + return ResourceManager.GetString("checkingLast", resourceCulture); + } + } + /// /// 查找类似 获取: 的本地化字符串。 /// @@ -105,6 +114,15 @@ namespace N_m3u8DL_RE.Common.Resource { } } + /// + /// 查找类似 内容匹配: [white on mediumorchid1]Dynamic Adaptive Streaming over HTTP[/] 的本地化字符串。 + /// + public static string matchDASH { + get { + return ResourceManager.GetString("matchDASH", resourceCulture); + } + } + /// /// 查找类似 内容匹配: [white on deepskyblue1]HTTP Live Streaming[/] 的本地化字符串。 /// @@ -123,6 +141,15 @@ namespace N_m3u8DL_RE.Common.Resource { } } + /// + /// 查找类似 正在解析媒体信息... 的本地化字符串。 + /// + public static string parsingStream { + get { + return ResourceManager.GetString("parsingStream", resourceCulture); + } + } + /// /// 查找类似 [grey](按键盘上下键以浏览更多内容)[/] 的本地化字符串。 /// diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx b/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx index 6efdea9..335f124 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.en-US.resx @@ -136,4 +136,13 @@ Writing meta.json + + Content Matched: [white on mediumorchid1]Dynamic Adaptive Streaming over HTTP[/] + + + Verifying the validity of the last segment + + + Parsing streams... + \ No newline at end of file diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.resx b/src/N_m3u8DL-RE.Common/Resource/ResString.resx index 34dd309..a7e8d14 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.resx +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.resx @@ -136,4 +136,13 @@ 写出meta.json + + 内容匹配: [white on mediumorchid1]Dynamic Adaptive Streaming over HTTP[/] + + + 验证最后一个分片有效性 + + + 正在解析媒体信息... + \ No newline at end of file diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx b/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx index 4f5c1de..8f07b5c 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.zh-TW.resx @@ -136,4 +136,13 @@ 寫出meta.json + + 內容匹配: [white on mediumorchid1]Dynamic Adaptive Streaming over HTTP[/] + + + 驗證最後一個分片有效性 + + + 正在解析媒體信息... + \ No newline at end of file diff --git a/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs b/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs index ddd6abf..da64b33 100644 --- a/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs +++ b/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs @@ -34,7 +34,7 @@ namespace N_m3u8DL_RE.Common.Util Timeout = TimeSpan.FromMinutes(2) }; - public static async Task DoGetAsync(string url, Dictionary? headers = null) + private static async Task DoGetAsync(string url, Dictionary? headers = null) { Logger.Debug(ResString.fetch + url); using var webRequest = new HttpRequestMessage(HttpMethod.Get, url); diff --git a/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor.cs b/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor.cs new file mode 100644 index 0000000..e65180d --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Extractor/DASHExtractor.cs @@ -0,0 +1,740 @@ +using N_m3u8DL_RE.Common.Config; +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Common.Enum; +using N_m3u8DL_RE.Common.Log; +using N_m3u8DL_RE.Common.Resource; +using N_m3u8DL_RE.Common.Util; +using N_m3u8DL_RE.Parser.Util; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Xml; + +namespace N_m3u8DL_RE.Parser.Extractor +{ + //code from https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py#L2076 + internal class DASHExtractor : IExtractor + { + private string MpdUrl = string.Empty; + private string BaseUrl = string.Empty; + private string MpdContent = string.Empty; + + public ParserConfig ParserConfig { get; set; } + + public DASHExtractor(ParserConfig parserConfig) + { + this.ParserConfig = parserConfig; + this.MpdUrl = parserConfig.Url ?? string.Empty; + if (!string.IsNullOrEmpty(parserConfig.BaseUrl)) + this.BaseUrl = parserConfig.BaseUrl; + else + this.BaseUrl = this.MpdUrl; + } + + + public async Task> ExtractStreamsAsync(string rawText) + { + this.MpdContent = rawText; + this.PreProcessContent(); + + + XmlDocument mpdDoc = new XmlDocument(); + mpdDoc.LoadXml(MpdContent); + + XmlNode xn = null; + //Select MPD node + foreach (XmlNode node in mpdDoc.ChildNodes) + { + if (node.NodeType == XmlNodeType.Element && node.Name == "MPD") + { + xn = node; + break; + } + } + + var mediaPresentationDuration = ((XmlElement)xn).GetAttribute("mediaPresentationDuration"); + var ns = ((XmlElement)xn).GetAttribute("xmlns"); + + XmlNamespaceManager nsMgr = new XmlNamespaceManager(mpdDoc.NameTable); + nsMgr.AddNamespace("ns", ns); + + TimeSpan ts = XmlConvert.ToTimeSpan(mediaPresentationDuration); //时长 + + //读取在MPD开头定义的，并替换本身的URL + var baseNode = xn.SelectSingleNode("ns:BaseURL", nsMgr); + if (baseNode != null) + { + if (MpdUrl.Contains("kkbox.com.tw/")) + { + var badUrl = baseNode.InnerText; + var goodUrl = badUrl.Replace("//https:%2F%2F", "//"); + BaseUrl = goodUrl; + } + else + { + BaseUrl = baseNode.InnerText; + } + } + + var formatList = new List(); //存放所有音视频清晰度 + var periodIndex = 0; //解决同一个period且同id导致被重复添加分片 + + + foreach (XmlElement period in xn.SelectNodes("ns:Period", nsMgr)) + { + periodIndex++; + var periodDuration = string.IsNullOrEmpty(period.GetAttribute("duration")) ? XmlConvert.ToTimeSpan(mediaPresentationDuration) : XmlConvert.ToTimeSpan(period.GetAttribute("duration")); + var periodMsInfo = ExtractMultisegmentInfo(period, nsMgr, new JsonObject() + { + ["StartNumber"] = 1, + ["Timescale"] = 1 + }); + foreach (XmlElement adaptationSet in period.SelectNodes("ns:AdaptationSet", nsMgr)) + { + var adaptionSetMsInfo = ExtractMultisegmentInfo(adaptationSet, nsMgr, periodMsInfo); + foreach (XmlElement representation in adaptationSet.SelectNodes("ns:Representation", nsMgr)) + { + string GetAttribute(string key) + { + var v1 = representation.GetAttribute(key); + if (string.IsNullOrEmpty(v1)) + return adaptationSet.GetAttribute(key); + return v1; + } + + var mimeType = GetAttribute("mimeType"); + var contentType = mimeType.Split('/')[0]; + if (contentType == "video" || contentType == "audio" || contentType == "text") + { + var baseUrl = ""; + bool CheckBaseUrl() + { + return Regex.IsMatch(baseUrl, @"^https?://"); + } + + var list = new List() + { + representation.ChildNodes, + adaptationSet.ChildNodes, + period.ChildNodes, + mpdDoc.ChildNodes + }; + + foreach (XmlNodeList xmlNodeList in list) + { + foreach (XmlNode node in xmlNodeList) + { + if (node.Name == "BaseURL") + { + baseUrl = node.InnerText + baseUrl; + if (CheckBaseUrl()) break; + } + } + if (CheckBaseUrl()) break; + } + + string GetBaseUrl(string url) + { + if (url.Contains("?")) + url = url.Remove(url.LastIndexOf('?')); + url = url.Substring(0, url.LastIndexOf('/') + 1); + return url; + } + + var mpdBaseUrl = string.IsNullOrEmpty(BaseUrl) ? GetBaseUrl(MpdUrl) : BaseUrl; + if (!string.IsNullOrEmpty(mpdBaseUrl) && !CheckBaseUrl()) + { + if (!mpdBaseUrl.EndsWith("/") && !baseUrl.StartsWith("/")) + { + mpdBaseUrl += "/"; + } + baseUrl = ParserUtil.CombineURL(mpdBaseUrl, baseUrl); + } + var representationId = GetAttribute("id"); + var lang = GetAttribute("lang"); + var bandwidth = IntOrNull(GetAttribute("bandwidth")); + var frameRate = GetAttribute("frameRate"); + if (frameRate.Contains("/")) + { + var d = Convert.ToDouble(frameRate.Split('/')[0]) / Convert.ToDouble(frameRate.Split('/')[1]); + frameRate = d.ToString("0.000"); + } + var f = new JsonObject() + { + ["PeriodIndex"] = periodIndex, + ["ContentType"] = contentType, + ["FormatId"] = representationId, + ["ManifestUrl"] = MpdUrl, + ["Width"] = IntOrNull(GetAttribute("width")), + ["Height"] = IntOrNull(GetAttribute("height")), + ["Tbr"] = (int)DoubleOrNull(bandwidth), + ["Asr"] = IntOrNull(GetAttribute("audioSamplingRate")), + ["Fps"] = DoubleOrNull(frameRate), + ["Language"] = lang, + ["Codecs"] = GetAttribute("codecs") + }; + + var representationMsInfo = ExtractMultisegmentInfo(representation, nsMgr, adaptionSetMsInfo); + + string PrepareTemplate(string templateName, string[] identifiers) + { + var tmpl = representationMsInfo?[templateName]?.GetValue(); + var t = new StringBuilder(); + var inTemplate = false; + foreach (var ch in tmpl) + { + t.Append(ch); + if (ch == '$') + { + inTemplate = !inTemplate; + } + else if (ch == '%' && !inTemplate) + { + t.Append(ch); + } + } + var str = t.ToString(); + str = str.Replace("$RepresentationID$", representationId); + str = Regex.Replace(str, "\\$(" + string.Join("|", identifiers) + ")\\$", "{{$1}}"); + str = Regex.Replace(str, "\\$(" + string.Join("|", identifiers) + ")%([^$]+)d\\$", "{{$1}}{0:D$2}"); + str = str.Replace("$$", "$"); + return str; + } + + string PadNumber(string template, string key, long value) + { + string ReplaceFirst(string text, string search, string replace) + { + int pos = text.IndexOf(search); + if (pos < 0) + { + return text; + } + return text.Substring(0, pos) + replace + text.Substring(pos + search.Length); + } + + template = template.Replace("{{" + key + "}}", ""); + var m = Regex.Match(template, "{0:D(\\d+)}"); + return ReplaceFirst(template, m.Value, value.ToString("0".PadRight(Convert.ToInt32(m.Groups[1].Value), '0'))); + } + + if (representationMsInfo.ContainsKey("Initialization")) + { + var initializationTemplate = PrepareTemplate("Initialization", new string[] { "Bandwidth" }); + var initializationUrl = ""; + if (initializationTemplate.Contains("{0:D")) + { + if (initializationTemplate.Contains("{{Bandwidth}}")) + initializationUrl = PadNumber(initializationTemplate, "Bandwidth", bandwidth); + } + else + { + initializationUrl = initializationTemplate.Replace("{{Bandwidth}}", bandwidth.ToString()); + } + representationMsInfo["InitializationUrl"] = ParserUtil.CombineURL(baseUrl, initializationUrl); + } + + string LocationKey(string location) + { + return Regex.IsMatch(location, "^https?://") ? "url" : "path"; + } + + if (!representationMsInfo.ContainsKey("SegmentUrls") && representationMsInfo.ContainsKey("Media")) + { + var mediaTemplate = PrepareTemplate("Media", new string[] { "Number", "Bandwidth", "Time" }); + var mediaLocationKey = LocationKey(mediaTemplate); + + if (mediaTemplate.Contains("{{Number") && !representationMsInfo.ContainsKey("S")) + { + var segmentDuration = 0.0; + if (!representationMsInfo.ContainsKey("TotalNumber") && representationMsInfo.ContainsKey("SegmentDuration")) + { + segmentDuration = DoubleOrNull(representationMsInfo["SegmentDuration"].GetValue(), representationMsInfo["Timescale"].GetValue()); + representationMsInfo["TotalNumber"] = (int)Math.Ceiling(periodDuration.TotalSeconds / segmentDuration); + } + var fragments = new JsonArray(); + for (int i = representationMsInfo["StartNumber"].GetValue(); i < representationMsInfo["StartNumber"].GetValue() + representationMsInfo["TotalNumber"].GetValue(); i++) + { + var segUrl = ""; + if (mediaTemplate.Contains("{0:D")) + { + if (mediaTemplate.Contains("{{Bandwidth}}")) + segUrl = PadNumber(mediaTemplate, "Bandwidth", bandwidth); + if (mediaTemplate.Contains("{{Number}}")) + segUrl = PadNumber(mediaTemplate, "Number", i); + } + else + { + segUrl = mediaTemplate.Replace("{{Bandwidth}}", bandwidth.ToString()); + segUrl = segUrl.Replace("{{Number}}", i.ToString()); + } + fragments.Add(new JsonObject() + { + [mediaLocationKey] = ParserUtil.CombineURL(baseUrl, segUrl), + ["duration"] = segmentDuration + }); + } + representationMsInfo["Fragments"] = fragments; + } + else + { + var fragments = new JsonArray(); + + var segmentTime = 0L; + var segmentD = 0L; + var segmentNumber = representationMsInfo["StartNumber"].GetValue(); + + void addSegmentUrl() + { + var segUrl = ""; + if (mediaTemplate.Contains("{0:D")) + { + if (mediaTemplate.Contains("{{Bandwidth}}")) + segUrl = PadNumber(mediaTemplate, "Bandwidth", bandwidth); + if (mediaTemplate.Contains("{{Number}}")) + segUrl = PadNumber(mediaTemplate, "Number", segmentNumber); + if (mediaTemplate.Contains("{{Time}}")) + segUrl = PadNumber(mediaTemplate, "Time", segmentTime); + } + else + { + segUrl = mediaTemplate.Replace("{{Bandwidth}}", bandwidth.ToString()); + segUrl = segUrl.Replace("{{Number}}", segmentNumber.ToString()); + segUrl = segUrl.Replace("{{Time}}", segmentTime.ToString()); + } + fragments.Add(new JsonObject() + { + [mediaLocationKey] = ParserUtil.CombineURL(baseUrl, segUrl), + ["duration"] = DoubleOrNull(segmentD, representationMsInfo["Timescale"].GetValue()) + }); + } + + if (representationMsInfo.ContainsKey("S")) + { + var array = representationMsInfo["S"].GetValue(); + for (int i = 0; i < array.Count; i++) + { + var s = array[i]; + segmentTime = s["t"].GetValue() == 0L ? segmentTime : s["t"].GetValue(); + segmentD = s["d"].GetValue(); + addSegmentUrl(); + segmentNumber++; + for (int j = 0; j < s["r"].GetValue(); j++) + { + segmentTime += segmentD; + addSegmentUrl(); + segmentNumber++; + } + segmentTime += segmentD; + } + } + representationMsInfo["Fragments"] = fragments; + } + } + else if (representationMsInfo.ContainsKey("SegmentUrls") && representationMsInfo.ContainsKey("S")) + { + var fragments = new JsonArray(); + + var segmentIndex = 0; + var timescale = representationMsInfo["Timescale"].GetValue(); + foreach (var s in representationMsInfo["S"].GetValue()) + { + var duration = DoubleOrNull(s["d"], timescale); + for (int j = 0; j < s["r"].GetValue() + 1; j++) + { + var segmentUri = representationMsInfo["SegmentUrls"][segmentIndex].GetValue(); + fragments.Add(new JsonObject() + { + [LocationKey(segmentUri)] = ParserUtil.CombineURL(baseUrl, segmentUri), + ["duration"] = duration + }); + segmentIndex++; + } + } + + representationMsInfo["Fragments"] = fragments; + } + else if (representationMsInfo.ContainsKey("SegmentUrls")) + { + var fragments = new JsonArray(); + + var segmentDuration = DoubleOrNull(representationMsInfo["SegmentDuration"].GetValue(), representationMsInfo.ContainsKey("SegmentDuration") ? representationMsInfo["Timescale"].GetValue() : 1); + foreach (var jsonNode in representationMsInfo["SegmentUrls"].GetValue()) + { + var segmentUrl = jsonNode.GetValue(); + if (segmentDuration != -1) + { + fragments.Add(new JsonObject() + { + [LocationKey(segmentUrl)] = ParserUtil.CombineURL(baseUrl, segmentUrl), + ["duration"] = segmentDuration + }); + } + else + { + fragments.Add(new JsonObject() + { + [LocationKey(segmentUrl)] = ParserUtil.CombineURL(baseUrl, segmentUrl) + }); + } + } + + representationMsInfo["Fragments"] = fragments; + } + + if (representationMsInfo.ContainsKey("Fragments")) + { + f["Url"] = string.IsNullOrEmpty(MpdUrl) ? baseUrl : MpdUrl; + f["FragmentBaseUrl"] = baseUrl; + if (representationMsInfo.ContainsKey("InitializationUrl")) + { + f["InitializationUrl"] = ParserUtil.CombineURL(baseUrl, representationMsInfo["InitializationUrl"].GetValue()); + if (f["InitializationUrl"].GetValue().StartsWith("$$Range")) + { + f["InitializationUrl"] = ParserUtil.CombineURL(baseUrl, f["InitializationUrl"].GetValue()); + } + f["Fragments"] = JsonArray.Parse(representationMsInfo["Fragments"].ToJsonString()); + } + } + else + { + //整段mp4 + f["Fragments"] = new JsonArray() { + new JsonObject() + { + ["url"] = baseUrl, + ["duration"] = ts.TotalSeconds + } + }; + } + + //处理同一ID分散在不同Period的情况 + if (formatList.Any(_f => _f["FormatId"].ToJsonString() == f["FormatId"].ToJsonString() && _f["Width"].ToJsonString() == f["Width"].ToJsonString() && _f["ContentType"].ToJsonString() == f["ContentType"].ToJsonString())) + { + for (int i = 0; i < formatList.Count; i++) + { + //参数相同但不在同一个Period才可以 + if (formatList[i]["FormatId"].ToJsonString() == f["FormatId"].ToJsonString() && formatList[i]["Width"].ToJsonString() == f["Width"].ToJsonString() && formatList[i]["ContentType"].ToJsonString() == f["ContentType"].ToJsonString() && formatList[i]["PeriodIndex"].ToJsonString() != f["PeriodIndex"].ToJsonString()) + { + var array = formatList[i]["Fragments"].AsArray(); + foreach (var item in f["Fragments"].AsArray()) + { + array.Add(item); + } + formatList[i]["Fragments"] = array; + break; + } + } + } + else + { + formatList.Add(f); + } + } + } + } + } + + var streamList = new List(); + foreach (var item in formatList) + { + //基本信息 + StreamSpec streamSpec = new(); + streamSpec.Name = item["FormatId"].GetValue(); + streamSpec.Bandwidth = item["Tbr"].GetValue(); + streamSpec.Codecs = item["Codecs"].GetValue(); + streamSpec.Language = item["Language"].GetValue(); + streamSpec.FrameRate = item["Fps"].GetValue(); + streamSpec.Resolution = item["Width"].GetValue() != -1 ? $"{item["Width"]}x{item["Height"]}" : ""; + streamSpec.Url = MpdUrl; + streamSpec.MediaType = item["ContentType"].GetValue() switch + { + "text" => MediaType.SUBTITLES, + "audio" => MediaType.AUDIO, + _ => null + }; + + //组装分片 + Playlist playlist = new(); + List segments = new(); + //Initial URL + if (item.ContainsKey("InitializationUrl")) + { + var initUrl = item["InitializationUrl"].GetValue(); + if (Regex.IsMatch(initUrl, "\\$\\$Range=(\\d+)-(\\d+)")) + { + var match = Regex.Match(initUrl, "\\$\\$Range=(\\d+)-(\\d+)"); + string rangeStr = match.Value; + long start = Convert.ToInt64(match.Groups[1].Value); + long end = Convert.ToInt64(match.Groups[2].Value); + playlist.MediaInit = new MediaSegment() + { + EncryptInfo = new EncryptInfo() + { + Method = EncryptMethod.UNKNOWN + }, + Url = PreProcessUrl(initUrl.Replace(rangeStr, "")), + StartRange = start, + ExpectLength = end - start + 1 + }; + } + else + { + playlist.MediaInit = new MediaSegment() + { + Url = PreProcessUrl(initUrl) + }; + } + } + //分片地址 + var fragments = item["Fragments"].AsArray(); + var index = 0; + foreach (var fragment in fragments) + { + var seg = fragment.AsObject(); + var dur = seg.ContainsKey("duration") ? seg["duration"].GetValue() : 0.0; + var url = seg.ContainsKey("url") ? seg["url"].GetValue() : seg["path"].GetValue(); + url = PreProcessUrl(url); + MediaSegment mediaSegment = new() + { + Index = index, + Duration = dur, + Url = url, + }; + + if (Regex.IsMatch(url, "\\$\\$Range=(\\d+)-(\\d+)")) + { + var match = Regex.Match(url, "\\$\\$Range=(\\d+)-(\\d+)"); + string rangeStr = match.Value; + long start = Convert.ToInt64(match.Groups[1].Value); + long end = Convert.ToInt64(match.Groups[2].Value); + mediaSegment.StartRange = start; + mediaSegment.ExpectLength = end - start + 1; + } + + segments.Add(mediaSegment); + index++; + } + playlist.MediaParts = new List() + { + new MediaPart() + { + MediaSegments = segments + } + }; + + //统一添加EncryptInfo + playlist.MediaInit.EncryptInfo = new EncryptInfo() + { + Method = EncryptMethod.UNKNOWN + }; + foreach (var seg in playlist.MediaParts[0].MediaSegments) + { + seg.EncryptInfo = new EncryptInfo() + { + Method = EncryptMethod.UNKNOWN + }; + } + streamSpec.Playlist = playlist; + streamList.Add(streamSpec); + } + return streamList; + } + + static bool CheckValid(string url) + { + try + { + HttpWebRequest request = (HttpWebRequest)WebRequest.Create(new Uri(url)); + request.Timeout = 120000; + HttpWebResponse response = (HttpWebResponse)request.GetResponse(); + if (((int)response.StatusCode).ToString().StartsWith("2")) return true; + else return false; + } + catch (Exception) { return false; } + } + + static double DoubleOrNull(object text, int scale = 1) + { + try + { + return Convert.ToDouble(text) / scale; + } + catch (Exception) + { + return -1; + } + } + + static int IntOrNull(string text, int scale = 1) + { + try + { + return Convert.ToInt32(text) / scale; + } + catch (Exception) + { + return -1; + } + } + + private JsonObject ExtractMultisegmentInfo(XmlElement period, XmlNamespaceManager nsMgr, JsonObject jsonObject) + { + var MultisegmentInfo = new JsonObject(); + foreach (var item in jsonObject) + { + MultisegmentInfo[item.Key] = JsonNode.Parse(item.Value.ToJsonString()); + } + void ExtractCommon(XmlNode source) + { + var sourceE = (XmlElement)source; + var segmentTimeline = source.SelectSingleNode("ns:SegmentTimeline", nsMgr); + if (segmentTimeline != null) + { + var sE = segmentTimeline.SelectNodes("ns:S", nsMgr); + if (sE?.Count > 0) + { + MultisegmentInfo["TotalNumber"] = 0; + var SList = new JsonArray(); + foreach (XmlElement s in sE) + { + var r = string.IsNullOrEmpty(s.GetAttribute("r")) ? 0 : Convert.ToInt64(s.GetAttribute("r")); + MultisegmentInfo["TotalNumber"] = MultisegmentInfo["TotalNumber"]?.GetValue() + 1 + r; + SList.Add(new JsonObject() + { + ["t"] = string.IsNullOrEmpty(s.GetAttribute("t")) ? 0 : Convert.ToInt64(s.GetAttribute("t")), + ["d"] = Convert.ToInt64(s.GetAttribute("d")), + ["r"] = r + }); + } + MultisegmentInfo.Add("S", SList); + } + } + var startNumber = sourceE.GetAttribute("startNumber"); + if (!string.IsNullOrEmpty(startNumber)) + { + MultisegmentInfo["StartNumber"] = Convert.ToInt32(startNumber); + } + var timescale = sourceE.GetAttribute("timescale"); + if (!string.IsNullOrEmpty(timescale)) + { + MultisegmentInfo["Timescale"] = Convert.ToInt32(timescale); + } + var segmentDuration = sourceE.GetAttribute("duration"); + if (!string.IsNullOrEmpty(segmentDuration)) + { + MultisegmentInfo["SegmentDuration"] = Convert.ToDouble(segmentDuration); + } + } + + void ExtractInitialization(XmlNode source) + { + var initialization = source.SelectSingleNode("ns:Initialization", nsMgr); + if (initialization != null) + { + MultisegmentInfo["InitializationUrl"] = ((XmlElement)initialization).GetAttribute("sourceURL"); + if (((XmlElement)initialization).HasAttribute("range")) + { + MultisegmentInfo["InitializationUrl"] += "$$Range=" + ((XmlElement)initialization).GetAttribute("range"); + } + } + } + + var segmentList = period.SelectSingleNode("ns:SegmentList", nsMgr); + if (segmentList != null) + { + ExtractCommon(segmentList); + ExtractInitialization(segmentList); + var segmentUrlsE = segmentList.SelectNodes("ns:SegmentURL", nsMgr); + var urls = new JsonArray(); + foreach (XmlElement segment in segmentUrlsE) + { + if (segment.HasAttribute("mediaRange")) + { + urls.Add("$$Range=" + segment.GetAttribute("mediaRange")); + } + else + { + urls.Add(segment.GetAttribute("media")); + } + } + MultisegmentInfo["SegmentUrls"] = urls; + } + else + { + var segmentTemplate = period.SelectSingleNode("ns:SegmentTemplate", nsMgr); + if (segmentTemplate != null) + { + ExtractCommon(segmentTemplate); + var media = ((XmlElement)segmentTemplate).GetAttribute("media"); + if (!string.IsNullOrEmpty(media)) + { + MultisegmentInfo["Media"] = media; + } + var initialization = ((XmlElement)segmentTemplate).GetAttribute("initialization"); + if (!string.IsNullOrEmpty(initialization)) + { + MultisegmentInfo["Initialization"] = initialization; + } + else + { + ExtractInitialization(segmentTemplate); + } + } + } + + return MultisegmentInfo; + } + + /// + /// 预处理URL + /// + private string PreProcessUrl(string url) + { + if (ParserConfig.AppendUrlParams) + { + url += new Regex("\\?.*").Match(MpdUrl).Value; + } + + return url; + } + + private void PreProcessContent() + { + //XiGua + if (this.MpdContent.Contains(" streamSpecs) + { + //MPD不需要重新去读取内容，只判断一下最后的分片是否有效即可 + foreach (var item in streamSpecs) + { + //检测最后一片的有效性 + if (item.Playlist.MediaParts[0].MediaSegments.Count > 1) + { + var last = item.Playlist.MediaParts[0].MediaSegments.Last(); + var secondToLast = item.Playlist.MediaParts[0].MediaSegments[item.Playlist.MediaParts[0].MediaSegments.Count - 2]; + var urlLast = last.Url; + var urlSecondToLast = secondToLast.Url; + //普通分段才判断 + if (urlLast.StartsWith("http") && !Regex.IsMatch(urlLast, "\\$\\$Range=(\\d+)-(\\d+)")) + { + Logger.Warn(ResString.checkingLast + $"({(item.MediaType == MediaType.AUDIO ? "Audio" : (item.MediaType == MediaType.SUBTITLES ? "Sub" : "Video"))})"); + //倒数第二段正常，倒数第一段不正常 + if (CheckValid(urlSecondToLast) && !CheckValid(urlLast)) + item.Playlist.MediaParts[0].MediaSegments.RemoveAt(item.Playlist.MediaParts[0].MediaSegments.Count - 1); + } + } + } + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/Extractor/HLSExtractor.cs b/src/N_m3u8DL-RE.Parser/Extractor/HLSExtractor.cs index d74407f..c522fb5 100644 --- a/src/N_m3u8DL-RE.Parser/Extractor/HLSExtractor.cs +++ b/src/N_m3u8DL-RE.Parser/Extractor/HLSExtractor.cs @@ -107,14 +107,11 @@ namespace N_m3u8DL_RE.Parser.Extractor /// private string PreProcessUrl(string url) { - if (url.Contains("?__gda__")) - { - url += new Regex("\\?__gda__.*").Match(M3u8Url).Value; - } - if (M3u8Url.Contains("//dlsc.hcs.cmvideo.cn") && (url.EndsWith(".ts") || url.EndsWith(".mp4"))) + if (ParserConfig.AppendUrlParams) { url += new Regex("\\?.*").Match(M3u8Url).Value; } + return url; } @@ -480,9 +477,16 @@ namespace N_m3u8DL_RE.Parser.Extractor private async Task ParseKeyAsync(string uri) { - var segUrl = PreProcessUrl(ParserUtil.CombineURL(BaseUrl, uri)); - var bytes = await HTTPUtil.GetBytesAsync(segUrl, ParserConfig.Headers); - return bytes; + if (uri.ToLower().StartsWith("base64:")) + { + return Convert.FromBase64String(uri.Substring(7)); + } + else + { + var segUrl = PreProcessUrl(ParserUtil.CombineURL(BaseUrl, uri)); + var bytes = await HTTPUtil.GetBytesAsync(segUrl, ParserConfig.Headers); + return bytes; + } } public async Task> ExtractStreamsAsync(string rawText) @@ -494,12 +498,6 @@ namespace N_m3u8DL_RE.Parser.Extractor Logger.Warn(ResString.masterM3u8Found); var lists = await ParseMasterListAsync(); lists = lists.DistinctBy(p => p.Url).ToList(); - for (int i = 0; i < lists.Count; i++) - { - //重新加载m3u8 - await LoadM3u8FromUrlAsync(lists[i].Url); - lists[i].Playlist = await ParseListAsync(); - } return lists; } else @@ -530,5 +528,15 @@ namespace N_m3u8DL_RE.Parser.Extractor this.M3u8Url = this.BaseUrl = url; this.PreProcessContent(); } + + public async Task FetchPlayListAsync(List lists) + { + for (int i = 0; i < lists.Count; i++) + { + //重新加载m3u8 + await LoadM3u8FromUrlAsync(lists[i].Url); + lists[i].Playlist = await ParseListAsync(); + } + } } } diff --git a/src/N_m3u8DL-RE.Parser/Extractor/IExtractor.cs b/src/N_m3u8DL-RE.Parser/Extractor/IExtractor.cs index 50b84e6..9c40a7b 100644 --- a/src/N_m3u8DL-RE.Parser/Extractor/IExtractor.cs +++ b/src/N_m3u8DL-RE.Parser/Extractor/IExtractor.cs @@ -13,5 +13,7 @@ namespace N_m3u8DL_RE.Parser.Extractor ParserConfig ParserConfig { get; set; } Task> ExtractStreamsAsync(string rawText); + + Task FetchPlayListAsync(List streamSpecs); } } diff --git a/src/N_m3u8DL-RE.Parser/StreamExtractor.cs b/src/N_m3u8DL-RE.Parser/StreamExtractor.cs index 99a46f6..a2360e2 100644 --- a/src/N_m3u8DL-RE.Parser/StreamExtractor.cs +++ b/src/N_m3u8DL-RE.Parser/StreamExtractor.cs @@ -38,6 +38,11 @@ namespace N_m3u8DL_RE.Parser this.rawText = HTTPUtil.GetWebSourceAsync(url, parserConfig.Headers).Result; parserConfig.Url = url; } + else if (File.Exists(url)) + { + this.rawText = File.ReadAllText(url); + parserConfig.Url = new Uri(url).AbsoluteUri; + } this.rawText = rawText.Trim(); LoadSourceFromText(this.rawText); } @@ -51,9 +56,10 @@ namespace N_m3u8DL_RE.Parser Logger.InfoMarkUp(ResString.matchHLS); extractor = new HLSExtractor(parserConfig); } - else if (rawText.StartsWith("..")) + else if (rawText.Contains("> ExtractStreamsAsync() + /// + /// 开始解析流媒体信息 + /// + /// + public async Task> ExtractStreamsAsync() { - return extractor.ExtractStreamsAsync(rawText); + Logger.Info(ResString.parsingStream); + return await extractor.ExtractStreamsAsync(rawText); + } + + /// + /// 根据规格说明填充媒体播放列表信息 + /// + /// + public async Task FetchPlayListAsync(List streamSpecs) + { + Logger.Info(ResString.parsingStream); + await extractor.FetchPlayListAsync(streamSpecs); } } } diff --git a/src/N_m3u8DL-RE.Parser/Util/PromptUtil.cs b/src/N_m3u8DL-RE.Parser/Util/PromptUtil.cs index ce4026b..4755bcd 100644 --- a/src/N_m3u8DL-RE.Parser/Util/PromptUtil.cs +++ b/src/N_m3u8DL-RE.Parser/Util/PromptUtil.cs @@ -14,6 +14,9 @@ namespace N_m3u8DL_RE.Parser.Util { public static List SelectStreams(IEnumerable lists) { + if (lists.Count() == 1) + return new List(lists); + //基本流 var basicStreams = lists.Where(x => x.MediaType == null); //可选音频轨道 diff --git a/src/N_m3u8DL-RE/Program.cs b/src/N_m3u8DL-RE/Program.cs index 0ab34f8..f2d9b0b 100644 --- a/src/N_m3u8DL-RE/Program.cs +++ b/src/N_m3u8DL-RE/Program.cs @@ -32,8 +32,8 @@ namespace N_m3u8DL_RE //Logger.LogLevel = LogLevel.DEBUG; var config = new ParserConfig(); var url = string.Empty; - //url = "http://playertest.longtailvideo.com/adaptive/oceans_aes/oceans_aes.m3u8"; - url = "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8"; + url = "http://playertest.longtailvideo.com/adaptive/oceans_aes/oceans_aes.m3u8"; + //url = "https://bitmovin-a.akamaihd.net/content/MI201109210084_1/mpds/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.mpd"; if (string.IsNullOrEmpty(url)) { @@ -44,7 +44,14 @@ namespace N_m3u8DL_RE var extractor = new StreamExtractor(config); extractor.LoadSourceFromUrl(url); + //解析流信息 var streams = await extractor.ExtractStreamsAsync(); + + if (streams.Count == 0) + { + throw new Exception("解析失败"); + } + //全部媒体 var lists = streams.OrderByDescending(p => p.Bandwidth); //基本流 @@ -59,33 +66,23 @@ namespace N_m3u8DL_RE Logger.Info(ResString.streamsInfo, lists.Count(), basicStreams.Count(), audios.Count(), subs.Count()); - if (streams.Count > 1) + foreach (var item in lists) { - - foreach (var item in lists) Logger.InfoMarkUp(item.ToString()); - - var selectedStreams = PromptUtil.SelectStreams(lists); - - Logger.Info(ResString.selectedStream); - await File.WriteAllTextAsync("meta_selected.json", GlobalUtil.ConvertToJson(selectedStreams), Encoding.UTF8); - foreach (var item in selectedStreams) - { - Logger.InfoMarkUp(item.ToString()); - } + Logger.InfoMarkUp(item.ToString()); } - else if (streams.Count == 1) + + //展示交互式选择框 + var selectedStreams = PromptUtil.SelectStreams(lists); + //加载playlist + Logger.Info(ResString.selectedStream); + await extractor.FetchPlayListAsync(selectedStreams); + Logger.Warn(ResString.writeJson); + await File.WriteAllTextAsync("meta_selected.json", GlobalUtil.ConvertToJson(selectedStreams), Encoding.UTF8); + foreach (var item in selectedStreams) { - var playlist = streams.First().Playlist; - if (playlist.IsLive) - { - Logger.Warn(ResString.liveFound); - } - //Print(playlist); - } - else - { - throw new Exception("解析失败"); + Logger.InfoMarkUp(item.ToString()); } + } catch (Exception ex) {