From 96b58e93845972cd6e575f0d72ff1813d3413b5e Mon Sep 17 00:00:00 2001 From: nilaoda Date: Thu, 17 Nov 2022 00:02:52 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E6=94=AF=E6=8C=81HTTP-TS?= =?UTF-8?q?=E5=BD=95=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/N_m3u8DL-RE.Common/Enum/ExtractorType.cs | 4 +- src/N_m3u8DL-RE.Common/Resource/ResString.cs | 2 + src/N_m3u8DL-RE.Common/Resource/StaticText.cs | 6 + src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs | 12 + src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs | 33 ++- .../Extractor/LiveTSExtractor.cs | 53 ++++ src/N_m3u8DL-RE.Parser/StreamExtractor.cs | 5 + src/N_m3u8DL-RE/Column/RecordingSizeColumn.cs | 39 +++ src/N_m3u8DL-RE/CommandLine/CommandInvoker.cs | 2 +- .../DownloadManager/HTTPLiveRecordManager.cs | 248 ++++++++++++++++++ src/N_m3u8DL-RE/Program.cs | 8 +- 11 files changed, 391 insertions(+), 21 deletions(-) create mode 100644 src/N_m3u8DL-RE.Parser/Extractor/LiveTSExtractor.cs create mode 100644 src/N_m3u8DL-RE/Column/RecordingSizeColumn.cs create mode 100644 src/N_m3u8DL-RE/DownloadManager/HTTPLiveRecordManager.cs diff --git a/src/N_m3u8DL-RE.Common/Enum/ExtractorType.cs b/src/N_m3u8DL-RE.Common/Enum/ExtractorType.cs index a50090a..27e2616 100644 --- a/src/N_m3u8DL-RE.Common/Enum/ExtractorType.cs +++ b/src/N_m3u8DL-RE.Common/Enum/ExtractorType.cs @@ -9,6 +9,8 @@ namespace N_m3u8DL_RE.Common.Enum public enum ExtractorType { MPEG_DASH, - HLS + HLS, + HTTP_LIVE, + MSS } } diff --git a/src/N_m3u8DL-RE.Common/Resource/ResString.cs b/src/N_m3u8DL-RE.Common/Resource/ResString.cs index 84a8631..4ee53fc 100644 --- a/src/N_m3u8DL-RE.Common/Resource/ResString.cs +++ b/src/N_m3u8DL-RE.Common/Resource/ResString.cs @@ -9,6 +9,7 @@ namespace N_m3u8DL_RE.Common.Resource { public class ResString { + public readonly static string ReLiveTs = ""; public static string autoBinaryMerge { get => GetText("autoBinaryMerge"); } public static string autoBinaryMerge2 { get => GetText("autoBinaryMerge2"); } public static string autoBinaryMerge3 { get => GetText("autoBinaryMerge3"); } @@ -88,6 +89,7 @@ namespace N_m3u8DL_RE.Common.Resource public static string loadingUrl { get => GetText("loadingUrl"); } public static string masterM3u8Found { get => GetText("masterM3u8Found"); } public static string matchDASH { get => GetText("matchDASH"); } + public static string matchTS { get => GetText("matchTS"); } public static string matchHLS { get => GetText("matchHLS"); } public static string notSupported { get => GetText("notSupported"); } public static string parsingStream { get => GetText("parsingStream"); } diff --git a/src/N_m3u8DL-RE.Common/Resource/StaticText.cs b/src/N_m3u8DL-RE.Common/Resource/StaticText.cs index 3ffd9de..89ccafc 100644 --- a/src/N_m3u8DL-RE.Common/Resource/StaticText.cs +++ b/src/N_m3u8DL-RE.Common/Resource/StaticText.cs @@ -601,6 +601,12 @@ namespace N_m3u8DL_RE.Common.Resource zhTW: "檢測到Master列表，開始解析全部流訊息", enUS: "Master List detected, try parse all streams" ), + ["matchTS"] = new TextContainer + ( + zhCN: "内容匹配: [white on green3]HTTP Live MPEG2-TS[/]", + zhTW: "內容匹配: [white on green3]HTTP Live MPEG2-TS[/]", + enUS: "Content Matched: [white on green3]HTTP Live MPEG2-TS[/]" + ), ["matchDASH"] = new TextContainer ( zhCN: "内容匹配: [white on mediumorchid1]Dynamic Adaptive Streaming over HTTP[/]", diff --git a/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs b/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs index e535f53..f25bc97 100644 --- a/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs +++ b/src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs @@ -42,6 +42,18 @@ namespace N_m3u8DL_RE.Common.Util return "{NOT SUPPORTED}"; } + public static string FormatFileSize(double fileSize) + { + return fileSize switch + { + < 0 => throw new ArgumentOutOfRangeException(nameof(fileSize)), + >= 1024 * 1024 * 1024 => string.Format("{0:########0.00}GB", (double)fileSize / (1024 * 1024 * 1024)), + >= 1024 * 1024 => string.Format("{0:####0.00}MB", (double)fileSize / (1024 * 1024)), + >= 1024 => string.Format("{0:####0.00}KB", (double)fileSize / 1024), + _ => string.Format("{0:####0.00}B", fileSize) + }; + } + //此函数用于格式化输出时长 public static string FormatTime(int time) { diff --git a/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs b/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs index 5b473fd..2fedae8 100644 --- a/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs +++ b/src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs @@ -1,21 +1,5 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Collections.Specialized; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Cache; -using System.Net.Http; +using System.Net; using System.Net.Http.Headers; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using System.Web; using N_m3u8DL_RE.Common.Log; using N_m3u8DL_RE.Common.Resource; @@ -110,6 +94,12 @@ namespace N_m3u8DL_RE.Common.Util return htmlCode; } + private static bool CheckMPEG2TS(HttpResponseMessage? webResponse) + { + var mediaType = webResponse?.Content.Headers.ContentType?.MediaType; + return mediaType == "video/ts" || mediaType == "video/mp2t"; + } + /// /// 获取网页源码和跳转后的URL /// @@ -120,7 +110,14 @@ namespace N_m3u8DL_RE.Common.Util { string htmlCode = string.Empty; var webResponse = await DoGetAsync(url, headers); - htmlCode = await webResponse.Content.ReadAsStringAsync(); + if (CheckMPEG2TS(webResponse)) + { + htmlCode = ResString.ReLiveTs; + } + else + { + htmlCode = await webResponse.Content.ReadAsStringAsync(); + } Logger.Debug(htmlCode); return (htmlCode, webResponse.Headers.Location != null ? webResponse.Headers.Location.AbsoluteUri : url); } diff --git a/src/N_m3u8DL-RE.Parser/Extractor/LiveTSExtractor.cs b/src/N_m3u8DL-RE.Parser/Extractor/LiveTSExtractor.cs new file mode 100644 index 0000000..eae7cc0 --- /dev/null +++ b/src/N_m3u8DL-RE.Parser/Extractor/LiveTSExtractor.cs @@ -0,0 +1,53 @@ +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Common.Enum; +using N_m3u8DL_RE.Common.Resource; +using N_m3u8DL_RE.Parser.Config; + +namespace N_m3u8DL_RE.Parser.Extractor +{ + internal class LiveTSExtractor : IExtractor + { + public ExtractorType ExtractorType => ExtractorType.HTTP_LIVE; + + public ParserConfig ParserConfig {get; set;} + + public LiveTSExtractor(ParserConfig parserConfig) + { + this.ParserConfig = parserConfig; + } + + public async Task> ExtractStreamsAsync(string rawText) + { + return new List() + { + new StreamSpec() + { + OriginalUrl = ParserConfig.OriginalUrl, + Url = ParserConfig.Url, + Playlist = new Playlist(), + GroupId = ResString.ReLiveTs + } + }; + } + + public async Task FetchPlayListAsync(List streamSpecs) + { + throw new NotImplementedException(); + } + + public async void PreProcessContent() + { + throw new NotImplementedException(); + } + + public string PreProcessUrl(string url) + { + throw new NotImplementedException(); + } + + public Task RefreshPlayListAsync(List streamSpecs) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/N_m3u8DL-RE.Parser/StreamExtractor.cs b/src/N_m3u8DL-RE.Parser/StreamExtractor.cs index 06e7aa9..0617dd6 100644 --- a/src/N_m3u8DL-RE.Parser/StreamExtractor.cs +++ b/src/N_m3u8DL-RE.Parser/StreamExtractor.cs @@ -68,6 +68,11 @@ namespace N_m3u8DL_RE.Parser //extractor = new DASHExtractor(parserConfig); extractor = new DASHExtractor2(parserConfig); } + else if (rawText == ResString.ReLiveTs) + { + Logger.InfoMarkUp(ResString.matchTS); + extractor = new LiveTSExtractor(parserConfig); + } else { throw new NotSupportedException(ResString.notSupported); diff --git a/src/N_m3u8DL-RE/Column/RecordingSizeColumn.cs b/src/N_m3u8DL-RE/Column/RecordingSizeColumn.cs new file mode 100644 index 0000000..72b3fe8 --- /dev/null +++ b/src/N_m3u8DL-RE/Column/RecordingSizeColumn.cs @@ -0,0 +1,39 @@ +using N_m3u8DL_RE.Common.Util; +using N_m3u8DL_RE.Entity; +using Spectre.Console; +using Spectre.Console.Rendering; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace N_m3u8DL_RE.Column +{ + internal class RecordingSizeColumn : ProgressColumn + { + protected override bool NoWrap => true; + private ConcurrentDictionary RecodingSizeDic = new(); //临时的大小 每秒刷新用 + private ConcurrentDictionary _recodingSizeDic; + private ConcurrentDictionary DateTimeStringDic = new(); + public Style MyStyle { get; set; } = new Style(foreground: Color.DarkCyan); + public RecordingSizeColumn(ConcurrentDictionary recodingSizeDic) + { + _recodingSizeDic = recodingSizeDic; + } + public override IRenderable Render(RenderContext context, ProgressTask task, TimeSpan deltaTime) + { + var now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"); + var taskId = task.Id; + //一秒汇报一次即可 + if (DateTimeStringDic.TryGetValue(taskId, out var oldTime) && oldTime != now) + { + RecodingSizeDic[task.Id] = _recodingSizeDic[task.Id]; + } + DateTimeStringDic[taskId] = now; + var flag = RecodingSizeDic.TryGetValue(taskId, out var size); + return new Text(GlobalUtil.FormatFileSize(flag ? size : 0), MyStyle).LeftAligned(); + } + } +} diff --git a/src/N_m3u8DL-RE/CommandLine/CommandInvoker.cs b/src/N_m3u8DL-RE/CommandLine/CommandInvoker.cs index b841966..16ee31f 100644 --- a/src/N_m3u8DL-RE/CommandLine/CommandInvoker.cs +++ b/src/N_m3u8DL-RE/CommandLine/CommandInvoker.cs @@ -18,7 +18,7 @@ namespace N_m3u8DL_RE.CommandLine { internal partial class CommandInvoker { - public const string VERSION_INFO = "N_m3u8DL-RE (Beta version) 20221115"; + public const string VERSION_INFO = "N_m3u8DL-RE (Beta version) 20221116"; [GeneratedRegex("((best|worst)\\d*|all)")] private static partial Regex ForStrRegex(); diff --git a/src/N_m3u8DL-RE/DownloadManager/HTTPLiveRecordManager.cs b/src/N_m3u8DL-RE/DownloadManager/HTTPLiveRecordManager.cs new file mode 100644 index 0000000..a2b9bb3 --- /dev/null +++ b/src/N_m3u8DL-RE/DownloadManager/HTTPLiveRecordManager.cs @@ -0,0 +1,248 @@ +using Mp4SubtitleParser; +using N_m3u8DL_RE.Column; +using N_m3u8DL_RE.Common.Entity; +using N_m3u8DL_RE.Common.Enum; +using N_m3u8DL_RE.Common.Log; +using N_m3u8DL_RE.Common.Resource; +using N_m3u8DL_RE.Common.Util; +using N_m3u8DL_RE.Config; +using N_m3u8DL_RE.Downloader; +using N_m3u8DL_RE.Entity; +using N_m3u8DL_RE.Parser; +using N_m3u8DL_RE.Util; +using Spectre.Console; +using Spectre.Console.Rendering; +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Net.Http.Headers; +using System.Reflection.PortableExecutable; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using System.Xml.Linq; + +namespace N_m3u8DL_RE.DownloadManager +{ + internal class HTTPLiveRecordManager + { + IDownloader Downloader; + DownloaderConfig DownloaderConfig; + StreamExtractor StreamExtractor; + List SelectedSteams; + List OutputFiles = new(); + DateTime NowDateTime; + DateTime? PublishDateTime; + bool STOP_FLAG = false; + bool READ_IFO = false; + ConcurrentDictionary RecordingDurDic = new(); //已录制时长 + ConcurrentDictionary RecordingSizeDic = new(); //已录制大小 + CancellationTokenSource CancellationTokenSource = new(); //取消Wait + List InfoBuffer = new List(188 * 5000); //5000个分包中解析信息，没有就算了 + + public HTTPLiveRecordManager(DownloaderConfig downloaderConfig, List selectedSteams, StreamExtractor streamExtractor) + { + this.DownloaderConfig = downloaderConfig; + Downloader = new SimpleDownloader(DownloaderConfig); + NowDateTime = DateTime.Now; + PublishDateTime = selectedSteams.FirstOrDefault()?.PublishTime; + StreamExtractor = streamExtractor; + SelectedSteams = selectedSteams; + } + + private async Task RecordStreamAsync(StreamSpec streamSpec, ProgressTask task, SpeedContainer speedContainer) + { + task.MaxValue = 1; + task.StartTask(); + + var name = streamSpec.ToShortString(); + var dirName = $"{DownloaderConfig.MyOptions.SaveName ?? NowDateTime.ToString("yyyy-MM-dd_HH-mm-ss")}_{task.Id}_{streamSpec.GroupId}_{streamSpec.Codecs}_{streamSpec.Bandwidth}_{streamSpec.Language}"; + var saveDir = DownloaderConfig.MyOptions.SaveDir ?? Environment.CurrentDirectory; + var saveName = DownloaderConfig.MyOptions.SaveName != null ? $"{DownloaderConfig.MyOptions.SaveName}.{streamSpec.Language}".TrimEnd('.') : dirName; + + Logger.Debug($"dirName: {dirName}; saveDir: {saveDir}; saveName: {saveName}"); + + //创建文件夹 + if (!Directory.Exists(saveDir)) Directory.CreateDirectory(saveDir); + + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(streamSpec.Url)); + request.Headers.ConnectionClose = false; + foreach (var item in DownloaderConfig.Headers) + { + request.Headers.TryAddWithoutValidation(item.Key, item.Value); + } + Logger.Debug(request.Headers.ToString()); + + using var response = await HTTPUtil.AppHttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancellationTokenSource.Token); + response.EnsureSuccessStatusCode(); + + var output = Path.Combine(saveDir, saveName + ".ts"); + using var stream = new FileStream(output, FileMode.Create, FileAccess.Write, FileShare.None); + using var responseStream = await response.Content.ReadAsStreamAsync(CancellationTokenSource.Token); + var buffer = new byte[16 * 1024]; + var size = 0; + + //计时器 + TimeCounterAsync(); + //读取INFO + ReadInfoAsync(); + + try + { + while ((size = await responseStream.ReadAsync(buffer, CancellationTokenSource.Token)) > 0) + { + if (!READ_IFO && InfoBuffer.Count < 188 * 5000) + { + InfoBuffer.AddRange(buffer); + } + speedContainer.Add(size); + RecordingSizeDic[task.Id] += size; + await stream.WriteAsync(buffer, 0, size); + } + } + catch (OperationCanceledException oce) when (oce.CancellationToken == CancellationTokenSource.Token) + { + ; + } + + Logger.InfoMarkUp("File Size: " + GlobalUtil.FormatFileSize(RecordingSizeDic[task.Id])); + + return true; + } + + public async Task ReadInfoAsync() + { + while (!STOP_FLAG && !READ_IFO) + { + await Task.Delay(200); + if (InfoBuffer.Count < 188 * 5000) continue; + + UInt16 ConvertToUint16(IEnumerable bytes) + { + if (BitConverter.IsLittleEndian) + bytes = bytes.Reverse(); + return BitConverter.ToUInt16(bytes.ToArray()); + } + + var data = InfoBuffer.ToArray(); + var programId = ""; + var serviceProvider = ""; + var serviceName = ""; + for (int i = 0; i < data.Length; i++) + { + if (data[i] == 0x47 && (i + 188) < data.Length && data[i + 188] == 0x47) + { + var tsData = data.Skip(i).Take(188); + var tsHeaderInt = BitConverter.ToUInt32(BitConverter.IsLittleEndian ? tsData.Take(4).Reverse().ToArray() : tsData.Take(4).ToArray(), 0); + var pid = (tsHeaderInt & 0x1fff00) >> 8; + var tsPayload = tsData.Skip(4); + //PAT + if (pid == 0x0000) + { + programId = ConvertToUint16(tsPayload.Skip(9).Take(2)).ToString(); + } + //SDT, BAT, ST + else if (pid == 0x0011) + { + var tableId = (int)tsPayload.Skip(1).First(); + //Current TS Info + if (tableId == 0x42) + { + var sectionLength = ConvertToUint16(tsPayload.Skip(2).Take(2)) & 0xfff; + var sectionData = tsPayload.Skip(4).Take(sectionLength); + var dscripData = sectionData.Skip(8); + var descriptorsLoopLength = (ConvertToUint16(dscripData.Skip(3).Take(2))) & 0xfff; + var descriptorsData = dscripData.Skip(5).Take(descriptorsLoopLength); + var serviceProviderLength = (int)descriptorsData.Skip(3).First(); + serviceProvider = Encoding.UTF8.GetString(descriptorsData.Skip(4).Take(serviceProviderLength).ToArray()); + var serviceNameLength = (int)descriptorsData.Skip(4 + serviceProviderLength).First(); + serviceName = Encoding.UTF8.GetString(descriptorsData.Skip(5 + serviceProviderLength).Take(serviceNameLength).ToArray()); + } + } + if (programId != "" && (serviceName != "" || serviceProvider != "")) + break; + } + } + + if (!string.IsNullOrEmpty(programId)) + { + Logger.InfoMarkUp($"Program Id: [cyan]{programId.EscapeMarkup()}[/]"); + if (!string.IsNullOrEmpty(serviceName)) Logger.InfoMarkUp($"Service Name: [cyan]{serviceName.EscapeMarkup()}[/]"); + if (!string.IsNullOrEmpty(serviceProvider)) Logger.InfoMarkUp($"Service Provider: [cyan]{serviceProvider.EscapeMarkup()}[/]"); + READ_IFO = true; + } + } + } + + public async Task TimeCounterAsync() + { + while (!STOP_FLAG) + { + await Task.Delay(1000); + RecordingDurDic[0]++; + + //检测时长限制 + if (RecordingDurDic.All(d => d.Value >= DownloaderConfig.MyOptions.LiveRecordLimit?.TotalSeconds)) + { + Logger.WarnMarkUp($"[darkorange3_1]{ResString.liveLimitReached}[/]"); + STOP_FLAG = true; + CancellationTokenSource.Cancel(); + } + } + } + + public async Task StartRecordAsync() + { + ConcurrentDictionary SpeedContainerDic = new(); //速度计算 + ConcurrentDictionary Results = new(); + + var progress = AnsiConsole.Progress().AutoClear(true); + + //进度条的列定义 + progress.Columns(new ProgressColumn[] + { + new TaskDescriptionColumn() { Alignment = Justify.Left }, + new RecordingDurationColumn(RecordingDurDic), //时长显示 + new RecordingSizeColumn(RecordingSizeDic), //大小显示 + new RecordingStatusColumn(), + new DownloadSpeedColumn(SpeedContainerDic), //速度计算 + new SpinnerColumn(), + }); + + await progress.StartAsync(async ctx => + { + //创建任务 + var dic = SelectedSteams.Select(item => + { + var task = ctx.AddTask(item.ToShortString(), autoStart: false, maxValue: 0); + SpeedContainerDic[task.Id] = new SpeedContainer(); //速度计算 + RecordingDurDic[task.Id] = 0; + RecordingSizeDic[task.Id] = 0; + return (item, task); + }).ToDictionary(item => item.item, item => item.task); + + DownloaderConfig.MyOptions.LiveRecordLimit = DownloaderConfig.MyOptions.LiveRecordLimit ?? TimeSpan.MaxValue; + var limit = DownloaderConfig.MyOptions.LiveRecordLimit; + if (limit != TimeSpan.MaxValue) + Logger.WarnMarkUp($"[darkorange3_1]{ResString.liveLimit}{GlobalUtil.FormatTime((int)limit.Value.TotalSeconds)}[/]"); + //录制直播时，用户选了几个流就并发录几个 + var options = new ParallelOptions() + { + MaxDegreeOfParallelism = SelectedSteams.Count + }; + //并发下载 + await Parallel.ForEachAsync(dic, options, async (kp, _) => + { + var task = kp.Value; + var consumerTask = RecordStreamAsync(kp.Key, task, SpeedContainerDic[task.Id]); + Results[kp.Key] = await consumerTask; + }); + }); + + var success = Results.Values.All(v => v == true); + + return success; + } + } +} diff --git a/src/N_m3u8DL-RE/Program.cs b/src/N_m3u8DL-RE/Program.cs index 9d89bee..535421f 100644 --- a/src/N_m3u8DL-RE/Program.cs +++ b/src/N_m3u8DL-RE/Program.cs @@ -323,7 +323,13 @@ namespace N_m3u8DL_RE }; var result = false; - if (!livingFlag) + + if (extractor.ExtractorType == ExtractorType.HTTP_LIVE) + { + var sldm = new HTTPLiveRecordManager(downloadConfig, selectedStreams, extractor); + result = await sldm.StartRecordAsync(); + } + else if(!livingFlag) { //开始下载 var sdm = new SimpleDownloadManager(downloadConfig);