初步支持HTTP-TS录制

src/N_m3u8DL-RE.Common/Enum/ExtractorType.cs

@@ -9,6 +9,8 @@ namespace N_m3u8DL_RE.Common.Enum
     public enum ExtractorType
     {
         MPEG_DASH,
-        HLS
+        HLS,
+        HTTP_LIVE,
+        MSS
     }
 }

src/N_m3u8DL-RE.Common/Resource/ResString.cs

@@ -9,6 +9,7 @@ namespace N_m3u8DL_RE.Common.Resource
 {
     public class ResString
     {
+        public readonly static string ReLiveTs = "<RE_LIVE_TS>";
         public static string autoBinaryMerge { get => GetText("autoBinaryMerge"); }
         public static string autoBinaryMerge2 { get => GetText("autoBinaryMerge2"); }
         public static string autoBinaryMerge3 { get => GetText("autoBinaryMerge3"); }
@@ -88,6 +89,7 @@ namespace N_m3u8DL_RE.Common.Resource
         public static string loadingUrl { get => GetText("loadingUrl"); }
         public static string masterM3u8Found { get => GetText("masterM3u8Found"); }
         public static string matchDASH { get => GetText("matchDASH"); }
+        public static string matchTS { get => GetText("matchTS"); }
         public static string matchHLS { get => GetText("matchHLS"); }
         public static string notSupported { get => GetText("notSupported"); }
         public static string parsingStream { get => GetText("parsingStream"); }

src/N_m3u8DL-RE.Common/Resource/StaticText.cs

@@ -601,6 +601,12 @@ namespace N_m3u8DL_RE.Common.Resource
                 zhTW: "檢測到Master列表，開始解析全部流訊息",
                 enUS: "Master List detected, try parse all streams"
             ),
+            ["matchTS"] = new TextContainer
+            (
+                zhCN: "内容匹配: [white on green3]HTTP Live MPEG2-TS[/]",
+                zhTW: "內容匹配: [white on green3]HTTP Live MPEG2-TS[/]",
+                enUS: "Content Matched: [white on green3]HTTP Live MPEG2-TS[/]"
+            ),
             ["matchDASH"] = new TextContainer
             (
                 zhCN: "内容匹配: [white on mediumorchid1]Dynamic Adaptive Streaming over HTTP[/]",

src/N_m3u8DL-RE.Common/Util/GlobalUtil.cs

@@ -42,6 +42,18 @@ namespace N_m3u8DL_RE.Common.Util
             return "{NOT SUPPORTED}";
         }
 
+        public static string FormatFileSize(double fileSize)
+        {
+            return fileSize switch
+            {
+                < 0 => throw new ArgumentOutOfRangeException(nameof(fileSize)),
+                >= 1024 * 1024 * 1024 => string.Format("{0:########0.00}GB", (double)fileSize / (1024 * 1024 * 1024)),
+                >= 1024 * 1024 => string.Format("{0:####0.00}MB", (double)fileSize / (1024 * 1024)),
+                >= 1024 => string.Format("{0:####0.00}KB", (double)fileSize / 1024),
+                _ => string.Format("{0:####0.00}B", fileSize)
+            };
+        }
+
         //此函数用于格式化输出时长  
         public static string FormatTime(int time)
         {

src/N_m3u8DL-RE.Common/Util/HTTPUtil.cs

@@ -1,21 +1,5 @@
-using System;
+using System.Net;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Collections.Specialized;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Net;
-using System.Net.Cache;
-using System.Net.Http;
 using System.Net.Http.Headers;
-using System.Security.Cryptography;
-using System.Text;
-using System.Text.Json;
-using System.Text.RegularExpressions;
-using System.Threading;
-using System.Threading.Tasks;
-using System.Web;
 using N_m3u8DL_RE.Common.Log;
 using N_m3u8DL_RE.Common.Resource;
 
@@ -110,6 +94,12 @@ namespace N_m3u8DL_RE.Common.Util
             return htmlCode;
         }
 
+        private static bool CheckMPEG2TS(HttpResponseMessage? webResponse)
+        {
+            var mediaType = webResponse?.Content.Headers.ContentType?.MediaType;
+            return mediaType == "video/ts" || mediaType == "video/mp2t";
+        }
+
         /// <summary>
         /// 获取网页源码和跳转后的URL
         /// </summary>
@@ -120,7 +110,14 @@ namespace N_m3u8DL_RE.Common.Util
         {
             string htmlCode = string.Empty;
             var webResponse = await DoGetAsync(url, headers);
+            if (CheckMPEG2TS(webResponse))
+            {
+                htmlCode = ResString.ReLiveTs;
+            }
+            else
+            {
                 htmlCode = await webResponse.Content.ReadAsStringAsync();
+            }
             Logger.Debug(htmlCode);
             return (htmlCode, webResponse.Headers.Location != null ? webResponse.Headers.Location.AbsoluteUri : url);
         }

src/N_m3u8DL-RE.Parser/Extractor/LiveTSExtractor.cs

@@ -0,0 +1,53 @@
+using N_m3u8DL_RE.Common.Entity;
+using N_m3u8DL_RE.Common.Enum;
+using N_m3u8DL_RE.Common.Resource;
+using N_m3u8DL_RE.Parser.Config;
+
+namespace N_m3u8DL_RE.Parser.Extractor
+{
+    internal class LiveTSExtractor : IExtractor
+    {
+        public ExtractorType ExtractorType => ExtractorType.HTTP_LIVE;
+
+        public ParserConfig ParserConfig {get; set;}
+
+        public LiveTSExtractor(ParserConfig parserConfig)
+        {
+            this.ParserConfig = parserConfig;
+        }
+
+        public async Task<List<StreamSpec>> ExtractStreamsAsync(string rawText)
+        {
+            return new List<StreamSpec>()
+            {
+                new StreamSpec()
+                {
+                    OriginalUrl = ParserConfig.OriginalUrl,
+                    Url = ParserConfig.Url,
+                    Playlist = new Playlist(),
+                    GroupId = ResString.ReLiveTs
+                }
+            };
+        }
+
+        public async Task FetchPlayListAsync(List<StreamSpec> streamSpecs)
+        {
+            throw new NotImplementedException();
+        }
+
+        public async void PreProcessContent()
+        {
+            throw new NotImplementedException();
+        }
+
+        public string PreProcessUrl(string url)
+        {
+            throw new NotImplementedException();
+        }
+
+        public Task RefreshPlayListAsync(List<StreamSpec> streamSpecs)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

src/N_m3u8DL-RE.Parser/StreamExtractor.cs

@@ -68,6 +68,11 @@ namespace N_m3u8DL_RE.Parser
                 //extractor = new DASHExtractor(parserConfig);
                 extractor = new DASHExtractor2(parserConfig);
             }
+            else if (rawText == ResString.ReLiveTs)
+            {
+                Logger.InfoMarkUp(ResString.matchTS);
+                extractor = new LiveTSExtractor(parserConfig);
+            }
             else
             {
                 throw new NotSupportedException(ResString.notSupported);

src/N_m3u8DL-RE/Column/RecordingSizeColumn.cs

@@ -0,0 +1,39 @@
+using N_m3u8DL_RE.Common.Util;
+using N_m3u8DL_RE.Entity;
+using Spectre.Console;
+using Spectre.Console.Rendering;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace N_m3u8DL_RE.Column
+{
+    internal class RecordingSizeColumn : ProgressColumn
+    {
+        protected override bool NoWrap => true;
+        private ConcurrentDictionary<int, double> RecodingSizeDic = new(); //临时的大小 每秒刷新用
+        private ConcurrentDictionary<int, double> _recodingSizeDic;
+        private ConcurrentDictionary<int, string> DateTimeStringDic = new();
+        public Style MyStyle { get; set; } = new Style(foreground: Color.DarkCyan);
+        public RecordingSizeColumn(ConcurrentDictionary<int, double> recodingSizeDic)
+        {
+            _recodingSizeDic = recodingSizeDic;
+        }
+        public override IRenderable Render(RenderContext context, ProgressTask task, TimeSpan deltaTime)
+        {
+            var now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss");
+            var taskId = task.Id;
+            //一秒汇报一次即可
+            if (DateTimeStringDic.TryGetValue(taskId, out var oldTime) && oldTime != now)
+            {
+                RecodingSizeDic[task.Id] = _recodingSizeDic[task.Id];
+            }
+            DateTimeStringDic[taskId] = now;
+            var flag = RecodingSizeDic.TryGetValue(taskId, out var size);
+            return new Text(GlobalUtil.FormatFileSize(flag ? size : 0), MyStyle).LeftAligned();
+        }
+    }
+}

src/N_m3u8DL-RE/CommandLine/CommandInvoker.cs

@@ -18,7 +18,7 @@ namespace N_m3u8DL_RE.CommandLine
 {
     internal partial class CommandInvoker
     {
-        public const string VERSION_INFO = "N_m3u8DL-RE (Beta version) 20221115";
+        public const string VERSION_INFO = "N_m3u8DL-RE (Beta version) 20221116";
 
         [GeneratedRegex("((best|worst)\\d*|all)")]
         private static partial Regex ForStrRegex();

src/N_m3u8DL-RE/DownloadManager/HTTPLiveRecordManager.cs

@@ -0,0 +1,248 @@
+using Mp4SubtitleParser;
+using N_m3u8DL_RE.Column;
+using N_m3u8DL_RE.Common.Entity;
+using N_m3u8DL_RE.Common.Enum;
+using N_m3u8DL_RE.Common.Log;
+using N_m3u8DL_RE.Common.Resource;
+using N_m3u8DL_RE.Common.Util;
+using N_m3u8DL_RE.Config;
+using N_m3u8DL_RE.Downloader;
+using N_m3u8DL_RE.Entity;
+using N_m3u8DL_RE.Parser;
+using N_m3u8DL_RE.Util;
+using Spectre.Console;
+using Spectre.Console.Rendering;
+using System;
+using System.Collections.Concurrent;
+using System.IO;
+using System.Net.Http.Headers;
+using System.Reflection.PortableExecutable;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using System.Xml.Linq;
+
+namespace N_m3u8DL_RE.DownloadManager
+{
+    internal class HTTPLiveRecordManager
+    {
+        IDownloader Downloader;
+        DownloaderConfig DownloaderConfig;
+        StreamExtractor StreamExtractor;
+        List<StreamSpec> SelectedSteams;
+        List<OutputFile> OutputFiles = new();
+        DateTime NowDateTime;
+        DateTime? PublishDateTime;
+        bool STOP_FLAG = false;
+        bool READ_IFO = false;
+        ConcurrentDictionary<int, int> RecordingDurDic = new(); //已录制时长
+        ConcurrentDictionary<int, double> RecordingSizeDic = new(); //已录制大小
+        CancellationTokenSource CancellationTokenSource = new(); //取消Wait
+        List<byte> InfoBuffer = new List<byte>(188 * 5000); //5000个分包中解析信息，没有就算了
+
+        public HTTPLiveRecordManager(DownloaderConfig downloaderConfig, List<StreamSpec> selectedSteams, StreamExtractor streamExtractor)
+        {
+            this.DownloaderConfig = downloaderConfig;
+            Downloader = new SimpleDownloader(DownloaderConfig);
+            NowDateTime = DateTime.Now;
+            PublishDateTime = selectedSteams.FirstOrDefault()?.PublishTime;
+            StreamExtractor = streamExtractor;
+            SelectedSteams = selectedSteams;
+        }
+
+        private async Task<bool> RecordStreamAsync(StreamSpec streamSpec, ProgressTask task, SpeedContainer speedContainer)
+        {
+            task.MaxValue = 1;
+            task.StartTask();
+
+            var name = streamSpec.ToShortString();
+            var dirName = $"{DownloaderConfig.MyOptions.SaveName ?? NowDateTime.ToString("yyyy-MM-dd_HH-mm-ss")}_{task.Id}_{streamSpec.GroupId}_{streamSpec.Codecs}_{streamSpec.Bandwidth}_{streamSpec.Language}";
+            var saveDir = DownloaderConfig.MyOptions.SaveDir ?? Environment.CurrentDirectory;
+            var saveName = DownloaderConfig.MyOptions.SaveName != null ? $"{DownloaderConfig.MyOptions.SaveName}.{streamSpec.Language}".TrimEnd('.') : dirName;
+
+            Logger.Debug($"dirName: {dirName}; saveDir: {saveDir}; saveName: {saveName}");
+
+            //创建文件夹
+            if (!Directory.Exists(saveDir)) Directory.CreateDirectory(saveDir);
+
+            using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(streamSpec.Url));
+            request.Headers.ConnectionClose = false;
+            foreach (var item in DownloaderConfig.Headers)
+            {
+                request.Headers.TryAddWithoutValidation(item.Key, item.Value);
+            }
+            Logger.Debug(request.Headers.ToString());
+
+            using var response = await HTTPUtil.AppHttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancellationTokenSource.Token);
+            response.EnsureSuccessStatusCode();
+
+            var output = Path.Combine(saveDir, saveName + ".ts");
+            using var stream = new FileStream(output, FileMode.Create, FileAccess.Write, FileShare.None);
+            using var responseStream = await response.Content.ReadAsStreamAsync(CancellationTokenSource.Token);
+            var buffer = new byte[16 * 1024];
+            var size = 0;
+
+            //计时器
+            TimeCounterAsync();
+            //读取INFO
+            ReadInfoAsync();
+
+            try
+            {
+                while ((size = await responseStream.ReadAsync(buffer, CancellationTokenSource.Token)) > 0)
+                {
+                    if (!READ_IFO && InfoBuffer.Count < 188 * 5000)
+                    {
+                        InfoBuffer.AddRange(buffer);
+                    }
+                    speedContainer.Add(size);
+                    RecordingSizeDic[task.Id] += size;
+                    await stream.WriteAsync(buffer, 0, size);
+                }
+            }
+            catch (OperationCanceledException oce) when (oce.CancellationToken == CancellationTokenSource.Token)
+            {
+                ;
+            }
+
+            Logger.InfoMarkUp("File Size: " + GlobalUtil.FormatFileSize(RecordingSizeDic[task.Id]));
+
+            return true;
+        }
+
+        public async Task ReadInfoAsync()
+        {
+            while (!STOP_FLAG && !READ_IFO)
+            {
+                await Task.Delay(200);
+                if (InfoBuffer.Count < 188 * 5000) continue;
+
+                UInt16 ConvertToUint16(IEnumerable<byte> bytes)
+                {
+                    if (BitConverter.IsLittleEndian)
+                        bytes = bytes.Reverse();
+                    return BitConverter.ToUInt16(bytes.ToArray());
+                }
+
+                var data = InfoBuffer.ToArray();
+                var programId = "";
+                var serviceProvider = "";
+                var serviceName = "";
+                for (int i = 0; i < data.Length; i++)
+                {
+                    if (data[i] == 0x47 && (i + 188) < data.Length && data[i + 188] == 0x47)
+                    {
+                        var tsData = data.Skip(i).Take(188);
+                        var tsHeaderInt = BitConverter.ToUInt32(BitConverter.IsLittleEndian ? tsData.Take(4).Reverse().ToArray() : tsData.Take(4).ToArray(), 0);
+                        var pid = (tsHeaderInt & 0x1fff00) >> 8;
+                        var tsPayload = tsData.Skip(4);
+                        //PAT
+                        if (pid == 0x0000)
+                        {
+                            programId = ConvertToUint16(tsPayload.Skip(9).Take(2)).ToString();
+                        }
+                        //SDT, BAT, ST
+                        else if (pid == 0x0011)
+                        {
+                            var tableId = (int)tsPayload.Skip(1).First();
+                            //Current TS Info
+                            if (tableId == 0x42)
+                            {
+                                var sectionLength = ConvertToUint16(tsPayload.Skip(2).Take(2)) & 0xfff;
+                                var sectionData = tsPayload.Skip(4).Take(sectionLength);
+                                var dscripData = sectionData.Skip(8);
+                                var descriptorsLoopLength = (ConvertToUint16(dscripData.Skip(3).Take(2))) & 0xfff;
+                                var descriptorsData = dscripData.Skip(5).Take(descriptorsLoopLength);
+                                var serviceProviderLength = (int)descriptorsData.Skip(3).First();
+                                serviceProvider = Encoding.UTF8.GetString(descriptorsData.Skip(4).Take(serviceProviderLength).ToArray());
+                                var serviceNameLength = (int)descriptorsData.Skip(4 + serviceProviderLength).First();
+                                serviceName = Encoding.UTF8.GetString(descriptorsData.Skip(5 + serviceProviderLength).Take(serviceNameLength).ToArray());
+                            }
+                        }
+                        if (programId != "" && (serviceName != "" || serviceProvider != ""))
+                            break;
+                    }
+                }
+
+                if (!string.IsNullOrEmpty(programId))
+                {
+                    Logger.InfoMarkUp($"Program Id: [cyan]{programId.EscapeMarkup()}[/]");
+                    if (!string.IsNullOrEmpty(serviceName)) Logger.InfoMarkUp($"Service Name: [cyan]{serviceName.EscapeMarkup()}[/]");
+                    if (!string.IsNullOrEmpty(serviceProvider)) Logger.InfoMarkUp($"Service Provider: [cyan]{serviceProvider.EscapeMarkup()}[/]");
+                    READ_IFO = true;
+                }
+            }
+        }
+
+        public async Task TimeCounterAsync()
+        {
+            while (!STOP_FLAG)
+            {
+                await Task.Delay(1000);
+                RecordingDurDic[0]++;
+
+                //检测时长限制
+                if (RecordingDurDic.All(d => d.Value >= DownloaderConfig.MyOptions.LiveRecordLimit?.TotalSeconds))
+                {
+                    Logger.WarnMarkUp($"[darkorange3_1]{ResString.liveLimitReached}[/]");
+                    STOP_FLAG = true;
+                    CancellationTokenSource.Cancel();
+                }
+            }
+        }
+
+        public async Task<bool> StartRecordAsync()
+        {
+            ConcurrentDictionary<int, SpeedContainer> SpeedContainerDic = new(); //速度计算
+            ConcurrentDictionary<StreamSpec, bool?> Results = new();
+
+            var progress = AnsiConsole.Progress().AutoClear(true);
+
+            //进度条的列定义
+            progress.Columns(new ProgressColumn[]
+            {
+                new TaskDescriptionColumn() { Alignment = Justify.Left },
+                new RecordingDurationColumn(RecordingDurDic), //时长显示
+                new RecordingSizeColumn(RecordingSizeDic), //大小显示
+                new RecordingStatusColumn(),
+                new DownloadSpeedColumn(SpeedContainerDic), //速度计算
+                new SpinnerColumn(),
+            });
+
+            await progress.StartAsync(async ctx =>
+            {
+                //创建任务
+                var dic = SelectedSteams.Select(item =>
+                {
+                    var task = ctx.AddTask(item.ToShortString(), autoStart: false, maxValue: 0);
+                    SpeedContainerDic[task.Id] = new SpeedContainer(); //速度计算
+                    RecordingDurDic[task.Id] = 0;
+                    RecordingSizeDic[task.Id] = 0;
+                    return (item, task);
+                }).ToDictionary(item => item.item, item => item.task);
+
+                DownloaderConfig.MyOptions.LiveRecordLimit = DownloaderConfig.MyOptions.LiveRecordLimit ?? TimeSpan.MaxValue;
+                var limit = DownloaderConfig.MyOptions.LiveRecordLimit;
+                if (limit != TimeSpan.MaxValue)
+                    Logger.WarnMarkUp($"[darkorange3_1]{ResString.liveLimit}{GlobalUtil.FormatTime((int)limit.Value.TotalSeconds)}[/]");
+                //录制直播时，用户选了几个流就并发录几个
+                var options = new ParallelOptions()
+                {
+                    MaxDegreeOfParallelism = SelectedSteams.Count
+                };
+                //并发下载
+                await Parallel.ForEachAsync(dic, options, async (kp, _) =>
+                {
+                    var task = kp.Value;
+                    var consumerTask = RecordStreamAsync(kp.Key, task, SpeedContainerDic[task.Id]);
+                    Results[kp.Key] = await consumerTask;
+                });
+            });
+
+            var success = Results.Values.All(v => v == true);
+
+            return success;
+        }
+    }
+}

src/N_m3u8DL-RE/Program.cs

@@ -323,7 +323,13 @@ namespace N_m3u8DL_RE
             };
 
             var result = false;
-            if (!livingFlag)
+            
+            if (extractor.ExtractorType == ExtractorType.HTTP_LIVE)
+            {
+                var sldm = new HTTPLiveRecordManager(downloadConfig, selectedStreams, extractor);
+                result = await sldm.StartRecordAsync();
+            }
+            else if(!livingFlag)
             {
                 //开始下载
                 var sdm = new SimpleDownloadManager(downloadConfig);