初步支持HTTP-TS录制
This commit is contained in:
nilaoda
parent
9e9a307f7c
commit
96b58e9384
|
|
@ -9,6 +9,8 @@ namespace N_m3u8DL_RE.Common.Enum
|
|||
public enum ExtractorType
|
||||
{
|
||||
MPEG_DASH,
|
||||
HLS
|
||||
HLS,
|
||||
HTTP_LIVE,
|
||||
MSS
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ namespace N_m3u8DL_RE.Common.Resource
|
|||
{
|
||||
public class ResString
|
||||
{
|
||||
public readonly static string ReLiveTs = "<RE_LIVE_TS>";
|
||||
public static string autoBinaryMerge { get => GetText("autoBinaryMerge"); }
|
||||
public static string autoBinaryMerge2 { get => GetText("autoBinaryMerge2"); }
|
||||
public static string autoBinaryMerge3 { get => GetText("autoBinaryMerge3"); }
|
||||
|
|
@ -88,6 +89,7 @@ namespace N_m3u8DL_RE.Common.Resource
|
|||
public static string loadingUrl { get => GetText("loadingUrl"); }
|
||||
public static string masterM3u8Found { get => GetText("masterM3u8Found"); }
|
||||
public static string matchDASH { get => GetText("matchDASH"); }
|
||||
public static string matchTS { get => GetText("matchTS"); }
|
||||
public static string matchHLS { get => GetText("matchHLS"); }
|
||||
public static string notSupported { get => GetText("notSupported"); }
|
||||
public static string parsingStream { get => GetText("parsingStream"); }
|
||||
|
|
|
|||
|
|
@ -601,6 +601,12 @@ namespace N_m3u8DL_RE.Common.Resource
|
|||
zhTW: "檢測到Master列表，開始解析全部流訊息",
|
||||
enUS: "Master List detected, try parse all streams"
|
||||
),
|
||||
["matchTS"] = new TextContainer
|
||||
(
|
||||
zhCN: "内容匹配: [white on green3]HTTP Live MPEG2-TS[/]",
|
||||
zhTW: "內容匹配: [white on green3]HTTP Live MPEG2-TS[/]",
|
||||
enUS: "Content Matched: [white on green3]HTTP Live MPEG2-TS[/]"
|
||||
),
|
||||
["matchDASH"] = new TextContainer
|
||||
(
|
||||
zhCN: "内容匹配: [white on mediumorchid1]Dynamic Adaptive Streaming over HTTP[/]",
|
||||
|
|
|
|||
|
|
@ -42,6 +42,18 @@ namespace N_m3u8DL_RE.Common.Util
|
|||
return "{NOT SUPPORTED}";
|
||||
}
|
||||
|
||||
public static string FormatFileSize(double fileSize)
|
||||
{
|
||||
return fileSize switch
|
||||
{
|
||||
< 0 => throw new ArgumentOutOfRangeException(nameof(fileSize)),
|
||||
>= 1024 * 1024 * 1024 => string.Format("{0:########0.00}GB", (double)fileSize / (1024 * 1024 * 1024)),
|
||||
>= 1024 * 1024 => string.Format("{0:####0.00}MB", (double)fileSize / (1024 * 1024)),
|
||||
>= 1024 => string.Format("{0:####0.00}KB", (double)fileSize / 1024),
|
||||
_ => string.Format("{0:####0.00}B", fileSize)
|
||||
};
|
||||
}
|
||||
|
||||
//此函数用于格式化输出时长
|
||||
public static string FormatTime(int time)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,21 +1,5 @@
|
|||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Specialized;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Cache;
|
||||
using System.Net.Http;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Web;
|
||||
using N_m3u8DL_RE.Common.Log;
|
||||
using N_m3u8DL_RE.Common.Resource;
|
||||
|
||||
|
|
@ -110,6 +94,12 @@ namespace N_m3u8DL_RE.Common.Util
|
|||
return htmlCode;
|
||||
}
|
||||
|
||||
private static bool CheckMPEG2TS(HttpResponseMessage? webResponse)
|
||||
{
|
||||
var mediaType = webResponse?.Content.Headers.ContentType?.MediaType;
|
||||
return mediaType == "video/ts" || mediaType == "video/mp2t";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 获取网页源码和跳转后的URL
|
||||
/// </summary>
|
||||
|
|
@ -120,7 +110,14 @@ namespace N_m3u8DL_RE.Common.Util
|
|||
{
|
||||
string htmlCode = string.Empty;
|
||||
var webResponse = await DoGetAsync(url, headers);
|
||||
htmlCode = await webResponse.Content.ReadAsStringAsync();
|
||||
if (CheckMPEG2TS(webResponse))
|
||||
{
|
||||
htmlCode = ResString.ReLiveTs;
|
||||
}
|
||||
else
|
||||
{
|
||||
htmlCode = await webResponse.Content.ReadAsStringAsync();
|
||||
}
|
||||
Logger.Debug(htmlCode);
|
||||
return (htmlCode, webResponse.Headers.Location != null ? webResponse.Headers.Location.AbsoluteUri : url);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
using N_m3u8DL_RE.Common.Entity;
|
||||
using N_m3u8DL_RE.Common.Enum;
|
||||
using N_m3u8DL_RE.Common.Resource;
|
||||
using N_m3u8DL_RE.Parser.Config;
|
||||
|
||||
namespace N_m3u8DL_RE.Parser.Extractor
|
||||
{
|
||||
internal class LiveTSExtractor : IExtractor
|
||||
{
|
||||
public ExtractorType ExtractorType => ExtractorType.HTTP_LIVE;
|
||||
|
||||
public ParserConfig ParserConfig {get; set;}
|
||||
|
||||
public LiveTSExtractor(ParserConfig parserConfig)
|
||||
{
|
||||
this.ParserConfig = parserConfig;
|
||||
}
|
||||
|
||||
public async Task<List<StreamSpec>> ExtractStreamsAsync(string rawText)
|
||||
{
|
||||
return new List<StreamSpec>()
|
||||
{
|
||||
new StreamSpec()
|
||||
{
|
||||
OriginalUrl = ParserConfig.OriginalUrl,
|
||||
Url = ParserConfig.Url,
|
||||
Playlist = new Playlist(),
|
||||
GroupId = ResString.ReLiveTs
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public async Task FetchPlayListAsync(List<StreamSpec> streamSpecs)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public async void PreProcessContent()
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public string PreProcessUrl(string url)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public Task RefreshPlayListAsync(List<StreamSpec> streamSpecs)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -68,6 +68,11 @@ namespace N_m3u8DL_RE.Parser
|
|||
//extractor = new DASHExtractor(parserConfig);
|
||||
extractor = new DASHExtractor2(parserConfig);
|
||||
}
|
||||
else if (rawText == ResString.ReLiveTs)
|
||||
{
|
||||
Logger.InfoMarkUp(ResString.matchTS);
|
||||
extractor = new LiveTSExtractor(parserConfig);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new NotSupportedException(ResString.notSupported);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
using N_m3u8DL_RE.Common.Util;
|
||||
using N_m3u8DL_RE.Entity;
|
||||
using Spectre.Console;
|
||||
using Spectre.Console.Rendering;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace N_m3u8DL_RE.Column
|
||||
{
|
||||
internal class RecordingSizeColumn : ProgressColumn
|
||||
{
|
||||
protected override bool NoWrap => true;
|
||||
private ConcurrentDictionary<int, double> RecodingSizeDic = new(); //临时的大小 每秒刷新用
|
||||
private ConcurrentDictionary<int, double> _recodingSizeDic;
|
||||
private ConcurrentDictionary<int, string> DateTimeStringDic = new();
|
||||
public Style MyStyle { get; set; } = new Style(foreground: Color.DarkCyan);
|
||||
public RecordingSizeColumn(ConcurrentDictionary<int, double> recodingSizeDic)
|
||||
{
|
||||
_recodingSizeDic = recodingSizeDic;
|
||||
}
|
||||
public override IRenderable Render(RenderContext context, ProgressTask task, TimeSpan deltaTime)
|
||||
{
|
||||
var now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss");
|
||||
var taskId = task.Id;
|
||||
//一秒汇报一次即可
|
||||
if (DateTimeStringDic.TryGetValue(taskId, out var oldTime) && oldTime != now)
|
||||
{
|
||||
RecodingSizeDic[task.Id] = _recodingSizeDic[task.Id];
|
||||
}
|
||||
DateTimeStringDic[taskId] = now;
|
||||
var flag = RecodingSizeDic.TryGetValue(taskId, out var size);
|
||||
return new Text(GlobalUtil.FormatFileSize(flag ? size : 0), MyStyle).LeftAligned();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -18,7 +18,7 @@ namespace N_m3u8DL_RE.CommandLine
|
|||
{
|
||||
internal partial class CommandInvoker
|
||||
{
|
||||
public const string VERSION_INFO = "N_m3u8DL-RE (Beta version) 20221115";
|
||||
public const string VERSION_INFO = "N_m3u8DL-RE (Beta version) 20221116";
|
||||
|
||||
[GeneratedRegex("((best|worst)\\d*|all)")]
|
||||
private static partial Regex ForStrRegex();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,248 @@
|
|||
using Mp4SubtitleParser;
|
||||
using N_m3u8DL_RE.Column;
|
||||
using N_m3u8DL_RE.Common.Entity;
|
||||
using N_m3u8DL_RE.Common.Enum;
|
||||
using N_m3u8DL_RE.Common.Log;
|
||||
using N_m3u8DL_RE.Common.Resource;
|
||||
using N_m3u8DL_RE.Common.Util;
|
||||
using N_m3u8DL_RE.Config;
|
||||
using N_m3u8DL_RE.Downloader;
|
||||
using N_m3u8DL_RE.Entity;
|
||||
using N_m3u8DL_RE.Parser;
|
||||
using N_m3u8DL_RE.Util;
|
||||
using Spectre.Console;
|
||||
using Spectre.Console.Rendering;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
using System.Xml.Linq;
|
||||
|
||||
namespace N_m3u8DL_RE.DownloadManager
|
||||
{
|
||||
internal class HTTPLiveRecordManager
|
||||
{
|
||||
IDownloader Downloader;
|
||||
DownloaderConfig DownloaderConfig;
|
||||
StreamExtractor StreamExtractor;
|
||||
List<StreamSpec> SelectedSteams;
|
||||
List<OutputFile> OutputFiles = new();
|
||||
DateTime NowDateTime;
|
||||
DateTime? PublishDateTime;
|
||||
bool STOP_FLAG = false;
|
||||
bool READ_IFO = false;
|
||||
ConcurrentDictionary<int, int> RecordingDurDic = new(); //已录制时长
|
||||
ConcurrentDictionary<int, double> RecordingSizeDic = new(); //已录制大小
|
||||
CancellationTokenSource CancellationTokenSource = new(); //取消Wait
|
||||
List<byte> InfoBuffer = new List<byte>(188 * 5000); //5000个分包中解析信息，没有就算了
|
||||
|
||||
public HTTPLiveRecordManager(DownloaderConfig downloaderConfig, List<StreamSpec> selectedSteams, StreamExtractor streamExtractor)
|
||||
{
|
||||
this.DownloaderConfig = downloaderConfig;
|
||||
Downloader = new SimpleDownloader(DownloaderConfig);
|
||||
NowDateTime = DateTime.Now;
|
||||
PublishDateTime = selectedSteams.FirstOrDefault()?.PublishTime;
|
||||
StreamExtractor = streamExtractor;
|
||||
SelectedSteams = selectedSteams;
|
||||
}
|
||||
|
||||
private async Task<bool> RecordStreamAsync(StreamSpec streamSpec, ProgressTask task, SpeedContainer speedContainer)
|
||||
{
|
||||
task.MaxValue = 1;
|
||||
task.StartTask();
|
||||
|
||||
var name = streamSpec.ToShortString();
|
||||
var dirName = $"{DownloaderConfig.MyOptions.SaveName ?? NowDateTime.ToString("yyyy-MM-dd_HH-mm-ss")}_{task.Id}_{streamSpec.GroupId}_{streamSpec.Codecs}_{streamSpec.Bandwidth}_{streamSpec.Language}";
|
||||
var saveDir = DownloaderConfig.MyOptions.SaveDir ?? Environment.CurrentDirectory;
|
||||
var saveName = DownloaderConfig.MyOptions.SaveName != null ? $"{DownloaderConfig.MyOptions.SaveName}.{streamSpec.Language}".TrimEnd('.') : dirName;
|
||||
|
||||
Logger.Debug($"dirName: {dirName}; saveDir: {saveDir}; saveName: {saveName}");
|
||||
|
||||
//创建文件夹
|
||||
if (!Directory.Exists(saveDir)) Directory.CreateDirectory(saveDir);
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(streamSpec.Url));
|
||||
request.Headers.ConnectionClose = false;
|
||||
foreach (var item in DownloaderConfig.Headers)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation(item.Key, item.Value);
|
||||
}
|
||||
Logger.Debug(request.Headers.ToString());
|
||||
|
||||
using var response = await HTTPUtil.AppHttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancellationTokenSource.Token);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var output = Path.Combine(saveDir, saveName + ".ts");
|
||||
using var stream = new FileStream(output, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
using var responseStream = await response.Content.ReadAsStreamAsync(CancellationTokenSource.Token);
|
||||
var buffer = new byte[16 * 1024];
|
||||
var size = 0;
|
||||
|
||||
//计时器
|
||||
TimeCounterAsync();
|
||||
//读取INFO
|
||||
ReadInfoAsync();
|
||||
|
||||
try
|
||||
{
|
||||
while ((size = await responseStream.ReadAsync(buffer, CancellationTokenSource.Token)) > 0)
|
||||
{
|
||||
if (!READ_IFO && InfoBuffer.Count < 188 * 5000)
|
||||
{
|
||||
InfoBuffer.AddRange(buffer);
|
||||
}
|
||||
speedContainer.Add(size);
|
||||
RecordingSizeDic[task.Id] += size;
|
||||
await stream.WriteAsync(buffer, 0, size);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException oce) when (oce.CancellationToken == CancellationTokenSource.Token)
|
||||
{
|
||||
;
|
||||
}
|
||||
|
||||
Logger.InfoMarkUp("File Size: " + GlobalUtil.FormatFileSize(RecordingSizeDic[task.Id]));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task ReadInfoAsync()
|
||||
{
|
||||
while (!STOP_FLAG && !READ_IFO)
|
||||
{
|
||||
await Task.Delay(200);
|
||||
if (InfoBuffer.Count < 188 * 5000) continue;
|
||||
|
||||
UInt16 ConvertToUint16(IEnumerable<byte> bytes)
|
||||
{
|
||||
if (BitConverter.IsLittleEndian)
|
||||
bytes = bytes.Reverse();
|
||||
return BitConverter.ToUInt16(bytes.ToArray());
|
||||
}
|
||||
|
||||
var data = InfoBuffer.ToArray();
|
||||
var programId = "";
|
||||
var serviceProvider = "";
|
||||
var serviceName = "";
|
||||
for (int i = 0; i < data.Length; i++)
|
||||
{
|
||||
if (data[i] == 0x47 && (i + 188) < data.Length && data[i + 188] == 0x47)
|
||||
{
|
||||
var tsData = data.Skip(i).Take(188);
|
||||
var tsHeaderInt = BitConverter.ToUInt32(BitConverter.IsLittleEndian ? tsData.Take(4).Reverse().ToArray() : tsData.Take(4).ToArray(), 0);
|
||||
var pid = (tsHeaderInt & 0x1fff00) >> 8;
|
||||
var tsPayload = tsData.Skip(4);
|
||||
//PAT
|
||||
if (pid == 0x0000)
|
||||
{
|
||||
programId = ConvertToUint16(tsPayload.Skip(9).Take(2)).ToString();
|
||||
}
|
||||
//SDT, BAT, ST
|
||||
else if (pid == 0x0011)
|
||||
{
|
||||
var tableId = (int)tsPayload.Skip(1).First();
|
||||
//Current TS Info
|
||||
if (tableId == 0x42)
|
||||
{
|
||||
var sectionLength = ConvertToUint16(tsPayload.Skip(2).Take(2)) & 0xfff;
|
||||
var sectionData = tsPayload.Skip(4).Take(sectionLength);
|
||||
var dscripData = sectionData.Skip(8);
|
||||
var descriptorsLoopLength = (ConvertToUint16(dscripData.Skip(3).Take(2))) & 0xfff;
|
||||
var descriptorsData = dscripData.Skip(5).Take(descriptorsLoopLength);
|
||||
var serviceProviderLength = (int)descriptorsData.Skip(3).First();
|
||||
serviceProvider = Encoding.UTF8.GetString(descriptorsData.Skip(4).Take(serviceProviderLength).ToArray());
|
||||
var serviceNameLength = (int)descriptorsData.Skip(4 + serviceProviderLength).First();
|
||||
serviceName = Encoding.UTF8.GetString(descriptorsData.Skip(5 + serviceProviderLength).Take(serviceNameLength).ToArray());
|
||||
}
|
||||
}
|
||||
if (programId != "" && (serviceName != "" || serviceProvider != ""))
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(programId))
|
||||
{
|
||||
Logger.InfoMarkUp($"Program Id: [cyan]{programId.EscapeMarkup()}[/]");
|
||||
if (!string.IsNullOrEmpty(serviceName)) Logger.InfoMarkUp($"Service Name: [cyan]{serviceName.EscapeMarkup()}[/]");
|
||||
if (!string.IsNullOrEmpty(serviceProvider)) Logger.InfoMarkUp($"Service Provider: [cyan]{serviceProvider.EscapeMarkup()}[/]");
|
||||
READ_IFO = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task TimeCounterAsync()
|
||||
{
|
||||
while (!STOP_FLAG)
|
||||
{
|
||||
await Task.Delay(1000);
|
||||
RecordingDurDic[0]++;
|
||||
|
||||
//检测时长限制
|
||||
if (RecordingDurDic.All(d => d.Value >= DownloaderConfig.MyOptions.LiveRecordLimit?.TotalSeconds))
|
||||
{
|
||||
Logger.WarnMarkUp($"[darkorange3_1]{ResString.liveLimitReached}[/]");
|
||||
STOP_FLAG = true;
|
||||
CancellationTokenSource.Cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> StartRecordAsync()
|
||||
{
|
||||
ConcurrentDictionary<int, SpeedContainer> SpeedContainerDic = new(); //速度计算
|
||||
ConcurrentDictionary<StreamSpec, bool?> Results = new();
|
||||
|
||||
var progress = AnsiConsole.Progress().AutoClear(true);
|
||||
|
||||
//进度条的列定义
|
||||
progress.Columns(new ProgressColumn[]
|
||||
{
|
||||
new TaskDescriptionColumn() { Alignment = Justify.Left },
|
||||
new RecordingDurationColumn(RecordingDurDic), //时长显示
|
||||
new RecordingSizeColumn(RecordingSizeDic), //大小显示
|
||||
new RecordingStatusColumn(),
|
||||
new DownloadSpeedColumn(SpeedContainerDic), //速度计算
|
||||
new SpinnerColumn(),
|
||||
});
|
||||
|
||||
await progress.StartAsync(async ctx =>
|
||||
{
|
||||
//创建任务
|
||||
var dic = SelectedSteams.Select(item =>
|
||||
{
|
||||
var task = ctx.AddTask(item.ToShortString(), autoStart: false, maxValue: 0);
|
||||
SpeedContainerDic[task.Id] = new SpeedContainer(); //速度计算
|
||||
RecordingDurDic[task.Id] = 0;
|
||||
RecordingSizeDic[task.Id] = 0;
|
||||
return (item, task);
|
||||
}).ToDictionary(item => item.item, item => item.task);
|
||||
|
||||
DownloaderConfig.MyOptions.LiveRecordLimit = DownloaderConfig.MyOptions.LiveRecordLimit ?? TimeSpan.MaxValue;
|
||||
var limit = DownloaderConfig.MyOptions.LiveRecordLimit;
|
||||
if (limit != TimeSpan.MaxValue)
|
||||
Logger.WarnMarkUp($"[darkorange3_1]{ResString.liveLimit}{GlobalUtil.FormatTime((int)limit.Value.TotalSeconds)}[/]");
|
||||
//录制直播时，用户选了几个流就并发录几个
|
||||
var options = new ParallelOptions()
|
||||
{
|
||||
MaxDegreeOfParallelism = SelectedSteams.Count
|
||||
};
|
||||
//并发下载
|
||||
await Parallel.ForEachAsync(dic, options, async (kp, _) =>
|
||||
{
|
||||
var task = kp.Value;
|
||||
var consumerTask = RecordStreamAsync(kp.Key, task, SpeedContainerDic[task.Id]);
|
||||
Results[kp.Key] = await consumerTask;
|
||||
});
|
||||
});
|
||||
|
||||
var success = Results.Values.All(v => v == true);
|
||||
|
||||
return success;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -323,7 +323,13 @@ namespace N_m3u8DL_RE
|
|||
};
|
||||
|
||||
var result = false;
|
||||
if (!livingFlag)
|
||||
|
||||
if (extractor.ExtractorType == ExtractorType.HTTP_LIVE)
|
||||
{
|
||||
var sldm = new HTTPLiveRecordManager(downloadConfig, selectedStreams, extractor);
|
||||
result = await sldm.StartRecordAsync();
|
||||
}
|
||||
else if(!livingFlag)
|
||||
{
|
||||
//开始下载
|
||||
var sdm = new SimpleDownloadManager(downloadConfig);
|
||||
|
|
|
|||
Loading…
Reference in New Issue