From 0f15ce149be87733e15590d11e9aabc81939f8d5 Mon Sep 17 00:00:00 2001 From: KongQun Yang Date: Sun, 22 Sep 2019 23:47:07 -0700 Subject: [PATCH] [HLS] Support FRAME-RATE attribute Calculate FRAME-RATE from sample duration as in the DASH solution. Right now calculated frame-rate may not be accurate in some scenarios, so we avoid setting the attribute in HLS unless it is more than 30 fps. We will set it unconditionally once it is fixed. Fixes #634. Change-Id: I87b6e9a047d959ae88dd4dcb2b4786527ba5c9fc --- packager/hls/base/hls_notifier.h | 9 ++++++ packager/hls/base/master_playlist.cc | 8 ++++++ packager/hls/base/master_playlist_unittest.cc | 28 +++++++++++++++++++ packager/hls/base/media_playlist.cc | 12 ++++++++ packager/hls/base/media_playlist.h | 9 ++++++ packager/hls/base/mock_media_playlist.h | 1 + packager/hls/base/simple_hls_notifier.cc | 13 +++++++++ packager/hls/base/simple_hls_notifier.h | 2 ++ packager/hls/base/tag.cc | 5 ++++ packager/hls/base/tag.h | 3 ++ .../media/event/hls_notify_muxer_listener.cc | 20 ++++++++++++- .../hls_notify_muxer_listener_unittest.cc | 14 +++++++++- .../media/event/mpd_notify_muxer_listener.cc | 3 -- 13 files changed, 122 insertions(+), 5 deletions(-) diff --git a/packager/hls/base/hls_notifier.h b/packager/hls/base/hls_notifier.h index 50e12e1fb9..8e2c81875f 100644 --- a/packager/hls/base/hls_notifier.h +++ b/packager/hls/base/hls_notifier.h @@ -41,6 +41,15 @@ class HlsNotifier { const std::string& group_id, uint32_t* stream_id) = 0; + /// Change the sample duration of stream with @a stream_id. + /// @param stream_id is the value set by NotifyNewStream(). + /// @param sample_duration is the duration of a sample in timescale of the + /// media. + /// @return true on success, false otherwise. This may fail if the stream + /// specified by @a stream_id does not exist. + virtual bool NotifySampleDuration(uint32_t stream_id, + uint32_t sample_duration) = 0; + /// @param stream_id is the value set by NotifyNewStream(). /// @param segment_name is the name of the new segment. /// @param start_time is the start time of the segment in timescale units diff --git a/packager/hls/base/master_playlist.cc b/packager/hls/base/master_playlist.cc index 8856007e37..ea64e96d96 100644 --- a/packager/hls/base/master_playlist.cc +++ b/packager/hls/base/master_playlist.cc @@ -222,6 +222,14 @@ void BuildStreamInfTag(const MediaPlaylist& playlist, uint32_t height; if (playlist.GetDisplayResolution(&width, &height)) { tag.AddNumberPair("RESOLUTION", width, 'x', height); + // Per HLS specification, The FRAME-RATE attribute SHOULD be included if any + // video in a Variant Stream exceeds 30 frames per second. + // Right now the frame-rate returned may not be accurate in some scenario. + // TODO(kqyang): Set frame-rate unconditionally once it is fixed. + const double frame_rate = playlist.GetFrameRate(); + if (frame_rate >= 30.5) + tag.AddFloat("FRAME-RATE", frame_rate); + const std::string video_range = playlist.GetVideoRange(); if (!video_range.empty()) tag.AddString("VIDEO-RANGE", video_range); diff --git a/packager/hls/base/master_playlist_unittest.cc b/packager/hls/base/master_playlist_unittest.cc index 4e46d6d755..22efb50a72 100644 --- a/packager/hls/base/master_playlist_unittest.cc +++ b/packager/hls/base/master_playlist_unittest.cc @@ -167,6 +167,34 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistOneVideo) { ASSERT_EQ(expected, actual); } +TEST_F(MasterPlaylistTest, WriteMasterPlaylistOneVideoWithFrameRate) { + const uint64_t kMaxBitrate = 435889; + const uint64_t kAvgBitrate = 235889; + const double kFrameRate = 60; + + std::unique_ptr mock_playlist = + CreateVideoPlaylist("media1.m3u8", "avc1", kMaxBitrate, kAvgBitrate); + EXPECT_CALL(*mock_playlist, GetFrameRate()).WillOnce(Return(kFrameRate)); + + const char kBaseUrl[] = "http://myplaylistdomain.com/"; + EXPECT_TRUE(master_playlist_.WriteMasterPlaylist(kBaseUrl, test_output_dir_, + {mock_playlist.get()})); + + std::string actual; + ASSERT_TRUE(File::ReadFileToString(master_playlist_path_.c_str(), &actual)); + + const std::string expected = + "#EXTM3U\n" + "## Generated with https://github.com/google/shaka-packager version " + "test\n" + "\n" + "#EXT-X-STREAM-INF:BANDWIDTH=435889,AVERAGE-BANDWIDTH=235889," + "CODECS=\"avc1\",RESOLUTION=800x600,FRAME-RATE=60.000\n" + "http://myplaylistdomain.com/media1.m3u8\n"; + + ASSERT_EQ(expected, actual); +} + TEST_F(MasterPlaylistTest, WriteMasterPlaylistOneIframePlaylist) { const uint64_t kMaxBitrate = 435889; const uint64_t kAvgBitrate = 235889; diff --git a/packager/hls/base/media_playlist.cc b/packager/hls/base/media_playlist.cc index 2b3166794d..bf2b61d98c 100644 --- a/packager/hls/base/media_playlist.cc +++ b/packager/hls/base/media_playlist.cc @@ -393,6 +393,11 @@ bool MediaPlaylist::SetMediaInfo(const MediaInfo& media_info) { return true; } +void MediaPlaylist::SetSampleDuration(uint32_t sample_duration) { + if (media_info_.has_video_info()) + media_info_.mutable_video_info()->set_frame_duration(sample_duration); +} + void MediaPlaylist::AddSegment(const std::string& file_name, int64_t start_time, int64_t duration, @@ -543,6 +548,13 @@ std::string MediaPlaylist::GetVideoRange() const { } } +double MediaPlaylist::GetFrameRate() const { + if (media_info_.video_info().frame_duration() == 0) + return 0; + return static_cast(time_scale_) / + media_info_.video_info().frame_duration(); +} + void MediaPlaylist::AddSegmentInfoEntry(const std::string& segment_file_name, int64_t start_time, int64_t duration, diff --git a/packager/hls/base/media_playlist.h b/packager/hls/base/media_playlist.h index fef00baad0..2f595dc2ab 100644 --- a/packager/hls/base/media_playlist.h +++ b/packager/hls/base/media_playlist.h @@ -99,6 +99,12 @@ class MediaPlaylist { /// @return true on success, false otherwise. virtual bool SetMediaInfo(const MediaInfo& media_info); + /// Set the sample duration. Sample duration is used to generate frame rate. + /// Sample duration is not available right away especially. This allows + /// setting the sample duration after the Media Playlist has been initialized. + /// @param sample_duration is the duration of a sample. + virtual void SetSampleDuration(uint32_t sample_duration); + /// Segments must be added in order. /// @param file_name is the file name of the segment. /// @param start_time is in terms of the timescale of the media. @@ -189,6 +195,9 @@ class MediaPlaylist { /// @return The video range of the stream. virtual std::string GetVideoRange() const; + /// @return the frame rate. + virtual double GetFrameRate() const; + /// @return the language of the media, as an ISO language tag in its shortest /// form. May be an empty string for video. const std::string& language() const { return language_; } diff --git a/packager/hls/base/mock_media_playlist.h b/packager/hls/base/mock_media_playlist.h index 761162dad7..cf9f483448 100644 --- a/packager/hls/base/mock_media_playlist.h +++ b/packager/hls/base/mock_media_playlist.h @@ -50,6 +50,7 @@ class MockMediaPlaylist : public MediaPlaylist { MOCK_CONST_METHOD0(GetNumChannels, int()); MOCK_CONST_METHOD2(GetDisplayResolution, bool(uint32_t* width, uint32_t* height)); + MOCK_CONST_METHOD0(GetFrameRate, double()); }; } // namespace hls diff --git a/packager/hls/base/simple_hls_notifier.cc b/packager/hls/base/simple_hls_notifier.cc index ba21f61744..d6d8363080 100644 --- a/packager/hls/base/simple_hls_notifier.cc +++ b/packager/hls/base/simple_hls_notifier.cc @@ -343,6 +343,19 @@ bool SimpleHlsNotifier::NotifyNewStream(const MediaInfo& media_info, return true; } +bool SimpleHlsNotifier::NotifySampleDuration(uint32_t stream_id, + uint32_t sample_duration) { + base::AutoLock auto_lock(lock_); + auto stream_iterator = stream_map_.find(stream_id); + if (stream_iterator == stream_map_.end()) { + LOG(ERROR) << "Cannot find stream with ID: " << stream_id; + return false; + } + auto& media_playlist = stream_iterator->second->media_playlist; + media_playlist->SetSampleDuration(sample_duration); + return true; +} + bool SimpleHlsNotifier::NotifyNewSegment(uint32_t stream_id, const std::string& segment_name, uint64_t start_time, diff --git a/packager/hls/base/simple_hls_notifier.h b/packager/hls/base/simple_hls_notifier.h index e0632293b2..abfdc08185 100644 --- a/packager/hls/base/simple_hls_notifier.h +++ b/packager/hls/base/simple_hls_notifier.h @@ -49,6 +49,8 @@ class SimpleHlsNotifier : public HlsNotifier { const std::string& stream_name, const std::string& group_id, uint32_t* stream_id) override; + bool NotifySampleDuration(uint32_t stream_id, + uint32_t sample_duration) override; bool NotifyNewSegment(uint32_t stream_id, const std::string& segment_name, uint64_t start_time, diff --git a/packager/hls/base/tag.cc b/packager/hls/base/tag.cc index 4a34b064f9..ad37c349e3 100644 --- a/packager/hls/base/tag.cc +++ b/packager/hls/base/tag.cc @@ -32,6 +32,11 @@ void Tag::AddNumber(const std::string& key, uint64_t value) { base::StringAppendF(buffer_, "%s=%" PRIu64, key.c_str(), value); } +void Tag::AddFloat(const std::string& key, float value) { + NextField(); + base::StringAppendF(buffer_, "%s=%.3f", key.c_str(), value); +} + void Tag::AddNumberPair(const std::string& key, uint64_t number1, char separator, diff --git a/packager/hls/base/tag.h b/packager/hls/base/tag.h index 7319b65962..481fdc8126 100644 --- a/packager/hls/base/tag.h +++ b/packager/hls/base/tag.h @@ -27,6 +27,9 @@ class Tag { /// Add a non-quoted numeric value to the argument list. void AddNumber(const std::string& key, uint64_t value); + /// Add a non-quoted float value to the argument list. + void AddFloat(const std::string& key, float value); + /// Add a pair of numbers with a symbol separating them. void AddNumberPair(const std::string& key, uint64_t number1, diff --git a/packager/media/event/hls_notify_muxer_listener.cc b/packager/media/event/hls_notify_muxer_listener.cc index ea7da62d2d..fdd326c4b6 100644 --- a/packager/media/event/hls_notify_muxer_listener.cc +++ b/packager/media/event/hls_notify_muxer_listener.cc @@ -125,7 +125,25 @@ void HlsNotifyMuxerListener::OnMediaStart(const MuxerOptions& muxer_options, } } -void HlsNotifyMuxerListener::OnSampleDurationReady(uint32_t sample_duration) {} +void HlsNotifyMuxerListener::OnSampleDurationReady(uint32_t sample_duration) { + if (stream_id_) { + // This happens in live mode. + hls_notifier_->NotifySampleDuration(stream_id_.value(), sample_duration); + return; + } + + if (!media_info_) { + LOG(WARNING) << "Got sample duration " << sample_duration + << " but no media was specified."; + return; + } + if (!media_info_->has_video_info()) { + // If non video, don't worry about it (at the moment). + return; + } + + media_info_->mutable_video_info()->set_frame_duration(sample_duration); +} void HlsNotifyMuxerListener::OnMediaEnd(const MediaRanges& media_ranges, float duration_seconds) { diff --git a/packager/media/event/hls_notify_muxer_listener_unittest.cc b/packager/media/event/hls_notify_muxer_listener_unittest.cc index da1636d44f..d0923a8f44 100644 --- a/packager/media/event/hls_notify_muxer_listener_unittest.cc +++ b/packager/media/event/hls_notify_muxer_listener_unittest.cc @@ -38,6 +38,8 @@ class MockHlsNotifier : public hls::HlsNotifier { const std::string& name, const std::string& group_id, uint32_t* stream_id)); + MOCK_METHOD2(NotifySampleDuration, + bool(uint32_t stream_id, uint32_t sample_duration)); MOCK_METHOD6(NotifyNewSegment, bool(uint32_t stream_id, const std::string& segment_name, @@ -310,8 +312,18 @@ TEST_F(HlsNotifyMuxerListenerTest, OnEncryptionInfoReadyWithProtectionScheme) { MuxerListener::kContainerMpeg2ts); } -// Make sure it doesn't crash. TEST_F(HlsNotifyMuxerListenerTest, OnSampleDurationReady) { + ON_CALL(mock_notifier_, NotifyNewStream(_, _, _, _, _)) + .WillByDefault(Return(true)); + VideoStreamInfoParameters video_params = GetDefaultVideoStreamInfoParams(); + std::shared_ptr video_stream_info = + CreateVideoStreamInfo(video_params); + MuxerOptions muxer_options; + muxer_options.segment_template = "$Number$.mp4"; + listener_.OnMediaStart(muxer_options, *video_stream_info, 90000, + MuxerListener::kContainerMpeg2ts); + + EXPECT_CALL(mock_notifier_, NotifySampleDuration(_, 2340)); listener_.OnSampleDurationReady(2340); } diff --git a/packager/media/event/mpd_notify_muxer_listener.cc b/packager/media/event/mpd_notify_muxer_listener.cc index aee82b60ba..b4b2f15633 100644 --- a/packager/media/event/mpd_notify_muxer_listener.cc +++ b/packager/media/event/mpd_notify_muxer_listener.cc @@ -118,9 +118,6 @@ void MpdNotifyMuxerListener::OnSampleDurationReady( // If non video, don't worry about it (at the moment). return; } - if (media_info_->video_info().has_frame_duration()) { - return; - } media_info_->mutable_video_info()->set_frame_duration(sample_duration); }