diff --git a/packager/app/test/packager_test.py b/packager/app/test/packager_test.py index 9ad469dae1..77aecde6cd 100755 --- a/packager/app/test/packager_test.py +++ b/packager/app/test/packager_test.py @@ -721,6 +721,11 @@ class PackagerFunctionalTest(PackagerAppTest): self._GetFlags(output_dash=True)) self._CheckTestResults('video-audio-text') + def testVideoNoEditList(self): + stream = self._GetStream('video', test_file='bear-640x360-no_edit_list.mp4') + self.assertPackageSuccess([stream], self._GetFlags(output_dash=True)) + self._CheckTestResults('video-no-edit-list') + def testAvcAacTs(self): # Currently we only support live packaging for ts. self.assertPackageSuccess( diff --git a/packager/app/test/testdata/video-no-edit-list/bear-640x360-no_edit_list-video.mp4 b/packager/app/test/testdata/video-no-edit-list/bear-640x360-no_edit_list-video.mp4 new file mode 100644 index 0000000000..37528cb505 Binary files /dev/null and b/packager/app/test/testdata/video-no-edit-list/bear-640x360-no_edit_list-video.mp4 differ diff --git a/packager/app/test/testdata/video-no-edit-list/output.mpd b/packager/app/test/testdata/video-no-edit-list/output.mpd new file mode 100644 index 0000000000..61df73f51c --- /dev/null +++ b/packager/app/test/testdata/video-no-edit-list/output.mpd @@ -0,0 +1,14 @@ + + + + + + + bear-640x360-no_edit_list-video.mp4 + + + + + + + diff --git a/packager/media/formats/mp4/mp4.gyp b/packager/media/formats/mp4/mp4.gyp index 9ee4c6638e..0707fd5aff 100644 --- a/packager/media/formats/mp4/mp4.gyp +++ b/packager/media/formats/mp4/mp4.gyp @@ -46,6 +46,7 @@ ], 'dependencies': [ '../../../third_party/boringssl/boringssl.gyp:boringssl', + '../../../third_party/gflags/gflags.gyp:gflags', '../../base/media_base.gyp:media_base', '../../codecs/codecs.gyp:codecs', '../../event/media_event.gyp:media_event', @@ -68,6 +69,7 @@ '../../../file/file.gyp:file', '../../../testing/gtest.gyp:gtest', '../../../testing/gmock.gyp:gmock', + '../../../third_party/gflags/gflags.gyp:gflags', '../../test/media_test.gyp:media_test_support', 'mp4', ] diff --git a/packager/media/formats/mp4/track_run_iterator.cc b/packager/media/formats/mp4/track_run_iterator.cc index dde7ebdb33..2a0dbd1eaa 100644 --- a/packager/media/formats/mp4/track_run_iterator.cc +++ b/packager/media/formats/mp4/track_run_iterator.cc @@ -4,6 +4,13 @@ #include "packager/media/formats/mp4/track_run_iterator.h" +#include + +DEFINE_bool(mp4_reset_initial_composition_offset_to_zero, + true, + "MP4 only. If it is true, reset the initial composition offset to " + "zero, i.e. by assuming that there is a missing EditList."); + #include #include @@ -180,7 +187,7 @@ bool TrackRunIterator::Init() { // dts is directly adjusted, which then propagates to pts as pts is encoded // as difference (composition offset) to dts in mp4. - int64_t run_start_dts = GetTimestampAdjustment(*moov_, *trak); + int64_t run_start_dts = GetTimestampAdjustment(*moov_, *trak, nullptr); uint32_t num_samples = sample_size.sample_count; uint32_t num_chunks = static_cast(chunk_offset_vector.size()); @@ -349,7 +356,7 @@ bool TrackRunIterator::Init(const MovieFragment& moof) { // dts is directly adjusted, which then propagates to pts as pts is encoded // as difference (composition offset) to dts in mp4. - run_start_dts += GetTimestampAdjustment(*moov_, *trak); + run_start_dts += GetTimestampAdjustment(*moov_, *trak, &traf); int sample_count_sum = 0; @@ -634,8 +641,14 @@ std::unique_ptr TrackRunIterator::GetDecryptConfig() { } int64_t TrackRunIterator::GetTimestampAdjustment(const Movie& movie, - const Track& track) { - int64_t edit_list_offset = 0; + const Track& track, + const TrackFragment* traf) { + const uint32_t track_id = track.header.track_id; + const auto iter = timestamp_adjustment_map_.find(track_id); + if (iter != timestamp_adjustment_map_.end()) + return iter->second; + + int64_t timestamp_adjustment = 0; const std::vector& edits = track.edit.list.edits; if (!edits.empty()) { // ISO/IEC 14496-12:2015 8.6.6 Edit List Box. @@ -651,13 +664,48 @@ int64_t TrackRunIterator::GetTimestampAdjustment(const Movie& movie, const int64_t scaled_time = Rescale(edit.segment_duration, movie.header.timescale, track.media.header.timescale); - edit_list_offset += scaled_time; + timestamp_adjustment += scaled_time; } else { - edit_list_offset -= edit.media_time; + timestamp_adjustment -= edit.media_time; } } } - return edit_list_offset; + + if (timestamp_adjustment == 0) { + int64_t composition_offset = 0; + if (traf && !traf->runs.empty()) { + const auto& cts_offsets = + traf->runs.front().sample_composition_time_offsets; + if (!cts_offsets.empty()) + composition_offset = cts_offsets.front(); + } else { + CompositionOffsetIterator composition_offset_iter( + track.media.information.sample_table.composition_time_to_sample); + if (composition_offset_iter.IsValid()) + composition_offset = composition_offset_iter.sample_offset(); + } + + int64_t decode_time = 0; + if (traf) + decode_time = traf->decode_time.decode_time; + if (composition_offset != 0 && decode_time == 0) { + LOG(WARNING) << "Seeing non-zero composition offset " + << composition_offset + << ". An EditList is probably missing."; + if (FLAGS_mp4_reset_initial_composition_offset_to_zero) { + LOG(WARNING) + << "Adjusting timestamps by " << -composition_offset + << ". Please file a bug to " + "https://github.com/google/shaka-packager/issues if you " + "do not think it is right or if you are seeing any problems."; + timestamp_adjustment = -composition_offset; + } + } + } + + timestamp_adjustment_map_.insert( + std::make_pair(track_id, timestamp_adjustment)); + return timestamp_adjustment; } } // namespace mp4 diff --git a/packager/media/formats/mp4/track_run_iterator.h b/packager/media/formats/mp4/track_run_iterator.h index 15cdf84080..e5b920c486 100644 --- a/packager/media/formats/mp4/track_run_iterator.h +++ b/packager/media/formats/mp4/track_run_iterator.h @@ -5,6 +5,7 @@ #ifndef PACKAGER_MEDIA_FORMATS_MP4_TRACK_RUN_ITERATOR_H_ #define PACKAGER_MEDIA_FORMATS_MP4_TRACK_RUN_ITERATOR_H_ +#include #include #include @@ -103,7 +104,9 @@ class TrackRunIterator { private: void ResetRun(); const TrackEncryption& track_encryption() const; - int64_t GetTimestampAdjustment(const Movie& movie, const Track& track); + int64_t GetTimestampAdjustment(const Movie& movie, + const Track& track, + const TrackFragment* traf); const Movie* moov_; @@ -118,6 +121,9 @@ class TrackRunIterator { int64_t sample_dts_; int64_t sample_offset_; + // TrackId => adjustment map. + std::map timestamp_adjustment_map_; + DISALLOW_COPY_AND_ASSIGN(TrackRunIterator); }; diff --git a/packager/media/formats/mp4/track_run_iterator_unittest.cc b/packager/media/formats/mp4/track_run_iterator_unittest.cc index 46bbf14f41..b57a2ef7fd 100644 --- a/packager/media/formats/mp4/track_run_iterator_unittest.cc +++ b/packager/media/formats/mp4/track_run_iterator_unittest.cc @@ -2,12 +2,16 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "packager/media/formats/mp4/track_run_iterator.h" + +#include #include #include #include #include "packager/base/logging.h" #include "packager/media/formats/mp4/box_definitions.h" -#include "packager/media/formats/mp4/track_run_iterator.h" + +DECLARE_bool(mp4_reset_initial_composition_offset_to_zero); namespace { @@ -15,6 +19,7 @@ namespace { // less the value of the last element. const int kSumAscending1 = 45; +const int kMovieScale = 1000; const int kAudioScale = 48000; const int kVideoScale = 25; @@ -108,7 +113,7 @@ class TrackRunIteratorTest : public testing::Test { std::unique_ptr iter_; void CreateMovie() { - moov_.header.timescale = 1000; + moov_.header.timescale = kMovieScale; moov_.tracks.resize(3); moov_.extends.tracks.resize(2); moov_.tracks[0].header.track_id = 1; @@ -384,7 +389,49 @@ TEST_F(TrackRunIteratorTest, FirstSampleFlagTest) { EXPECT_FALSE(iter_->is_keyframe()); } +TEST_F(TrackRunIteratorTest, EmptyEditTest) { + iter_.reset(new TrackRunIterator(&moov_)); + + EditListEntry entry; + entry.segment_duration = 2 * kMovieScale; + entry.media_time = -1; + entry.media_rate_integer = 1; + entry.media_rate_fraction = 0; + moov_.tracks[1].edit.list.edits.push_back(entry); + + MovieFragment moof = CreateFragment(); + moof.tracks[1].decode_time.decode_time = 0; + + ASSERT_TRUE(iter_->Init(moof)); + iter_->AdvanceRun(); + EXPECT_EQ(iter_->dts(), 2 * kVideoScale); + EXPECT_EQ(iter_->cts(), 2 * kVideoScale); +} + +TEST_F(TrackRunIteratorTest, NormalEditTest) { + iter_.reset(new TrackRunIterator(&moov_)); + + const int kMediaTime = 5; + + EditListEntry entry; + entry.segment_duration = 0; + entry.media_time = kMediaTime; + entry.media_rate_integer = 1; + entry.media_rate_fraction = 0; + moov_.tracks[1].edit.list.edits.push_back(entry); + + MovieFragment moof = CreateFragment(); + moof.tracks[1].decode_time.decode_time = 0; + + ASSERT_TRUE(iter_->Init(moof)); + iter_->AdvanceRun(); + EXPECT_EQ(iter_->dts(), -kMediaTime); + EXPECT_EQ(iter_->cts(), -kMediaTime); +} + TEST_F(TrackRunIteratorTest, ReorderingTest) { + FLAGS_mp4_reset_initial_composition_offset_to_zero = false; + // Test frame reordering. The frames have the following // decode timestamps: // @@ -424,6 +471,71 @@ TEST_F(TrackRunIteratorTest, ReorderingTest) { EXPECT_EQ(iter_->duration(), 3); } +TEST_F(TrackRunIteratorTest, ReorderingTest_WithEditList) { + FLAGS_mp4_reset_initial_composition_offset_to_zero = false; + + // See the test above for background. + iter_.reset(new TrackRunIterator(&moov_)); + MovieFragment moof = CreateFragment(); + std::vector& cts_offsets = + moof.tracks[1].runs[0].sample_composition_time_offsets; + cts_offsets.resize(10); + cts_offsets[0] = 2; + cts_offsets[1] = 5; + cts_offsets[2] = 0; + moof.tracks[1].decode_time.decode_time = 0; + + EditListEntry entry; + entry.segment_duration = 0; + entry.media_time = 2; + entry.media_rate_integer = 1; + entry.media_rate_fraction = 0; + moov_.tracks[1].edit.list.edits.push_back(entry); + + ASSERT_TRUE(iter_->Init(moof)); + iter_->AdvanceRun(); + EXPECT_EQ(iter_->dts(), -2); + EXPECT_EQ(iter_->cts(), 0); + EXPECT_EQ(iter_->duration(), 1); + iter_->AdvanceSample(); + EXPECT_EQ(iter_->dts(), -1); + EXPECT_EQ(iter_->cts(), 4); + EXPECT_EQ(iter_->duration(), 2); + iter_->AdvanceSample(); + EXPECT_EQ(iter_->dts(), 1); + EXPECT_EQ(iter_->cts(), 1); + EXPECT_EQ(iter_->duration(), 3); +} + +TEST_F(TrackRunIteratorTest, ReorderingTest_ResetInitialCompositionOffset) { + FLAGS_mp4_reset_initial_composition_offset_to_zero = true; + + // See the test above for background. + iter_.reset(new TrackRunIterator(&moov_)); + MovieFragment moof = CreateFragment(); + std::vector& cts_offsets = + moof.tracks[1].runs[0].sample_composition_time_offsets; + cts_offsets.resize(10); + cts_offsets[0] = 2; + cts_offsets[1] = 5; + cts_offsets[2] = 0; + moof.tracks[1].decode_time.decode_time = 0; + + ASSERT_TRUE(iter_->Init(moof)); + iter_->AdvanceRun(); + EXPECT_EQ(iter_->dts(), -2); + EXPECT_EQ(iter_->cts(), 0); + EXPECT_EQ(iter_->duration(), 1); + iter_->AdvanceSample(); + EXPECT_EQ(iter_->dts(), -1); + EXPECT_EQ(iter_->cts(), 4); + EXPECT_EQ(iter_->duration(), 2); + iter_->AdvanceSample(); + EXPECT_EQ(iter_->dts(), 1); + EXPECT_EQ(iter_->cts(), 1); + EXPECT_EQ(iter_->duration(), 3); +} + TEST_F(TrackRunIteratorTest, IgnoreUnknownAuxInfoTest) { iter_.reset(new TrackRunIterator(&moov_)); MovieFragment moof = CreateFragment(); diff --git a/packager/media/test/data/README b/packager/media/test/data/README index 0311df8c15..d4bce77108 100644 --- a/packager/media/test/data/README +++ b/packager/media/test/data/README @@ -45,6 +45,7 @@ bear-flac.mp4 - Unfragmented audio-only 44.1kHz FLAC in MP4 file, created using: ffmpeg -i bear-1280x720.mp4 -map 0:0 -acodec flac -strict -2 bear-flac.mp4 Note, "-strict -2" was required because current ffmpeg libavformat version 57.75.100 indicates that flac in MP4 support is experimental. +bear-640x360-no_edit_list.mp4 - Same content, but with EditLists removed. // Non square pixels. bear-640x360-non_square_pixel-with_pasp.mp4 - A non-square pixel version of the video track of bear-640x360.mp4 with PixelAspectRatio box. diff --git a/packager/media/test/data/bear-640x360-no_edit_list.mp4 b/packager/media/test/data/bear-640x360-no_edit_list.mp4 new file mode 100644 index 0000000000..8981306b68 Binary files /dev/null and b/packager/media/test/data/bear-640x360-no_edit_list.mp4 differ