Fix for gap size warning in Low Latency mode (#985)

## The issue
- With LL-DASH mode enabled, the gap size warning was hit and printed to the console every time a new segment was registered to the manifest.
- This occurred because the first chunk's size and duration were being stored for each segment, rather than the full segment size and duration. Note, only the first chunk's metrics are known at first because in low latency mode, the segment is registered to the manifest before it is finished being processed and written.
- Because of this, the gap size check was comparing the end time of the first chunk in the previous segment to the beginning time of the current segment, causing the check to fail every time.

## The Fix
- Update a low latency segment's duration and size once the segment file has been fully written.
- The full segment size and duration will be used to update the bandwidth estimator and the segment info list. 
- Updating the segment info list to hold the full duration is necessary for satisfying [the gap size check found in Represenation.cc](https://github.com/google/shaka-packager/blob/master/packager/mpd/base/representation.cc#L391).
- NOTE: bandwidth estimation is currently only used in HLS

packager/media/event/combined_muxer_listener.cc

@@ -77,6 +77,13 @@ void CombinedMuxerListener::OnNewSegment(const std::string& file_name,
   }
 }
 
+void CombinedMuxerListener::OnCompletedSegment(int64_t duration,
+                                               uint64_t segment_file_size) {
+  for (auto& listener : muxer_listeners_) {
+    listener->OnCompletedSegment(duration, segment_file_size);
+  }
+}
+
 void CombinedMuxerListener::OnKeyFrame(int64_t timestamp,
                                        uint64_t start_byte_offset,
                                        uint64_t size) {

packager/media/event/combined_muxer_listener.h

@@ -45,6 +45,8 @@ class CombinedMuxerListener : public MuxerListener {
                     int64_t start_time,
                     int64_t duration,
                     uint64_t segment_file_size) override;
+  void OnCompletedSegment(int64_t duration,
+                          uint64_t segment_file_size) override;
   void OnKeyFrame(int64_t timestamp, uint64_t start_byte_offset, uint64_t size);
   void OnCueEvent(int64_t timestamp, const std::string& cue_data) override;
   /// @}

packager/media/event/mpd_notify_muxer_listener.cc

@@ -204,6 +204,12 @@ void MpdNotifyMuxerListener::OnNewSegment(const std::string& file_name,
   }
 }
 
+void MpdNotifyMuxerListener::OnCompletedSegment(int64_t duration,
+                                                uint64_t segment_file_size) {
+  mpd_notifier_->NotifyCompletedSegment(notification_id_.value(), duration,
+                                        segment_file_size);
+}
+
 void MpdNotifyMuxerListener::OnKeyFrame(int64_t timestamp,
                                         uint64_t start_byte_offset,
                                         uint64_t size) {

packager/media/event/mpd_notify_muxer_listener.h

@@ -53,6 +53,8 @@ class MpdNotifyMuxerListener : public MuxerListener {
                     int64_t start_time,
                     int64_t duration,
                     uint64_t segment_file_size) override;
+  void OnCompletedSegment(int64_t duration,
+                          uint64_t segment_file_size) override;
   void OnKeyFrame(int64_t timestamp, uint64_t start_byte_offset, uint64_t size);
   void OnCueEvent(int64_t timestamp, const std::string& cue_data) override;
   /// @}

packager/media/event/muxer_listener.h

@@ -120,9 +120,11 @@ class MuxerListener {
                           float duration_seconds) = 0;
 
   /// Called when a segment has been muxed and the file has been written.
-  /// Note: For some implementations, this is used to signal new subsegments.
-  /// For example, for generating video on demand (VOD) MPD manifest, this is
-  /// called to signal subsegments.
+  /// Note: For some implementations, this is used to signal new subsegments
+  /// or chunks. For example, for generating video on demand (VOD) MPD manifest,
+  /// this is called to signal subsegments. In the low latency case, this
+  /// indicates the start of a new segment and will contain info about the
+  /// segment's first chunk.
   /// @param segment_name is the name of the new segment. Note that some
   ///        implementations may not require this, e.g. if this is a subsegment.
   /// @param start_time is the start time of the segment, relative to the
@@ -135,6 +137,15 @@ class MuxerListener {
                             int64_t duration,
                             uint64_t segment_file_size) = 0;
 
+  /// Called when a segment has been muxed and the entire file has been written.
+  /// For Low Latency only. Note that it should be called after OnNewSegment.
+  /// When the low latency segment is initally added to the manifest, the size
+  /// and duration are not known, because the segment is still being processed.
+  /// This will update the segment's duration and size after the segment is
+  /// fully written and these values are known.
+  virtual void OnCompletedSegment(int64_t duration,
+                                  uint64_t segment_file_size) {}
+
   /// Called when there is a new key frame. For Video only. Note that it should
   /// be called before OnNewSegment is called on the containing segment.
   /// @param timestamp is in terms of the timescale of the media.

packager/media/formats/mp4/low_latency_segment_segmenter.cc

@@ -128,8 +128,8 @@ Status LowLatencySegmentSegmenter::WriteInitialChunk() {
   styp_->Write(buffer.get());
 
   const size_t segment_header_size = buffer->Size();
-  const size_t segment_size = segment_header_size + fragment_buffer()->Size();
-  DCHECK_NE(segment_size, 0u);
+  segment_size_ = segment_header_size + fragment_buffer()->Size();
+  DCHECK_NE(segment_size_, 0u);
 
   RETURN_IF_ERROR(buffer->WriteToFile(segment_file_.get()));
   if (muxer_listener()) {
@@ -160,7 +160,7 @@ Status LowLatencySegmentSegmenter::WriteInitialChunk() {
     // Following chunks will be appended to the open segment file.
     muxer_listener()->OnNewSegment(file_name_,
                                    sidx()->earliest_presentation_time,
-                                   segment_duration, segment_size);
+                                   segment_duration, segment_size_);
     is_initial_chunk_in_seg_ = false;
   }
 
@@ -179,6 +179,9 @@ Status LowLatencySegmentSegmenter::WriteChunk() {
 }
 
 Status LowLatencySegmentSegmenter::FinalizeSegment() {
+  if (muxer_listener()) {
+    muxer_listener()->OnCompletedSegment(GetSegmentDuration(), segment_size_);
+  }
   // Close the file now that the final chunk has been written
   if (!segment_file_.release()->Close()) {
     return Status(
@@ -190,6 +193,7 @@ Status LowLatencySegmentSegmenter::FinalizeSegment() {
   // Current segment is complete. Reset state in preparation for the next
   // segment.
   is_initial_chunk_in_seg_ = true;
+  segment_size_ = 0u;
   num_segments_++;
 
   return Status::OK;

packager/media/formats/mp4/low_latency_segment_segmenter.h

@@ -61,6 +61,7 @@ class LowLatencySegmentSegmenter : public Segmenter {
   bool ll_dash_mpd_values_initialized_ = false;
   std::unique_ptr<File, FileCloser> segment_file_;
   std::string file_name_;
+  size_t segment_size_ = 0u;
 
   DISALLOW_COPY_AND_ASSIGN(LowLatencySegmentSegmenter);
 };

packager/mpd/base/mock_mpd_notifier.h

@@ -31,6 +31,8 @@ class MockMpdNotifier : public MpdNotifier {
                     int64_t start_time,
                     int64_t duration,
                     uint64_t size));
+  MOCK_METHOD3(NotifyCompletedSegment,
+               bool(uint32_t container_id, int64_t duration, uint64_t size));
   MOCK_METHOD1(NotifyAvailabilityTimeOffset, bool(uint32_t container_id));
   MOCK_METHOD1(NotifySegmentDuration, bool(uint32_t container_id));
   MOCK_METHOD2(NotifyCueEvent, bool(uint32_t container_id, int64_t timestamp));

packager/mpd/base/mpd_notifier.h

@@ -73,7 +73,8 @@ class MpdNotifier {
   virtual bool NotifySegmentDuration(uint32_t container_id) { return true; }
 
   /// Notifies MpdBuilder that there is a new segment ready. For live, this
-  /// is usually a new segment, for VOD this is usually a subsegment.
+  /// is usually a new segment, for VOD this is usually a subsegment, for low
+  /// latency this is the first chunk.
   /// @param container_id Container ID obtained from calling
   ///        NotifyNewContainer().
   /// @param start_time is the start time of the new segment, in units of the
@@ -87,6 +88,23 @@ class MpdNotifier {
                                 int64_t duration,
                                 uint64_t size) = 0;
 
+  /// Notifies MpdBuilder that a segment is fully written and provides the
+  /// segment's complete duration and size. For Low Latency only. Note, size and
+  /// duration are not known when the low latency segment is first registered
+  /// with the MPD, so we must update these values after the segment is
+  /// complete.
+  /// @param container_id Container ID obtained from calling
+  ///        NotifyNewContainer().
+  /// @param duration is the duration of the complete segment, in units of the
+  ///        stream's time scale.
+  /// @param size is the complete segment size in bytes.
+  /// @return true on success, false otherwise.
+  virtual bool NotifyCompletedSegment(uint32_t container_id,
+                                      int64_t duration,
+                                      uint64_t size) {
+    return true;
+  }
+
   /// Notifies MpdBuilder that there is a new CueEvent.
   /// @param container_id Container ID obtained from calling
   ///        NotifyNewContainer().

packager/mpd/base/representation.cc

@@ -187,6 +187,29 @@ void Representation::AddNewSegment(int64_t start_time,
     state_change_listener_->OnNewSegmentForRepresentation(start_time, duration);
 
   AddSegmentInfo(start_time, duration);
+
+  // Only update the buffer depth and bandwidth estimator when the full segment
+  // is completed. In the low latency case, only the first chunk in the segment
+  // has been written at this point. Therefore, we must wait until the entire
+  // segment has been written before updating buffer depth and bandwidth
+  // estimator.
+  if (!mpd_options_.mpd_params.low_latency_dash_mode) {
+    current_buffer_depth_ += segment_infos_.back().duration;
+
+    bandwidth_estimator_.AddBlock(size, static_cast<double>(duration) /
+                                            media_info_.reference_time_scale());
+  }
+}
+
+void Representation::UpdateCompletedSegment(int64_t duration, uint64_t size) {
+  if (!mpd_options_.mpd_params.low_latency_dash_mode) {
+    LOG(WARNING)
+        << "UpdateCompletedSegment is only applicable to low latency mode.";
+    return;
+  }
+
+  UpdateSegmentInfo(duration);
+
   current_buffer_depth_ += segment_infos_.back().duration;
 
   bandwidth_estimator_.AddBlock(
@@ -410,6 +433,13 @@ void Representation::AddSegmentInfo(int64_t start_time, int64_t duration) {
   segment_infos_.push_back({start_time, adjusted_duration, kNoRepeat});
 }
 
+void Representation::UpdateSegmentInfo(int64_t duration) {
+  if (!segment_infos_.empty()) {
+    // Update the duration in the current segment.
+    segment_infos_.back().duration = duration;
+  }
+}
+
 bool Representation::ApproximiatelyEqual(int64_t time1, int64_t time2) const {
   if (!allow_approximate_segment_timeline_)
     return time1 == time2;

packager/mpd/base/representation.h

@@ -95,12 +95,25 @@ class Representation {
   /// @param start_time is the start time for the (sub)segment, in units of the
   ///        stream's time scale.
   /// @param duration is the duration of the segment, in units of the stream's
-  ///        time scale.
-  /// @param size of the segment in bytes.
+  ///        time scale. In the low latency case, this duration is that of the
+  ///        first chunk because the full duration is not yet known.
+  /// @param size of the segment in bytes. In the low latency case, this size is
+  /// that of the
+  ///        first chunk because the full size is not yet known.
   virtual void AddNewSegment(int64_t start_time,
                              int64_t duration,
                              uint64_t size);
 
+  /// Update a media segment in the Representation.
+  /// In the low latency case, the segment duration will not be ready until the
+  /// entire segment has been processed. This allows setting the full duration
+  /// after the segment has been completed and the true duratio is known.
+  /// @param duration is the duration of the complete segment, in units of the
+  /// stream's
+  ///        time scale.
+  /// @param size of the complete segment in bytes.
+  virtual void UpdateCompletedSegment(int64_t duration, uint64_t size);
+
   /// Set the sample duration of this Representation.
   /// Sample duration is not available right away especially for live. This
   /// allows setting the sample duration after the Representation has been
@@ -188,6 +201,11 @@ class Representation {
   // |allow_approximate_segment_timeline_| is set.
   void AddSegmentInfo(int64_t start_time, int64_t duration);
 
+  // Update the current SegmentInfo. This method is used to update the duration
+  // value after a low latency segment is complete, and the full segment
+  // duration is known.
+  void UpdateSegmentInfo(int64_t duration);
+
   // Check if two timestamps are approximately equal if
   // |allow_approximate_segment_timeline_| is set; Otherwise check whether the
   // two times match.

packager/mpd/base/representation_unittest.cc

@@ -444,6 +444,8 @@ class SegmentTemplateTest : public RepresentationTest {
  public:
   void SetUp() override {
     mpd_options_.mpd_type = MpdType::kDynamic;
+    mpd_options_.mpd_params.low_latency_dash_mode = false;
+
     representation_ =
         CreateRepresentation(ConvertToMediaInfo(GetDefaultMediaInfo()),
                              kAnyRepresentationId, NoListener());
@@ -458,6 +460,16 @@ class SegmentTemplateTest : public RepresentationTest {
 
     SegmentInfo s = {start_time, duration, repeat};
     segment_infos_for_expected_out_.push_back(s);
+
+    if (mpd_options_.mpd_params.low_latency_dash_mode) {
+      // Low latency segments do not repeat, so create 1 new segment and return.
+      // At this point, only the first chunk of the low latency segment has been
+      // written. The bandwidth will be updated once the segment is fully
+      // written and the segment duration and size are known.
+      representation_->AddNewSegment(start_time, duration, size);
+      return;
+    }
+
     if (repeat == 0) {
       expected_s_elements_ +=
           base::StringPrintf(kSElementTemplateWithoutR, start_time, duration);
@@ -474,6 +486,16 @@ class SegmentTemplateTest : public RepresentationTest {
     }
   }
 
+  void UpdateSegment(int64_t duration, uint64_t size) {
+    DCHECK(representation_);
+    DCHECK(!segment_infos_for_expected_out_.empty());
+
+    segment_infos_for_expected_out_.back().duration = duration;
+    representation_->UpdateCompletedSegment(duration, size);
+    bandwidth_estimator_.AddBlock(
+        size, static_cast<double>(duration) / kDefaultTimeScale);
+  }
+
  protected:
   std::string ExpectedXml() {
     const char kOutputTemplate[] =
@@ -510,6 +532,39 @@ TEST_F(SegmentTemplateTest, OneSegmentNormal) {
   EXPECT_THAT(representation_->GetXml(), XmlNodeEqual(ExpectedXml()));
 }
 
+TEST_F(SegmentTemplateTest, OneSegmentLowLatency) {
+  const int64_t kStartTime = 0;
+  const int64_t kChunkDuration = 5;
+  const uint64_t kChunkSize = 128;
+  const int64_t kSegmentDuration = kChunkDuration * 1000;
+  const uint64_t kSegmentSize = kChunkSize * 1000;
+
+  mpd_options_.mpd_params.low_latency_dash_mode = true;
+  mpd_options_.mpd_params.target_segment_duration =
+      kSegmentDuration / representation_->GetMediaInfo().reference_time_scale();
+
+  // Set values used in LL-DASH MPD attributes
+  representation_->SetSampleDuration(kChunkDuration);
+  representation_->SetAvailabilityTimeOffset();
+  representation_->SetSegmentDuration();
+
+  // Register segment after the first chunk is complete
+  AddSegments(kStartTime, kChunkDuration, kChunkSize, 0);
+  // Update SegmentInfo after the segment is complete
+  UpdateSegment(kSegmentDuration, kSegmentSize);
+
+  const char kOutputTemplate[] =
+      "<Representation id=\"1\" bandwidth=\"204800\" "
+      " codecs=\"avc1.010101\" mimeType=\"video/mp4\" sar=\"1:1\" "
+      " width=\"720\" height=\"480\" frameRate=\"10/5\">\n"
+      "  <SegmentTemplate timescale=\"1000\" "
+      "   duration=\"5000\" availabilityTimeOffset=\"4.995\" "
+      "   initialization=\"init.mp4\" media=\"$Time$.mp4\" "
+      "   startNumber=\"1\"/>\n"
+      "</Representation>\n";
+  EXPECT_THAT(representation_->GetXml(), XmlNodeEqual(kOutputTemplate));
+}
+
 TEST_F(SegmentTemplateTest, RepresentationClone) {
   MediaInfo media_info = ConvertToMediaInfo(GetDefaultMediaInfo());
   media_info.set_segment_template_url("$Number$.mp4");

packager/mpd/base/simple_mpd_notifier.cc

@@ -119,6 +119,19 @@ bool SimpleMpdNotifier::NotifyNewSegment(uint32_t container_id,
   return true;
 }
 
+bool SimpleMpdNotifier::NotifyCompletedSegment(uint32_t container_id,
+                                               int64_t duration,
+                                               uint64_t size) {
+  base::AutoLock auto_lock(lock_);
+  auto it = representation_map_.find(container_id);
+  if (it == representation_map_.end()) {
+    LOG(ERROR) << "Unexpected container_id: " << container_id;
+    return false;
+  }
+  it->second->UpdateCompletedSegment(duration, size);
+  return true;
+}
+
 bool SimpleMpdNotifier::NotifyCueEvent(uint32_t container_id,
                                        int64_t timestamp) {
   base::AutoLock auto_lock(lock_);

packager/mpd/base/simple_mpd_notifier.h

@@ -44,6 +44,9 @@ class SimpleMpdNotifier : public MpdNotifier {
                         int64_t start_time,
                         int64_t duration,
                         uint64_t size) override;
+  bool NotifyCompletedSegment(uint32_t container_id,
+                              int64_t duration,
+                              uint64_t size) override;
   bool NotifyCueEvent(uint32_t container_id, int64_t timestamp) override;
   bool NotifyEncryptionUpdate(uint32_t container_id,
                               const std::string& drm_uuid,