Support removing segments outside of live window in DASH

Issue: #223

Change-Id: Ie2d98a8f1553862532f1bdd074ca022d8904f0ea

packager/media/base/muxer_util.h

@@ -11,9 +11,7 @@
 
 #include <stdint.h>
 
-#include <string>
+#include "packager/status.h"
-
-#include "packager/media/base/key_source.h"
 
 namespace shaka {
 namespace media {

packager/mpd/base/representation.cc

@@ -9,6 +9,8 @@
 #include <gflags/gflags.h>
 
 #include "packager/base/logging.h"
+#include "packager/file/file.h"
+#include "packager/media/base/muxer_util.h"
 #include "packager/mpd/base/mpd_options.h"
 #include "packager/mpd/base/mpd_utils.h"
 #include "packager/mpd/base/xml/xml_node.h"
@@ -93,8 +95,8 @@ uint64_t LatestSegmentStartTime(const std::list<SegmentInfo>& segments) {
 
 // Given |timeshift_limit|, finds out the number of segments that are no longer
 // valid and should be removed from |segment_info|.
-int SearchTimedOutRepeatIndex(uint64_t timeshift_limit,
+uint64_t SearchTimedOutRepeatIndex(uint64_t timeshift_limit,
-                              const SegmentInfo& segment_info) {
+                                   const SegmentInfo& segment_info) {
   DCHECK_LE(timeshift_limit, LastSegmentEndTime(segment_info));
   if (timeshift_limit < segment_info.start_time)
     return 0;
@@ -427,34 +429,52 @@ void Representation::SlideWindow() {
   // looking at the very last segment's end time.
   std::list<SegmentInfo>::iterator first = segment_infos_.begin();
   std::list<SegmentInfo>::iterator last = first;
-  size_t num_segments_removed = 0;
   for (; last != segment_infos_.end(); ++last) {
     const uint64_t last_segment_end_time = LastSegmentEndTime(*last);
     if (timeshift_limit < last_segment_end_time)
       break;
-    num_segments_removed += last->repeat + 1;
+    RemoveSegments(last->start_time, last->duration, last->repeat + 1);
+    start_number_ += last->repeat + 1;
   }
   segment_infos_.erase(first, last);
-  start_number_ += num_segments_removed;
 
   // Now some segment in the first SegmentInfo should be left in the list.
   SegmentInfo* first_segment_info = &segment_infos_.front();
   DCHECK_LE(timeshift_limit, LastSegmentEndTime(*first_segment_info));
 
   // Identify which segments should still be in the SegmentInfo.
-  const int repeat_index =
+  const uint64_t repeat_index =
       SearchTimedOutRepeatIndex(timeshift_limit, *first_segment_info);
-  CHECK_GE(repeat_index, 0);
   if (repeat_index == 0)
     return;
 
+  RemoveSegments(first_segment_info->start_time, first_segment_info->duration,
+                 repeat_index);
+
   first_segment_info->start_time = first_segment_info->start_time +
                                    first_segment_info->duration * repeat_index;
-
   first_segment_info->repeat = first_segment_info->repeat - repeat_index;
   start_number_ += repeat_index;
 }
 
+void Representation::RemoveSegments(uint64_t start_time,
+                                    uint64_t duration,
+                                    uint64_t num_segments) {
+  if (mpd_options_.mpd_params.preserved_segments_outside_live_window == 0)
+    return;
+
+  for (size_t i = 0; i < num_segments; ++i) {
+    segments_to_be_removed_.push_back(media::GetSegmentName(
+        media_info_.segment_template(), start_time + i * duration,
+        start_number_ - 1 + i, media_info_.bandwidth()));
+  }
+  while (segments_to_be_removed_.size() >
+         mpd_options_.mpd_params.preserved_segments_outside_live_window) {
+    File::Delete(segments_to_be_removed_.front().c_str());
+    segments_to_be_removed_.pop_front();
+  }
+}
+
 std::string Representation::GetVideoMimeType() const {
   return GetMimeType("video", media_info_.container_type());
 }

packager/mpd/base/representation.h

@@ -188,6 +188,11 @@ class Representation {
   // |start_number_| by the number of segments removed.
   void SlideWindow();
 
+  // Remove |num_segments| starting from |start_time| with |duration|.
+  void RemoveSegments(uint64_t start_time,
+                      uint64_t duration,
+                      uint64_t num_segments);
+
   // Note: Because 'mimeType' is a required field for a valid MPD, these return
   // strings.
   std::string GetVideoMimeType() const;
@@ -200,6 +205,8 @@ class Representation {
   std::list<ContentProtectionElement> content_protection_elements_;
   // TODO(kqyang): Address sliding window issue with multiple periods.
   std::list<SegmentInfo> segment_infos_;
+  // A temporary list to hold the file names of segments to be removed.
+  std::list<std::string> segments_to_be_removed_;
 
   const uint32_t id_;
   std::string mime_type_;

packager/mpd/base/representation_unittest.cc

@@ -11,6 +11,8 @@
 #include <inttypes.h>
 
 #include "packager/base/strings/stringprintf.h"
+#include "packager/file/file.h"
+#include "packager/file/file_closer.h"
 #include "packager/mpd/base/mpd_options.h"
 #include "packager/mpd/test/mpd_builder_test_helper.h"
 #include "packager/mpd/test/xml_compare.h"
@@ -22,6 +24,12 @@ namespace {
 
 const uint32_t kAnyRepresentationId = 1;
 
+bool SegmentDeleted(const std::string& segment_name) {
+  std::unique_ptr<File, FileCloser> file_closer(
+      File::Open(segment_name.c_str(), "r"));
+  return file_closer.get() == nullptr;
+}
+
 class MockRepresentationStateChangeListener
     : public RepresentationStateChangeListener {
  public:
@@ -1006,4 +1014,82 @@ TEST_F(TimeShiftBufferDepthTest, ManySegments) {
           expected_s_element, kDefaultStartNumber + kExpectedRemovedSegments)));
 }
 
+TEST_F(TimeShiftBufferDepthTest, DeleteSegmentsOutsideOfLiveWindow) {
+  const char kSegmentTemplate[] = "memory://$Number$.mp4";
+  const char kStringPrintTemplate[] = "memory://%d.mp4";
+
+  // Create 100 files with the template.
+  for (int i = 1; i <= 100; ++i) {
+    File::WriteStringToFile(base::StringPrintf(kStringPrintTemplate, i).c_str(),
+                            "dummy content");
+  }
+
+  MediaInfo media_info = ConvertToMediaInfo(GetDefaultMediaInfo());
+  media_info.set_segment_template(kSegmentTemplate);
+  representation_ =
+      CreateRepresentation(media_info, kAnyRepresentationId, NoListener());
+  ASSERT_TRUE(representation_->Init());
+
+  const int kTimeShiftBufferDepth = 2;
+  const int kNumPreservedSegmentsOutsideLiveWindow = 3;
+  const int kMaxNumSegmentsAvailable =
+      kTimeShiftBufferDepth + 1 + kNumPreservedSegmentsOutsideLiveWindow;
+
+  mutable_mpd_options()->mpd_params.time_shift_buffer_depth =
+      kTimeShiftBufferDepth;
+  mutable_mpd_options()->mpd_params.preserved_segments_outside_live_window =
+      kNumPreservedSegmentsOutsideLiveWindow;
+
+  const uint64_t kInitialStartTime = 0;
+  const uint64_t kDuration = kDefaultTimeScale;
+  const uint64_t kSize = 10;
+  const uint64_t kNoRepeat = 0;
+
+  // Verify that no segments are deleted initially until there are more than
+  // |kMaxNumSegmentsAvailable| segments.
+  uint64_t next_start_time = kInitialStartTime;
+  int num_total_segments = 0;
+  int last_available_segment_index = 1;  // No segments are deleted.
+  for (int i = 0; i < kMaxNumSegmentsAvailable; ++i) {
+    AddSegments(next_start_time, kDuration, kSize, kNoRepeat);
+    next_start_time += kDuration;
+    ++num_total_segments;
+
+    EXPECT_FALSE(SegmentDeleted(base::StringPrintf(
+        kStringPrintTemplate, last_available_segment_index)));
+  }
+
+  AddSegments(next_start_time, kDuration, kSize, kNoRepeat);
+  next_start_time += kDuration;
+  ++num_total_segments;
+  last_available_segment_index = 2;
+
+  EXPECT_FALSE(SegmentDeleted(
+      base::StringPrintf(kStringPrintTemplate, last_available_segment_index)));
+  EXPECT_TRUE(SegmentDeleted(base::StringPrintf(
+      kStringPrintTemplate, last_available_segment_index - 1)));
+
+  // Verify that there are exactly |kMaxNumSegmentsAvailable| segments
+  // remaining.
+  const uint64_t kRepeat = 10;
+  AddSegments(next_start_time, kDuration, kSize, kRepeat);
+  next_start_time += (kRepeat + 1) * kDuration;
+  num_total_segments += kRepeat + 1;
+  last_available_segment_index =
+      num_total_segments - kMaxNumSegmentsAvailable + 1;
+
+  EXPECT_FALSE(SegmentDeleted(
+      base::StringPrintf(kStringPrintTemplate, last_available_segment_index)));
+  EXPECT_TRUE(SegmentDeleted(base::StringPrintf(
+      kStringPrintTemplate, last_available_segment_index - 1)));
+
+  AddSegments(next_start_time, kDuration, kSize, kNoRepeat);
+  ++last_available_segment_index;
+
+  EXPECT_FALSE(SegmentDeleted(
+      base::StringPrintf(kStringPrintTemplate, last_available_segment_index)));
+  EXPECT_TRUE(SegmentDeleted(base::StringPrintf(
+      kStringPrintTemplate, last_available_segment_index - 1)));
+}
+
 }  // namespace shaka

packager/mpd/public/mpd_params.h

@@ -37,6 +37,13 @@ struct MpdParams {
   /// Set MPD@timeShiftBufferDepth attribute, which is the guaranteed duration
   /// of the time shifting buffer for 'dynamic' media presentations, in seconds.
   double time_shift_buffer_depth = 0;
+  /// Segments outside live window (defined by 'time_shift_buffer_depth' above)
+  /// are automatically removed except the latest number of segments defined by
+  /// this parameter. This is needed to accommodate latencies in various stages
+  /// of content serving pipeline, so that the segments stay accessible as they
+  /// may still be accessed by the player.
+  /// The segments are not removed if the value is zero.
+  size_t preserved_segments_outside_live_window = 0;
   /// UTCTimings. For dynamic MPD only.
   struct UtcTiming {
     std::string scheme_id_uri;