diff --git a/packager/media/formats/webvtt/text_padder.cc b/packager/media/formats/webvtt/text_padder.cc
index 56219d6038..ac3eff01e4 100644
--- a/packager/media/formats/webvtt/text_padder.cc
+++ b/packager/media/formats/webvtt/text_padder.cc
@@ -16,7 +16,8 @@ namespace {
 const uint64_t kStreamIndex = 0;
 }  // namespace
 
-TextPadder::TextPadder(int64_t duration_ms) : duration_ms_(duration_ms) {}
+TextPadder::TextPadder(int64_t zero_start_bias_ms)
+    : zero_start_bias_ms_(zero_start_bias_ms) {}
 
 Status TextPadder::InitializeInternal() {
   return Status::OK;
@@ -30,20 +31,16 @@ Status TextPadder::Process(std::unique_ptr<StreamData> data) {
                         : Dispatch(std::move(data));
 }
 
-Status TextPadder::OnFlushRequest(size_t index) {
-  if (duration_ms_ > max_end_time_ms_) {
-    std::shared_ptr<TextSample> filler = std::make_shared<TextSample>();
-    filler->SetTime(max_end_time_ms_, duration_ms_);
-    RETURN_IF_ERROR(
-        MediaHandler::DispatchTextSample(kStreamIndex, std::move(filler)));
-  }
-
-  return FlushDownstream(index);
-}
-
 Status TextPadder::OnTextSample(std::unique_ptr<StreamData> data) {
   const TextSample& sample = *data->text_sample;
 
+  // If this is the first sample we have seen, we need to check if we should
+  // start at time zero.
+  if (max_end_time_ms_ < 0) {
+    max_end_time_ms_ =
+        sample.start_time() > zero_start_bias_ms_ ? sample.start_time() : 0;
+  }
+
   // Check if there will be a gap between samples if we just dispatch this
   // sample right away. If there will be one, create an empty sample that will
   // fill in that gap.
diff --git a/packager/media/formats/webvtt/text_padder.h b/packager/media/formats/webvtt/text_padder.h
index 30fba431cd..3d6bc9bc11 100644
--- a/packager/media/formats/webvtt/text_padder.h
+++ b/packager/media/formats/webvtt/text_padder.h
@@ -13,13 +13,18 @@ namespace shaka {
 namespace media {
 
 /// A media handler that will inject empty text samples to fill any gaps
-/// that may appear in the text stream. A min duration can be given to
-/// ensure that the stream will have samples up to the given duration.
+/// that may appear in the text stream.
 class TextPadder : public MediaHandler {
  public:
-  /// Create a new text padder that will ensure the stream's duration is
-  // at least |duration_ms| long.
-  explicit TextPadder(int64_t duration_ms);
+  /// Create a new text padder.
+  ///
+  /// |zero_start_bias_ms| is the threshold used to determine if we should
+  /// assume that the stream actually starts at time zero. If the first sample
+  /// comes before the |zero_start_bias_ms|, then the start will be padded as
+  /// the stream is assumed to start at zero. If the first sample comes after
+  /// |zero_start_bias_ms| then the start of the stream will not be padded as
+  /// we cannot assume the start time of the stream.
+  explicit TextPadder(int64_t zero_start_bias_ms);
   ~TextPadder() override = default;
 
  private:
@@ -29,11 +34,12 @@ class TextPadder : public MediaHandler {
   Status InitializeInternal() override;
 
   Status Process(std::unique_ptr<StreamData> data) override;
-  Status OnFlushRequest(size_t index) override;
   Status OnTextSample(std::unique_ptr<StreamData> data);
 
-  int64_t duration_ms_;
-  int64_t max_end_time_ms_ = 0;
+  const int64_t zero_start_bias_ms_;
+  // Will be set once we see our first sample. Using -1 to signal that we have
+  // not seen the first sample yet.
+  int64_t max_end_time_ms_ = -1;
 };
 
 }  // namespace media
diff --git a/packager/packager.cc b/packager/packager.cc
index ead61cdb33..97e5049594 100644
--- a/packager/packager.cc
+++ b/packager/packager.cc
@@ -66,6 +66,8 @@ namespace {
 
 const char kMediaInfoSuffix[] = ".media_info";
 
+const int64_t kDefaultTextZeroBiasMs = 10 * 60 * 1000;  // 10 minutes
+
 MuxerOptions CreateMuxerOptions(const StreamDescriptor& stream,
                                 const PackagingParams& params) {
   MuxerOptions options;
@@ -465,10 +467,9 @@ Status CreateHlsTextJob(const StreamDescriptor& stream,
   std::unique_ptr<FileReader> reader;
   RETURN_IF_ERROR(FileReader::Open(stream.input, &reader));
 
-  const int64_t kNoDuration = 0;
   auto parser =
       std::make_shared<WebVttParser>(std::move(reader), stream.language);
-  auto padder = std::make_shared<TextPadder>(kNoDuration);
+  auto padder = std::make_shared<TextPadder>(kDefaultTextZeroBiasMs);
   auto cue_aligner = sync_points
                          ? std::make_shared<CueAlignmentHandler>(sync_points)
                          : nullptr;
@@ -490,10 +491,9 @@ Status CreateWebVttToMp4TextJob(const StreamDescriptor& stream,
   std::unique_ptr<FileReader> reader;
   RETURN_IF_ERROR(FileReader::Open(stream.input, &reader));
 
-  const int64_t kNoDuration = 0;
   auto parser =
       std::make_shared<WebVttParser>(std::move(reader), stream.language);
-  auto padder = std::make_shared<TextPadder>(kNoDuration);
+  auto padder = std::make_shared<TextPadder>(kDefaultTextZeroBiasMs);
 
   auto text_to_mp4 = std::make_shared<WebVttToMp4Handler>();
   auto muxer = muxer_factory->CreateMuxer(GetOutputFormat(stream), stream);