Support parameter sets in stream (avc3, hev1 etc)

- Parses parameter set NAL units in the samples.
- Calculate pixel width and height from track width and height.

Fixes #621, #627.

Change-Id: Ic1e120dccbd220b01168f7bf4effeaa43f95b055

packager/media/codecs/avc_decoder_configuration_record.cc

@@ -43,8 +43,7 @@ bool AVCDecoderConfigurationRecord::ParseInternal() {
   RCHECK(reader.Read1(&num_sps));
   num_sps &= 0x1f;
   if (num_sps < 1) {
-    LOG(ERROR) << "No SPS found.";
+    VLOG(1) << "No SPS found.";
-    return false;
   }
 
   for (uint8_t i = 0; i < num_sps; i++) {

packager/media/codecs/video_slice_header_parser.cc

@@ -44,6 +44,18 @@ bool H264VideoSliceHeaderParser::Initialize(
   return true;
 }
 
+bool H264VideoSliceHeaderParser::ProcessNalu(const Nalu& nalu) {
+  int id;
+  switch (nalu.type()) {
+    case Nalu::H264_SPS:
+      return parser_.ParseSps(nalu, &id) == H264Parser::kOk;
+    case Nalu::H264_PPS:
+      return parser_.ParsePps(nalu, &id) == H264Parser::kOk;
+    default:
+      return true;
+  }
+}
+
 int64_t H264VideoSliceHeaderParser::GetHeaderSize(const Nalu& nalu) {
   DCHECK(nalu.is_video_slice());
   H264SliceHeader slice_header;
@@ -79,6 +91,21 @@ bool H265VideoSliceHeaderParser::Initialize(
   return true;
 }
 
+bool H265VideoSliceHeaderParser::ProcessNalu(const Nalu& nalu) {
+  int id;
+  switch (nalu.type()) {
+    case Nalu::H265_SPS:
+      return parser_.ParseSps(nalu, &id) == H265Parser::kOk;
+    case Nalu::H265_PPS:
+      return parser_.ParsePps(nalu, &id) == H265Parser::kOk;
+    case Nalu::H265_VPS:
+      // Ignore since it does not affect video slice header parsing.
+      return true;
+    default:
+      return true;
+  }
+}
+
 int64_t H265VideoSliceHeaderParser::GetHeaderSize(const Nalu& nalu) {
   DCHECK(nalu.is_video_slice());
   H265SliceHeader slice_header;

packager/media/codecs/video_slice_header_parser.h

@@ -26,6 +26,14 @@ class VideoSliceHeaderParser {
   virtual bool Initialize(
       const std::vector<uint8_t>& decoder_configuration) = 0;
 
+  /// Process NAL unit, in particular parameter set NAL units.  Non parameter
+  /// set NAL unit is allowed but the function always returns true.
+  /// Returns false if there is any problem processing the parameter set NAL
+  /// unit.
+  /// This function is needed to handle parameter set NAL units not in decoder
+  /// configuration record, i.e. in the samples.
+  virtual bool ProcessNalu(const Nalu& nalu) = 0;
+
   /// Gets the header size of the given NALU.  Returns < 0 on error.
   virtual int64_t GetHeaderSize(const Nalu& nalu) = 0;
 
@@ -41,6 +49,7 @@ class H264VideoSliceHeaderParser : public VideoSliceHeaderParser {
   /// @name VideoSliceHeaderParser implementation overrides.
   /// @{
   bool Initialize(const std::vector<uint8_t>& decoder_configuration) override;
+  bool ProcessNalu(const Nalu& nalu) override;
   int64_t GetHeaderSize(const Nalu& nalu) override;
   /// @}
 
@@ -58,6 +67,7 @@ class H265VideoSliceHeaderParser : public VideoSliceHeaderParser {
   /// @name VideoSliceHeaderParser implementation overrides.
   /// @{
   bool Initialize(const std::vector<uint8_t>& decoder_configuration) override;
+  bool ProcessNalu(const Nalu& nalu) override;
   int64_t GetHeaderSize(const Nalu& nalu) override;
   /// @}
 
@@ -71,4 +81,3 @@ class H265VideoSliceHeaderParser : public VideoSliceHeaderParser {
 }  // namespace shaka
 
 #endif  // PACKAGER_MEDIA_CODECS_VIDEO_SLICE_HEADER_PARSER_H_
-

packager/media/crypto/subsample_generator.cc

@@ -306,6 +306,13 @@ Status SubsampleGenerator::GenerateSubsamplesFromH26xFrame(
   Nalu nalu;
   NaluReader::Result result;
   while ((result = reader.Advance(&nalu)) == NaluReader::kOk) {
+    // |header_parser_| is only used if |leading_clear_bytes_size_| is not
+    // availble. See lines below.
+    if (leading_clear_bytes_size_ == 0 && !header_parser_->ProcessNalu(nalu)) {
+      LOG(ERROR) << "Failed to process NAL unit: NAL type = " << nalu.type();
+      return Status(error::ENCRYPTION_FAILURE, "Failed to process NAL unit.");
+    }
+
     const size_t nalu_total_size = nalu.header_size() + nalu.payload_size();
     size_t clear_bytes = 0;
     if (nalu.is_video_slice() && nalu_total_size >= min_protected_data_size_) {

packager/media/crypto/subsample_generator_unittest.cc

@@ -21,6 +21,7 @@ namespace media {
 namespace {
 
 using ::testing::_;
+using ::testing::AtLeast;
 using ::testing::DoAll;
 using ::testing::ElementsAre;
 using ::testing::ElementsAreArray;
@@ -123,6 +124,7 @@ class MockVideoSliceHeaderParser : public VideoSliceHeaderParser {
  public:
   MOCK_METHOD1(Initialize,
                bool(const std::vector<uint8_t>& decoder_configuration));
+  MOCK_METHOD1(ProcessNalu, bool(const Nalu& nalu));
   MOCK_METHOD1(GetHeaderSize, int64_t(const Nalu& nalu));
 };
 
@@ -290,6 +292,8 @@ TEST_P(SubsampleGeneratorTest, H264ParseFailed) {
 
   std::unique_ptr<MockVideoSliceHeaderParser> mock_video_slice_header_parser(
       new MockVideoSliceHeaderParser);
+  EXPECT_CALL(*mock_video_slice_header_parser, ProcessNalu(_))
+      .WillOnce(Return(true));
   EXPECT_CALL(*mock_video_slice_header_parser, GetHeaderSize(_))
       .WillOnce(Return(-1));
 
@@ -342,6 +346,9 @@ TEST_P(SubsampleGeneratorTest, H264SubsampleEncryption) {
 
   std::unique_ptr<MockVideoSliceHeaderParser> mock_video_slice_header_parser(
       new MockVideoSliceHeaderParser);
+  EXPECT_CALL(*mock_video_slice_header_parser, ProcessNalu(_))
+      .Times(AtLeast(2))
+      .WillRepeatedly(Return(true));
   EXPECT_CALL(*mock_video_slice_header_parser, GetHeaderSize(_))
       .WillOnce(Return(kSliceHeaderSize[0]))
       .WillOnce(Return(kSliceHeaderSize[1]));

packager/media/formats/mp4/mp4_media_parser.cc

@@ -115,6 +115,56 @@ Codec ObjectTypeToCodec(ObjectType object_type) {
   }
 }
 
+uint64_t CalculateGCD(uint64_t a, uint64_t b) {
+  while (b != 0) {
+    uint64_t temp = a;
+    a = b;
+    b = temp % b;
+  }
+  return a;
+}
+
+void ReducePixelWidthHeight(uint64_t* pixel_width, uint64_t* pixel_height) {
+  if (*pixel_width == 0 || *pixel_height == 0)
+    return;
+  const uint64_t kMaxUint32 = std::numeric_limits<uint32_t>::max();
+  while (true) {
+    uint64_t gcd = CalculateGCD(*pixel_width, *pixel_height);
+    *pixel_width /= gcd;
+    *pixel_height /= gcd;
+    // Both width and height needs to be 32 bit or less.
+    if (*pixel_width <= kMaxUint32 && *pixel_height <= kMaxUint32)
+      break;
+    *pixel_width >>= 1;
+    *pixel_height >>= 1;
+  }
+}
+
+// Derive pixel aspect ratio from Display Aspect Ratio and Frame Aspect Ratio.
+//   DAR = PAR * FAR => PAR = DAR / FAR.
+//   Thus:
+//     pixel_width             display_width            frame_width
+//     -----------      =      -------------      /     -----------
+//     pixel_height            display_height           frame_height
+//   So:
+//     pixel_width             display_width  x  frame_width
+//     -----------      =      ------------------------------
+//     pixel_height            display_height x  frame_height
+void DerivePixelWidthHeight(uint32_t frame_width,
+                            uint32_t frame_height,
+                            uint32_t display_width,
+                            uint32_t display_height,
+                            uint32_t* pixel_width,
+                            uint32_t* pixel_height) {
+  uint64_t pixel_width_unreduced =
+      static_cast<uint64_t>(display_width) * frame_height;
+  uint64_t pixel_height_unreduced =
+      static_cast<uint64_t>(display_height) * frame_width;
+  ReducePixelWidthHeight(&pixel_width_unreduced, &pixel_height_unreduced);
+  *pixel_width = pixel_width_unreduced;
+  *pixel_height = pixel_height_unreduced;
+}
+
 const uint64_t kNanosecondsPerSecond = 1000000000ull;
 
 }  // namespace
@@ -515,8 +565,9 @@ bool MP4MediaParser::ParseMoov(BoxReader* reader) {
       uint32_t pixel_width = entry.pixel_aspect.h_spacing;
       uint32_t pixel_height = entry.pixel_aspect.v_spacing;
       if (pixel_width == 0 && pixel_height == 0) {
-        pixel_width = 1;
+        DerivePixelWidthHeight(coded_width, coded_height, track->header.width,
-        pixel_height = 1;
+                               track->header.height, &pixel_width,
+                               &pixel_height);
       }
       std::string codec_string;
       uint8_t nalu_length_size = 0;
@@ -543,6 +594,9 @@ bool MP4MediaParser::ParseMoov(BoxReader* reader) {
           codec_string = avc_config.GetCodecString(actual_format);
           nalu_length_size = avc_config.nalu_length_size();
 
+          // Use configurations from |avc_config| if it is valid.
+          if (avc_config.coded_width() != 0) {
+            DCHECK_NE(avc_config.coded_height(), 0u);
             if (coded_width != avc_config.coded_width() ||
                 coded_height != avc_config.coded_height()) {
               LOG(WARNING) << "Resolution in VisualSampleEntry (" << coded_width
@@ -556,18 +610,23 @@ bool MP4MediaParser::ParseMoov(BoxReader* reader) {
               coded_height = avc_config.coded_height();
             }
 
+            DCHECK_NE(avc_config.pixel_width(), 0u);
+            DCHECK_NE(avc_config.pixel_height(), 0u);
             if (pixel_width != avc_config.pixel_width() ||
                 pixel_height != avc_config.pixel_height()) {
               LOG_IF(WARNING, pixel_width != 1 || pixel_height != 1)
                   << "Pixel aspect ratio in PASP box (" << pixel_width << ","
                   << pixel_height
-                << ") does not match with SAR in AVCDecoderConfigurationRecord "
+                  << ") does not match with SAR in "
+                     "AVCDecoderConfigurationRecord "
                      "("
-                << avc_config.pixel_width() << "," << avc_config.pixel_height()
+                  << avc_config.pixel_width() << ","
+                  << avc_config.pixel_height()
                   << "). Use AVCDecoderConfigurationRecord.";
               pixel_width = avc_config.pixel_width();
               pixel_height = avc_config.pixel_height();
             }
+          }
           break;
         }
         case FOURCC_hev1: