Add pixel_{width,height} to VideoStreamInfo

- Added logic to container parsers to extract pixel width and height.
- If the container doesn't contain the pixel width and height of the
  stream and the stream is H264, this parses the SPS to get the
  sar_width and sar_height.
- If extracted sar_width and sar_height are 0s then they imply 1s.
- Add hb2_v_frag.mp4 for test media. This does not have a
  'pasp' box and has sar_width = 8 sar_height = 9.

Change-Id: I4a06ce95582547bec19adb7905e7612c5a1f359e

packager/media/base/media_base.gyp

@@ -78,6 +78,7 @@
         '../../base/base.gyp:base',
         '../../third_party/curl/curl.gyp:libcurl',
         '../../third_party/openssl/openssl.gyp:openssl',
+        '../filters/filters.gyp:filters',
       ],
     },
     {

packager/media/base/stream_info.cc

@@ -39,14 +39,12 @@ StreamInfo::~StreamInfo() {}
 
 std::string StreamInfo::ToString() const {
   return base::StringPrintf(
-      "type: %s\n codec_string: %s\n time_scale: %d\n duration: %" PRIu64 " "
+      "type: %s\n codec_string: %s\n time_scale: %d\n duration: "
-      "(%.1f seconds)\n language: %s\n is_encrypted: %s\n",
+      "%" PRIu64 " (%.1f seconds)\n language: %s\n is_encrypted: %s\n",
       (stream_type_ == kStreamAudio ? "Audio" : "Video"),
       codec_string_.c_str(),
-      time_scale_,
+      time_scale_, duration_,
-      duration_,
+      static_cast<double>(duration_) / time_scale_, language_.c_str(),
-      static_cast<double>(duration_) / time_scale_,
-      language_.c_str(),
       is_encrypted_ ? "true" : "false");
 }
 

packager/media/base/video_stream_info.cc

@@ -6,10 +6,13 @@
 
 #include "packager/media/base/video_stream_info.h"
 
+#include "base/logging.h"
+#include "base/stl_util.h"
 #include "packager/base/strings/string_number_conversions.h"
 #include "packager/base/strings/string_util.h"
 #include "packager/base/strings/stringprintf.h"
 #include "packager/media/base/limits.h"
+#include "packager/media/filters/h264_parser.h"
 
 namespace edash_packager {
 namespace media {
@@ -36,6 +39,7 @@ std::string VideoCodecToString(VideoCodec video_codec) {
       return "UnknownVideoCodec";
   }
 }
+
 }  // namespace
 
 VideoStreamInfo::VideoStreamInfo(int track_id,
@@ -46,6 +50,8 @@ VideoStreamInfo::VideoStreamInfo(int track_id,
                                  const std::string& language,
                                  uint16_t width,
                                  uint16_t height,
+                                 uint32_t pixel_width,
+                                 uint32_t pixel_height,
                                  int16_t trick_play_rate,
                                  uint8_t nalu_length_size,
                                  const uint8_t* extra_data,
@@ -63,8 +69,20 @@ VideoStreamInfo::VideoStreamInfo(int track_id,
       codec_(codec),
       width_(width),
       height_(height),
+      pixel_width_(pixel_width),
+      pixel_height_(pixel_height),
       trick_play_rate_(trick_play_rate),
       nalu_length_size_(nalu_length_size) {
+  // If H264 and the pixel width and height were not passed in, parse the extra
+  // data to get the sar width and height.
+  if ((pixel_width_ == 0 || pixel_height_ == 0) &&
+      codec == kCodecH264 &&
+      extra_data && extra_data_size > 0) {
+    ExtractSarFromDecoderConfig(extra_data, extra_data_size, &pixel_width_,
+                                &pixel_height_);
+    DVLOG_IF(2, pixel_width_ == 0 || pixel_height_ == 0)
+        << "Failed to extract sar_width and sar_height.";
+  }
 }
 
 VideoStreamInfo::~VideoStreamInfo() {}
@@ -78,14 +96,13 @@ bool VideoStreamInfo::IsValidConfig() const {
 
 std::string VideoStreamInfo::ToString() const {
   return base::StringPrintf(
-      "%s codec: %s\n width: %d\n height: %d\n trick_play_rate: %d\n"
+      "%s codec: %s\n width: %d\n height: %d\n pixel_width: %d\n pixel_height: "
-      " nalu_length_size: %d\n",
+      "%d\n trick_play_rate: %d\n nalu_length_size: %d\n",
       StreamInfo::ToString().c_str(),
       VideoCodecToString(codec_).c_str(),
-      width_,
+      width_, height_,
-      height_,
+      pixel_width_, pixel_height_,
-      trick_play_rate_,
+      trick_play_rate_, nalu_length_size_);
-      nalu_length_size_);
 }
 
 std::string VideoStreamInfo::GetCodecString(VideoCodec codec,

packager/media/base/video_stream_info.h

@@ -28,6 +28,11 @@ enum VideoCodec {
 class VideoStreamInfo : public StreamInfo {
  public:
   /// Construct an initialized video stream info object.
+  /// If @a codec is @a kCodecH264 and either @pixel_width and @pixel_height is
+  /// 0 (unknown), then this tries to parse @extra_data to extract the pixel
+  /// width and height from it.
+  /// @param pixel_width is the width of the pixel. 0 if unknown.
+  /// @param pixel_height is the height of the pixels. 0 if unknown.
   VideoStreamInfo(int track_id,
                   uint32_t time_scale,
                   uint64_t duration,
@@ -36,6 +41,8 @@ class VideoStreamInfo : public StreamInfo {
                   const std::string& language,
                   uint16_t width,
                   uint16_t height,
+                  uint32_t pixel_width,
+                  uint32_t pixel_height,
                   int16_t trick_play_rate,
                   uint8_t nalu_length_size,
                   const uint8_t* extra_data,
@@ -51,6 +58,12 @@ class VideoStreamInfo : public StreamInfo {
   VideoCodec codec() const { return codec_; }
   uint16_t width() const { return width_; }
   uint16_t height() const { return height_; }
+  /// Returns the pixel width.
+  /// @return 0 if unknown.
+  uint32_t pixel_width() const { return pixel_width_; }
+  /// Returns the pixel height.
+  /// @return 0 if unknown.
+  uint32_t pixel_height() const { return pixel_height_; }
   uint8_t nalu_length_size() const { return nalu_length_size_; }
   int16_t trick_play_rate() const { return trick_play_rate_; }
 
@@ -67,6 +80,11 @@ class VideoStreamInfo : public StreamInfo {
   VideoCodec codec_;
   uint16_t width_;
   uint16_t height_;
+
+  // pixel_width_:pixel_height_ is the sample aspect ratio.
+  // 0 means unknown.
+  uint32_t pixel_width_;
+  uint32_t pixel_height_;
   int16_t trick_play_rate_;  // Non-zero for trick-play streams.
 
   // Specifies the normalized size of the NAL unit length field. Can be 1, 2 or

packager/media/event/vod_media_info_dump_muxer_listener_unittest.cc

@@ -63,6 +63,10 @@ scoped_refptr<StreamInfo> CreateVideoStreamInfo(
                           param.language,
                           param.width,
                           param.height,
+                          // TODO(rkuroiwa): Once MedianInfo proto change that
+                          // adds pizel_{width,height} lands, add tests.
+                          0,  // No pixel width.
+                          0,  // No pixel height.
                           0,  // trick_play_rate
                           param.nalu_length_size,
                           vector_as_array(&param.extra_data),

packager/media/filters/h264_parser.cc

@@ -7,10 +7,73 @@
 #include "packager/base/logging.h"
 #include "packager/base/memory/scoped_ptr.h"
 #include "packager/base/stl_util.h"
+#include "packager/media/base/buffer_reader.h"
 
 namespace edash_packager {
 namespace media {
 
+#define RCHECK(x)                                                     \
+  do {                                                                \
+    if (!(x)) {                                                       \
+      LOG(ERROR) << "Failure while parsing AVCDecoderConfig: " << #x; \
+      return;                                                         \
+    }                                                                 \
+  } while (0)
+
+void ExtractSarFromDecoderConfig(
+    const uint8_t* avc_decoder_config_data, size_t avc_decoder_config_data_size,
+    uint32_t* sar_width, uint32_t* sar_height) {
+  BufferReader reader(avc_decoder_config_data, avc_decoder_config_data_size);
+  uint8_t value = 0;
+  // version check, must be 1.
+  RCHECK(reader.Read1(&value));
+  RCHECK(value == 1);
+
+  // avc profile. No value check.
+  RCHECK(reader.Read1(&value));
+
+  // profile compatibility. No value check.
+  RCHECK(reader.Read1(&value));
+
+  // avc level indication. No value check.
+  RCHECK(reader.Read1(&value));
+
+  // reserved and length sized minus one.
+  RCHECK(reader.Read1(&value));
+  // upper 6 bits are reserved and must be 111111.
+  RCHECK((value & 0xFC) == 0xFC);
+
+  // reserved and num sps.
+  RCHECK(reader.Read1(&value));
+  // upper 3 bits are reserved for 0b111.
+  RCHECK((value & 0xE0) == 0xE0);
+
+  const uint8_t num_sps = value & 0x1F;
+  if (num_sps < 1) {
+    LOG(ERROR) << "No SPS found.";
+    return;
+  }
+  uint16_t sps_length = 0;
+  RCHECK(reader.Read2(&sps_length));
+
+  H264Parser parser;
+  int sps_id;
+  RCHECK(parser.ParseSPSFromArray(reader.data() + reader.pos(), sps_length,
+                                  &sps_id) == H264Parser::kOk);
+  const H264SPS& sps = *parser.GetSPS(sps_id);
+  // 0 means it wasn't in the SPS and therefore assume 1.
+  *sar_width = sps.sar_width == 0 ? 1 : sps.sar_width;
+  *sar_height = sps.sar_height == 0 ? 1 : sps.sar_height;
+  DVLOG(2) << "Found sar_width: " << *sar_width
+           << " sar_height: " << *sar_height;
+
+  // It is unlikely to have more than one SPS in practice. Also there's
+  // no way to change the sar_{width,height} dynamically from VideoStreamInfo.
+  // So skip the rest (if there are any).
+}
+
+#undef RCHECK
+
 bool H264SliceHeader::IsPSlice() const {
   return (slice_type % 5 == kPSlice);
 }
@@ -888,6 +951,28 @@ H264Parser::Result H264Parser::ParsePPS(int* pps_id) {
   return kOk;
 }
 
+H264Parser::Result H264Parser::ParseSPSFromArray(
+    const uint8_t* sps_data,
+    size_t sps_data_length,
+    int* sps_id) {
+  br_.Initialize(sps_data, sps_data_length);
+
+  int data;
+  READ_BITS_OR_RETURN(1, &data);
+  // First bit must be 0.
+  TRUE_OR_RETURN(data == 0);
+  int nal_ref_idc;
+  READ_BITS_OR_RETURN(2, &nal_ref_idc);
+  // From the spec "nal_ref_idc shall not be equal to 0 for sequence parameter
+  // set".
+  TRUE_OR_RETURN(nal_ref_idc != 0);
+  int nal_unit_type;
+  READ_BITS_OR_RETURN(5, &nal_unit_type);
+  TRUE_OR_RETURN(nal_unit_type == H264NALU::kSPS);
+
+  return ParseSPS(sps_id);
+}
+
 H264Parser::Result H264Parser::ParseRefPicListModification(
     int num_ref_idx_active_minus1,
     H264ModificationOfPicNum* ref_list_mods) {

packager/media/filters/h264_parser.h

@@ -17,6 +17,16 @@
 namespace edash_packager {
 namespace media {
 
+// |avc_decoder_config_data| must be AVCDecoderConfigurationRecord specified
+// in ISO/IEC 14496-15.
+// On success, |sar_width| and |sar_height| are set to the data specified in
+// |avc_decoder_config_data|.
+// If the value is 0 or ommited in |avc_decoder_config_data|, then 1 is
+// assigned.
+void ExtractSarFromDecoderConfig(
+    const uint8_t* avc_decoder_config_data, size_t avc_decoder_config_data_size,
+    uint32_t* sar_width, uint32_t* sar_height);
+
 // For explanations of each struct and its members, see H.264 specification
 // at http://www.itu.int/rec/T-REC-H.264.
 struct H264NALU {
@@ -314,6 +324,11 @@ class H264Parser {
   Result ParseSPS(int* sps_id);
   Result ParsePPS(int* pps_id);
 
+  // Samme as ParseSPS but instead uses |sps_data|.
+  Result ParseSPSFromArray(const uint8_t* sps_data,
+                           size_t sps_data_size,
+                           int* sps_id);
+
   // Return a pointer to SPS/PPS with given |sps_id|/|pps_id| or NULL if not
   // present.
   const H264SPS* GetSPS(int sps_id);

packager/media/formats/mp2t/es_parser_h264.cc

@@ -363,6 +363,8 @@ bool EsParserH264::UpdateVideoDecoderConfig(const H264SPS* sps) {
           std::string(),
           width,
           height,
+          sps->sar_width == 0 ? 1 : sps->sar_width,
+          sps->sar_height == 0 ? 1 : sps->sar_height,
           0,
           H264ByteToUnitStreamConverter::kUnitStreamNaluLengthSize,
           decoder_config_record.data(),

packager/media/formats/mp2t/es_parser_h264_unittest.cc

@@ -12,6 +12,7 @@
 #include "packager/base/stl_util.h"
 #include "packager/media/base/media_sample.h"
 #include "packager/media/base/timestamp.h"
+#include "packager/media/base/video_stream_info.h"
 #include "packager/media/filters/h264_parser.h"
 #include "packager/media/formats/mp2t/es_parser_h264.h"
 #include "packager/media/test/test_data_util.h"
@@ -131,6 +132,8 @@ class EsParserH264Test : public testing::Test {
   }
 
   void NewVideoConfig(scoped_refptr<StreamInfo>& config) {
+    DVLOG(1) << config->ToString();
+    stream_map_[config->track_id()] = config;
   }
 
   size_t sample_count() const { return sample_count_; }
@@ -143,6 +146,8 @@ class EsParserH264Test : public testing::Test {
   std::vector<Packet> access_units_;
 
  protected:
+  typedef std::map<int, scoped_refptr<StreamInfo> > StreamMap;
+  StreamMap stream_map_;
   size_t sample_count_;
   bool first_frame_is_key_frame_;
 };
@@ -198,7 +203,6 @@ void EsParserH264Test::ProcessPesPackets(
   es_parser.Flush();
 }
 
-
 TEST_F(EsParserH264Test, OneAccessUnitPerPes) {
   LoadStream("bear.h264");
 
@@ -280,6 +284,21 @@ TEST_F(EsParserH264Test, NonIFrameStart) {
   EXPECT_TRUE(first_frame_is_key_frame());
 }
 
+// Verify that the parser can get the the sar width and height.
+TEST_F(EsParserH264Test, PixelWidthPixelHeight) {
+  LoadStream("bear.h264");
+  std::vector<Packet> pes_packets(access_units_);
+  ProcessPesPackets(pes_packets);
+
+  const int kVideoTrackId = 0;
+  EXPECT_EQ(1u,
+            reinterpret_cast<VideoStreamInfo*>(stream_map_[kVideoTrackId].get())
+                ->pixel_width());
+  EXPECT_EQ(1u,
+            reinterpret_cast<VideoStreamInfo*>(stream_map_[kVideoTrackId].get())
+                ->pixel_height());
+}
+
 }  // namespace mp2t
 }  // namespace media
 }  // namespace edash_packager

packager/media/formats/mp4/mp4_media_parser.cc

@@ -378,6 +378,8 @@ bool MP4MediaParser::ParseMoov(BoxReader* reader) {
                                             track->media.header.language,
                                             entry.width,
                                             entry.height,
+                                            entry.pixel_aspect.h_spacing,
+                                            entry.pixel_aspect.v_spacing,
                                             0,  // trick_play_rate
                                             entry.avcc.length_size,
                                             &entry.avcc.data[0],

packager/media/formats/mp4/mp4_media_parser_unittest.cc

@@ -12,6 +12,7 @@
 #include "packager/media/base/key_source.h"
 #include "packager/media/base/media_sample.h"
 #include "packager/media/base/stream_info.h"
+#include "packager/media/base/video_stream_info.h"
 #include "packager/media/formats/mp4/mp4_media_parser.h"
 #include "packager/media/test/test_data_util.h"
 
@@ -41,6 +42,8 @@ class MP4MediaParserTest : public testing::Test {
   }
 
  protected:
+  typedef std::map<int, scoped_refptr<StreamInfo> > StreamMap;
+  StreamMap stream_map_;
   scoped_ptr<MP4MediaParser> parser_;
   size_t num_streams_;
   size_t num_samples_;
@@ -70,6 +73,7 @@ class MP4MediaParserTest : public testing::Test {
          iter != streams.end();
          ++iter) {
       DVLOG(2) << (*iter)->ToString();
+      stream_map_[(*iter)->track_id()] = *iter;
     }
     num_streams_ = streams.size();
     num_samples_ = 0;
@@ -106,6 +110,65 @@ TEST_F(MP4MediaParserTest, UnalignedAppend) {
   EXPECT_EQ(201u, num_samples_);
 }
 
+// Verify that the pixel width and pixel height are extracted correctly if
+// the container has a 'pasp' box.
+TEST_F(MP4MediaParserTest, PixelWidthPixelHeightFromPaspBox) {
+  // This content has a 'pasp' box that has the aspect ratio.
+  EXPECT_TRUE(ParseMP4File("bear-1280x720.mp4", 512));
+
+  // Track ID 2 has the video stream which should have pixel width and height
+  // both 1.
+  const int kVideoTrackId = 2;
+  EXPECT_EQ(1u,
+            reinterpret_cast<VideoStreamInfo*>(stream_map_[kVideoTrackId].get())
+                ->pixel_width());
+  EXPECT_EQ(1u,
+            reinterpret_cast<VideoStreamInfo*>(stream_map_[kVideoTrackId].get())
+                ->pixel_height());
+}
+
+// Verify that pixel width and height can be extracted from the
+// extra data (AVCDecoderConfigurationRecord) for H264.
+// No 'pasp' box.
+TEST_F(MP4MediaParserTest,
+       PixelWidthPixelHeightFromAVCDecoderConfigurationRecord) {
+  // This file doesn't have pasp. SPS for the video has
+  // sar_width = sar_height = 0. So the stream info should return 1 for both
+  // pixel_width and pixel_height.
+  EXPECT_TRUE(ParseMP4File("hb2_v_frag.mp4", 512));
+
+  // Track ID 1 has the video stream which should have pixel width and height
+  // both 1.
+  const int kVideoTrackId = 1;
+  EXPECT_EQ(8u,
+            reinterpret_cast<VideoStreamInfo*>(stream_map_[kVideoTrackId].get())
+                ->pixel_width());
+  EXPECT_EQ(9u,
+            reinterpret_cast<VideoStreamInfo*>(stream_map_[kVideoTrackId].get())
+                ->pixel_height());
+}
+
+// Verify that pixel width and height can be extracted from the
+// extra data (AVCDecoderConfigurationRecord) for H264.
+// If sar_width and sar_height are not set, then they should both be 1.
+TEST_F(MP4MediaParserTest,
+       PixelWidthPixelHeightFromAVCDecoderConfigurationRecordNotSet) {
+  // This file doesn't have pasp. SPS for the video has
+  // sar_width = sar_height = 0. So the stream info should return 1 for both
+  // pixel_width and pixel_height.
+  EXPECT_TRUE(ParseMP4File("bear-1280x720-av_frag.mp4", 512));
+
+  // Track ID 1 has the video stream which should have pixel width and height
+  // both 1.
+  const int kVideoTrackId = 1;
+  EXPECT_EQ(1u,
+            reinterpret_cast<VideoStreamInfo*>(stream_map_[kVideoTrackId].get())
+                ->pixel_width());
+  EXPECT_EQ(1u,
+            reinterpret_cast<VideoStreamInfo*>(stream_map_[kVideoTrackId].get())
+                ->pixel_height());
+}
+
 TEST_F(MP4MediaParserTest, BytewiseAppend) {
   // Ensure no incremental errors occur when parsing
   EXPECT_TRUE(ParseMP4File("bear-1280x720-av_frag.mp4", 1));

packager/media/formats/wvm/wvm_media_parser.cc

@@ -569,6 +569,8 @@ bool WvmMediaParser::ParseIndexEntry() {
     uint32_t time_scale = kMpeg2ClockRate;
     uint16_t video_width = 0;
     uint16_t video_height = 0;
+    uint32_t pixel_width = 0;
+    uint32_t pixel_height = 0;
     uint8_t nalu_length_size = kNaluLengthSize;
     uint8_t num_channels = 0;
     int audio_pes_stream_id = 0;
@@ -596,7 +598,7 @@ bool WvmMediaParser::ParseIndexEntry() {
       if (index_metadata_max_size < length) {
         return false;
       }
-     int value = 0;
+      int64_t value = 0;
       Tag tagtype = Unset;
       std::vector<uint8_t> binary_data(length);
       switch (Type(type)) {
@@ -693,6 +695,12 @@ bool WvmMediaParser::ParseIndexEntry() {
         case AudioType:
           has_audio = true;
           break;
+        case VideoPixelWidth:
+          pixel_width = static_cast<uint32_t>(value);
+          break;
+        case VideoPixelHeight:
+          pixel_height = static_cast<uint32_t>(value);
+          break;
         default:
           break;
       }
@@ -710,7 +718,8 @@ bool WvmMediaParser::ParseIndexEntry() {
       stream_infos_.push_back(new VideoStreamInfo(
           stream_id_count_, time_scale, track_duration, video_codec,
           video_codec_string, std::string(), video_width, video_height,
-         trick_play_rate, nalu_length_size, NULL, 0, true));
+          pixel_width, pixel_height, trick_play_rate, nalu_length_size, NULL, 0,
+          true));
       program_demux_stream_map_[base::UintToString(index_program_id_) + ":" +
                                 base::UintToString(video_pes_stream_id)] =
           stream_id_count_++;

packager/media/formats/wvm/wvm_media_parser_unittest.cc

@@ -163,6 +163,18 @@ TEST_F(WvmMediaParserTest, ParseWvmWithoutKeySource) {
   EXPECT_EQ(kExpectedVideoFrameCount, video_frame_count_);
   EXPECT_EQ(kExpectedAudioFrameCount, audio_frame_count_);
   EXPECT_EQ(kExpectedEncryptedSampleCount, encrypted_sample_count_);
+
+  // Also verify that the pixel width and height have the right values.
+  // Track 0 and 2 are videos and they both have pixel_width = 8 and
+  // pixel_height = 9.
+  EXPECT_EQ(8u, reinterpret_cast<VideoStreamInfo*>(stream_map_[0].get())
+                   ->pixel_width());
+  EXPECT_EQ(8u, reinterpret_cast<VideoStreamInfo*>(stream_map_[2].get())
+                   ->pixel_width());
+  EXPECT_EQ(9u, reinterpret_cast<VideoStreamInfo*>(stream_map_[0].get())
+                   ->pixel_height());
+  EXPECT_EQ(9u, reinterpret_cast<VideoStreamInfo*>(stream_map_[2].get())
+                   ->pixel_height());
 }
 
 TEST_F(WvmMediaParserTest, ParseWvmInitWithoutKeySource) {