Add instructions to build on Alpine Linux

- Also fixed compilations in Alpine Linux and other flavors of Linux.
- Added container versions in docker files to always use a verified
  version.

Closes #164.

Change-Id: I949a8709e4d70c49129c9c2e8608dd78193d964c

docs/source/build_instructions.md

@@ -232,6 +232,31 @@ $ ninja -C out/Release
 
 ## Notes for other linux distros
 
+### Alpine Linux
+
+Use `apk` command to install dependencies:
+
+```shell
+$apk add --no-cache bash build-base curl findutils git ninja python \
+                    bsd-compat-headers linux-headers libexecinfo-dev
+```
+
+Alpine uses musl which does not have mallinfo defined in malloc.h. It is
+required by one of Shaka Packager's dependency. To workaround the problem, a
+dummy structure has to be defined in /usr/include/malloc.h, e.g.
+
+```shell
+$ sed -i \
+  '/malloc_usable_size/a \\nstruct mallinfo {\n  int arena;\n  int hblkhd;\n  int uordblks;\n};' \
+  /usr/include/malloc.h
+```
+
+We also need to disable clang and some other features to make it work with musl:
+
+```shell
+export GYP_DEFINES='linux_use_bundled_binutils=0 linux_use_bundled_gold=0 clang=0 use_experimental_allocator_shim=0 use_allocator=none musl=1'
+```
+
 ### Arch Linux
 
 Instead of running `sudo apt-get install` to install build dependencies, run:
@@ -251,6 +276,12 @@ $ gpg --keyserver pgp.mit.edu --recv-keys F7E48EDB
 $ makepkg -si
 ```
 
+Optionally, disable clang to build with gcc:
+
+```shell
+$ export GYP_DEFINES='clang=0'
+```
+
 ### Debian
 
 Same as Ubuntu.

packager/common.gypi

@@ -10,8 +10,11 @@
   'variables': {
     'variables': {
       'shaka_code%': 0,
+      # musl is a lightweight C standard library used in Alpine Linux.
+      'musl%': 0,
     },
     'shaka_code%': '<(shaka_code)',
+    'musl%': '<(musl)',
     'libpackager_type%': 'static_library',
     'conditions': [
       ['shaka_code==1', {
@@ -75,6 +78,20 @@
           }],
         ],
       }],
+      ['musl==1', {
+        'defines': [
+          # musl is not uClibc but is similar to uClibc that a minimal feature
+          # set is supported. One of Shaka Packager's dependencies, Chromium
+          # base uses __UCLIBC__ flag to disable some features, which needs to
+          # be disabled for musl too.
+          '__UCLIBC__',
+        ],
+        'cflags!': [
+          # Do not treat warnings as errors on musl as there is a hard-coded
+          # warning in musl's sys/errno.h.
+          '-Werror',
+        ],
+      }]
     ],
   },
 }

packager/file/udp_file.cc

@@ -18,6 +18,7 @@
 #include <errno.h>
 #include <strings.h>
 #include <sys/socket.h>
+#include <sys/time.h>
 #include <unistd.h>
 #define INVALID_SOCKET -1
 

packager/media/base/aes_cryptor_unittest.cc

@@ -53,12 +53,6 @@ const uint8_t kAesCtrCiphertext[] = {
     0x1e, 0x03, 0x1d, 0xda, 0x2f, 0xbe, 0x03, 0xd1,
     0x79, 0x21, 0x70, 0xa0, 0xf3, 0x00, 0x9c, 0xee};
 
-// Subsample test cases.
-struct SubsampleTestCase {
-  const uint8_t* subsample_sizes;
-  uint32_t subsample_count;
-};
-
 const uint8_t kSubsampleTest1[] = {64};
 const uint8_t kSubsampleTest2[] = {13, 51};
 const uint8_t kSubsampleTest3[] = {52, 12};
@@ -70,18 +64,6 @@ const uint8_t kSubsampleTest8[] = {10, 1, 33, 20};
 const uint8_t kSubsampleTest9[] = {7, 19, 6, 32};
 const uint8_t kSubsampleTest10[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9};
 
-const SubsampleTestCase kSubsampleTestCases[] = {
-    {kSubsampleTest1, arraysize(kSubsampleTest1)},
-    {kSubsampleTest2, arraysize(kSubsampleTest2)},
-    {kSubsampleTest3, arraysize(kSubsampleTest3)},
-    {kSubsampleTest4, arraysize(kSubsampleTest4)},
-    {kSubsampleTest5, arraysize(kSubsampleTest5)},
-    {kSubsampleTest6, arraysize(kSubsampleTest6)},
-    {kSubsampleTest7, arraysize(kSubsampleTest7)},
-    {kSubsampleTest8, arraysize(kSubsampleTest8)},
-    {kSubsampleTest9, arraysize(kSubsampleTest9)},
-    {kSubsampleTest10, arraysize(kSubsampleTest10)}};
-
 // IV test values.
 const uint32_t kTextSizeInBytes = 60;  // 3 full blocks + 1 partial block.
 
@@ -102,26 +84,6 @@ const uint8_t kIv64MaxMinusOne[] = {0xff, 0xff, 0xff, 0xff,
                                     0xff, 0xff, 0xff, 0xfe};
 const uint8_t kIv64Max[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 
-struct IvTestCase {
-  const uint8_t* iv_test;
-  uint32_t iv_size;
-  const uint8_t* iv_expected;
-};
-
-// As recommended in ISO/IEC FDIS 23001-7: CENC spec,
-// For 64-bit (8-byte) IV_Sizes, initialization vectors for subsequent samples
-// can be created by incrementing the initialization vector of the previous
-// sample. For 128-bit (16-byte) IV_Sizes, initialization vectors for subsequent
-// samples should be created by adding the block count of the previous sample to
-// the initialization vector of the previous sample.
-const IvTestCase kIvTestCases[] = {
-    {kIv128Zero, arraysize(kIv128Zero), kIv128Four},
-    {kIv128Max64, arraysize(kIv128Max64), kIv128OneAndThree},
-    {kIv128MaxMinusOne, arraysize(kIv128MaxMinusOne), kIv128Two},
-    {kIv64Zero, arraysize(kIv64Zero), kIv64One},
-    {kIv64MaxMinusOne, arraysize(kIv64MaxMinusOne), kIv64Max},
-    {kIv64Max, arraysize(kIv64Max), kIv64Zero}};
-
 // We support AES 128, i.e. 16 bytes key only.
 const uint8_t kInvalidKey[] = {0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2,
                                0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09};
@@ -253,6 +215,12 @@ TEST_F(AesCtrEncryptorTest, UnsupportedIV) {
   ASSERT_FALSE(encryptor_.InitializeWithIv(key_, iv));
 }
 
+// Subsample test cases.
+struct SubsampleTestCase {
+  const uint8_t* subsample_sizes;
+  uint32_t subsample_count;
+};
+
 class AesCtrEncryptorSubsampleTest
     : public AesCtrEncryptorTest,
       public ::testing::WithParamInterface<SubsampleTestCase> {};
@@ -280,10 +248,30 @@ TEST_P(AesCtrEncryptorSubsampleTest, NistTestCaseSubsamples) {
   EXPECT_EQ(plaintext_, decrypted);
 }
 
+namespace {
+const SubsampleTestCase kSubsampleTestCases[] = {
+    {kSubsampleTest1, arraysize(kSubsampleTest1)},
+    {kSubsampleTest2, arraysize(kSubsampleTest2)},
+    {kSubsampleTest3, arraysize(kSubsampleTest3)},
+    {kSubsampleTest4, arraysize(kSubsampleTest4)},
+    {kSubsampleTest5, arraysize(kSubsampleTest5)},
+    {kSubsampleTest6, arraysize(kSubsampleTest6)},
+    {kSubsampleTest7, arraysize(kSubsampleTest7)},
+    {kSubsampleTest8, arraysize(kSubsampleTest8)},
+    {kSubsampleTest9, arraysize(kSubsampleTest9)},
+    {kSubsampleTest10, arraysize(kSubsampleTest10)}};
+}  // namespace
+
 INSTANTIATE_TEST_CASE_P(SubsampleTestCases,
                         AesCtrEncryptorSubsampleTest,
                         ::testing::ValuesIn(kSubsampleTestCases));
 
+struct IvTestCase {
+  const uint8_t* iv_test;
+  uint32_t iv_size;
+  const uint8_t* iv_expected;
+};
+
 class AesCtrEncryptorIvTest : public ::testing::TestWithParam<IvTestCase> {};
 
 TEST_P(AesCtrEncryptorIvTest, IvTest) {
@@ -305,6 +293,22 @@ TEST_P(AesCtrEncryptorIvTest, IvTest) {
   EXPECT_EQ(iv_expected, encryptor.iv());
 }
 
+namespace {
+// As recommended in ISO/IEC FDIS 23001-7: CENC spec,
+// For 64-bit (8-byte) IV_Sizes, initialization vectors for subsequent samples
+// can be created by incrementing the initialization vector of the previous
+// sample. For 128-bit (16-byte) IV_Sizes, initialization vectors for subsequent
+// samples should be created by adding the block count of the previous sample to
+// the initialization vector of the previous sample.
+const IvTestCase kIvTestCases[] = {
+    {kIv128Zero, arraysize(kIv128Zero), kIv128Four},
+    {kIv128Max64, arraysize(kIv128Max64), kIv128OneAndThree},
+    {kIv128MaxMinusOne, arraysize(kIv128MaxMinusOne), kIv128Two},
+    {kIv64Zero, arraysize(kIv64Zero), kIv64One},
+    {kIv64MaxMinusOne, arraysize(kIv64MaxMinusOne), kIv64Max},
+    {kIv64Max, arraysize(kIv64Max), kIv64Zero}};
+}  // namespace
+
 INSTANTIATE_TEST_CASE_P(IvTestCases,
                         AesCtrEncryptorIvTest,
                         ::testing::ValuesIn(kIvTestCases));

packager/media/base/aes_pattern_cryptor_unittest.cc

@@ -60,13 +60,44 @@ TEST_F(AesPatternCryptorTest, InitializeWithIv) {
   EXPECT_EQ(iv, pattern_cryptor_.iv());
 }
 
-namespace {
-
 struct PatternTestCase {
   const char* text_hex;
   const char* expected_crypt_text_hex;
 };
 
+class AesPatternCryptorVerificationTest
+    : public AesPatternCryptorTest,
+      public ::testing::WithParamInterface<PatternTestCase> {};
+
+TEST_P(AesPatternCryptorVerificationTest, PatternTest) {
+  std::vector<uint8_t> text;
+  std::string text_hex(GetParam().text_hex);
+  if (!text_hex.empty()) {
+    ASSERT_TRUE(base::HexStringToBytes(text_hex, &text));
+  }
+  std::vector<uint8_t> expected_crypt_text;
+  std::string expected_crypt_text_hex(GetParam().expected_crypt_text_hex);
+  if (!expected_crypt_text_hex.empty()) {
+    ASSERT_TRUE(
+        base::HexStringToBytes(expected_crypt_text_hex, &expected_crypt_text));
+  }
+
+  ON_CALL(*mock_cryptor_, CryptInternal(_, _, _, _))
+      .WillByDefault(Invoke([](const uint8_t* text, size_t text_size,
+                               uint8_t* crypt_text, size_t* crypt_text_size) {
+        *crypt_text_size = text_size;
+        for (size_t i = 0; i < text_size; ++i) {
+          *crypt_text++ = *text++ + 0x10;
+        }
+        return true;
+      }));
+
+  std::vector<uint8_t> crypt_text;
+  ASSERT_TRUE(pattern_cryptor_.Crypt(text, &crypt_text));
+  EXPECT_EQ(expected_crypt_text, crypt_text);
+}
+
+namespace {
 const PatternTestCase kPatternTestCases[] = {
     // Empty.
     {"", ""},
@@ -102,41 +133,8 @@ const PatternTestCase kPatternTestCases[] = {
      "81828384858687888990919293949596"
      "97989900010203040506070809101112"},
 };
-
 }  // namespace
 
-class AesPatternCryptorVerificationTest
-    : public AesPatternCryptorTest,
-      public ::testing::WithParamInterface<PatternTestCase> {};
-
-TEST_P(AesPatternCryptorVerificationTest, PatternTest) {
-  std::vector<uint8_t> text;
-  std::string text_hex(GetParam().text_hex);
-  if (!text_hex.empty()) {
-    ASSERT_TRUE(base::HexStringToBytes(text_hex, &text));
-  }
-  std::vector<uint8_t> expected_crypt_text;
-  std::string expected_crypt_text_hex(GetParam().expected_crypt_text_hex);
-  if (!expected_crypt_text_hex.empty()) {
-    ASSERT_TRUE(
-        base::HexStringToBytes(expected_crypt_text_hex, &expected_crypt_text));
-  }
-
-  ON_CALL(*mock_cryptor_, CryptInternal(_, _, _, _))
-      .WillByDefault(Invoke([](const uint8_t* text, size_t text_size,
-                               uint8_t* crypt_text, size_t* crypt_text_size) {
-        *crypt_text_size = text_size;
-        for (size_t i = 0; i < text_size; ++i) {
-          *crypt_text++ = *text++ + 0x10;
-        }
-        return true;
-      }));
-
-  std::vector<uint8_t> crypt_text;
-  ASSERT_TRUE(pattern_cryptor_.Crypt(text, &crypt_text));
-  EXPECT_EQ(expected_crypt_text, crypt_text);
-}
-
 INSTANTIATE_TEST_CASE_P(PatternTestCases,
                         AesPatternCryptorVerificationTest,
                         ::testing::ValuesIn(kPatternTestCases));

packager/media/base/producer_consumer_queue_unittest.cc

@@ -278,13 +278,11 @@ TEST_F(MultiThreadProducerConsumerQueueTest, PeekOnLargePosition) {
   queue_.Stop();
 }
 
-namespace {
 enum Operation {
   kPush,
   kPop,
   kPeek,
 };
-}  // namespace
 
 class MultiThreadProducerConsumerQueueStopTest
     : public ::testing::TestWithParam<Operation> {

packager/media/codecs/h264_parser.cc

@@ -988,7 +988,7 @@ H264Parser::Result H264Parser::ParseSliceHeader(const Nalu& nalu,
   reader.Initialize(nalu.data() + nalu.header_size(), nalu.payload_size());
   H26xBitReader* br = &reader;
 
-  memset(shdr, 0, sizeof(*shdr));
+  memset(reinterpret_cast<void*>(shdr), 0, sizeof(*shdr));
 
   shdr->idr_pic_flag = (nalu.type() == 5);
   shdr->nal_ref_idc = nalu.ref_idc();
@@ -1134,7 +1134,7 @@ H264Parser::Result H264Parser::ParseSEI(const Nalu& nalu,
   reader.Initialize(nalu.data() + nalu.header_size(), nalu.payload_size());
   H26xBitReader* br = &reader;
 
-  memset(sei_msg, 0, sizeof(*sei_msg));
+  memset(reinterpret_cast<void*>(sei_msg), 0, sizeof(*sei_msg));
 
   READ_BITS_OR_RETURN(8, &byte);
   while (byte == 0xff) {

packager/media/formats/webm/encryptor_unittest.cc

@@ -57,13 +57,6 @@ TEST(EncryptionUtilTest, SampleNotEncrypted) {
 
 namespace {
 
-struct EncryptionTestCase {
-  const SubsampleEntry* subsamples;
-  size_t num_subsamples;
-  const uint8_t* subsample_partition_data;
-  size_t subsample_partition_data_size;
-};
-
 const SubsampleEntry kSubsamples1[] = {
     SubsampleEntry(0x12, 0x100),
 };
@@ -94,21 +87,15 @@ const uint8_t kSubsamplePartitionData4[] = {
     0x6B, 0x00, 0x09, 0x03, 0x6B, 0x00, 0x09, 0x03, 0x6D,
 };
 
-EncryptionTestCase kEncryptionTestCases[] = {
-    // Special case with no subsamples.
-    {nullptr, 0, nullptr, 0},
-    {kSubsamples1, arraysize(kSubsamples1), kSubsamplePartitionData1,
-     arraysize(kSubsamplePartitionData1)},
-    {kSubsamples2, arraysize(kSubsamples2), kSubsamplePartitionData2,
-     arraysize(kSubsamplePartitionData2)},
-    {kSubsamples3, arraysize(kSubsamples3), kSubsamplePartitionData3,
-     arraysize(kSubsamplePartitionData3)},
-    {kSubsamples4, arraysize(kSubsamples4), kSubsamplePartitionData4,
-     arraysize(kSubsamplePartitionData4)},
-};
-
 }  // namespace
 
+struct EncryptionTestCase {
+  const SubsampleEntry* subsamples;
+  size_t num_subsamples;
+  const uint8_t* subsample_partition_data;
+  size_t subsample_partition_data_size;
+};
+
 class EncryptionUtilEncryptedTest
     : public ::testing::TestWithParam<EncryptionTestCase> {};
 
@@ -148,6 +135,21 @@ TEST_P(EncryptionUtilEncryptedTest, SampleEncrypted) {
                                  sample->data() + sample->data_size()));
 }
 
+namespace {
+EncryptionTestCase kEncryptionTestCases[] = {
+    // Special case with no subsamples.
+    {nullptr, 0, nullptr, 0},
+    {kSubsamples1, arraysize(kSubsamples1), kSubsamplePartitionData1,
+     arraysize(kSubsamplePartitionData1)},
+    {kSubsamples2, arraysize(kSubsamples2), kSubsamplePartitionData2,
+     arraysize(kSubsamplePartitionData2)},
+    {kSubsamples3, arraysize(kSubsamples3), kSubsamplePartitionData3,
+     arraysize(kSubsamplePartitionData3)},
+    {kSubsamples4, arraysize(kSubsamples4), kSubsamplePartitionData4,
+     arraysize(kSubsamplePartitionData4)},
+};
+}  // namespace
+
 INSTANTIATE_TEST_CASE_P(Encryption,
                         EncryptionUtilEncryptedTest,
                         testing::ValuesIn(kEncryptionTestCases));

packager/testing/dockers/Alpine_Dockerfile

@@ -0,0 +1,21 @@
+FROM alpine:3.8
+
+# Install packages needed for Shaka Packager.
+RUN apk add --no-cache bash build-base curl findutils git ninja python \
+                       bsd-compat-headers linux-headers libexecinfo-dev
+
+# Install depot_tools.
+WORKDIR /
+RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
+ENV PATH $PATH:/depot_tools
+
+# Alpine uses musl which does not have mallinfo defined in malloc.h. Define the
+# structure to workaround a Chromium base bug.
+RUN sed -i \
+    '/malloc_usable_size/a \\nstruct mallinfo {\n  int arena;\n  int hblkhd;\n  int uordblks;\n};' \
+    /usr/include/malloc.h
+
+ENV GYP_DEFINES='linux_use_bundled_binutils=0 linux_use_bundled_gold=0 clang=0 use_experimental_allocator_shim=0 use_allocator=none musl=1'
+
+# Build and run this docker by mapping shaka-packager with
+# -v "shaka-packager:/shaka-packager".

packager/testing/dockers/ArchLinux_Dockerfile

@@ -1,27 +1,18 @@
-FROM pritunl/archlinux
+FROM pritunl/archlinux:2018-08-11
 
 # Update, and install basic packages.
-RUN pacman -S --needed --noconfirm python2 git curl gcc gcc-libs make fakeroot
+RUN pacman -S --needed --noconfirm python2 git curl gcc gcc-libs make
 
 # depot_tools uses python2 instead of python3.
 RUN ln -sf python2 /usr/bin/python
 
-# Install libtinfo.so.5 which is needed by clang.
-RUN git clone https://aur.archlinux.org/ncurses5-compat-libs.git /tmp/ncurses-libs
-# makepkg need to run from non root user.
-RUN useradd -m makepkg_user
-RUN chown makepkg_user /tmp/ncurses-libs
-USER makepkg_user
-WORKDIR /tmp/ncurses-libs
-RUN gpg --keyserver pgp.mit.edu --recv-keys F7E48EDB
-RUN makepkg
-USER root
-RUN pacman -U --noconfirm /tmp/ncurses-libs/ncurses5-compat-libs-6.0+20170527-1-x86_64.pkg.tar.xz
-
 # Install depot_tools.
 WORKDIR /
 RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
 ENV PATH /depot_tools:$PATH
 
+# Clang needs libtinfo.so.5 which is not available on ArchLinux.
+ENV GYP_DEFINES='clang=0'
+
 # Build and run this docker by mapping shaka-packager with
 # -v "shaka-packager:/shaka-packager".

packager/testing/dockers/CentOS_Dockerfile

@@ -1,4 +1,4 @@
-FROM centos
+FROM centos:7.5.1804
 
 # Update, and install basic packages.
 RUN yum install -y git python git curl gcc-c++ findutils bzip2 \

packager/testing/dockers/Debian_Dockerfile

@@ -1,4 +1,4 @@
-FROM debian
+FROM debian:9.5
 
 # Update, and install basic packages
 RUN apt-get update

packager/testing/dockers/Fedora_Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora
+FROM fedora:28
 
 # Update, and install basic packages.
 RUN yum install -y git python git curl gcc-c++ findutils bzip2 \

packager/testing/dockers/OpenSUSE_Dockerfile

@@ -1,4 +1,4 @@
-FROM opensuse
+FROM opensuse:42.3
 
 # Update, and install basic packages.
 RUN zypper in -y git python python-xml git curl gcc-c++ tar

packager/testing/dockers/Ubuntu17_Dockerfile

@@ -1,12 +0,0 @@
-FROM ubuntu:17.10
-
-# Update, and install basic packages.
-RUN apt-get update
-RUN apt-get install -y build-essential curl git python
-
-# Install depot_tools.
-RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
-ENV PATH /depot_tools:$PATH
-
-# Build and run this docker by mapping shaka-packager with
-# -v "shaka-packager:/shaka-packager".

packager/testing/dockers/Ubuntu_Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu
+FROM ubuntu:18.04
 
 # Update, and install basic packages.
 RUN apt-get update

packager/testing/dockers/test_dockers.sh

@@ -13,6 +13,4 @@ for docker_file in ${SCRIPT_DIR}/*_Dockerfile ; do
   docker build -t my_container -f ${docker_file} ${SCRIPT_DIR}
   docker_run gclient runhooks
   docker_run ninja -C out/Release
-  docker_run bash -c 'GYP_DEFINES="clang=0" gclient runhooks'
-  docker_run ninja -C out/Release
 done