diff --git a/docs/source/build_instructions.md b/docs/source/build_instructions.md index b0be3055ab..9b70c75ac9 100644 --- a/docs/source/build_instructions.md +++ b/docs/source/build_instructions.md @@ -232,6 +232,31 @@ $ ninja -C out/Release ## Notes for other linux distros +### Alpine Linux + +Use `apk` command to install dependencies: + +```shell +$apk add --no-cache bash build-base curl findutils git ninja python \ + bsd-compat-headers linux-headers libexecinfo-dev +``` + +Alpine uses musl which does not have mallinfo defined in malloc.h. It is +required by one of Shaka Packager's dependency. To workaround the problem, a +dummy structure has to be defined in /usr/include/malloc.h, e.g. + +```shell +$ sed -i \ + '/malloc_usable_size/a \\nstruct mallinfo {\n int arena;\n int hblkhd;\n int uordblks;\n};' \ + /usr/include/malloc.h +``` + +We also need to disable clang and some other features to make it work with musl: + +```shell +export GYP_DEFINES='linux_use_bundled_binutils=0 linux_use_bundled_gold=0 clang=0 use_experimental_allocator_shim=0 use_allocator=none musl=1' +``` + ### Arch Linux Instead of running `sudo apt-get install` to install build dependencies, run: @@ -251,6 +276,12 @@ $ gpg --keyserver pgp.mit.edu --recv-keys F7E48EDB $ makepkg -si ``` +Optionally, disable clang to build with gcc: + +```shell +$ export GYP_DEFINES='clang=0' +``` + ### Debian Same as Ubuntu. diff --git a/packager/common.gypi b/packager/common.gypi index b5c5dbe9c0..256368be72 100644 --- a/packager/common.gypi +++ b/packager/common.gypi @@ -10,8 +10,11 @@ 'variables': { 'variables': { 'shaka_code%': 0, + # musl is a lightweight C standard library used in Alpine Linux. + 'musl%': 0, }, 'shaka_code%': '<(shaka_code)', + 'musl%': '<(musl)', 'libpackager_type%': 'static_library', 'conditions': [ ['shaka_code==1', { @@ -75,6 +78,20 @@ }], ], }], + ['musl==1', { + 'defines': [ + # musl is not uClibc but is similar to uClibc that a minimal feature + # set is supported. One of Shaka Packager's dependencies, Chromium + # base uses __UCLIBC__ flag to disable some features, which needs to + # be disabled for musl too. + '__UCLIBC__', + ], + 'cflags!': [ + # Do not treat warnings as errors on musl as there is a hard-coded + # warning in musl's sys/errno.h. + '-Werror', + ], + }] ], }, } diff --git a/packager/file/udp_file.cc b/packager/file/udp_file.cc index d6a5dffadf..6904953af0 100644 --- a/packager/file/udp_file.cc +++ b/packager/file/udp_file.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #define INVALID_SOCKET -1 diff --git a/packager/media/base/aes_cryptor_unittest.cc b/packager/media/base/aes_cryptor_unittest.cc index 84c89aa8ce..8fe02e88dc 100644 --- a/packager/media/base/aes_cryptor_unittest.cc +++ b/packager/media/base/aes_cryptor_unittest.cc @@ -53,12 +53,6 @@ const uint8_t kAesCtrCiphertext[] = { 0x1e, 0x03, 0x1d, 0xda, 0x2f, 0xbe, 0x03, 0xd1, 0x79, 0x21, 0x70, 0xa0, 0xf3, 0x00, 0x9c, 0xee}; -// Subsample test cases. -struct SubsampleTestCase { - const uint8_t* subsample_sizes; - uint32_t subsample_count; -}; - const uint8_t kSubsampleTest1[] = {64}; const uint8_t kSubsampleTest2[] = {13, 51}; const uint8_t kSubsampleTest3[] = {52, 12}; @@ -70,18 +64,6 @@ const uint8_t kSubsampleTest8[] = {10, 1, 33, 20}; const uint8_t kSubsampleTest9[] = {7, 19, 6, 32}; const uint8_t kSubsampleTest10[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9}; -const SubsampleTestCase kSubsampleTestCases[] = { - {kSubsampleTest1, arraysize(kSubsampleTest1)}, - {kSubsampleTest2, arraysize(kSubsampleTest2)}, - {kSubsampleTest3, arraysize(kSubsampleTest3)}, - {kSubsampleTest4, arraysize(kSubsampleTest4)}, - {kSubsampleTest5, arraysize(kSubsampleTest5)}, - {kSubsampleTest6, arraysize(kSubsampleTest6)}, - {kSubsampleTest7, arraysize(kSubsampleTest7)}, - {kSubsampleTest8, arraysize(kSubsampleTest8)}, - {kSubsampleTest9, arraysize(kSubsampleTest9)}, - {kSubsampleTest10, arraysize(kSubsampleTest10)}}; - // IV test values. const uint32_t kTextSizeInBytes = 60; // 3 full blocks + 1 partial block. @@ -102,26 +84,6 @@ const uint8_t kIv64MaxMinusOne[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe}; const uint8_t kIv64Max[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; -struct IvTestCase { - const uint8_t* iv_test; - uint32_t iv_size; - const uint8_t* iv_expected; -}; - -// As recommended in ISO/IEC FDIS 23001-7: CENC spec, -// For 64-bit (8-byte) IV_Sizes, initialization vectors for subsequent samples -// can be created by incrementing the initialization vector of the previous -// sample. For 128-bit (16-byte) IV_Sizes, initialization vectors for subsequent -// samples should be created by adding the block count of the previous sample to -// the initialization vector of the previous sample. -const IvTestCase kIvTestCases[] = { - {kIv128Zero, arraysize(kIv128Zero), kIv128Four}, - {kIv128Max64, arraysize(kIv128Max64), kIv128OneAndThree}, - {kIv128MaxMinusOne, arraysize(kIv128MaxMinusOne), kIv128Two}, - {kIv64Zero, arraysize(kIv64Zero), kIv64One}, - {kIv64MaxMinusOne, arraysize(kIv64MaxMinusOne), kIv64Max}, - {kIv64Max, arraysize(kIv64Max), kIv64Zero}}; - // We support AES 128, i.e. 16 bytes key only. const uint8_t kInvalidKey[] = {0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09}; @@ -253,6 +215,12 @@ TEST_F(AesCtrEncryptorTest, UnsupportedIV) { ASSERT_FALSE(encryptor_.InitializeWithIv(key_, iv)); } +// Subsample test cases. +struct SubsampleTestCase { + const uint8_t* subsample_sizes; + uint32_t subsample_count; +}; + class AesCtrEncryptorSubsampleTest : public AesCtrEncryptorTest, public ::testing::WithParamInterface {}; @@ -280,10 +248,30 @@ TEST_P(AesCtrEncryptorSubsampleTest, NistTestCaseSubsamples) { EXPECT_EQ(plaintext_, decrypted); } +namespace { +const SubsampleTestCase kSubsampleTestCases[] = { + {kSubsampleTest1, arraysize(kSubsampleTest1)}, + {kSubsampleTest2, arraysize(kSubsampleTest2)}, + {kSubsampleTest3, arraysize(kSubsampleTest3)}, + {kSubsampleTest4, arraysize(kSubsampleTest4)}, + {kSubsampleTest5, arraysize(kSubsampleTest5)}, + {kSubsampleTest6, arraysize(kSubsampleTest6)}, + {kSubsampleTest7, arraysize(kSubsampleTest7)}, + {kSubsampleTest8, arraysize(kSubsampleTest8)}, + {kSubsampleTest9, arraysize(kSubsampleTest9)}, + {kSubsampleTest10, arraysize(kSubsampleTest10)}}; +} // namespace + INSTANTIATE_TEST_CASE_P(SubsampleTestCases, AesCtrEncryptorSubsampleTest, ::testing::ValuesIn(kSubsampleTestCases)); +struct IvTestCase { + const uint8_t* iv_test; + uint32_t iv_size; + const uint8_t* iv_expected; +}; + class AesCtrEncryptorIvTest : public ::testing::TestWithParam {}; TEST_P(AesCtrEncryptorIvTest, IvTest) { @@ -305,6 +293,22 @@ TEST_P(AesCtrEncryptorIvTest, IvTest) { EXPECT_EQ(iv_expected, encryptor.iv()); } +namespace { +// As recommended in ISO/IEC FDIS 23001-7: CENC spec, +// For 64-bit (8-byte) IV_Sizes, initialization vectors for subsequent samples +// can be created by incrementing the initialization vector of the previous +// sample. For 128-bit (16-byte) IV_Sizes, initialization vectors for subsequent +// samples should be created by adding the block count of the previous sample to +// the initialization vector of the previous sample. +const IvTestCase kIvTestCases[] = { + {kIv128Zero, arraysize(kIv128Zero), kIv128Four}, + {kIv128Max64, arraysize(kIv128Max64), kIv128OneAndThree}, + {kIv128MaxMinusOne, arraysize(kIv128MaxMinusOne), kIv128Two}, + {kIv64Zero, arraysize(kIv64Zero), kIv64One}, + {kIv64MaxMinusOne, arraysize(kIv64MaxMinusOne), kIv64Max}, + {kIv64Max, arraysize(kIv64Max), kIv64Zero}}; +} // namespace + INSTANTIATE_TEST_CASE_P(IvTestCases, AesCtrEncryptorIvTest, ::testing::ValuesIn(kIvTestCases)); diff --git a/packager/media/base/aes_pattern_cryptor_unittest.cc b/packager/media/base/aes_pattern_cryptor_unittest.cc index 7625119275..0ba87e18ec 100644 --- a/packager/media/base/aes_pattern_cryptor_unittest.cc +++ b/packager/media/base/aes_pattern_cryptor_unittest.cc @@ -60,13 +60,44 @@ TEST_F(AesPatternCryptorTest, InitializeWithIv) { EXPECT_EQ(iv, pattern_cryptor_.iv()); } -namespace { - struct PatternTestCase { const char* text_hex; const char* expected_crypt_text_hex; }; +class AesPatternCryptorVerificationTest + : public AesPatternCryptorTest, + public ::testing::WithParamInterface {}; + +TEST_P(AesPatternCryptorVerificationTest, PatternTest) { + std::vector text; + std::string text_hex(GetParam().text_hex); + if (!text_hex.empty()) { + ASSERT_TRUE(base::HexStringToBytes(text_hex, &text)); + } + std::vector expected_crypt_text; + std::string expected_crypt_text_hex(GetParam().expected_crypt_text_hex); + if (!expected_crypt_text_hex.empty()) { + ASSERT_TRUE( + base::HexStringToBytes(expected_crypt_text_hex, &expected_crypt_text)); + } + + ON_CALL(*mock_cryptor_, CryptInternal(_, _, _, _)) + .WillByDefault(Invoke([](const uint8_t* text, size_t text_size, + uint8_t* crypt_text, size_t* crypt_text_size) { + *crypt_text_size = text_size; + for (size_t i = 0; i < text_size; ++i) { + *crypt_text++ = *text++ + 0x10; + } + return true; + })); + + std::vector crypt_text; + ASSERT_TRUE(pattern_cryptor_.Crypt(text, &crypt_text)); + EXPECT_EQ(expected_crypt_text, crypt_text); +} + +namespace { const PatternTestCase kPatternTestCases[] = { // Empty. {"", ""}, @@ -102,41 +133,8 @@ const PatternTestCase kPatternTestCases[] = { "81828384858687888990919293949596" "97989900010203040506070809101112"}, }; - } // namespace -class AesPatternCryptorVerificationTest - : public AesPatternCryptorTest, - public ::testing::WithParamInterface {}; - -TEST_P(AesPatternCryptorVerificationTest, PatternTest) { - std::vector text; - std::string text_hex(GetParam().text_hex); - if (!text_hex.empty()) { - ASSERT_TRUE(base::HexStringToBytes(text_hex, &text)); - } - std::vector expected_crypt_text; - std::string expected_crypt_text_hex(GetParam().expected_crypt_text_hex); - if (!expected_crypt_text_hex.empty()) { - ASSERT_TRUE( - base::HexStringToBytes(expected_crypt_text_hex, &expected_crypt_text)); - } - - ON_CALL(*mock_cryptor_, CryptInternal(_, _, _, _)) - .WillByDefault(Invoke([](const uint8_t* text, size_t text_size, - uint8_t* crypt_text, size_t* crypt_text_size) { - *crypt_text_size = text_size; - for (size_t i = 0; i < text_size; ++i) { - *crypt_text++ = *text++ + 0x10; - } - return true; - })); - - std::vector crypt_text; - ASSERT_TRUE(pattern_cryptor_.Crypt(text, &crypt_text)); - EXPECT_EQ(expected_crypt_text, crypt_text); -} - INSTANTIATE_TEST_CASE_P(PatternTestCases, AesPatternCryptorVerificationTest, ::testing::ValuesIn(kPatternTestCases)); diff --git a/packager/media/base/producer_consumer_queue_unittest.cc b/packager/media/base/producer_consumer_queue_unittest.cc index 0275c69833..15394d9186 100644 --- a/packager/media/base/producer_consumer_queue_unittest.cc +++ b/packager/media/base/producer_consumer_queue_unittest.cc @@ -278,13 +278,11 @@ TEST_F(MultiThreadProducerConsumerQueueTest, PeekOnLargePosition) { queue_.Stop(); } -namespace { enum Operation { kPush, kPop, kPeek, }; -} // namespace class MultiThreadProducerConsumerQueueStopTest : public ::testing::TestWithParam { diff --git a/packager/media/codecs/h264_parser.cc b/packager/media/codecs/h264_parser.cc index 77cb9e9b33..aa45d039ed 100644 --- a/packager/media/codecs/h264_parser.cc +++ b/packager/media/codecs/h264_parser.cc @@ -988,7 +988,7 @@ H264Parser::Result H264Parser::ParseSliceHeader(const Nalu& nalu, reader.Initialize(nalu.data() + nalu.header_size(), nalu.payload_size()); H26xBitReader* br = &reader; - memset(shdr, 0, sizeof(*shdr)); + memset(reinterpret_cast(shdr), 0, sizeof(*shdr)); shdr->idr_pic_flag = (nalu.type() == 5); shdr->nal_ref_idc = nalu.ref_idc(); @@ -1134,7 +1134,7 @@ H264Parser::Result H264Parser::ParseSEI(const Nalu& nalu, reader.Initialize(nalu.data() + nalu.header_size(), nalu.payload_size()); H26xBitReader* br = &reader; - memset(sei_msg, 0, sizeof(*sei_msg)); + memset(reinterpret_cast(sei_msg), 0, sizeof(*sei_msg)); READ_BITS_OR_RETURN(8, &byte); while (byte == 0xff) { diff --git a/packager/media/formats/webm/encryptor_unittest.cc b/packager/media/formats/webm/encryptor_unittest.cc index 13e2e275e4..ee8ece6621 100644 --- a/packager/media/formats/webm/encryptor_unittest.cc +++ b/packager/media/formats/webm/encryptor_unittest.cc @@ -57,13 +57,6 @@ TEST(EncryptionUtilTest, SampleNotEncrypted) { namespace { -struct EncryptionTestCase { - const SubsampleEntry* subsamples; - size_t num_subsamples; - const uint8_t* subsample_partition_data; - size_t subsample_partition_data_size; -}; - const SubsampleEntry kSubsamples1[] = { SubsampleEntry(0x12, 0x100), }; @@ -94,21 +87,15 @@ const uint8_t kSubsamplePartitionData4[] = { 0x6B, 0x00, 0x09, 0x03, 0x6B, 0x00, 0x09, 0x03, 0x6D, }; -EncryptionTestCase kEncryptionTestCases[] = { - // Special case with no subsamples. - {nullptr, 0, nullptr, 0}, - {kSubsamples1, arraysize(kSubsamples1), kSubsamplePartitionData1, - arraysize(kSubsamplePartitionData1)}, - {kSubsamples2, arraysize(kSubsamples2), kSubsamplePartitionData2, - arraysize(kSubsamplePartitionData2)}, - {kSubsamples3, arraysize(kSubsamples3), kSubsamplePartitionData3, - arraysize(kSubsamplePartitionData3)}, - {kSubsamples4, arraysize(kSubsamples4), kSubsamplePartitionData4, - arraysize(kSubsamplePartitionData4)}, -}; - } // namespace +struct EncryptionTestCase { + const SubsampleEntry* subsamples; + size_t num_subsamples; + const uint8_t* subsample_partition_data; + size_t subsample_partition_data_size; +}; + class EncryptionUtilEncryptedTest : public ::testing::TestWithParam {}; @@ -148,6 +135,21 @@ TEST_P(EncryptionUtilEncryptedTest, SampleEncrypted) { sample->data() + sample->data_size())); } +namespace { +EncryptionTestCase kEncryptionTestCases[] = { + // Special case with no subsamples. + {nullptr, 0, nullptr, 0}, + {kSubsamples1, arraysize(kSubsamples1), kSubsamplePartitionData1, + arraysize(kSubsamplePartitionData1)}, + {kSubsamples2, arraysize(kSubsamples2), kSubsamplePartitionData2, + arraysize(kSubsamplePartitionData2)}, + {kSubsamples3, arraysize(kSubsamples3), kSubsamplePartitionData3, + arraysize(kSubsamplePartitionData3)}, + {kSubsamples4, arraysize(kSubsamples4), kSubsamplePartitionData4, + arraysize(kSubsamplePartitionData4)}, +}; +} // namespace + INSTANTIATE_TEST_CASE_P(Encryption, EncryptionUtilEncryptedTest, testing::ValuesIn(kEncryptionTestCases)); diff --git a/packager/testing/dockers/Alpine_Dockerfile b/packager/testing/dockers/Alpine_Dockerfile new file mode 100644 index 0000000000..bfce90bab1 --- /dev/null +++ b/packager/testing/dockers/Alpine_Dockerfile @@ -0,0 +1,21 @@ +FROM alpine:3.8 + +# Install packages needed for Shaka Packager. +RUN apk add --no-cache bash build-base curl findutils git ninja python \ + bsd-compat-headers linux-headers libexecinfo-dev + +# Install depot_tools. +WORKDIR / +RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +ENV PATH $PATH:/depot_tools + +# Alpine uses musl which does not have mallinfo defined in malloc.h. Define the +# structure to workaround a Chromium base bug. +RUN sed -i \ + '/malloc_usable_size/a \\nstruct mallinfo {\n int arena;\n int hblkhd;\n int uordblks;\n};' \ + /usr/include/malloc.h + +ENV GYP_DEFINES='linux_use_bundled_binutils=0 linux_use_bundled_gold=0 clang=0 use_experimental_allocator_shim=0 use_allocator=none musl=1' + +# Build and run this docker by mapping shaka-packager with +# -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/ArchLinux_Dockerfile b/packager/testing/dockers/ArchLinux_Dockerfile index 7648de7d78..5bb8fe4481 100644 --- a/packager/testing/dockers/ArchLinux_Dockerfile +++ b/packager/testing/dockers/ArchLinux_Dockerfile @@ -1,27 +1,18 @@ -FROM pritunl/archlinux +FROM pritunl/archlinux:2018-08-11 # Update, and install basic packages. -RUN pacman -S --needed --noconfirm python2 git curl gcc gcc-libs make fakeroot +RUN pacman -S --needed --noconfirm python2 git curl gcc gcc-libs make # depot_tools uses python2 instead of python3. RUN ln -sf python2 /usr/bin/python -# Install libtinfo.so.5 which is needed by clang. -RUN git clone https://aur.archlinux.org/ncurses5-compat-libs.git /tmp/ncurses-libs -# makepkg need to run from non root user. -RUN useradd -m makepkg_user -RUN chown makepkg_user /tmp/ncurses-libs -USER makepkg_user -WORKDIR /tmp/ncurses-libs -RUN gpg --keyserver pgp.mit.edu --recv-keys F7E48EDB -RUN makepkg -USER root -RUN pacman -U --noconfirm /tmp/ncurses-libs/ncurses5-compat-libs-6.0+20170527-1-x86_64.pkg.tar.xz - # Install depot_tools. WORKDIR / RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ENV PATH /depot_tools:$PATH +# Clang needs libtinfo.so.5 which is not available on ArchLinux. +ENV GYP_DEFINES='clang=0' + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/CentOS_Dockerfile b/packager/testing/dockers/CentOS_Dockerfile index 78308fef58..05f09aeeff 100644 --- a/packager/testing/dockers/CentOS_Dockerfile +++ b/packager/testing/dockers/CentOS_Dockerfile @@ -1,4 +1,4 @@ -FROM centos +FROM centos:7.5.1804 # Update, and install basic packages. RUN yum install -y git python git curl gcc-c++ findutils bzip2 \ diff --git a/packager/testing/dockers/Debian_Dockerfile b/packager/testing/dockers/Debian_Dockerfile index a07da02817..39941d24ec 100644 --- a/packager/testing/dockers/Debian_Dockerfile +++ b/packager/testing/dockers/Debian_Dockerfile @@ -1,4 +1,4 @@ -FROM debian +FROM debian:9.5 # Update, and install basic packages RUN apt-get update diff --git a/packager/testing/dockers/Fedora_Dockerfile b/packager/testing/dockers/Fedora_Dockerfile index b7892b6f0f..ca4a0ff99a 100644 --- a/packager/testing/dockers/Fedora_Dockerfile +++ b/packager/testing/dockers/Fedora_Dockerfile @@ -1,4 +1,4 @@ -FROM fedora +FROM fedora:28 # Update, and install basic packages. RUN yum install -y git python git curl gcc-c++ findutils bzip2 \ diff --git a/packager/testing/dockers/OpenSUSE_Dockerfile b/packager/testing/dockers/OpenSUSE_Dockerfile index 1686317d78..af4767a516 100644 --- a/packager/testing/dockers/OpenSUSE_Dockerfile +++ b/packager/testing/dockers/OpenSUSE_Dockerfile @@ -1,4 +1,4 @@ -FROM opensuse +FROM opensuse:42.3 # Update, and install basic packages. RUN zypper in -y git python python-xml git curl gcc-c++ tar diff --git a/packager/testing/dockers/Ubuntu17_Dockerfile b/packager/testing/dockers/Ubuntu17_Dockerfile deleted file mode 100644 index ad358f59e0..0000000000 --- a/packager/testing/dockers/Ubuntu17_Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:17.10 - -# Update, and install basic packages. -RUN apt-get update -RUN apt-get install -y build-essential curl git python - -# Install depot_tools. -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git -ENV PATH /depot_tools:$PATH - -# Build and run this docker by mapping shaka-packager with -# -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/Ubuntu_Dockerfile b/packager/testing/dockers/Ubuntu_Dockerfile index 35350e88ee..f526dbaa5a 100644 --- a/packager/testing/dockers/Ubuntu_Dockerfile +++ b/packager/testing/dockers/Ubuntu_Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu +FROM ubuntu:18.04 # Update, and install basic packages. RUN apt-get update diff --git a/packager/testing/dockers/test_dockers.sh b/packager/testing/dockers/test_dockers.sh index 4f97027334..028365804d 100755 --- a/packager/testing/dockers/test_dockers.sh +++ b/packager/testing/dockers/test_dockers.sh @@ -13,6 +13,4 @@ for docker_file in ${SCRIPT_DIR}/*_Dockerfile ; do docker build -t my_container -f ${docker_file} ${SCRIPT_DIR} docker_run gclient runhooks docker_run ninja -C out/Release - docker_run bash -c 'GYP_DEFINES="clang=0" gclient runhooks' - docker_run ninja -C out/Release done