From 0afda1725c9db2b06c85389d056c2f14e05d50ad Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 13 Oct 2021 11:00:37 -0700 Subject: [PATCH] test: Update Dockerfiles This updates the main Dockerfile and all the docker-based distro-specific tests. The base OS versions have been updated to versions that have not reached end-of-life status yet, and the list of dependencies required has been updated and pruned. Change-Id: Ibcff2f60e739fd5d999af100af76c40aa91a75bc --- Dockerfile | 26 ++--- docs/source/build_instructions.md | 103 +++++++++--------- packager/testing/dockers/Alpine_Dockerfile | 16 ++- packager/testing/dockers/ArchLinux_Dockerfile | 11 +- packager/testing/dockers/CentOS_Dockerfile | 13 ++- packager/testing/dockers/Debian_Dockerfile | 12 +- packager/testing/dockers/Fedora_Dockerfile | 13 ++- packager/testing/dockers/OpenSUSE_Dockerfile | 6 +- packager/testing/dockers/Ubuntu_Dockerfile | 8 +- 9 files changed, 112 insertions(+), 96 deletions(-) diff --git a/Dockerfile b/Dockerfile index 264a0091aa..75a4a0d8da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,13 @@ -FROM alpine:3.8 as builder +FROM alpine:3.11 as builder -# Install packages needed for Shaka Packager. -RUN apk add --no-cache bash build-base curl findutils git ninja python \ - bsd-compat-headers linux-headers libexecinfo-dev +# Install utilities, libraries, and dev tools. +RUN apk add --no-cache \ + bash curl \ + bsd-compat-headers linux-headers \ + build-base git ninja python2 python3 # Install depot_tools. +WORKDIR / RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ENV PATH $PATH:/depot_tools @@ -13,24 +16,21 @@ ENV PATH $PATH:/depot_tools RUN sed -i \ '/malloc_usable_size/a \\nstruct mallinfo {\n int arena;\n int hblkhd;\n int uordblks;\n};' \ /usr/include/malloc.h - ENV GYP_DEFINES='musl=1' -# Alpine does not support python3 yet, but depot_tools enabled python3 -# by default. Disable python3 explicitly for now. -# See https://github.com/google/shaka-packager/issues/763 for details. -ENV GCLIENT_PY3=0 + # Bypass VPYTHON included by depot_tools, which no longer works in Alpine. ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" -# Build shaka-packager +# Build shaka-packager from the current directory, rather than what has been +# merged. WORKDIR shaka_packager RUN gclient config https://github.com/google/shaka-packager.git --name=src --unmanaged COPY . src -RUN gclient sync -RUN cd src && ninja -C out/Release +RUN gclient sync --force +RUN ninja -C src/out/Release # Copy only result binaries to our final image. -FROM alpine:3.8 +FROM alpine:3.11 RUN apk add --no-cache libstdc++ python COPY --from=builder /shaka_packager/src/out/Release/packager \ /shaka_packager/src/out/Release/mpd_generator \ diff --git a/docs/source/build_instructions.md b/docs/source/build_instructions.md index 210de16df5..33d99acbab 100644 --- a/docs/source/build_instructions.md +++ b/docs/source/build_instructions.md @@ -10,7 +10,9 @@ for [other distros below](#notes-for-other-linux-distros). ```shell sudo apt-get update -sudo apt-get install build-essential curl git python +sudo apt-get install -y \ + curl \ + build-essential git python python3 ``` Note that `Git` must be v1.7.5 or above. @@ -21,7 +23,7 @@ Note that `Git` must be v1.7.5 or above. * The OS X 10.10 SDK or later. Run ```shell - $ ls `xcode-select -p`/Platforms/MacOSX.platform/Developer/SDKs + ls `xcode-select -p`/Platforms/MacOSX.platform/Developer/SDKs ``` to check whether you have it. @@ -58,7 +60,7 @@ GYP_MSVS_OVERRIDE_PATH="C:/Program Files (x86)/Microsoft Visual Studio/2019/Comm Clone the `depot_tools` repository from Chromium: ```shell -$ git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ``` Add `depot_tools` to the end of your PATH (you will probably want to put this @@ -66,7 +68,7 @@ in your `~/.bashrc` or `~/.zshrc`). Assuming you cloned `depot_tools` to `/path/to/depot_tools`: ```shell -$ export PATH="$PATH:/path/to/depot_tools" +export PATH="$PATH:/path/to/depot_tools" ``` ### Windows @@ -111,26 +113,26 @@ call this whatever you like and put it wherever you like, as long as the full path has no spaces): ```shell -$ mkdir shaka_packager && cd shaka_packager +mkdir shaka_packager && cd shaka_packager ``` Run the `gclient` tool from `depot_tools` to check out the code and its dependencies. ```shell -$ gclient config https://www.github.com/google/shaka-packager.git --name=src --unmanaged -$ gclient sync +gclient config https://www.github.com/google/shaka-packager.git --name=src --unmanaged +gclient sync ``` To sync to a particular commit or version, add the '-r \' flag to `gclient sync`, e.g. ```shell -$ gclient sync -r 4cb5326355e1559d60b46167740e04624d0d2f51 +gclient sync -r 4cb5326355e1559d60b46167740e04624d0d2f51 ``` ```shell -$ gclient sync -r v1.2.0 +gclient sync -r v1.2.0 ``` If you don't want the full repo history, you can save some time by adding the @@ -141,7 +143,7 @@ and a directory called `src` in the working directory. The remaining instructions assume you have switched to the `src` directory: ```shell -$ cd src +cd src ``` ### Build Shaka Packager @@ -154,7 +156,7 @@ which is bundled in depot_tools. To build the code, run `ninja` command: ```shell -$ ninja -C out/Release +ninja -C out/Release ``` If you want to build debug code, replace `Release` above with `Debug`. @@ -163,7 +165,7 @@ We also provide a mechanism to change build settings, for example, you can change build system to `make` by overriding `GYP_GENERATORS`: ```shell -$ GYP_GENERATORS='make' gclient runhooks +GYP_GENERATORS='make' gclient runhooks ``` #### Windows @@ -172,8 +174,8 @@ The instructions are similar, except that Windows allows using either `/` or `\` as path separator: ```shell -$ ninja -C out/Release -$ ninja -C out\Release +ninja -C out/Release +ninja -C out\Release ``` Also, unlike Linux / Mac, 32-bit is chosen by default even if the system is @@ -181,9 +183,9 @@ Also, unlike Linux / Mac, 32-bit is chosen by default even if the system is configured to `out/%CONFIGURATION%_x64`, i.e.: ```shell -$ SET GYP_DEFINES='target_arch=x64' -$ gclient runhooks -$ ninja -C out/Release_x64 +SET GYP_DEFINES='target_arch=x64' +gclient runhooks +ninja -C out/Release_x64 ``` ### Build artifacts @@ -200,8 +202,8 @@ on how to use `Shaka Packager`. To update an existing checkout, you can run ```shell -$ git pull origin master --rebase -$ gclient sync +git pull origin master --rebase +gclient sync ``` The first command updates the primary Packager source repository and rebases on @@ -217,19 +219,19 @@ The install-build-deps script can be used to install all the compiler and library dependencies directly from Ubuntu: ```shell -$ ./packager/build/install-build-deps.sh +./packager/build/install-build-deps.sh ``` Install sysroot image and others using `gclient`: ```shell -$ GYP_CROSSCOMPILE=1 GYP_DEFINES="target_arch=arm" gclient runhooks +GYP_CROSSCOMPILE=1 GYP_DEFINES="target_arch=arm" gclient runhooks ``` The build command is the same as in Ubuntu: ```shell -$ ninja -C out/Release +ninja -C out/Release ``` ## Notes for other linux distros @@ -239,8 +241,10 @@ $ ninja -C out/Release Use `apk` command to install dependencies: ```shell -$ apk add --no-cache bash build-base curl findutils git ninja python \ - bsd-compat-headers linux-headers libexecinfo-dev +apk add --no-cache \ + bash curl \ + bsd-compat-headers linux-headers \ + build-base git ninja python2 python3 ``` Alpine uses musl which does not have mallinfo defined in malloc.h. It is @@ -248,7 +252,7 @@ required by one of Shaka Packager's dependency. To workaround the problem, a dummy structure has to be defined in /usr/include/malloc.h, e.g. ```shell -$ sed -i \ +sed -i \ '/malloc_usable_size/a \\nstruct mallinfo {\n int arena;\n int hblkhd;\n int uordblks;\n};' \ /usr/include/malloc.h ``` @@ -264,18 +268,9 @@ export GYP_DEFINES='musl=1' Instead of running `sudo apt-get install` to install build dependencies, run: ```shell -$ sudo pacman -Sy --needed python2 git curl gcc gcc-libs make -$ sudo ln -sf python2 /usr/bin/python -``` - -Clang requires libtinfo.so.5 which is not available by default on Arch Linux. -You can get libtinfo from ncurses5-compat-libs in AUR: - -```shell -$ git clone https://aur.archlinux.org/ncurses5-compat-libs.git -$ cd ncurses5-compat-libs -$ gpg --keyserver pgp.mit.edu --recv-keys F7E48EDB -$ makepkg -si +sudo pacman -Sy --needed \ + core/which \ + gcc git python2 python3 ``` ### Debian @@ -287,16 +282,24 @@ Same as Ubuntu. Instead of running `sudo apt-get install` to install build dependencies, run: ```shell -$ su -c 'yum install -y git python git curl gcc-c++ findutils bzip2 \ - ncurses-compat-libs' +su -c 'yum install -y \ + which \ + libatomic \ + gcc-c++ git python2' ``` +### CentOS + +Same as Fedora. + ### OpenSUSE Use `zypper` command to install dependencies: ```shell -sudo zypper in git python python-xml git curl gcc-c++ tar libncurses5 +sudo zypper in -y \ + curl which \ + gcc-c++ git python python3 ``` ## Tips, tricks, and troubleshooting @@ -312,13 +315,13 @@ the Xcode license has not been accepted yet which (contrary to the message) any user can do by running: ```shell -$ xcodebuild -license +xcodebuild -license ``` Only accepting for all users of the machine requires root: ```shell -$ sudo xcodebuild -license +sudo xcodebuild -license ``` ### Missing curl CA bundle @@ -331,7 +334,7 @@ curl CA bundle is not able to be located. Installing curl with openssl should resolve the issue: ```shell -$ brew install curl --with-openssl +brew install curl --with-openssl ``` ### Using an IDE @@ -344,17 +347,17 @@ Manual editing might be necessary. To generate CMakeLists.txt in out/Release and out/Debug use: ```shell -$ GYP_GENERATORS=cmake gclient runhooks +GYP_GENERATORS=cmake gclient runhooks ``` To generate IDE project files in out/Release and out/Debug use: ```shell -$ GYP_GENERATORS=eclipse gclient runhooks -$ GYP_GENERATORS=xcode gclient runhooks -$ GYP_GENERATORS=xcode_test gclient runhooks -$ GYP_GENERATORS=msvs gclient runhooks -$ GYP_GENERATORS=msvs_test gclient runhooks +GYP_GENERATORS=eclipse gclient runhooks +GYP_GENERATORS=xcode gclient runhooks +GYP_GENERATORS=xcode_test gclient runhooks +GYP_GENERATORS=msvs gclient runhooks +GYP_GENERATORS=msvs_test gclient runhooks ``` ## Contributing @@ -370,7 +373,7 @@ If you know which tests are affected by your change, you can limit which tests are run using the `--gtest_filter` arg, e.g.: ```shell -$ out/Debug/mp4_unittest --gtest_filter="MP4MediaParserTest.*" +out/Debug/mp4_unittest --gtest_filter="MP4MediaParserTest.*" ``` You can find out more about GoogleTest at its diff --git a/packager/testing/dockers/Alpine_Dockerfile b/packager/testing/dockers/Alpine_Dockerfile index 460ba550b7..0ef955596f 100644 --- a/packager/testing/dockers/Alpine_Dockerfile +++ b/packager/testing/dockers/Alpine_Dockerfile @@ -1,8 +1,10 @@ -FROM alpine:3.8 +FROM alpine:3.11 -# Install packages needed for Shaka Packager. -RUN apk add --no-cache bash build-base curl findutils git ninja python \ - bsd-compat-headers linux-headers libexecinfo-dev +# Install utilities, libraries, and dev tools. +RUN apk add --no-cache \ + bash curl \ + bsd-compat-headers linux-headers \ + build-base git ninja python2 python3 # Install depot_tools. WORKDIR / @@ -14,12 +16,8 @@ ENV PATH $PATH:/depot_tools RUN sed -i \ '/malloc_usable_size/a \\nstruct mallinfo {\n int arena;\n int hblkhd;\n int uordblks;\n};' \ /usr/include/malloc.h - ENV GYP_DEFINES='musl=1' -# Alpine does not support python3 yet, but depot_tools enabled python3 -# by default. Disable python3 explicitly for now. -# See https://github.com/google/shaka-packager/issues/763 for details. -ENV GCLIENT_PY3=0 + # Bypass VPYTHON included by depot_tools, which no longer works in Alpine. ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" diff --git a/packager/testing/dockers/ArchLinux_Dockerfile b/packager/testing/dockers/ArchLinux_Dockerfile index 5227c9a31d..a461161fb3 100644 --- a/packager/testing/dockers/ArchLinux_Dockerfile +++ b/packager/testing/dockers/ArchLinux_Dockerfile @@ -1,10 +1,9 @@ -FROM archlinux:20200908 +FROM archlinux:latest -# Update, and install basic packages. -RUN pacman -Sy --needed --noconfirm python2 git curl gcc gcc-libs make - -# depot_tools uses python2 instead of python3. -RUN ln -sf python2 /usr/bin/python +# Install utilities, libraries, and dev tools. +RUN pacman -Sy --needed --noconfirm \ + core/which \ + gcc git python2 python3 # Install depot_tools. WORKDIR / diff --git a/packager/testing/dockers/CentOS_Dockerfile b/packager/testing/dockers/CentOS_Dockerfile index 05f09aeeff..364945ad30 100644 --- a/packager/testing/dockers/CentOS_Dockerfile +++ b/packager/testing/dockers/CentOS_Dockerfile @@ -1,8 +1,13 @@ -FROM centos:7.5.1804 +FROM centos:8 -# Update, and install basic packages. -RUN yum install -y git python git curl gcc-c++ findutils bzip2 \ - ncurses-compat-libs libatomic +# Install utilities, libraries, and dev tools. +RUN yum install -y \ + which \ + libatomic \ + gcc-c++ git python2 python3 + +# Default to python3. +RUN alternatives --set python /usr/bin/python3 # Install depot_tools. WORKDIR / diff --git a/packager/testing/dockers/Debian_Dockerfile b/packager/testing/dockers/Debian_Dockerfile index 39941d24ec..2299a46318 100644 --- a/packager/testing/dockers/Debian_Dockerfile +++ b/packager/testing/dockers/Debian_Dockerfile @@ -1,10 +1,12 @@ -FROM debian:9.5 +FROM debian:9 -# Update, and install basic packages -RUN apt-get update -RUN apt-get install -y build-essential curl git python +# Install utilities, libraries, and dev tools. +RUN apt-get update && apt-get install -y apt-utils +RUN apt-get install -y \ + curl \ + build-essential git python python3 -# install depot_tools http://www.chromium.org/developers/how-tos/install-depot-tools +# Install depot_tools. RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ENV PATH /depot_tools:$PATH diff --git a/packager/testing/dockers/Fedora_Dockerfile b/packager/testing/dockers/Fedora_Dockerfile index ca4a0ff99a..8438319fc3 100644 --- a/packager/testing/dockers/Fedora_Dockerfile +++ b/packager/testing/dockers/Fedora_Dockerfile @@ -1,8 +1,13 @@ -FROM fedora:28 +FROM fedora:34 -# Update, and install basic packages. -RUN yum install -y git python git curl gcc-c++ findutils bzip2 \ - ncurses-compat-libs libatomic +# Install utilities, libraries, and dev tools. +RUN yum install -y \ + which \ + libatomic \ + gcc-c++ git python2 + +# Default to python3. +RUN alternatives --install /usr/bin/python python /usr/bin/python3 3 # Install depot_tools. WORKDIR / diff --git a/packager/testing/dockers/OpenSUSE_Dockerfile b/packager/testing/dockers/OpenSUSE_Dockerfile index 3987235915..2a99f2e233 100644 --- a/packager/testing/dockers/OpenSUSE_Dockerfile +++ b/packager/testing/dockers/OpenSUSE_Dockerfile @@ -1,7 +1,9 @@ FROM opensuse/leap:15 -# Update, and install basic packages. -RUN zypper in -y git python python-xml git curl gcc-c++ tar libncurses5 +# Install utilities, libraries, and dev tools. +RUN zypper in -y \ + curl which \ + gcc-c++ git python python3 # Install depot_tools. WORKDIR / diff --git a/packager/testing/dockers/Ubuntu_Dockerfile b/packager/testing/dockers/Ubuntu_Dockerfile index f526dbaa5a..a7ccf3cb2a 100644 --- a/packager/testing/dockers/Ubuntu_Dockerfile +++ b/packager/testing/dockers/Ubuntu_Dockerfile @@ -1,8 +1,10 @@ FROM ubuntu:18.04 -# Update, and install basic packages. -RUN apt-get update -RUN apt-get install -y build-essential curl git python +# Install utilities, libraries, and dev tools. +RUN apt-get update && apt-get install -y apt-utils +RUN apt-get install -y \ + curl \ + build-essential git python python3 # Install depot_tools. RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git