From 052fb640685deb99928e194d571331cdbc43c69c Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 17 Jul 2023 13:16:03 -0700 Subject: [PATCH] ci: Enable parallel builds (#1241) Our workflows and Dockerfiles now use Ninja on Linux & Mac, which enables safe parallel builds. This significantly speeds up our workflows. GitHub Actions typical compilation times (build step only): - Linux 19m => 9m - macOS 23m => 8m - Windows 12m => 10m - Linux arm64 (self-hosted) 72m => 29m - Docker build 25m => 14m Overall workflow time: 84m => 33m Compilation time on my workstation (12 CPUs @3.3GHz): 15m => 3m This also adds a new environment variable "PACKAGER_LOW_MEMORY_BUILD". If defined when CMake is first run, this will configure the build to disable parallel linking to reduce memory usage. This helps us avoid failures on our self-hosted arm64 machines, where 6 CPUs share 4GB of RAM. NOTE: Parallel builds are **NOT** recommended with Unix Makefiles due to the use of excessive RAM during parallel linking. Unix Makefiles, unlike Ninja, cannot be configured to restrict parallel linking during a parallel build. Anecdotally, parallel builds with Makefiles have exhausted a system with 32GB RAM. (My workstation.) In a follow-up, I will update the build documentation to refer to CMake and recommend all of the flags now used in our workflows. --- .github/workflows/build-matrix.json | 13 +++++++--- .github/workflows/build.yaml | 25 ++++++++++++++++--- Dockerfile | 6 ++--- packager/policies.cmake | 12 +++++++++ packager/testing/dockers/Alpine_Dockerfile | 2 +- packager/testing/dockers/ArchLinux_Dockerfile | 2 +- packager/testing/dockers/CentOS_Dockerfile | 8 +++++- packager/testing/dockers/Debian_Dockerfile | 2 +- packager/testing/dockers/Fedora_Dockerfile | 2 +- packager/testing/dockers/OpenSUSE_Dockerfile | 2 +- packager/testing/dockers/Ubuntu_Dockerfile | 2 +- packager/testing/test_dockers.sh | 4 +-- 12 files changed, 61 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-matrix.json b/.github/workflows/build-matrix.json index 5cb053dc37..71c87f1833 100644 --- a/.github/workflows/build-matrix.json +++ b/.github/workflows/build-matrix.json @@ -5,19 +5,22 @@ "os": "ubuntu-latest", "os_name": "linux", "target_arch": "x64", - "exe_ext": "" + "exe_ext": "", + "generator": "Ninja" }, { "os": "macos-latest", "os_name": "osx", "target_arch": "x64", - "exe_ext": "" + "exe_ext": "", + "generator": "Ninja" }, { "os": "windows-latest", "os_name": "win", "target_arch": "x64", - "exe_ext": ".exe" + "exe_ext": ".exe", + "generator": "" } ], @@ -27,7 +30,9 @@ "os": "self-hosted-linux-arm64", "os_name": "linux", "target_arch": "arm64", - "exe_ext": "" + "exe_ext": "", + "generator": "Ninja", + "low_mem": "yes" } ] } diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b49dce6e8d..af437b5295 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -100,12 +100,19 @@ jobs: - name: Install Linux deps if: runner.os == 'Linux' - # NOTE: cmake is already installed in GitHub Actions VMs, but not + # NOTE: CMake is already installed in GitHub Actions VMs, but not # necessarily in a self-hosted runner. run: | sudo apt update && sudo apt install -y \ cmake \ - libc-ares-dev + libc-ares-dev \ + ninja-build + + - name: Install Mac deps + if: runner.os == 'macOS' + # NOTE: GitHub Action VMs do not install ninja by default. + run: | + brew install ninja - name: Generate build files run: | @@ -117,6 +124,18 @@ jobs: LIBPACKAGER_SHARED="OFF" fi + # If set, override the default generator for the platform. + # Not every entry in the build matrix config defines this. + # If this is blank, CMake will choose the default generator. + export CMAKE_GENERATOR="${{ matrix.generator }}" + + # If set, configure the build to restrict parallel operations. + # This helps us avoid the compiler failing due to a lack of RAM + # on our arm64 build devices (4GB RAM shared among 6 CPUs). + if [[ "${{ matrix.low_mem }}" != "" ]]; then + export PACKAGER_LOW_MEMORY_BUILD=yes + fi + cmake \ -DCMAKE_BUILD_TYPE="${{ matrix.build_type }}" \ -DLIBPACKAGER_SHARED="$LIBPACKAGER_SHARED" \ @@ -128,7 +147,7 @@ jobs: # Visual Studio on Windows. Note that the VS generator is what cmake # calls a "multi-configuration" generator, and so the desired build # type must be specified for Windows. - run: cmake --build build/ --config "${{ matrix.build_type }}" + run: cmake --build build/ --config "${{ matrix.build_type }}" --parallel - name: Test run: ctest -C "${{ matrix.build_type }}" -V --test-dir build/ diff --git a/Dockerfile b/Dockerfile index b5baae9f52..f6e5efeb07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,15 +4,15 @@ FROM alpine:3.12 as builder RUN apk add --no-cache \ bash curl \ bsd-compat-headers c-ares-dev linux-headers \ - build-base cmake git python3 + build-base cmake git ninja python3 # Build shaka-packager from the current directory, rather than what has been # merged. WORKDIR shaka-packager COPY . /shaka-packager/ RUN mkdir build -RUN cmake -S . -B build -RUN make -C build +RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -G Ninja +RUN cmake --build build/ --config Debug --parallel # Copy only result binaries to our final image. FROM alpine:3.12 diff --git a/packager/policies.cmake b/packager/policies.cmake index c6ba4451b3..573c842967 100644 --- a/packager/policies.cmake +++ b/packager/policies.cmake @@ -29,3 +29,15 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") message(FATAL_ERROR "GCC version must be at least 9! (Found ${CMAKE_CXX_COMPILER_VERSION})") endif() endif() + +# If the environment variable PACKAGER_LOW_MEMORY_BUILD is defined, limit the +# number of parallel processes. This is used in our workflow to keep our arm64 +# builds from failing. There, we only have 4GB RAM to share among 6 CPUs. +# NOTE: This only affects CMake's Ninja generator. +if(DEFINED ENV{PACKAGER_LOW_MEMORY_BUILD}) + set_property(GLOBAL PROPERTY JOB_POOLS + compile_jobs=4 + link_jobs=1) + set(CMAKE_JOB_POOL_COMPILE compile_jobs) + set(CMAKE_JOB_POOL_LINK link_jobs) +endif() diff --git a/packager/testing/dockers/Alpine_Dockerfile b/packager/testing/dockers/Alpine_Dockerfile index 6a8f983dc7..e9ad169fe5 100644 --- a/packager/testing/dockers/Alpine_Dockerfile +++ b/packager/testing/dockers/Alpine_Dockerfile @@ -4,7 +4,7 @@ FROM alpine:3.12 RUN apk add --no-cache \ bash curl \ bsd-compat-headers c-ares-dev linux-headers \ - build-base cmake git python3 + build-base cmake git ninja python3 # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/ArchLinux_Dockerfile b/packager/testing/dockers/ArchLinux_Dockerfile index 72f1129022..91311b04a4 100644 --- a/packager/testing/dockers/ArchLinux_Dockerfile +++ b/packager/testing/dockers/ArchLinux_Dockerfile @@ -4,7 +4,7 @@ FROM archlinux:latest RUN pacman -Sy --needed --noconfirm \ core/which \ c-ares \ - cmake gcc git make python3 + cmake gcc git ninja python3 # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/CentOS_Dockerfile b/packager/testing/dockers/CentOS_Dockerfile index 0f56e5e79c..dacb1ffa25 100644 --- a/packager/testing/dockers/CentOS_Dockerfile +++ b/packager/testing/dockers/CentOS_Dockerfile @@ -1,10 +1,16 @@ FROM tgagor/centos:stream9 +# For CentOS, Ninja is only available from the "CRB" ("Code Ready Builder") +# repo. Enable that first. +RUN dnf update +RUN dnf install -y yum-utils +RUN dnf config-manager --set-enabled crb + # Install utilities, libraries, and dev tools. RUN yum install -y \ which \ c-ares-devel libatomic \ - cmake gcc-c++ git python3 + cmake gcc-c++ git ninja-build python3 # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/Debian_Dockerfile b/packager/testing/dockers/Debian_Dockerfile index 3c3b38a108..acf552ccf9 100644 --- a/packager/testing/dockers/Debian_Dockerfile +++ b/packager/testing/dockers/Debian_Dockerfile @@ -5,7 +5,7 @@ RUN apt-get update && apt-get install -y apt-utils RUN apt-get install -y \ curl \ libc-ares-dev \ - build-essential cmake git python3 + build-essential cmake git ninja-build python3 # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/Fedora_Dockerfile b/packager/testing/dockers/Fedora_Dockerfile index 12e6be8816..a7b8956555 100644 --- a/packager/testing/dockers/Fedora_Dockerfile +++ b/packager/testing/dockers/Fedora_Dockerfile @@ -4,7 +4,7 @@ FROM fedora:34 RUN yum install -y \ which \ c-ares-devel libatomic \ - cmake gcc-c++ git python3 + cmake gcc-c++ git ninja-build python3 # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/OpenSUSE_Dockerfile b/packager/testing/dockers/OpenSUSE_Dockerfile index 8d4aa12145..efdedc3783 100644 --- a/packager/testing/dockers/OpenSUSE_Dockerfile +++ b/packager/testing/dockers/OpenSUSE_Dockerfile @@ -4,7 +4,7 @@ FROM opensuse/leap:15.5 RUN zypper in -y \ curl which \ c-ares-devel \ - cmake gcc9-c++ git python3 + cmake gcc9-c++ git ninja python3 # OpenSuse 15 doesn't have the required gcc 9+ by default, but we can install # it as gcc9 and symlink it. diff --git a/packager/testing/dockers/Ubuntu_Dockerfile b/packager/testing/dockers/Ubuntu_Dockerfile index f6335c0395..469add4f7a 100644 --- a/packager/testing/dockers/Ubuntu_Dockerfile +++ b/packager/testing/dockers/Ubuntu_Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y apt-utils RUN apt-get install -y \ curl \ libc-ares-dev \ - build-essential cmake git python3 + build-essential cmake git ninja-build python3 # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/test_dockers.sh b/packager/testing/test_dockers.sh index 3a02a87e2d..69574433eb 100755 --- a/packager/testing/test_dockers.sh +++ b/packager/testing/test_dockers.sh @@ -85,8 +85,8 @@ for DOCKER_FILE in ${SCRIPT_DIR}/dockers/*; do RAN_SOMETHING=1 docker build --pull -t ${CONTAINER} -f ${DOCKER_FILE} ${SCRIPT_DIR}/dockers/ mkdir -p "${TEMP_BUILD_DIR}" - docker_run cmake -S . -B build/ -DCMAKE_BUILD_TYPE=Debug - docker_run cmake --build build/ --config Debug + docker_run cmake -S . -B build/ -DCMAKE_BUILD_TYPE=Debug -G Ninja + docker_run cmake --build build/ --config Debug --parallel docker_run bash -c "cd build && ctest -C Debug -V" rm -rf "${TEMP_BUILD_DIR}" done