From 01e7f3bd194d15c40d16b395e5bbdf7fc4ee4ba0 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 30 Nov 2023 13:17:01 -0800 Subject: [PATCH] ci: Refactor release workflows (#1309) This adopts release-please to manage releases and changelogs, similar to other shaka-project repos. All release workflows can be run by forks by configuring repo secrets. See docs in .github/workflows/ for details. - Use release-please for releases, changelogs - Convert publication jobs (docs, docker, npm) into reusable workflows - Update workflow documentation - Modernize docker commands - Fix doc permissions for publication - Update artifact handling in build workflow - Fix paths in Dockerfile - Fix paths and arm64 support in NPM package - Fix install paths for PSSH tools - Fix warnings in NPM & Docker actions - Delete custom changelog tooling --- .github/workflows/README.md | 44 ++++-- .../{docker-image.yaml => build-docker.yaml} | 4 +- .github/workflows/build-docs.yaml | 9 ++ .github/workflows/build.yaml | 119 +++++++------- .github/workflows/docker-hub-release.yaml | 52 ------- .github/workflows/github-release.yaml | 133 ---------------- .github/workflows/lint.yaml | 4 +- .github/workflows/npm-release.yaml | 60 ------- .github/workflows/pr.yaml | 17 +- .github/workflows/publish-docker.yaml | 68 ++++++++ .github/workflows/publish-docs.yaml | 51 ++++++ .github/workflows/publish-npm.yaml | 100 ++++++++++++ .github/workflows/release-please.yaml | 146 ++++++++++++++++++ .github/workflows/update-docs.yaml | 56 ------- CHANGELOG.md | 3 + Dockerfile | 15 +- npm/index.js | 26 +++- npm/package.json | 1 + npm/prepublish.js | 39 +++-- packager/CMakeLists.txt | 8 +- packager/testing/test_dockers.sh | 2 +- packager/tools/extract_from_changelog.py | 50 ------ packager/tools/pssh/CMakeLists.txt | 14 +- 23 files changed, 541 insertions(+), 480 deletions(-) rename .github/workflows/{docker-image.yaml => build-docker.yaml} (89%) delete mode 100644 .github/workflows/docker-hub-release.yaml delete mode 100644 .github/workflows/github-release.yaml delete mode 100644 .github/workflows/npm-release.yaml create mode 100644 .github/workflows/publish-docker.yaml create mode 100644 .github/workflows/publish-docs.yaml create mode 100644 .github/workflows/publish-npm.yaml create mode 100644 .github/workflows/release-please.yaml delete mode 100644 .github/workflows/update-docs.yaml delete mode 100755 packager/tools/extract_from_changelog.py diff --git a/.github/workflows/README.md b/.github/workflows/README.md index dc30ebc0d6..acc2296fe6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -8,12 +8,21 @@ - `build-docs.yaml`: Build Packager docs. Runs only on Linux. - - `docker-image.yaml`: + - `build-docker.yaml`: Build the official Docker image. - `lint.yaml`: Lint Shaka Packager. + - `publish-docs.yaml`: + Publish Packager docs. Runs on the latest release. + + - `publish-docker.yaml`: + Publish the official docker image. Runs on all releases. + + - `publish-npm.yaml`: + Publish binaries to NPM. Runs on all releases. + - `test-linux-distros.yaml`: Test the build on all Linux distros via docker. @@ -22,23 +31,23 @@ - `lint.yaml` - `build.yaml` - `build-docs.yaml` - - `docker-image.yaml` + - `build-docker.yaml` - `test-linux-distros.yaml` - - On release tag (`github-release.yaml`): - - Create a draft release - - Invoke: - - `lint.yaml` - - `build.yaml` - - `test-linux-distros.yaml` - - Publish the release with binaries from `build.yaml` attached - - - On release published: - - `docker-hub-release.yaml`, publishes the official Docker image - - `npm-release.yaml`, publishes the official NPM package - - `update-docs.yaml`: - - Invoke `build-docs.yaml` - - Push the output to the `gh-pages` branch +## Release workflow + - `release-please.yaml` + - Updates changelogs, version numbers based on conventional commits syntax + and semantic versioning + - https://conventionalcommits.org/ + - https://semver.org/ + - Generates/updates a PR on each push + - When the PR is merged, runs additional steps: + - Creates a GitHub release + - Invokes `publish-docs.yaml` to publish the docs + - Invokes `publish-docker.yaml` to publish the docker image + - Invokes `build.yaml` + - Attaches the binaries from `build.yaml` to the GitHub release + - Invokes `publish-npm.yaml` to publish the binaries to NPM ## Common workflows from shaka-project - `sync-labels.yaml` @@ -46,6 +55,9 @@ - `validate-pr-title.yaml` ## Required Repo Secrets + - `RELEASE_PLEASE_TOKEN`: A PAT for `shaka-bot` to run the `release-please` + action. If missing, the release workflow will use the default + `GITHUB_TOKEN` - `DOCKERHUB_CI_USERNAME`: The username of the Docker Hub CI account - `DOCKERHUB_CI_TOKEN`: An access token for Docker Hub - To generate, visit https://hub.docker.com/settings/security diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/build-docker.yaml similarity index 89% rename from .github/workflows/docker-image.yaml rename to .github/workflows/build-docker.yaml index a48c940da8..f70cccb113 100644 --- a/.github/workflows/docker-image.yaml +++ b/.github/workflows/build-docker.yaml @@ -29,9 +29,9 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - ref: ${{ github.event.inputs.ref || github.ref }} + ref: ${{ inputs.ref }} submodules: recursive - name: Build shell: bash - run: docker build . + run: docker buildx build . diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml index 76d3e9862d..33d4d02406 100644 --- a/.github/workflows/build-docs.yaml +++ b/.github/workflows/build-docs.yaml @@ -61,6 +61,15 @@ jobs: cp -a build/doxygen/html gh-pages/docs cp docs/index.html gh-pages/index.html + # Now set permissions on the generated docs. + # https://github.com/actions/upload-pages-artifact#file-permissions + chmod -R +rX gh-pages/ + + - name: Upload docs artifacts + uses: actions/upload-pages-artifact@v2 + with: + path: gh-pages + - name: Debug uses: mxschmitt/action-tmate@v3.6 with: diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f069a97479..50d452258a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -26,12 +26,6 @@ on: type: boolean default: false - secrets: - # The GITHUB_TOKEN name is reserved, but not passed through implicitly. - # So we call our secret parameter simply TOKEN. - TOKEN: - required: false - # By default, run all commands in a bash shell. On Windows, the default would # otherwise be powershell. defaults: @@ -114,7 +108,7 @@ jobs: - name: Install Mac deps if: runner.os == 'macOS' - # NOTE: GitHub Action VMs do not install ninja by default. + # NOTE: GitHub Actions VMs on Mac do not install ninja by default. run: | brew install ninja @@ -162,66 +156,59 @@ jobs: with: report_paths: 'junit-reports/TEST-*.xml' - # TODO(joeyparrish): Prepare artifacts when build system is complete again -# - name: Prepare artifacts (static release only) -# run: | -# BUILD_CONFIG="${{ matrix.build_type }}-${{ matrix.lib_type }}" -# if [[ "$BUILD_CONFIG" != "Release-static" ]]; then -# echo "Skipping artifacts for $BUILD_CONFIG." -# exit 0 -# fi -# if [[ "${{ runner.os }}" == "Linux" ]]; then -# echo "::group::Check for static executables" -# ( -# cd build/Release -# # Prove that we built static executables on Linux. First, check that -# # the executables exist, and fail if they do not. Then check "ldd", -# # which will fail if the executable is not dynamically linked. If -# # "ldd" succeeds, we fail the workflow. Finally, we call "true" so -# # that the last executed statement will be a success, and the step -# # won't be failed if we get that far. -# ls packager mpd_generator >/dev/null || exit 1 -# ldd packager 2>&1 && exit 1 -# ldd mpd_generator 2>&1 && exit 1 -# true -# ) -# echo "::endgroup::" -# fi -# echo "::group::Prepare artifacts folder" -# mkdir artifacts -# ARTIFACTS="$GITHUB_WORKSPACE/artifacts" -# cd build/Release -# echo "::endgroup::" -# echo "::group::Strip executables" -# strip packager${{ matrix.exe_ext }} -# strip mpd_generator${{ matrix.exe_ext }} -# echo "::endgroup::" -# SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}" -# EXE_SUFFIX="$SUFFIX${{ matrix.exe_ext}}" -# echo "::group::Copy packager" -# cp packager${{ matrix.exe_ext }} $ARTIFACTS/packager$EXE_SUFFIX -# echo "::endgroup::" -# echo "::group::Copy mpd_generator" -# cp mpd_generator${{ matrix.exe_ext }} $ARTIFACTS/mpd_generator$EXE_SUFFIX -# echo "::endgroup::" -# # The pssh-box bundle is OS and architecture independent. So only do -# # it on this one OS and architecture, and give it a more generic -# # filename. -# if [[ '${{ matrix.os_name }}' == 'linux' && '${{ matrix.target_arch }}' == 'x64' ]]; then -# echo "::group::Tar pssh-box" -# tar -czf $ARTIFACTS/pssh-box.py.tar.gz pyproto pssh-box.py -# echo "::endgroup::" -# fi + - name: Prepare artifacts (static release only) + run: | + BUILD_CONFIG="${{ matrix.build_type }}-${{ matrix.lib_type }}" + if [[ "$BUILD_CONFIG" != "Release-static" ]]; then + echo "Skipping artifacts for $BUILD_CONFIG." + exit 0 + fi - # TODO(joeyparrish): Attach artifacts when build system is complete again -# - name: Attach artifacts to release -# if: matrix.build_type == 'Release' && matrix.lib_type == 'static' -# uses: dwenegar/upload-release-assets@v1 -# env: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# with: -# release_id: ${{ needs.draft_release.outputs.release_id }} -# assets_path: artifacts + # TODO: Check static executables? + + echo "::group::Prepare artifacts folder" + mkdir artifacts + ARTIFACTS="$GITHUB_WORKSPACE/artifacts" + if [[ "${{ runner.os }}" == "Windows" ]]; then + cd build/packager/Release + else + cd build/packager + fi + echo "::endgroup::" + + echo "::group::Strip executables" + strip packager${{ matrix.exe_ext }} + strip mpd_generator${{ matrix.exe_ext }} + echo "::endgroup::" + + SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}" + EXE_SUFFIX="$SUFFIX${{ matrix.exe_ext}}" + + echo "::group::Copy packager" + cp packager${{ matrix.exe_ext }} $ARTIFACTS/packager$EXE_SUFFIX + echo "::endgroup::" + + echo "::group::Copy mpd_generator" + cp mpd_generator${{ matrix.exe_ext }} $ARTIFACTS/mpd_generator$EXE_SUFFIX + echo "::endgroup::" + + # The pssh-box bundle is OS and architecture independent. So only do + # it on this one OS and architecture, and give it a more generic + # filename. + if [[ '${{ matrix.os_name }}' == 'linux' && '${{ matrix.target_arch }}' == 'x64' ]]; then + echo "::group::Tar pssh-box" + tar -czf $ARTIFACTS/pssh-box.py.tar.gz pssh-box.py pssh-box-protos + echo "::endgroup::" + fi + + - name: Upload static release build artifacts + uses: actions/upload-artifact@v3 + if: matrix.build_type == 'Release' && matrix.lib_type == 'static' + with: + name: artifacts-${{ matrix.os_name }}-${{ matrix.target_arch }} + path: artifacts/* + if-no-files-found: error + retention-days: 5 - name: Debug uses: mxschmitt/action-tmate@v3.6 diff --git a/.github/workflows/docker-hub-release.yaml b/.github/workflows/docker-hub-release.yaml deleted file mode 100644 index 4f42639502..0000000000 --- a/.github/workflows/docker-hub-release.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2022 Google LLC -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd - -name: Docker Hub Release - -# Runs when a new release is published on GitHub. -# Creates a corresponding Docker Hub release and publishes it. -# -# Can also be run manually for debugging purposes. -on: - release: - types: [published] - # For manual debugging: - workflow_dispatch: - inputs: - ref: - description: "The tag to release to Docker Hub." - required: True - -jobs: - publish_docker_hub: - name: Publish to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Compute ref - id: ref - # We could be building from a workflow dispatch (manual run), or a - # release event. Subsequent steps can refer to $TARGET_REF to - # determine the correct ref in all cases. - run: | - echo "TARGET_REF=${{ github.event.inputs.ref || github.event.release.tag_name }}" >> $GITHUB_ENV - - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ env.TARGET_REF }} - submodules: recursive - - - name: Log in to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_CI_USERNAME }} - password: ${{ secrets.DOCKERHUB_CI_TOKEN }} - - - name: Push to Docker Hub - uses: docker/build-push-action@v2 - with: - push: true - tags: ${{ secrets.DOCKERHUB_PACKAGE_NAME }}:latest,${{ secrets.DOCKERHUB_PACKAGE_NAME }}:${{ env.TARGET_REF }} diff --git a/.github/workflows/github-release.yaml b/.github/workflows/github-release.yaml deleted file mode 100644 index 040b0e874b..0000000000 --- a/.github/workflows/github-release.yaml +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2022 Google LLC -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd - -name: GitHub Release - -# Runs when a new tag is created that looks like a version number. -# -# 1. Creates a draft release on GitHub with the latest release notes -# 2. On all combinations of OS, build type, and library type: -# a. builds Packager -# b. builds the docs -# c. runs all tests -# d. attaches build artifacts to the release -# 3. Fully publishes the release on GitHub -# -# Publishing the release then triggers additional workflows for NPM, Docker -# Hub, and GitHub Pages. -# -# Can also be run manually for debugging purposes. -on: - push: - tags: - - "v*.*" - # For manual debugging: - workflow_dispatch: - inputs: - tag: - description: "An existing tag to release." - required: True - -jobs: - # TODO(joeyparrish): Switch to release-please - setup: - name: Setup - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.compute_tag.outputs.tag }} - steps: - - name: Compute tag - id: compute_tag - # We could be building from a workflow dispatch (manual run) - # or from a pushed tag. If triggered from a pushed tag, we would like - # to strip refs/tags/ off of the incoming ref and just use the tag - # name. Subsequent jobs can refer to the "tag" output of this job to - # determine the correct tag name in all cases. - run: | - # Strip refs/tags/ from the input to get the tag name, then store - # that in output. - echo "::set-output name=tag::${{ github.event.inputs.tag || github.ref }}" \ - | sed -e 's@refs/tags/@@' - - # TODO(joeyparrish): Switch to release-please - draft_release: - name: Create GitHub release - needs: setup - runs-on: ubuntu-latest - outputs: - release_id: ${{ steps.draft_release.outputs.id }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ needs.setup.outputs.tag }} - - - name: Check changelog version - # This check prevents releases without appropriate changelog updates. - run: | - VERSION=$(packager/tools/extract_from_changelog.py --version) - if [[ "$VERSION" != "${{ needs.setup.outputs.tag }}" ]]; then - echo "" - echo "" - echo "***** ***** *****" - echo "" - echo "Version mismatch!" - echo "Workflow is targetting ${{ needs.setup.outputs.tag }}," - echo "but CHANGELOG.md contains $VERSION!" - exit 1 - fi - - - name: Extract release notes - run: | - packager/tools/extract_from_changelog.py --release_notes \ - | tee ../RELEASE_NOTES.md - - - name: Draft release - id: draft_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ needs.setup.outputs.tag }} - release_name: ${{ needs.setup.outputs.tag }} - body_path: RELEASE_NOTES.md - draft: true - - lint: - needs: setup - name: Lint - uses: ./.github/workflows/lint.yaml - with: - ref: ${{ needs.setup.outputs.tag }} - - build_and_test: - needs: [setup, lint, draft_release] - name: Build and test - uses: ./.github/workflows/build.yaml - with: - ref: ${{ needs.setup.outputs.tag }} - - test_supported_linux_distros: - # Doesn't really "need" it, but let's not waste time on a series of docker - # builds just to cancel it because of a linter error. - needs: lint - name: Test Linux distros - uses: ./.github/workflows/test-linux-distros.yaml - with: - ref: ${{ needs.setup.outputs.tag }} - - # TODO(joeyparrish): Switch to release-please - publish_release: - name: Publish GitHub release - needs: [draft_release, build_and_test] - runs-on: ubuntu-latest - steps: - - name: Publish release - uses: eregon/publish-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - release_id: ${{ needs.draft_release.outputs.release_id }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 81b9b1dd2d..e8d1fe9712 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -45,8 +45,8 @@ jobs: # NOTE: Must use base.sha instead of base.ref, since we don't have # access to the branch name that base.ref would give us. - # NOTE: Must also use fetch-depth: 2 in actions/checkout to have access - # to the base ref for comparison. + # NOTE: Must also use fetch-depth: 2 in actions/checkout to have + # access to the base ref for comparison. packager/tools/git/check_formatting.py \ --binary /usr/bin/clang-format \ ${{ github.event.pull_request.base.sha || 'HEAD^' }} diff --git a/.github/workflows/npm-release.yaml b/.github/workflows/npm-release.yaml deleted file mode 100644 index 70e467adc6..0000000000 --- a/.github/workflows/npm-release.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2022 Google LLC -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd - -# A workflow to release Shaka Packager to NPM. -name: NPM Release - -# Runs when a new release is published on GitHub. -# Creates a corresponding NPM release and publishes it. -# -# Can also be run manually for debugging purposes. -on: - release: - types: [published] - # For manual debugging: - workflow_dispatch: - inputs: - ref: - description: "The tag to release to NPM." - required: True - -jobs: - publish_npm: - name: Publish to NPM - runs-on: ubuntu-latest - steps: - - name: Compute ref - id: ref - # We could be building from a workflow dispatch (manual run), or a - # release event. Subsequent steps can refer to $TARGET_REF to - # determine the correct ref in all cases. - run: | - echo "TARGET_REF=${{ github.event.inputs.ref || github.event.release.tag_name }}" >> $GITHUB_ENV - - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ env.TARGET_REF }} - - - name: Setup NodeJS - uses: actions/setup-node@v1 - with: - node-version: 10 - - - name: Set package name and version - run: | - cd npm - sed package.json -i \ - -e 's/"name": ""/"name": "${{ secrets.NPM_PACKAGE_NAME }}"/' \ - -e 's/"version": ""/"version": "${{ env.TARGET_REF }}"/' - - - name: Publish NPM package - uses: JS-DevTools/npm-publish@v1 - with: - token: ${{ secrets.NPM_CI_TOKEN }} - package: npm/package.json - check-version: false - access: public diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0128c2376f..0c991ad87f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,13 +19,14 @@ on: inputs: ref: description: "The ref to build and test." - required: False + required: false + type: string # If another instance of this workflow is started for the same PR, cancel the # old one. If a PR is updated and a new test run is started, the old test run # will be cancelled automatically to conserve resources. concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.ref || github.ref }} + group: ${{ github.workflow }}-${{ inputs.ref || github.ref }} cancel-in-progress: true jobs: @@ -37,14 +38,14 @@ jobs: name: Lint uses: ./.github/workflows/lint.yaml with: - ref: ${{ github.event.inputs.ref || github.ref }} + ref: ${{ inputs.ref || github.ref }} build_and_test: needs: [lint, settings] name: Build and test uses: ./.github/workflows/build.yaml with: - ref: ${{ github.event.inputs.ref || github.ref }} + ref: ${{ inputs.ref || github.ref }} self_hosted: ${{ needs.settings.outputs.self_hosted != '' }} debug: ${{ needs.settings.outputs.debug != '' }} @@ -53,15 +54,15 @@ jobs: name: Build docs uses: ./.github/workflows/build-docs.yaml with: - ref: ${{ github.event.inputs.ref || github.ref }} + ref: ${{ inputs.ref || github.ref }} debug: ${{ needs.settings.outputs.debug != '' }} official_docker_image: needs: lint name: Official Docker image - uses: ./.github/workflows/docker-image.yaml + uses: ./.github/workflows/build-docker.yaml with: - ref: ${{ github.event.inputs.ref || github.ref }} + ref: ${{ inputs.ref || github.ref }} test_supported_linux_distros: # Doesn't really "need" it, but let's not waste time on a series of docker @@ -70,4 +71,4 @@ jobs: name: Test Linux distros uses: ./.github/workflows/test-linux-distros.yaml with: - ref: ${{ github.event.inputs.ref || github.ref }} + ref: ${{ inputs.ref || github.ref }} diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml new file mode 100644 index 0000000000..8d9c816f39 --- /dev/null +++ b/.github/workflows/publish-docker.yaml @@ -0,0 +1,68 @@ +# Copyright 2022 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# A workflow to publish the official docker image. +name: Publish to Docker Hub + +# Runs when called from another workflow. +# Can also be run manually for debugging purposes. +on: + workflow_call: + inputs: + tag: + required: true + type: string + latest: + required: true + type: boolean + secrets: + DOCKERHUB_CI_USERNAME: + required: true + DOCKERHUB_CI_TOKEN: + required: true + DOCKERHUB_PACKAGE_NAME: + required: true + # For manual debugging: + workflow_dispatch: + inputs: + tag: + description: The tag to build from and to push to. + required: true + type: string + latest: + description: If true, push to the "latest" tag. + required: true + type: boolean + +jobs: + publish_docker_hub: + name: Publish to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ inputs.tag }} + submodules: recursive + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_CI_USERNAME }} + password: ${{ secrets.DOCKERHUB_CI_TOKEN }} + + - name: Push to Docker Hub + uses: docker/build-push-action@v5 + with: + push: true + tags: ${{ secrets.DOCKERHUB_PACKAGE_NAME }}:${{ inputs.tag }} + + - name: Push to Docker Hub as "latest" + if: ${{ inputs.latest }} + uses: docker/build-push-action@v5 + with: + push: true + tags: ${{ secrets.DOCKERHUB_PACKAGE_NAME }}:latest diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml new file mode 100644 index 0000000000..7f1b7d8768 --- /dev/null +++ b/.github/workflows/publish-docs.yaml @@ -0,0 +1,51 @@ +# Copyright 2022 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# A workflow to publish the docs to GitHub Pages. +name: Publish Docs + +# Runs when called from another workflow. +# Can also be run manually for debugging purposes. +on: + workflow_call: + inputs: + ref: + required: true + type: string + # For manual debugging: + workflow_dispatch: + inputs: + ref: + description: "The ref to build docs from." + required: true + type: string + +jobs: + build_docs: + name: Build docs + uses: ./.github/workflows/build-docs.yaml + with: + ref: ${{ inputs.ref }} + + publish_docs: + name: Publish updated docs + needs: build_docs + runs-on: ubuntu-latest + + # Grant GITHUB_TOKEN the permissions required to deploy to Pages + permissions: + pages: write + id-token: write + + # Deploy to the github-pages environment + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 diff --git a/.github/workflows/publish-npm.yaml b/.github/workflows/publish-npm.yaml new file mode 100644 index 0000000000..97c7c54677 --- /dev/null +++ b/.github/workflows/publish-npm.yaml @@ -0,0 +1,100 @@ +# Copyright 2022 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# A workflow to publish the official NPM package. +name: Publish to NPM + +# Runs when called from another workflow. +# Can also be run manually for debugging purposes. +on: + workflow_call: + inputs: + tag: + required: true + type: string + latest: + required: true + type: boolean + secrets: + NPM_CI_TOKEN: + required: true + NPM_PACKAGE_NAME: + required: true + # For manual debugging: + workflow_dispatch: + inputs: + tag: + description: The tag to build from. + required: true + type: string + latest: + description: If true, push to the "latest" tag. + required: true + type: boolean + +jobs: + publish: + name: Publish + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ inputs.tag }} + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 16 + registry-url: 'https://registry.npmjs.org' + + - name: Compute tags + run: | + # NPM publish always sets a tag. If you don't provide an explicit + # tag, you get the "latest" tag by default, but we want "latest" to + # always point to the highest version number. So we set an explicit + # tag on every "publish" command, then follow up with a command to + # set "latest" only if this release was the highest version yet. + + # The explicit tag is based on the branch. If the git tag is v4.4.1, + # the branch was v4.4.x, and the explicit NPM tag should be + # v4.4-latest. + GIT_TAG_NAME=${{ inputs.tag }} + NPM_TAG=$(echo "$GIT_TAG_NAME" | cut -f 1-2 -d .)-latest + echo "NPM_TAG=$NPM_TAG" >> $GITHUB_ENV + + # Since we also set the package version on-the-fly during publication, + # compute that here. It's the tag without the leading "v". + NPM_VERSION=$(echo "$GIT_TAG_NAME" | sed -e 's/^v//') + echo "NPM_VERSION=$NPM_VERSION" >> $GITHUB_ENV + + # Debug the decisions made here. + echo "This release: $GIT_TAG_NAME" + echo "NPM tag: $NPM_TAG" + echo "NPM version: $NPM_VERSION" + + - name: Set package name and version + run: | + # These substitutions use | because the package name could contain + # both / and @, but | should not appear in package names. + sed npm/package.json -i \ + -e 's|"name": ""|"name": "${{ secrets.NPM_PACKAGE_NAME }}"|' \ + -e 's|"version": ""|"version": "${{ env.NPM_VERSION }}"|' + + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_CI_TOKEN }} + run: | + cd npm + + # Publish with an explicit tag. + # Also publish with explicit public access, to allow scoped packages. + npm publish --tag "$NPM_TAG" --access=public + + # Add the "latest" tag if needed. + if [[ "${{ inputs.latest }}" == "true" ]]; then + npm dist-tag add "${{ secrets.NPM_PACKAGE_NAME }}@$NPM_VERSION" latest + fi diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml new file mode 100644 index 0000000000..ff7338031f --- /dev/null +++ b/.github/workflows/release-please.yaml @@ -0,0 +1,146 @@ +# Copyright 2023 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +name: Release + +on: + push: + branches: + - main + - v[0-9]* + +jobs: + release: + runs-on: ubuntu-latest + outputs: + release_created: ${{ steps.release.outputs.release_created }} + tag_name: ${{ steps.release.outputs.tag_name }} + steps: + # Create/update release PR + - uses: google-github-actions/release-please-action@v3 + id: release + with: + # Required input to specify the release type. This is not really a + # go project, but go projects in release-please only update + # CHANGELOG.md and nothing else. This is what we want. + release-type: go + # Make sure we create the PR against the correct branch. + default-branch: ${{ github.ref_name }} + # Use a special shaka-bot access token for releases. + token: ${{ secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }} + # Temporary settings to bootstrap v3.0.0. + last-release-sha: 634af6591ce8c701587a78042ae7f81761725710 + bootstrap-sha: 634af6591ce8c701587a78042ae7f81761725710 + + # The jobs below are all conditional on a release having been created by + # someone merging the release PR. + + # Several actions either only run on the latest release or run with different + # options on the latest release. Here we compute if this is the highest + # version number (what we are calling "latest" for NPM, Docker, and the + # docs). You can have a more recent release from an older branch, but this + # would not qualify as "latest" here. + compute: + name: Compute latest release flag + runs-on: ubuntu-latest + needs: release + if: needs.release.outputs.release_created + outputs: + latest: ${{ steps.compute.outputs.latest }} + steps: + - uses: actions/checkout@v3 + with: + fetch-tags: true + persist-credentials: false + + - name: Compute latest + id: compute + run: | + GIT_TAG_NAME=${{ needs.release.outputs.tag_name }} + RELEASE_TAGS=$(git tag | grep ^v[0-9]) + LATEST_RELEASE=$(echo "$RELEASE_TAGS" | sort --version-sort | tail -1) + if [[ "$GIT_TAG_NAME" == "$LATEST_RELEASE" ]]; then + LATEST=true + else + LATEST=false + fi + echo latest=$LATEST >> $GITHUB_OUTPUT + + # Debug the decisions made here. + echo "This release: $GIT_TAG_NAME" + echo "Latest release: $LATEST_RELEASE" + echo "This release is latest: $LATEST" + + # Publish docs to GitHub Pages + docs: + name: Update docs + needs: [release, compute] + # Only if this is the latest release + if: needs.release.outputs.release_created && needs.compute.outputs.latest + uses: ./.github/workflows/publish-docs.yaml + with: + ref: ${{ github.ref }} + + # Publish official docker image + docker: + name: Update docker image + needs: [release, compute] + if: needs.release.outputs.release_created + uses: ./.github/workflows/publish-docker.yaml + with: + tag: ${{ needs.release.outputs.tag_name }} + latest: ${{ needs.compute.outputs.latest == 'true' }} + secrets: + DOCKERHUB_CI_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_CI_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + DOCKERHUB_PACKAGE_NAME: ${{ secrets.DOCKERHUB_PACKAGE_NAME }} + + # Do a complete build + build: + name: Build + needs: release + if: needs.release.outputs.release_created + uses: ./.github/workflows/build.yaml + with: + ref: ${{ github.ref }} + + # Attach build artifacts to the release + artifacts: + name: Artifacts + runs-on: ubuntu-latest + needs: [release, build] + if: needs.release.outputs.release_created + steps: + - uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Debug + run: find -ls + + - name: Attach packager to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ needs.release.outputs.tag_name }} + make_latest: false # Already set for the release + file_glob: true + file: artifacts/artifacts*/* + overwrite: true + + # Surprisingly, Shaka Packager binaries can be installed via npm. + # Publish NPM package updates. + npm: + name: Update NPM + needs: [release, compute, artifacts] + if: needs.release.outputs.release_created + uses: ./.github/workflows/publish-npm.yaml + with: + tag: ${{ needs.release.outputs.tag_name }} + latest: ${{ needs.compute.outputs.latest == 'true' }} + secrets: + NPM_CI_TOKEN: ${{ secrets.NPM_CI_TOKEN }} + NPM_PACKAGE_NAME: ${{ secrets.NPM_PACKAGE_NAME }} diff --git a/.github/workflows/update-docs.yaml b/.github/workflows/update-docs.yaml deleted file mode 100644 index 9cf5d8782b..0000000000 --- a/.github/workflows/update-docs.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2022 Google LLC -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd - -# A workflow to update the public docs. -name: Update Docs - -# Runs when a new release is published on GitHub. -# -# Pushes updated docs to GitHub Pages if triggered from a release workflow. -# -# Can also be run manually for debugging purposes. -on: - release: - types: [published] - # For manual debugging: - workflow_dispatch: - inputs: - ref: - description: "The ref to build docs from." - required: True - -jobs: - publish_docs: - name: Build updated docs - runs-on: ubuntu-latest - steps: - - name: Compute ref - id: ref - # We could be building from a workflow dispatch (manual run) or from a - # release event. Subsequent steps can refer to the "ref" output of - # this job to determine the correct ref in all cases. - run: | - echo "::set-output name=ref::${{ github.event.inputs.ref || github.event.release.tag_name }}" - - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Build docs - uses: ./.github/workflows/custom-actions/build-docs - - - name: Deploy to gh-pages branch - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: gh-pages - full_commit_message: Generate docs for ${{ steps.ref.outputs.ref }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 7af92af1d1..2a630bf5f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changelog + + ## [2.6.1] - 2021-10-14 ### Fixed - Fix crash in static-linked linux builds (#996) diff --git a/Dockerfile b/Dockerfile index 2b820f7eb7..c65c021689 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,21 +10,20 @@ RUN apk add --no-cache \ # merged. WORKDIR shaka-packager COPY . /shaka-packager/ -RUN mkdir build +RUN rm -rf build RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -G Ninja RUN cmake --build build/ --config Debug --parallel # Copy only result binaries to our final image. FROM alpine:3.12 RUN apk add --no-cache libstdc++ python3 -# TODO(joeyparrish): Copy binaries when build system is complete -#COPY --from=builder /shaka-packager/build/packager \ -# /shaka-packager/build/mpd_generator \ -# /shaka-packager/build/pssh-box.py \ -# /usr/bin/ +COPY --from=builder /shaka-packager/build/packager/packager \ + /shaka-packager/build/packager/mpd_generator \ + /shaka-packager/build/packager/pssh-box.py \ + /usr/bin/ # Copy pyproto directory, which is needed by pssh-box.py script. This line # cannot be combined with the line above as Docker's copy command skips the # directory itself. See https://github.com/moby/moby/issues/15858 for details. -# TODO(joeyparrish): Copy binaries when build system is complete -#COPY --from=builder /shaka-packager/build/pyproto /usr/bin/pyproto +COPY --from=builder /shaka-packager/build/packager/pssh-box-protos \ + /usr/bin/pssh-box-protos diff --git a/npm/index.js b/npm/index.js index 97ba60547e..549a8c24a0 100755 --- a/npm/index.js +++ b/npm/index.js @@ -4,15 +4,31 @@ var path = require('path'); var spawnSync = require('child_process').spawnSync; -// Command names per-platform: +// Command names per-platform (process.platform) and per-architecture +// (process.arch): var commandNames = { - linux: 'packager-linux', - darwin: 'packager-osx', - win32: 'packager-win.exe', + linux: { + 'x64': 'packager-linux-x64', + 'arm64': 'packager-linux-arm64', + }, + darwin: { + 'x64': 'packager-osx-x64', + }, + win32: { + 'x64': 'packager-win-x64.exe', + }, }; // Find the platform-specific binary: -var binaryPath = path.resolve(__dirname, 'bin', commandNames[process.platform]); +if (!(process.platform in commandNames)) { + throw new Error('Platform not supported: ' + process.platform); +} +if (!(process.arch in commandNames[process.platform])) { + throw new Error( + 'Architecture not supported: ' + process.platform + '/' + process.arch); +} +var commandName = commandNames[process.platform][process.arch]; +var binaryPath = path.resolve(__dirname, 'bin', commandName); // Find the args to pass to that binary: // argv[0] is node itself, and argv[1] is the script. diff --git a/npm/package.json b/npm/package.json index 15711fc160..4c27134356 100644 --- a/npm/package.json +++ b/npm/package.json @@ -2,6 +2,7 @@ "name": "", "description": "A media packaging tool and SDK.", "version": "", + "private": false, "homepage": "https://github.com/shaka-project/shaka-packager", "author": "Google", "maintainers": [ diff --git a/npm/prepublish.js b/npm/prepublish.js index dc758e4be0..57b1304984 100755 --- a/npm/prepublish.js +++ b/npm/prepublish.js @@ -5,11 +5,19 @@ var fs = require('fs'); var path = require('path'); var spawnSync = require('child_process').spawnSync; -// Command names per-platform: +// Command names per-platform (process.platform) and per-architecture +// (process.arch): var commandNames = { - linux: 'packager-linux', - darwin: 'packager-osx', - win32: 'packager-win.exe', + linux: { + 'x64': 'packager-linux-x64', + 'arm64': 'packager-linux-arm64', + }, + darwin: { + 'x64': 'packager-osx-x64', + }, + win32: { + 'x64': 'packager-win-x64.exe', + }, }; // Get the current package version: @@ -44,12 +52,23 @@ fs.readdirSync(binFolderPath).forEach(function(childName) { }); for (var platform in commandNames) { - // Find the destination for this binary: - var command = commandNames[platform]; - var binaryPath = path.resolve(binFolderPath, command); + for (var arch in commandNames[platform]) { + // Find the destination for this binary: + var command = commandNames[platform][arch]; + var binaryPath = path.resolve(binFolderPath, command); - download(urlBase + command, binaryPath); - fs.chmodSync(binaryPath, 0755); + try { + download(urlBase + command, binaryPath); + fs.chmodSync(binaryPath, 0755); + } catch (error) { + if (arch == 'arm64') { + // Optional. Forks may not have arm64 builds available. Ignore. + } else { + // Required. Re-throw and fail. + throw error; + } + } + } } // Fetch LICENSE and README files from the same tag, and include them in the @@ -83,6 +102,6 @@ function download(url, outputPath) { console.log('Downloading', url, 'to', outputPath); var returnValue = spawnSync('curl', args, options); if (returnValue.status != 0) { - process.exit(returnValue.status); + throw new Error('Download of ' + url + ' failed: ' + returnValue.status); } } diff --git a/packager/CMakeLists.txt b/packager/CMakeLists.txt index 4a06c1eebc..1381bb65e1 100644 --- a/packager/CMakeLists.txt +++ b/packager/CMakeLists.txt @@ -115,7 +115,7 @@ set(libpackager_deps add_library(libpackager_static STATIC ${libpackager_sources}) target_link_libraries(libpackager_static ${libpackager_deps}) -# And always installed as libpackager.a: +# And always installed as libpackager.a / libpackager.lib: if(NOT MSVC) set_property(TARGET libpackager_static PROPERTY OUTPUT_NAME packager) else() @@ -249,9 +249,9 @@ configure_file(packager.pc.in packager.pc @ONLY) install(TARGETS mpd_generator packager) # Always install the python tools. -install(PROGRAMS ${CMAKE_BINARY_DIR}/pssh-box.py +install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/pssh-box.py DESTINATION ${CMAKE_INSTALL_BINDIR}) -install(DIRECTORY ${CMAKE_BINARY_DIR}/pssh-box-protos +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/pssh-box-protos DESTINATION ${CMAKE_INSTALL_BINDIR}) # With shared libraries, also install the library, headers, and pkgconfig. @@ -261,6 +261,6 @@ if(BUILD_SHARED_LIBS) install(TARGETS libpackager_shared) install(DIRECTORY ../include/packager DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - install(FILES ${CMAKE_BINARY_DIR}/packager/packager.pc + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/packager.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) endif() diff --git a/packager/testing/test_dockers.sh b/packager/testing/test_dockers.sh index 69574433eb..8bb3d91c63 100755 --- a/packager/testing/test_dockers.sh +++ b/packager/testing/test_dockers.sh @@ -83,7 +83,7 @@ for DOCKER_FILE in ${SCRIPT_DIR}/dockers/*; do CONTAINER="$( echo "packager_test_${OS_NAME}" | tr A-Z a-z )" RAN_SOMETHING=1 - docker build --pull -t ${CONTAINER} -f ${DOCKER_FILE} ${SCRIPT_DIR}/dockers/ + docker buildx build --pull -t ${CONTAINER} -f ${DOCKER_FILE} ${SCRIPT_DIR}/dockers/ mkdir -p "${TEMP_BUILD_DIR}" docker_run cmake -S . -B build/ -DCMAKE_BUILD_TYPE=Debug -G Ninja docker_run cmake --build build/ --config Debug --parallel diff --git a/packager/tools/extract_from_changelog.py b/packager/tools/extract_from_changelog.py deleted file mode 100755 index 1ee6cec1d5..0000000000 --- a/packager/tools/extract_from_changelog.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright 2018 Google LLC. All rights reserved. -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd -"""This script extracts a version or release notes from the changelog.""" - -import argparse -import re -import sys - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--release_notes', action='store_true', - help='Print the latest release notes from the changelog') - parser.add_argument('--version', action='store_true', - help='Print the latest version from the changelog') - - args = parser.parse_args() - - with open('CHANGELOG.md', 'r', encoding='utf8') as f: - contents = f.read() - - # This excludes the header line with the release name and date, to match the - # style of releases done before the automation was introduced. - latest_entry = re.split(r'^(?=## \[)', contents, flags=re.M)[1] - - lines = latest_entry.strip().split('\n') - first_line = lines[0] - release_notes = '\n'.join(lines[1:]) - - match = re.match(r'^## \[(.*)\]', first_line) - if not match: - raise RuntimeError('Unable to parse first line of CHANGELOG.md!') - - version = match[1] - if not version.startswith('v'): - version = 'v' + version - - if args.version: - print(version) - if args.release_notes: - print(release_notes) - - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/packager/tools/pssh/CMakeLists.txt b/packager/tools/pssh/CMakeLists.txt index e2dd5885f7..8efa15f70d 100644 --- a/packager/tools/pssh/CMakeLists.txt +++ b/packager/tools/pssh/CMakeLists.txt @@ -7,10 +7,10 @@ # Copy the pssh-box.py script and its protos into the required directory # structure. set(PSSH_BOX_OUTPUTS - ${CMAKE_BINARY_DIR}/pssh-box.py - ${CMAKE_BINARY_DIR}/pssh-box-protos - ${CMAKE_BINARY_DIR}/pssh-box-protos/widevine_common_encryption_pb2.py - ${CMAKE_BINARY_DIR}/pssh-box-protos/widevine_pssh_data_pb2.py) + ${CMAKE_BINARY_DIR}/packager/pssh-box.py + ${CMAKE_BINARY_DIR}/packager/pssh-box-protos + ${CMAKE_BINARY_DIR}/packager/pssh-box-protos/widevine_common_encryption_pb2.py + ${CMAKE_BINARY_DIR}/packager/pssh-box-protos/widevine_pssh_data_pb2.py) add_custom_command( DEPENDS @@ -19,15 +19,15 @@ add_custom_command( OUTPUT ${PSSH_BOX_OUTPUTS} COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_BINARY_DIR}/pssh-box-protos/ + ${CMAKE_BINARY_DIR}/packager/pssh-box-protos/ COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/packager/media/base/widevine_common_encryption_pb2.py ${CMAKE_BINARY_DIR}/packager/media/base/widevine_pssh_data_pb2.py - ${CMAKE_BINARY_DIR}/pssh-box-protos/ + ${CMAKE_BINARY_DIR}/packager/pssh-box-protos/ COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pssh-box.py - ${CMAKE_BINARY_DIR}/) + ${CMAKE_BINARY_DIR}/packager/) add_custom_target(pssh_box_py ALL DEPENDS ${PSSH_BOX_OUTPUTS})