ci: Refactor release workflows (#1309)

This adopts release-please to manage releases and changelogs, similar to
other shaka-project repos.

All release workflows can be run by forks by configuring repo secrets.
See docs in .github/workflows/ for details.

 - Use release-please for releases, changelogs
 - Convert publication jobs (docs, docker, npm) into reusable workflows
 - Update workflow documentation
 - Modernize docker commands
 - Fix doc permissions for publication
 - Update artifact handling in build workflow
 - Fix paths in Dockerfile
 - Fix paths and arm64 support in NPM package
 - Fix install paths for PSSH tools
 - Fix warnings in NPM & Docker actions
 - Delete custom changelog tooling

.github/workflows/README.md

@@ -8,12 +8,21 @@
  - `build-docs.yaml`:
    Build Packager docs.  Runs only on Linux.
 
- - `docker-image.yaml`:
+ - `build-docker.yaml`:
    Build the official Docker image.
 
  - `lint.yaml`:
    Lint Shaka Packager.
 
+ - `publish-docs.yaml`:
+   Publish Packager docs.  Runs on the latest release.
+
+ - `publish-docker.yaml`:
+   Publish the official docker image.  Runs on all releases.
+
+ - `publish-npm.yaml`:
+   Publish binaries to NPM.  Runs on all releases.
+
  - `test-linux-distros.yaml`:
    Test the build on all Linux distros via docker.
 
@@ -22,23 +31,23 @@
    - `lint.yaml`
    - `build.yaml`
    - `build-docs.yaml`
-   - `docker-image.yaml`
+   - `build-docker.yaml`
    - `test-linux-distros.yaml`
 
- - On release tag (`github-release.yaml`):
+## Release workflow
-   - Create a draft release
+ - `release-please.yaml`
-   - Invoke:
+   - Updates changelogs, version numbers based on conventional commits syntax
-     - `lint.yaml`
+     and semantic versioning
-     - `build.yaml`
+     - https://conventionalcommits.org/
-     - `test-linux-distros.yaml`
+     - https://semver.org/
-   - Publish the release with binaries from `build.yaml` attached
+   - Generates/updates a PR on each push
-
+   - When the PR is merged, runs additional steps:
- - On release published:
+     - Creates a GitHub release
-   - `docker-hub-release.yaml`, publishes the official Docker image
+     - Invokes `publish-docs.yaml` to publish the docs
-   - `npm-release.yaml`, publishes the official NPM package
+     - Invokes `publish-docker.yaml` to publish the docker image
-   - `update-docs.yaml`:
+     - Invokes `build.yaml`
-     - Invoke `build-docs.yaml`
+     - Attaches the binaries from `build.yaml` to the GitHub release
-     - Push the output to the `gh-pages` branch
+     - Invokes `publish-npm.yaml` to publish the binaries to NPM
 
 ## Common workflows from shaka-project
  - `sync-labels.yaml`
@@ -46,6 +55,9 @@
  - `validate-pr-title.yaml`
 
 ## Required Repo Secrets
+ - `RELEASE_PLEASE_TOKEN`: A PAT for `shaka-bot` to run the `release-please`
+   action.  If missing, the release workflow will use the default
+   `GITHUB_TOKEN`
  - `DOCKERHUB_CI_USERNAME`: The username of the Docker Hub CI account
  - `DOCKERHUB_CI_TOKEN`: An access token for Docker Hub
    - To generate, visit https://hub.docker.com/settings/security

.github/workflows/build-docker.yaml

@@ -29,9 +29,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.inputs.ref || github.ref }}
+          ref: ${{ inputs.ref }}
           submodules: recursive
 
       - name: Build
         shell: bash
-        run: docker build .
+        run: docker buildx build .

.github/workflows/build-docs.yaml

@@ -61,6 +61,15 @@ jobs:
           cp -a build/doxygen/html gh-pages/docs
           cp docs/index.html gh-pages/index.html
 
+          # Now set permissions on the generated docs.
+          # https://github.com/actions/upload-pages-artifact#file-permissions
+          chmod -R +rX gh-pages/
+
+      - name: Upload docs artifacts
+        uses: actions/upload-pages-artifact@v2
+        with:
+          path: gh-pages
+
       - name: Debug
         uses: mxschmitt/action-tmate@v3.6
         with:

.github/workflows/build.yaml

@@ -26,12 +26,6 @@ on:
         type: boolean
         default: false
 
-    secrets:
-      # The GITHUB_TOKEN name is reserved, but not passed through implicitly.
-      # So we call our secret parameter simply TOKEN.
-      TOKEN:
-        required: false
-
 # By default, run all commands in a bash shell.  On Windows, the default would
 # otherwise be powershell.
 defaults:
@@ -114,7 +108,7 @@ jobs:
 
       - name: Install Mac deps
         if: runner.os == 'macOS'
-        # NOTE: GitHub Action VMs do not install ninja by default.
+        # NOTE: GitHub Actions VMs on Mac do not install ninja by default.
         run: |
           brew install ninja
 
@@ -162,66 +156,59 @@ jobs:
         with:
           report_paths: 'junit-reports/TEST-*.xml'
 
-      # TODO(joeyparrish): Prepare artifacts when build system is complete again
+      - name: Prepare artifacts (static release only)
-#      - name: Prepare artifacts (static release only)
+        run: |
-#        run: |
+          BUILD_CONFIG="${{ matrix.build_type }}-${{ matrix.lib_type }}"
-#          BUILD_CONFIG="${{ matrix.build_type }}-${{ matrix.lib_type }}"
+          if [[ "$BUILD_CONFIG" != "Release-static" ]]; then
-#          if [[ "$BUILD_CONFIG" != "Release-static" ]]; then
+            echo "Skipping artifacts for $BUILD_CONFIG."
-#            echo "Skipping artifacts for $BUILD_CONFIG."
+            exit 0
-#            exit 0
+          fi
-#          fi
-#          if [[ "${{ runner.os }}" == "Linux" ]]; then
-#            echo "::group::Check for static executables"
-#            (
-#              cd build/Release
-#              # Prove that we built static executables on Linux.  First, check that
-#              # the executables exist, and fail if they do not.  Then check "ldd",
-#              # which will fail if the executable is not dynamically linked.  If
-#              # "ldd" succeeds, we fail the workflow.  Finally, we call "true" so
-#              # that the last executed statement will be a success, and the step
-#              # won't be failed if we get that far.
-#              ls packager mpd_generator >/dev/null || exit 1
-#              ldd packager 2>&1 && exit 1
-#              ldd mpd_generator 2>&1 && exit 1
-#              true
-#            )
-#            echo "::endgroup::"
-#          fi
-#          echo "::group::Prepare artifacts folder"
-#          mkdir artifacts
-#          ARTIFACTS="$GITHUB_WORKSPACE/artifacts"
-#          cd build/Release
-#          echo "::endgroup::"
-#          echo "::group::Strip executables"
-#          strip packager${{ matrix.exe_ext }}
-#          strip mpd_generator${{ matrix.exe_ext }}
-#          echo "::endgroup::"
-#          SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}"
-#          EXE_SUFFIX="$SUFFIX${{ matrix.exe_ext}}"
-#          echo "::group::Copy packager"
-#          cp packager${{ matrix.exe_ext }} $ARTIFACTS/packager$EXE_SUFFIX
-#          echo "::endgroup::"
-#          echo "::group::Copy mpd_generator"
-#          cp mpd_generator${{ matrix.exe_ext }} $ARTIFACTS/mpd_generator$EXE_SUFFIX
-#          echo "::endgroup::"
-#          # The pssh-box bundle is OS and architecture independent.  So only do
-#          # it on this one OS and architecture, and give it a more generic
-#          # filename.
-#          if [[ '${{ matrix.os_name }}' == 'linux' && '${{ matrix.target_arch }}' == 'x64' ]]; then
-#            echo "::group::Tar pssh-box"
-#            tar -czf $ARTIFACTS/pssh-box.py.tar.gz pyproto pssh-box.py
-#            echo "::endgroup::"
-#          fi
 
-      # TODO(joeyparrish): Attach artifacts when build system is complete again
+          # TODO: Check static executables?
-#      - name: Attach artifacts to release
+
-#        if: matrix.build_type == 'Release' && matrix.lib_type == 'static'
+          echo "::group::Prepare artifacts folder"
-#        uses: dwenegar/upload-release-assets@v1
+          mkdir artifacts
-#        env:
+          ARTIFACTS="$GITHUB_WORKSPACE/artifacts"
-#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          if [[ "${{ runner.os }}" == "Windows" ]]; then
-#        with:
+            cd build/packager/Release
-#          release_id: ${{ needs.draft_release.outputs.release_id }}
+          else
-#          assets_path: artifacts
+            cd build/packager
+          fi
+          echo "::endgroup::"
+
+          echo "::group::Strip executables"
+          strip packager${{ matrix.exe_ext }}
+          strip mpd_generator${{ matrix.exe_ext }}
+          echo "::endgroup::"
+
+          SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}"
+          EXE_SUFFIX="$SUFFIX${{ matrix.exe_ext}}"
+
+          echo "::group::Copy packager"
+          cp packager${{ matrix.exe_ext }} $ARTIFACTS/packager$EXE_SUFFIX
+          echo "::endgroup::"
+
+          echo "::group::Copy mpd_generator"
+          cp mpd_generator${{ matrix.exe_ext }} $ARTIFACTS/mpd_generator$EXE_SUFFIX
+          echo "::endgroup::"
+
+          # The pssh-box bundle is OS and architecture independent.  So only do
+          # it on this one OS and architecture, and give it a more generic
+          # filename.
+          if [[ '${{ matrix.os_name }}' == 'linux' && '${{ matrix.target_arch }}' == 'x64' ]]; then
+            echo "::group::Tar pssh-box"
+            tar -czf $ARTIFACTS/pssh-box.py.tar.gz pssh-box.py pssh-box-protos
+            echo "::endgroup::"
+          fi
+
+      - name: Upload static release build artifacts
+        uses: actions/upload-artifact@v3
+        if: matrix.build_type == 'Release' && matrix.lib_type == 'static'
+        with:
+          name: artifacts-${{ matrix.os_name }}-${{ matrix.target_arch }}
+          path: artifacts/*
+          if-no-files-found: error
+          retention-days: 5
 
       - name: Debug
         uses: mxschmitt/action-tmate@v3.6

.github/workflows/docker-hub-release.yaml

@@ -1,52 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file or at
-# https://developers.google.com/open-source/licenses/bsd
-
-name: Docker Hub Release
-
-# Runs when a new release is published on GitHub.
-# Creates a corresponding Docker Hub release and publishes it.
-#
-# Can also be run manually for debugging purposes.
-on:
-  release:
-    types: [published]
-  # For manual debugging:
-  workflow_dispatch:
-    inputs:
-      ref:
-        description: "The tag to release to Docker Hub."
-        required: True
-
-jobs:
-  publish_docker_hub:
-    name: Publish to Docker Hub
-    runs-on: ubuntu-latest
-    steps:
-      - name: Compute ref
-        id: ref
-        # We could be building from a workflow dispatch (manual run), or a
-        # release event.  Subsequent steps can refer to $TARGET_REF to
-        # determine the correct ref in all cases.
-        run: |
-          echo "TARGET_REF=${{ github.event.inputs.ref || github.event.release.tag_name }}" >> $GITHUB_ENV
-
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ env.TARGET_REF }}
-          submodules: recursive
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_CI_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_CI_TOKEN }}
-
-      - name: Push to Docker Hub
-        uses: docker/build-push-action@v2
-        with:
-          push: true
-          tags: ${{ secrets.DOCKERHUB_PACKAGE_NAME }}:latest,${{ secrets.DOCKERHUB_PACKAGE_NAME }}:${{ env.TARGET_REF }}

.github/workflows/github-release.yaml

@@ -1,133 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file or at
-# https://developers.google.com/open-source/licenses/bsd
-
-name: GitHub Release
-
-# Runs when a new tag is created that looks like a version number.
-#
-# 1. Creates a draft release on GitHub with the latest release notes
-# 2. On all combinations of OS, build type, and library type:
-#    a. builds Packager
-#    b. builds the docs
-#    c. runs all tests
-#    d. attaches build artifacts to the release
-# 3. Fully publishes the release on GitHub
-#
-# Publishing the release then triggers additional workflows for NPM, Docker
-# Hub, and GitHub Pages.
-#
-# Can also be run manually for debugging purposes.
-on:
-  push:
-    tags:
-      - "v*.*"
-  # For manual debugging:
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: "An existing tag to release."
-        required: True
-
-jobs:
-  # TODO(joeyparrish): Switch to release-please
-  setup:
-    name: Setup
-    runs-on: ubuntu-latest
-    outputs:
-      tag: ${{ steps.compute_tag.outputs.tag }}
-    steps:
-      - name: Compute tag
-        id: compute_tag
-        # We could be building from a workflow dispatch (manual run)
-        # or from a pushed tag.  If triggered from a pushed tag, we would like
-        # to strip refs/tags/ off of the incoming ref and just use the tag
-        # name.  Subsequent jobs can refer to the "tag" output of this job to
-        # determine the correct tag name in all cases.
-        run: |
-          # Strip refs/tags/ from the input to get the tag name, then store
-          # that in output.
-          echo "::set-output name=tag::${{ github.event.inputs.tag || github.ref }}" \
-            | sed -e 's@refs/tags/@@'
-
-  # TODO(joeyparrish): Switch to release-please
-  draft_release:
-    name: Create GitHub release
-    needs: setup
-    runs-on: ubuntu-latest
-    outputs:
-      release_id: ${{ steps.draft_release.outputs.id }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ needs.setup.outputs.tag }}
-
-      - name: Check changelog version
-        # This check prevents releases without appropriate changelog updates.
-        run: |
-          VERSION=$(packager/tools/extract_from_changelog.py --version)
-          if [[ "$VERSION" != "${{ needs.setup.outputs.tag }}" ]]; then
-            echo ""
-            echo ""
-            echo "***** ***** *****"
-            echo ""
-            echo "Version mismatch!"
-            echo "Workflow is targetting ${{ needs.setup.outputs.tag }},"
-            echo "but CHANGELOG.md contains $VERSION!"
-            exit 1
-          fi
-
-      - name: Extract release notes
-        run: |
-          packager/tools/extract_from_changelog.py --release_notes \
-              | tee ../RELEASE_NOTES.md
-
-      - name: Draft release
-        id: draft_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ needs.setup.outputs.tag }}
-          release_name: ${{ needs.setup.outputs.tag }}
-          body_path: RELEASE_NOTES.md
-          draft: true
-
-  lint:
-    needs: setup
-    name: Lint
-    uses: ./.github/workflows/lint.yaml
-    with:
-      ref: ${{ needs.setup.outputs.tag }}
-
-  build_and_test:
-    needs: [setup, lint, draft_release]
-    name: Build and test
-    uses: ./.github/workflows/build.yaml
-    with:
-      ref: ${{ needs.setup.outputs.tag }}
-
-  test_supported_linux_distros:
-    # Doesn't really "need" it, but let's not waste time on a series of docker
-    # builds just to cancel it because of a linter error.
-    needs: lint
-    name: Test Linux distros
-    uses: ./.github/workflows/test-linux-distros.yaml
-    with:
-      ref: ${{ needs.setup.outputs.tag }}
-
-  # TODO(joeyparrish): Switch to release-please
-  publish_release:
-    name: Publish GitHub release
-    needs: [draft_release, build_and_test]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Publish release
-        uses: eregon/publish-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          release_id: ${{ needs.draft_release.outputs.release_id }}

.github/workflows/lint.yaml

@@ -45,8 +45,8 @@ jobs:
 
           # NOTE: Must use base.sha instead of base.ref, since we don't have
           # access to the branch name that base.ref would give us.
-          # NOTE: Must also use fetch-depth: 2 in actions/checkout to have access
+          # NOTE: Must also use fetch-depth: 2 in actions/checkout to have
-          # to the base ref for comparison.
+          # access to the base ref for comparison.
           packager/tools/git/check_formatting.py \
               --binary /usr/bin/clang-format \
               ${{ github.event.pull_request.base.sha || 'HEAD^' }}

.github/workflows/npm-release.yaml

@@ -1,60 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file or at
-# https://developers.google.com/open-source/licenses/bsd
-
-# A workflow to release Shaka Packager to NPM.
-name: NPM Release
-
-# Runs when a new release is published on GitHub.
-# Creates a corresponding NPM release and publishes it.
-#
-# Can also be run manually for debugging purposes.
-on:
-  release:
-    types: [published]
-  # For manual debugging:
-  workflow_dispatch:
-    inputs:
-      ref:
-        description: "The tag to release to NPM."
-        required: True
-
-jobs:
-  publish_npm:
-    name: Publish to NPM
-    runs-on: ubuntu-latest
-    steps:
-      - name: Compute ref
-        id: ref
-        # We could be building from a workflow dispatch (manual run), or a
-        # release event.  Subsequent steps can refer to $TARGET_REF to
-        # determine the correct ref in all cases.
-        run: |
-          echo "TARGET_REF=${{ github.event.inputs.ref || github.event.release.tag_name }}" >> $GITHUB_ENV
-
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ env.TARGET_REF }}
-
-      - name: Setup NodeJS
-        uses: actions/setup-node@v1
-        with:
-          node-version: 10
-
-      - name: Set package name and version
-        run: |
-          cd npm
-          sed package.json -i \
-              -e 's/"name": ""/"name": "${{ secrets.NPM_PACKAGE_NAME }}"/' \
-              -e 's/"version": ""/"version": "${{ env.TARGET_REF }}"/'
-
-      - name: Publish NPM package
-        uses: JS-DevTools/npm-publish@v1
-        with:
-          token: ${{ secrets.NPM_CI_TOKEN }}
-          package: npm/package.json
-          check-version: false
-          access: public

.github/workflows/pr.yaml

@@ -19,13 +19,14 @@ on:
     inputs:
       ref:
         description: "The ref to build and test."
-        required: False
+        required: false
+        type: string
 
 # If another instance of this workflow is started for the same PR, cancel the
 # old one.  If a PR is updated and a new test run is started, the old test run
 # will be cancelled automatically to conserve resources.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.inputs.ref || github.ref }}
+  group: ${{ github.workflow }}-${{ inputs.ref || github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -37,14 +38,14 @@ jobs:
     name: Lint
     uses: ./.github/workflows/lint.yaml
     with:
-      ref: ${{ github.event.inputs.ref || github.ref }}
+      ref: ${{ inputs.ref || github.ref }}
 
   build_and_test:
     needs: [lint, settings]
     name: Build and test
     uses: ./.github/workflows/build.yaml
     with:
-      ref: ${{ github.event.inputs.ref || github.ref }}
+      ref: ${{ inputs.ref || github.ref }}
       self_hosted: ${{ needs.settings.outputs.self_hosted != '' }}
       debug: ${{ needs.settings.outputs.debug != '' }}
 
@@ -53,15 +54,15 @@ jobs:
     name: Build docs
     uses: ./.github/workflows/build-docs.yaml
     with:
-      ref: ${{ github.event.inputs.ref || github.ref }}
+      ref: ${{ inputs.ref || github.ref }}
       debug: ${{ needs.settings.outputs.debug != '' }}
 
   official_docker_image:
     needs: lint
     name: Official Docker image
-    uses: ./.github/workflows/docker-image.yaml
+    uses: ./.github/workflows/build-docker.yaml
     with:
-      ref: ${{ github.event.inputs.ref || github.ref }}
+      ref: ${{ inputs.ref || github.ref }}
 
   test_supported_linux_distros:
     # Doesn't really "need" it, but let's not waste time on a series of docker
@@ -70,4 +71,4 @@ jobs:
     name: Test Linux distros
     uses: ./.github/workflows/test-linux-distros.yaml
     with:
-      ref: ${{ github.event.inputs.ref || github.ref }}
+      ref: ${{ inputs.ref || github.ref }}

.github/workflows/publish-docker.yaml

@@ -0,0 +1,68 @@
+# Copyright 2022 Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+# A workflow to publish the official docker image.
+name: Publish to Docker Hub
+
+# Runs when called from another workflow.
+# Can also be run manually for debugging purposes.
+on:
+  workflow_call:
+    inputs:
+      tag:
+        required: true
+        type: string
+      latest:
+        required: true
+        type: boolean
+    secrets:
+      DOCKERHUB_CI_USERNAME:
+        required: true
+      DOCKERHUB_CI_TOKEN:
+        required: true
+      DOCKERHUB_PACKAGE_NAME:
+        required: true
+  # For manual debugging:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: The tag to build from and to push to.
+        required: true
+        type: string
+      latest:
+        description: If true, push to the "latest" tag.
+        required: true
+        type: boolean
+
+jobs:
+  publish_docker_hub:
+    name: Publish to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.tag }}
+          submodules: recursive
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_CI_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_CI_TOKEN }}
+
+      - name: Push to Docker Hub
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: ${{ secrets.DOCKERHUB_PACKAGE_NAME }}:${{ inputs.tag }}
+
+      - name: Push to Docker Hub as "latest"
+        if: ${{ inputs.latest }}
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: ${{ secrets.DOCKERHUB_PACKAGE_NAME }}:latest

.github/workflows/publish-docs.yaml

@@ -0,0 +1,51 @@
+# Copyright 2022 Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+# A workflow to publish the docs to GitHub Pages.
+name: Publish Docs
+
+# Runs when called from another workflow.
+# Can also be run manually for debugging purposes.
+on:
+  workflow_call:
+    inputs:
+      ref:
+        required: true
+        type: string
+  # For manual debugging:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "The ref to build docs from."
+        required: true
+        type: string
+
+jobs:
+  build_docs:
+    name: Build docs
+    uses: ./.github/workflows/build-docs.yaml
+    with:
+      ref: ${{ inputs.ref }}
+
+  publish_docs:
+    name: Publish updated docs
+    needs: build_docs
+    runs-on: ubuntu-latest
+
+    # Grant GITHUB_TOKEN the permissions required to deploy to Pages
+    permissions:
+      pages: write
+      id-token: write
+
+    # Deploy to the github-pages environment
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.github/workflows/publish-npm.yaml

@@ -0,0 +1,100 @@
+# Copyright 2022 Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+# A workflow to publish the official NPM package.
+name: Publish to NPM
+
+# Runs when called from another workflow.
+# Can also be run manually for debugging purposes.
+on:
+  workflow_call:
+    inputs:
+      tag:
+        required: true
+        type: string
+      latest:
+        required: true
+        type: boolean
+    secrets:
+      NPM_CI_TOKEN:
+        required: true
+      NPM_PACKAGE_NAME:
+        required: true
+  # For manual debugging:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: The tag to build from.
+        required: true
+        type: string
+      latest:
+        description: If true, push to the "latest" tag.
+        required: true
+        type: boolean
+
+jobs:
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.tag }}
+          submodules: recursive
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 16
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Compute tags
+        run: |
+          # NPM publish always sets a tag.  If you don't provide an explicit
+          # tag, you get the "latest" tag by default, but we want "latest" to
+          # always point to the highest version number.  So we set an explicit
+          # tag on every "publish" command, then follow up with a command to
+          # set "latest" only if this release was the highest version yet.
+
+          # The explicit tag is based on the branch.  If the git tag is v4.4.1,
+          # the branch was v4.4.x, and the explicit NPM tag should be
+          # v4.4-latest.
+          GIT_TAG_NAME=${{ inputs.tag }}
+          NPM_TAG=$(echo "$GIT_TAG_NAME" | cut -f 1-2 -d .)-latest
+          echo "NPM_TAG=$NPM_TAG" >> $GITHUB_ENV
+
+          # Since we also set the package version on-the-fly during publication,
+          # compute that here.  It's the tag without the leading "v".
+          NPM_VERSION=$(echo "$GIT_TAG_NAME" | sed -e 's/^v//')
+          echo "NPM_VERSION=$NPM_VERSION" >> $GITHUB_ENV
+
+          # Debug the decisions made here.
+          echo "This release: $GIT_TAG_NAME"
+          echo "NPM tag: $NPM_TAG"
+          echo "NPM version: $NPM_VERSION"
+
+      - name: Set package name and version
+        run: |
+          # These substitutions use | because the package name could contain
+          # both / and @, but | should not appear in package names.
+          sed npm/package.json -i \
+              -e 's|"name": ""|"name": "${{ secrets.NPM_PACKAGE_NAME }}"|' \
+              -e 's|"version": ""|"version": "${{ env.NPM_VERSION }}"|'
+
+      - name: Publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_CI_TOKEN }}
+        run: |
+          cd npm
+
+          # Publish with an explicit tag.
+          # Also publish with explicit public access, to allow scoped packages.
+          npm publish --tag "$NPM_TAG" --access=public
+
+          # Add the "latest" tag if needed.
+          if [[ "${{ inputs.latest }}" == "true" ]]; then
+            npm dist-tag add "${{ secrets.NPM_PACKAGE_NAME }}@$NPM_VERSION" latest
+          fi

.github/workflows/release-please.yaml

@@ -0,0 +1,146 @@
+# Copyright 2023 Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+      - v[0-9]*
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    outputs:
+      release_created: ${{ steps.release.outputs.release_created }}
+      tag_name: ${{ steps.release.outputs.tag_name }}
+    steps:
+      # Create/update release PR
+      - uses: google-github-actions/release-please-action@v3
+        id: release
+        with:
+          # Required input to specify the release type.  This is not really a
+          # go project, but go projects in release-please only update
+          # CHANGELOG.md and nothing else.  This is what we want.
+          release-type: go
+          # Make sure we create the PR against the correct branch.
+          default-branch: ${{ github.ref_name }}
+          # Use a special shaka-bot access token for releases.
+          token: ${{ secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }}
+          # Temporary settings to bootstrap v3.0.0.
+          last-release-sha: 634af6591ce8c701587a78042ae7f81761725710
+          bootstrap-sha: 634af6591ce8c701587a78042ae7f81761725710
+
+  # The jobs below are all conditional on a release having been created by
+  # someone merging the release PR.
+
+  # Several actions either only run on the latest release or run with different
+  # options on the latest release.  Here we compute if this is the highest
+  # version number (what we are calling "latest" for NPM, Docker, and the
+  # docs).  You can have a more recent release from an older branch, but this
+  # would not qualify as "latest" here.
+  compute:
+    name: Compute latest release flag
+    runs-on: ubuntu-latest
+    needs: release
+    if: needs.release.outputs.release_created
+    outputs:
+      latest: ${{ steps.compute.outputs.latest }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-tags: true
+          persist-credentials: false
+
+      - name: Compute latest
+        id: compute
+        run: |
+          GIT_TAG_NAME=${{ needs.release.outputs.tag_name }}
+          RELEASE_TAGS=$(git tag | grep ^v[0-9])
+          LATEST_RELEASE=$(echo "$RELEASE_TAGS" | sort --version-sort | tail -1)
+          if [[ "$GIT_TAG_NAME" == "$LATEST_RELEASE" ]]; then
+            LATEST=true
+          else
+            LATEST=false
+          fi
+          echo latest=$LATEST >> $GITHUB_OUTPUT
+
+          # Debug the decisions made here.
+          echo "This release: $GIT_TAG_NAME"
+          echo "Latest release: $LATEST_RELEASE"
+          echo "This release is latest: $LATEST"
+
+  # Publish docs to GitHub Pages
+  docs:
+    name: Update docs
+    needs: [release, compute]
+    # Only if this is the latest release
+    if: needs.release.outputs.release_created && needs.compute.outputs.latest
+    uses: ./.github/workflows/publish-docs.yaml
+    with:
+      ref: ${{ github.ref }}
+
+  # Publish official docker image
+  docker:
+    name: Update docker image
+    needs: [release, compute]
+    if: needs.release.outputs.release_created
+    uses: ./.github/workflows/publish-docker.yaml
+    with:
+      tag: ${{ needs.release.outputs.tag_name }}
+      latest: ${{ needs.compute.outputs.latest == 'true' }}
+    secrets:
+      DOCKERHUB_CI_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
+      DOCKERHUB_CI_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
+      DOCKERHUB_PACKAGE_NAME: ${{ secrets.DOCKERHUB_PACKAGE_NAME }}
+
+  # Do a complete build
+  build:
+    name: Build
+    needs: release
+    if: needs.release.outputs.release_created
+    uses: ./.github/workflows/build.yaml
+    with:
+      ref: ${{ github.ref }}
+
+  # Attach build artifacts to the release
+  artifacts:
+    name: Artifacts
+    runs-on: ubuntu-latest
+    needs: [release, build]
+    if: needs.release.outputs.release_created
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+
+      - name: Debug
+        run: find -ls
+
+      - name: Attach packager to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ needs.release.outputs.tag_name }}
+          make_latest: false  # Already set for the release
+          file_glob: true
+          file: artifacts/artifacts*/*
+          overwrite: true
+
+  # Surprisingly, Shaka Packager binaries can be installed via npm.
+  # Publish NPM package updates.
+  npm:
+    name: Update NPM
+    needs: [release, compute, artifacts]
+    if: needs.release.outputs.release_created
+    uses: ./.github/workflows/publish-npm.yaml
+    with:
+      tag: ${{ needs.release.outputs.tag_name }}
+      latest: ${{ needs.compute.outputs.latest == 'true' }}
+    secrets:
+      NPM_CI_TOKEN: ${{ secrets.NPM_CI_TOKEN }}
+      NPM_PACKAGE_NAME: ${{ secrets.NPM_PACKAGE_NAME }}

.github/workflows/update-docs.yaml

@@ -1,56 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file or at
-# https://developers.google.com/open-source/licenses/bsd
-
-# A workflow to update the public docs.
-name: Update Docs
-
-# Runs when a new release is published on GitHub.
-#
-# Pushes updated docs to GitHub Pages if triggered from a release workflow.
-#
-# Can also be run manually for debugging purposes.
-on:
-  release:
-    types: [published]
-  # For manual debugging:
-  workflow_dispatch:
-    inputs:
-      ref:
-        description: "The ref to build docs from."
-        required: True
-
-jobs:
-  publish_docs:
-    name: Build updated docs
-    runs-on: ubuntu-latest
-    steps:
-      - name: Compute ref
-        id: ref
-        # We could be building from a workflow dispatch (manual run) or from a
-        # release event.  Subsequent steps can refer to the "ref" output of
-        # this job to determine the correct ref in all cases.
-        run: |
-          echo "::set-output name=ref::${{ github.event.inputs.ref || github.event.release.tag_name }}"
-
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ steps.ref.outputs.ref }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Build docs
-        uses: ./.github/workflows/custom-actions/build-docs
-
-      - name: Deploy to gh-pages branch
-        uses: peaceiris/actions-gh-pages@v3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: gh-pages
-          full_commit_message: Generate docs for ${{ steps.ref.outputs.ref }}

CHANGELOG.md

@@ -1,3 +1,6 @@
+# Changelog
+
+
 ## [2.6.1] - 2021-10-14
 ### Fixed
  - Fix crash in static-linked linux builds (#996)

Dockerfile

@@ -10,21 +10,20 @@ RUN apk add --no-cache \
 # merged.
 WORKDIR shaka-packager
 COPY . /shaka-packager/
-RUN mkdir build
+RUN rm -rf build
 RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -G Ninja
 RUN cmake --build build/ --config Debug --parallel
 
 # Copy only result binaries to our final image.
 FROM alpine:3.12
 RUN apk add --no-cache libstdc++ python3
-# TODO(joeyparrish): Copy binaries when build system is complete
+COPY --from=builder /shaka-packager/build/packager/packager \
-#COPY --from=builder /shaka-packager/build/packager \
+                    /shaka-packager/build/packager/mpd_generator \
-#                    /shaka-packager/build/mpd_generator \
+                    /shaka-packager/build/packager/pssh-box.py \
-#                    /shaka-packager/build/pssh-box.py \
+                    /usr/bin/
-#                    /usr/bin/
 
 # Copy pyproto directory, which is needed by pssh-box.py script. This line
 # cannot be combined with the line above as Docker's copy command skips the
 # directory itself. See https://github.com/moby/moby/issues/15858 for details.
-# TODO(joeyparrish): Copy binaries when build system is complete
+COPY --from=builder /shaka-packager/build/packager/pssh-box-protos \
-#COPY --from=builder /shaka-packager/build/pyproto /usr/bin/pyproto
+                    /usr/bin/pssh-box-protos

npm/index.js

@@ -4,15 +4,31 @@
 var path = require('path');
 var spawnSync = require('child_process').spawnSync;
 
-// Command names per-platform:
+// Command names per-platform (process.platform) and per-architecture
+// (process.arch):
 var commandNames = {
-  linux: 'packager-linux',
+  linux: {
-  darwin: 'packager-osx',
+    'x64': 'packager-linux-x64',
-  win32: 'packager-win.exe',
+    'arm64': 'packager-linux-arm64',
+  },
+  darwin: {
+    'x64': 'packager-osx-x64',
+  },
+  win32: {
+    'x64': 'packager-win-x64.exe',
+  },
 };
 
 // Find the platform-specific binary:
-var binaryPath = path.resolve(__dirname, 'bin', commandNames[process.platform]);
+if (!(process.platform in commandNames)) {
+  throw new Error('Platform not supported: ' + process.platform);
+}
+if (!(process.arch in commandNames[process.platform])) {
+  throw new Error(
+      'Architecture not supported: ' + process.platform + '/' + process.arch);
+}
+var commandName = commandNames[process.platform][process.arch];
+var binaryPath = path.resolve(__dirname, 'bin', commandName);
 
 // Find the args to pass to that binary:
 // argv[0] is node itself, and argv[1] is the script.

npm/package.json

@@ -2,6 +2,7 @@
   "name": "",
   "description": "A media packaging tool and SDK.",
   "version": "",
+  "private": false,
   "homepage": "https://github.com/shaka-project/shaka-packager",
   "author": "Google",
   "maintainers": [

npm/prepublish.js

@@ -5,11 +5,19 @@ var fs = require('fs');
 var path = require('path');
 var spawnSync = require('child_process').spawnSync;
 
-// Command names per-platform:
+// Command names per-platform (process.platform) and per-architecture
+// (process.arch):
 var commandNames = {
-  linux: 'packager-linux',
+  linux: {
-  darwin: 'packager-osx',
+    'x64': 'packager-linux-x64',
-  win32: 'packager-win.exe',
+    'arm64': 'packager-linux-arm64',
+  },
+  darwin: {
+    'x64': 'packager-osx-x64',
+  },
+  win32: {
+    'x64': 'packager-win-x64.exe',
+  },
 };
 
 // Get the current package version:
@@ -44,12 +52,23 @@ fs.readdirSync(binFolderPath).forEach(function(childName) {
 });
 
 for (var platform in commandNames) {
-  // Find the destination for this binary:
+  for (var arch in commandNames[platform]) {
-  var command = commandNames[platform];
+    // Find the destination for this binary:
-  var binaryPath = path.resolve(binFolderPath, command);
+    var command = commandNames[platform][arch];
+    var binaryPath = path.resolve(binFolderPath, command);
 
-  download(urlBase + command, binaryPath);
+    try {
-  fs.chmodSync(binaryPath, 0755);
+      download(urlBase + command, binaryPath);
+      fs.chmodSync(binaryPath, 0755);
+    } catch (error) {
+      if (arch == 'arm64') {
+        // Optional.  Forks may not have arm64 builds available.  Ignore.
+      } else {
+        // Required.  Re-throw and fail.
+        throw error;
+      }
+    }
+  }
 }
 
 // Fetch LICENSE and README files from the same tag, and include them in the
@@ -83,6 +102,6 @@ function download(url, outputPath) {
   console.log('Downloading', url, 'to', outputPath);
   var returnValue = spawnSync('curl', args, options);
   if (returnValue.status != 0) {
-    process.exit(returnValue.status);
+    throw new Error('Download of ' + url + ' failed: ' + returnValue.status);
   }
 }

packager/CMakeLists.txt

@@ -115,7 +115,7 @@ set(libpackager_deps
 add_library(libpackager_static STATIC ${libpackager_sources})
 target_link_libraries(libpackager_static ${libpackager_deps})
 
-# And always installed as libpackager.a:
+# And always installed as libpackager.a / libpackager.lib:
 if(NOT MSVC)
   set_property(TARGET libpackager_static PROPERTY OUTPUT_NAME packager)
 else()
@@ -249,9 +249,9 @@ configure_file(packager.pc.in packager.pc @ONLY)
 install(TARGETS mpd_generator packager)
 
 # Always install the python tools.
-install(PROGRAMS ${CMAKE_BINARY_DIR}/pssh-box.py
+install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/pssh-box.py
   DESTINATION ${CMAKE_INSTALL_BINDIR})
-install(DIRECTORY ${CMAKE_BINARY_DIR}/pssh-box-protos
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/pssh-box-protos
   DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 # With shared libraries, also install the library, headers, and pkgconfig.
@@ -261,6 +261,6 @@ if(BUILD_SHARED_LIBS)
   install(TARGETS libpackager_shared)
   install(DIRECTORY ../include/packager
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  install(FILES ${CMAKE_BINARY_DIR}/packager/packager.pc
+  install(FILES ${CMAKE_CURRENT_BINARY_DIR}/packager.pc
     DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 endif()

packager/testing/test_dockers.sh

@@ -83,7 +83,7 @@ for DOCKER_FILE in ${SCRIPT_DIR}/dockers/*; do
   CONTAINER="$( echo "packager_test_${OS_NAME}" | tr A-Z a-z )"
 
   RAN_SOMETHING=1
-  docker build --pull -t ${CONTAINER} -f ${DOCKER_FILE} ${SCRIPT_DIR}/dockers/
+  docker buildx build --pull -t ${CONTAINER} -f ${DOCKER_FILE} ${SCRIPT_DIR}/dockers/
   mkdir -p "${TEMP_BUILD_DIR}"
   docker_run cmake -S . -B build/ -DCMAKE_BUILD_TYPE=Debug -G Ninja
   docker_run cmake --build build/ --config Debug --parallel

packager/tools/extract_from_changelog.py

@@ -1,50 +0,0 @@
-#!/usr/bin/python3
-#
-# Copyright 2018 Google LLC. All rights reserved.
-#
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file or at
-# https://developers.google.com/open-source/licenses/bsd
-"""This script extracts a version or release notes from the changelog."""
-
-import argparse
-import re
-import sys
-
-def main():
-  parser = argparse.ArgumentParser(description=__doc__)
-  parser.add_argument('--release_notes', action='store_true',
-                      help='Print the latest release notes from the changelog')
-  parser.add_argument('--version', action='store_true',
-                      help='Print the latest version from the changelog')
-
-  args = parser.parse_args()
-
-  with open('CHANGELOG.md', 'r', encoding='utf8') as f:
-    contents = f.read()
-
-  # This excludes the header line with the release name and date, to match the
-  # style of releases done before the automation was introduced.
-  latest_entry = re.split(r'^(?=## \[)', contents, flags=re.M)[1]
-
-  lines = latest_entry.strip().split('\n')
-  first_line = lines[0]
-  release_notes = '\n'.join(lines[1:])
-
-  match = re.match(r'^## \[(.*)\]', first_line)
-  if not match:
-    raise RuntimeError('Unable to parse first line of CHANGELOG.md!')
-
-  version = match[1]
-  if not version.startswith('v'):
-    version = 'v' + version
-
-  if args.version:
-    print(version)
-  if args.release_notes:
-    print(release_notes)
-
-  return 0
-
-if __name__ == '__main__':
-  sys.exit(main())

packager/tools/pssh/CMakeLists.txt

@@ -7,10 +7,10 @@
 # Copy the pssh-box.py script and its protos into the required directory
 # structure.
 set(PSSH_BOX_OUTPUTS
-    ${CMAKE_BINARY_DIR}/pssh-box.py
+    ${CMAKE_BINARY_DIR}/packager/pssh-box.py
-    ${CMAKE_BINARY_DIR}/pssh-box-protos
+    ${CMAKE_BINARY_DIR}/packager/pssh-box-protos
-    ${CMAKE_BINARY_DIR}/pssh-box-protos/widevine_common_encryption_pb2.py
+    ${CMAKE_BINARY_DIR}/packager/pssh-box-protos/widevine_common_encryption_pb2.py
-    ${CMAKE_BINARY_DIR}/pssh-box-protos/widevine_pssh_data_pb2.py)
+    ${CMAKE_BINARY_DIR}/packager/pssh-box-protos/widevine_pssh_data_pb2.py)
 
 add_custom_command(
     DEPENDS
@@ -19,15 +19,15 @@ add_custom_command(
     OUTPUT ${PSSH_BOX_OUTPUTS}
     COMMAND
         ${CMAKE_COMMAND} -E make_directory
-        ${CMAKE_BINARY_DIR}/pssh-box-protos/
+        ${CMAKE_BINARY_DIR}/packager/pssh-box-protos/
     COMMAND
         ${CMAKE_COMMAND} -E copy
         ${CMAKE_BINARY_DIR}/packager/media/base/widevine_common_encryption_pb2.py
         ${CMAKE_BINARY_DIR}/packager/media/base/widevine_pssh_data_pb2.py
-        ${CMAKE_BINARY_DIR}/pssh-box-protos/
+        ${CMAKE_BINARY_DIR}/packager/pssh-box-protos/
     COMMAND
         ${CMAKE_COMMAND} -E copy
         ${CMAKE_CURRENT_SOURCE_DIR}/pssh-box.py
-        ${CMAKE_BINARY_DIR}/)
+        ${CMAKE_BINARY_DIR}/packager/)
 
 add_custom_target(pssh_box_py ALL DEPENDS ${PSSH_BOX_OUTPUTS})