diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml new file mode 100644 index 0000000000..53b31b0102 --- /dev/null +++ b/.github/workflows/build_and_test.yaml @@ -0,0 +1,180 @@ +name: Build and Test + +# Builds and tests on all combinations of OS, build type, and library type. +# +# Runs when a pull request is opened, or triggered from other workflows. +# +# If triggered from another workflow, optionally attaches release artifacts to a +# release, since those artifacts can't be accessed from another workflow later. +# +# If triggered from another workflow, optionally initiates another workflow +# afterward, allowing for chaining of workflows. This capability is used in the +# release process, so that the relatively-complex build_and_test workflow can be +# reused in the middle of the release process without duplication. +# +# Can also be run manually for debugging purposes. +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + inputs: + ref: + description: "The ref to build and test." + required: False + release_id: + description: "The ID of a release to attach release artifacts." + required: False + next_workflow: + description: "The workflow to trigger next." + required: False + next_workflow_payload: + description: "JSON input parameters to send to the next workflow." + required: False + default: "{}" + +jobs: + build_and_test: + strategy: + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + build_type: ["Debug", "Release"] + lib_type: ["static", "shared"] + include: + - os: ubuntu-latest + os_name: linux + exe_ext: "" + build_type_suffix: "" + - os: macos-latest + os_name: osx + exe_ext: "" + build_type_suffix: "" + - os: windows-latest + os_name: win + exe_ext: ".exe" + # 64-bit outputs on Windows go to a different folder name. + build_type_suffix: "_x64" + + name: Build and test ${{ matrix.os_name }} ${{ matrix.build_type }} ${{ matrix.lib_type }} + runs-on: ${{ matrix.os }} + + env: + MACOSX_DEPLOYMENT_TARGET: "10.10" + GYP_DEFINES: "target_arch=x64 libpackager_type=${{ matrix.lib_type }}_library" + DEPOT_TOOLS_WIN_TOOLCHAIN: "0" + GYP_MSVS_VERSION: "2019" + GYP_MSVS_OVERRIDE_PATH: "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise" + + steps: + - name: Compute ref + # We could be building from a workflow dispatch (manual run or triggered + # by another workflow), or from a pull request. Subsequent steps can + # refer to $TARGET_REF to determine the correct ref in all cases. + run: | + echo "TARGET_REF=${{ github.event.inputs.ref || github.ref }}" >> $GITHUB_ENV + + - name: Configure git to preserve line endings + # Fix issues with CRLF in test outputs on Windows by explicitly setting + # this core.autocrlf config. Without this, tests will later fail + # because the "golden" test outputs in the source code will not have + # the correct line endings. + run: | + git config --global core.autocrlf false + + - name: Checkout code + uses: actions/checkout@v2 + with: + path: src + ref: ${{ env.TARGET_REF }} + + - name: Install depot tools + # NOTE: can't use actions/checkout here because this is not hosted on + # GitHub. + run: | + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + echo "${GITHUB_WORKSPACE}/depot_tools" >> $GITHUB_PATH + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 2.7 + + - name: Install macOS SDK 10.3 (macOS only) + if: matrix.os_name == 'osx' + # NOTE: macOS 11 doesn't work with our (old) version of Chromium build, + # and the latest Chromium build doesn't work with Packager's build + # system. To work around this, we need an older SDK version, and to + # get that, we need an older XCode version. XCode 10.3 has SDK 10.14, + # which works. + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: 10.3 + + - name: Configure gclient + run: | + depot_tools/gclient config https://github.com/google/shaka-packager.git --name=src --unmanaged + + - name: Sync gclient + run: | + depot_tools/gclient sync + + - name: Build + run: | + depot_tools/ninja -C src/out/${{ matrix.build_type }}${{ matrix.build_type_suffix }} + + - name: Test + shell: bash + run: | + # NOTE: Some of these tests must be run from the "src" directory. + cd src + if [[ '${{ matrix.os_name }}' == 'osx' ]]; then + export DYLD_FALLBACK_LIBRARY_PATH=out/${{ matrix.build_type }}${{ matrix.build_type_suffix }} + fi + for i in out/${{ matrix.build_type }}${{ matrix.build_type_suffix }}/*test${{ matrix.exe_ext }}; do "$i" || exit 1; done + python out/${{ matrix.build_type }}${{ matrix.build_type_suffix }}/packager_test.py -v --libpackager_type=${{ matrix.lib_type }}_library + + - name: Prepare artifacts + if: matrix.build_type == 'Release' && matrix.lib_type == 'static' + shell: bash + run: | + mkdir artifacts + mv \ + src/out/Release${{ matrix.build_type_suffix }}/packager${{ matrix.exe_ext }} \ + artifacts/packager-${{ matrix.os_name }}${{ matrix.exe_ext }} + mv \ + src/out/Release${{ matrix.build_type_suffix }}/mpd_generator${{ matrix.exe_ext }} \ + artifacts/mpd_generator-${{ matrix.os_name }}${{ matrix.exe_ext }} + if [[ '${{ matrix.os_name }}' == 'win' ]]; then + ( + cd src/out/Release${{ matrix.build_type_suffix }} + 7z a ../../../pssh-box.py.zip pyproto pssh-box.py + ) + mv pssh-box.py.zip artifacts/pssh-box-${{ matrix.os_name }}.py.zip + else + tar -czf pssh-box.py.tar.gz -C src/out/Release${{ matrix.build_type_suffix }} pyproto pssh-box.py + mv pssh-box.py.tar.gz artifacts/pssh-box-${{ matrix.os_name }}.py.tar.gz + fi + + - name: Attach artifacts to release + if: matrix.build_type == 'Release' && matrix.lib_type == 'static' && github.event.inputs.release_id + uses: dwenegar/upload-release-assets@v1 + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} + assets_path: artifacts + + launch_next_workflow: + name: Launch next workflow + if: github.event_name == 'workflow_dispatch' && github.event.inputs.next_workflow + needs: build_and_test + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + steps: + - name: Launch next workflow + run: | + echo '${{ github.event.inputs.next_workflow_payload }}' | \ + gh workflow \ + -R '${{ github.repository }}' \ + run '${{ github.event.inputs.next_workflow }}' \ + --json diff --git a/.github/workflows/docker_hub_release.yaml b/.github/workflows/docker_hub_release.yaml new file mode 100644 index 0000000000..883d880650 --- /dev/null +++ b/.github/workflows/docker_hub_release.yaml @@ -0,0 +1,47 @@ +name: Docker Hub Release + +# Runs when a new release is published on GitHub. +# Creates a corresponding Docker Hub release and publishes it. +# +# Can also be run manually for debugging purposes. +on: + release: + types: [published] + # For manual debugging: + workflow_dispatch: + inputs: + ref: + description: "The tag to release to NPM." + required: True + +jobs: + publish_docker_hub: + name: Publish to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Compute ref + id: ref + # We could be building from a workflow dispatch (manual run), or a + # release event. Subsequent steps can refer to $TARGET_REF to + # determine the correct ref in all cases. + run: | + echo "TARGET_REF=${{ github.event.inputs.ref || github.event.release.tag_name }}" >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@v2 + with: + path: src + ref: ${{ env.TARGET_REF }} + + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_CI_USERNAME }} + password: ${{ secrets.DOCKERHUB_CI_TOKEN }} + + - name: Push to Docker Hub + uses: docker/build-push-action@v2 + with: + push: true + context: src/ + tags: google/shaka-packager:latest,google/shaka-packager:${{ env.TARGET_REF }} diff --git a/.github/workflows/draft_github_release.yaml b/.github/workflows/draft_github_release.yaml new file mode 100644 index 0000000000..7a894a26db --- /dev/null +++ b/.github/workflows/draft_github_release.yaml @@ -0,0 +1,85 @@ +name: Draft GitHub Release + +# Runs when a new tag is created that looks like a version number. +# +# Creates a draft release on GitHub with the latest release notes, then chains +# to the build_and_test workflow and the publish_github_release workflow. +# +# Collectively, this will build and tests on all OSes, attach release artifacts +# to the release, and fully publish the release. +# +# Publishing the release then triggers additional workflows for NPM, Docker +# Hub, and GitHub Pages. +# +# Can also be run manually for debugging purposes. +on: + push: + tags: + - "v*.*" + # For manual debugging: + workflow_dispatch: + inputs: + tag: + description: "An existing tag to release." + required: True + +jobs: + draft_release: + name: Draft GitHub release + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + path: src + + - name: Compute ref + # We could be building from a workflow dispatch (manual run) + # or from a pushed tag. If triggered from a pushed tag, we would like + # to strip refs/tags/ off of the incoming ref and just use the tag + # name. Subsequent steps can refer to $TARGET_REF to determine the + # correct ref in all cases. + run: | + echo "TARGET_REF=${{ github.event.inputs.tag || github.ref }}" | \ + sed -e 's@refs/tags/@@' >> $GITHUB_ENV + + - name: Extract release notes + run: | + cd src + packager/tools/extract_from_changelog.py --release_notes \ + | tee ../RELEASE_NOTES.md + # This check prevents releases without appropriate changelog updates. + VERSION=$(packager/tools/extract_from_changelog.py --version) + if [[ "$VERSION" != "$TARGET_REF" ]]; then + echo "" + echo "" + echo "***** ***** *****" + echo "" + echo "Version mismatch!" + echo "Workflow is targetting $TARGET_REF," + echo "but CHANGELOG.md contains $VERSION!" + exit 1 + fi + + - name: Draft release + id: draft_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + with: + tag_name: ${{ env.TARGET_REF }} + release_name: ${{ env.TARGET_REF }} + body_path: RELEASE_NOTES.md + draft: true + + - name: Start build and test, then publish + run: | + cd src + gh workflow run build_and_test.yaml \ + -f 'ref=${{ env.TARGET_REF }}' \ + -f 'release_id=${{ steps.draft_release.outputs.id }}' \ + -f 'next_workflow=publish_github_release.yaml' \ + -f 'next_workflow_payload={ "release_id": "${{ steps.draft_release.outputs.id }}" }' + echo "Triggered build_and_test workflow for release ID ${{ steps.draft_release.outputs.id }}." + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} diff --git a/.github/workflows/npm_release.yaml b/.github/workflows/npm_release.yaml new file mode 100644 index 0000000000..d08d7e27d5 --- /dev/null +++ b/.github/workflows/npm_release.yaml @@ -0,0 +1,51 @@ +name: NPM Release + +# Runs when a new release is published on GitHub. +# Creates a corresponding NPM release and publishes it. +# +# Can also be run manually for debugging purposes. +on: + release: + types: [published] + # For manual debugging: + workflow_dispatch: + inputs: + ref: + description: "The tag to release to NPM." + required: True + +jobs: + publish_npm: + name: Publish to NPM + runs-on: ubuntu-latest + steps: + - name: Compute ref + id: ref + # We could be building from a workflow dispatch (manual run), or a + # release event. Subsequent steps can refer to $TARGET_REF to + # determine the correct ref in all cases. + run: | + echo "TARGET_REF=${{ github.event.inputs.ref || github.event.release.tag_name }}" >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@v2 + with: + path: src + ref: ${{ env.TARGET_REF }} + + - name: Setup NodeJS + uses: actions/setup-node@v1 + with: + node-version: 10 + + - name: Set package version + run: | + cd src/npm + npm version ${{ env.TARGET_REF }} + + - name: Publish NPM package + uses: JS-DevTools/npm-publish@v1 + with: + token: ${{ secrets.NPM_CI_TOKEN }} + package: src/npm/package.json + check-version: false diff --git a/.github/workflows/publish_github_release.yaml b/.github/workflows/publish_github_release.yaml new file mode 100644 index 0000000000..c4f6f2991c --- /dev/null +++ b/.github/workflows/publish_github_release.yaml @@ -0,0 +1,30 @@ +name: Publish GitHub Release + +# Fully publishes a draft release on GitHub. +# +# Triggered in a chain initiated by the draft_github_release workflow, after +# the build_and_test workflow has finished attaching release artifacts to the +# draft release. +# +# Publishing the release then triggers additional workflows for NPM, Docker +# Hub, and GitHub Pages. +# +# Can also be run manually for debugging purposes. +on: + workflow_dispatch: + inputs: + release_id: + description: "The draft release ID." + required: True + +jobs: + publish_release: + name: Publish GitHub release + runs-on: ubuntu-latest + steps: + - name: Publish release + uses: eregon/publish-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} diff --git a/.github/workflows/update_docs.yaml b/.github/workflows/update_docs.yaml new file mode 100644 index 0000000000..86eadf49d5 --- /dev/null +++ b/.github/workflows/update_docs.yaml @@ -0,0 +1,76 @@ +name: Update Docs + +# Runs when a new release is published on GitHub, or when a pull request is +# opened. +# +# Pushes updated docs to GitHub Pages if triggered from a release workflow. +# +# Can also be run manually for debugging purposes. +on: + pull_request: + types: [opened, synchronize, reopened] + release: + types: [published] + # For manual debugging: + workflow_dispatch: + inputs: + ref: + description: "The ref to build docs from." + required: True + +jobs: + publish_docs: + name: Publish updated docs + runs-on: ubuntu-latest + steps: + - name: Compute ref + id: ref + # We could be building from a workflow dispatch (manual run), release + # event, or pull request. Subsequent steps can refer to $TARGET_REF to + # determine the correct ref in all cases. + run: | + echo "TARGET_REF=${{ github.event.inputs.ref || github.event.release.tag_name || github.ref }}" >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@v2 + with: + path: src + ref: ${{ env.TARGET_REF }} + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + pip install wheel + pip install sphinxcontrib.plantuml + pip install recommonmark + pip install cloud_sptheme + pip install breathe + sudo apt-get install -y doxygen + + - name: Generate docs + run: | + mkdir gh-pages + cd src + mkdir out + # Doxygen must run before Sphinx. Sphinx will refer to + # Doxygen-generated output when it builds its own docs. + doxygen docs/Doxyfile + # Now build the Sphinx-based docs. + make -C docs/ html + # Now move the generated outputs. + mv out/sphinx/html ../gh-pages/html + mv out/doxygen/html ../gh-pages/docs + cp docs/index.html ../gh-pages/index.html + + - name: Deploy to gh-pages branch (releases only) + # This is skipped when testing a PR + if: github.event_name != 'pull_request' + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.SHAKA_BOT_TOKEN }} + publish_dir: gh-pages + full_commit_message: Generate docs for ${{ env.TARGET_REF }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 56c618396e..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,63 +0,0 @@ -language: cpp - -os: - - linux - - osx - -env: - - BUILD_TYPE=Debug LIBPACKAGER_TYPE=static_library - - BUILD_TYPE=Debug LIBPACKAGER_TYPE=shared_library - - BUILD_TYPE=Release LIBPACKAGER_TYPE=static_library - - BUILD_TYPE=Release LIBPACKAGER_TYPE=shared_library - -before_install: - - test -n $CC && unset CC - - test -n $CXX && unset CXX - -install: - - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ../depot_tools/ - - export PATH="$PATH:$PWD/../depot_tools" - -before_script: - - mkdir src - - shopt -s extglob dotglob - - mv !(src) src - - gclient config https://github.com/google/shaka-packager.git --name=src --unmanaged - - export GYP_DEFINES="libpackager_type=${LIBPACKAGER_TYPE}" - - gclient sync - - cd src - - ninja -C out/${BUILD_TYPE} -k 100 - -script: - - if [ ${LIBPACKAGER_TYPE} == "shared_library" ] && [ ${TRAVIS_OS_NAME} == "osx" ]; then - export DYLD_FALLBACK_LIBRARY_PATH=./out/${BUILD_TYPE}; - fi - - ( find out/${BUILD_TYPE} -name "*_*test" | while read i ; do $i || exit ; done ) - - out/${BUILD_TYPE}/packager_test.py -v --libpackager_type=${LIBPACKAGER_TYPE} - -before_deploy: - - rm -rf deploy - - mkdir deploy - - | - if [ ${LIBPACKAGER_TYPE} == "static_library" ] ; then - mv out/${BUILD_TYPE}/packager deploy/packager-${TRAVIS_OS_NAME} - mv out/${BUILD_TYPE}/mpd_generator deploy/mpd_generator-${TRAVIS_OS_NAME} - tar -zcf pssh-box.py.tar.gz -C out/${BUILD_TYPE} pyproto pssh-box.py - mv pssh-box.py.tar.gz deploy/pssh-box-${TRAVIS_OS_NAME}.py.tar.gz - fi - -deploy: - provider: releases - api_key: - secure: Ya5ptzB+UXESFhM8YKVRGwtfksPqCEe3XxHLLCwSDRscvs9Cck6M7Y4IGbEVfvfM8Wpb7Eyt2UoYgAsrugj9K6Y103NZ/Vv2ZJrdWYIi52QTye7GFtSEgImHdlM8LOjXbbh/KroWZFCi+ZxRcFDROaXaTM+P/xvYHeopLwEWs7g= - file_glob: true - file: deploy/* - skip_cleanup: true - on: - tags: true - condition: ${BUILD_TYPE} = Release && ${LIBPACKAGER_TYPE} == static_library - -branches: - only: - - master - - "/^v\\d+\\./" diff --git a/Dockerfile b/Dockerfile index 98c45eaef1..ab55f93034 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" # Build shaka-packager WORKDIR shaka_packager -RUN gclient config https://www.github.com/google/shaka-packager.git --name=src --unmanaged +RUN gclient config https://github.com/google/shaka-packager.git --name=src --unmanaged COPY . src RUN gclient sync RUN cd src && ninja -C out/Release diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index bf81d2d54b..0000000000 --- a/appveyor.yml +++ /dev/null @@ -1,69 +0,0 @@ -platform: - - x64 - - x86 - -configuration: - - Debug - - Release - -environment: - matrix: - - language: cpp - libpackager_type: static_library - - language: cpp - libpackager_type: shared_library - -clone_folder: c:\projects\shaka-packager\src - -install: - - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ..\depot_tools\ - -before_build: - - cd .. - - depot_tools\gclient config https://github.com/google/shaka-packager.git --name=src --unmanaged - - set DEPOT_TOOLS_WIN_TOOLCHAIN=0 - - set GYP_DEFINES="target_arch=%PLATFORM% libpackager_type=%LIBPACKAGER_TYPE%" - - depot_tools\gclient sync - - if [%PLATFORM%] == [x64] ( - set "output_directory=%CONFIGURATION%_x64" - ) else ( - set "output_directory=%CONFIGURATION%" - ) - -build_script: - - cd src - - ..\depot_tools\ninja -C "out\%OUTPUT_DIRECTORY%" -k 100 - -test_script: - - for %%f in ("out\%OUTPUT_DIRECTORY%\*_*test.exe") do (%%f || exit /b 666) - - python "out\%OUTPUT_DIRECTORY%\packager_test.py" -v --libpackager_type="%LIBPACKAGER_TYPE%" - -after_build: - - if exist deploy rmdir /s /q deploy - - mkdir deploy - - ps: >- - If ($env:LIBPACKAGER_TYPE -eq "static_library") { - copy "out\$env:OUTPUT_DIRECTORY\packager.exe" deploy\packager-win.exe - copy "out\$env:OUTPUT_DIRECTORY\mpd_generator.exe" deploy\mpd_generator-win.exe - 7z a pssh-box.py.zip "$env:APPVEYOR_BUILD_FOLDER\out\$env:OUTPUT_DIRECTORY\pyproto" - 7z a pssh-box.py.zip "$env:APPVEYOR_BUILD_FOLDER\out\$env:OUTPUT_DIRECTORY\pssh-box.py" - copy pssh-box.py.zip deploy\pssh-box-win.py.zip - } - -artifacts: - - path: 'deploy\*' - -deploy: - provider: GitHub - auth_token: - secure: tKyFNS3wiKnZ5+P71RPzsAzySnZLQWUWXKR3XAeco4bOTEXcTRwuQZSgggfSj19O - on: - appveyor_repo_tag: true - platform: x64 - configuration: Release - libpackager_type: static_library - -branches: - only: - - master - - "/^v\\d+\\./" diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000000..3a5012badd --- /dev/null +++ b/docs/index.html @@ -0,0 +1,19 @@ + + + +
+ + +