Reworked CI builds for master/develop. (#23182)
This commit is contained in:
Nick Brassel
GitHub
parent
7fc552632e
commit
dca7c3f86f
2 changed files with 303 additions and 0 deletions
122
.github/workflows/ci_build_major_branch.yml
vendored
Normal file
122
.github/workflows/ci_build_major_branch.yml
vendored
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
name: CI Build Major Branch
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, develop]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
type: choice
|
||||
description: "Branch to build"
|
||||
options: [master, develop]
|
||||
|
||||
env:
|
||||
# https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits
|
||||
# We've decreased it from 20 to 15 to allow for other GHA to run unimpeded
|
||||
CONCURRENT_JOBS: 15
|
||||
|
||||
# Ensure we only have one build running at a time, cancelling any active builds if a new commit is pushed to the respective branch
|
||||
concurrency:
|
||||
group: ci_build-${{ github.event.inputs.branch || github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
determine_concurrency:
|
||||
name: "Determine concurrency"
|
||||
if: github.repository == 'qmk/qmk_firmware'
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/qmk/qmk_cli
|
||||
|
||||
outputs:
|
||||
slice_length: ${{ steps.generate_slice_length.outputs.slice_length }}
|
||||
|
||||
steps:
|
||||
- name: Install prerequisites
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y jq
|
||||
|
||||
- name: Disable safe.directory check
|
||||
run: |
|
||||
git config --global --add safe.directory '*'
|
||||
|
||||
- name: Checkout QMK Firmware
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Determine concurrency
|
||||
id: generate_slice_length
|
||||
run: |
|
||||
target_count=$( {
|
||||
qmk find -km default 2>/dev/null
|
||||
qmk find -km via 2>/dev/null
|
||||
} | sort | uniq | wc -l)
|
||||
slice_length=$((target_count / ($CONCURRENT_JOBS - 1))) # Err on the side of caution as we're splitting default and via
|
||||
echo "slice_length=$slice_length" >> $GITHUB_OUTPUT
|
||||
|
||||
build_targets:
|
||||
name: "Compile keymap ${{ matrix.keymap }}"
|
||||
needs: determine_concurrency
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
keymap: [default, via]
|
||||
uses: ./.github/workflows/ci_build_major_branch_keymap.yml
|
||||
with:
|
||||
branch: ${{ inputs.branch || github.ref_name }}
|
||||
keymap: ${{ matrix.keymap }}
|
||||
slice_length: ${{ needs.determine_concurrency.outputs.slice_length }}
|
||||
|
||||
rollup_tasks:
|
||||
name: "Housekeeping"
|
||||
needs: build_targets
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Download firmwares
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: firmware-*
|
||||
path: firmwares
|
||||
merge-multiple: true
|
||||
|
||||
- name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/${{ github.sha }}
|
||||
uses: jakejarvis/s3-sync-action@master
|
||||
with:
|
||||
args: --acl public-read --follow-symlinks --delete
|
||||
env:
|
||||
AWS_S3_BUCKET: qmk-ci
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
|
||||
AWS_REGION: nyc3
|
||||
AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com
|
||||
SOURCE_DIR: firmwares
|
||||
DEST_DIR: ${{ inputs.branch || github.ref_name }}/${{ github.sha }}
|
||||
|
||||
- name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/latest
|
||||
uses: jakejarvis/s3-sync-action@master
|
||||
with:
|
||||
args: --acl public-read --follow-symlinks --delete
|
||||
env:
|
||||
AWS_S3_BUCKET: qmk-ci
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
|
||||
AWS_REGION: nyc3
|
||||
AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com
|
||||
SOURCE_DIR: firmwares
|
||||
DEST_DIR: ${{ inputs.branch || github.ref_name }}/latest
|
||||
|
||||
- name: Check if failure marker file exists
|
||||
id: check_failure_marker
|
||||
uses: andstor/file-existence-action@v3
|
||||
with:
|
||||
files: firmwares/.failed
|
||||
|
||||
- name: Fail build if needed
|
||||
if: steps.check_failure_marker.outputs.exists == 'true'
|
||||
run: |
|
||||
# Exit with failure if the compilation stage failed
|
||||
exit 1
|
||||
181
.github/workflows/ci_build_major_branch_keymap.yml
vendored
Normal file
181
.github/workflows/ci_build_major_branch_keymap.yml
vendored
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
name: CI Build Major Branch Keymap
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
branch:
|
||||
type: string
|
||||
required: true
|
||||
keymap:
|
||||
type: string
|
||||
required: true
|
||||
slice_length:
|
||||
type: string
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
generate_targets:
|
||||
name: "Generate targets (${{ inputs.keymap }})"
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/qmk/qmk_cli
|
||||
|
||||
outputs:
|
||||
targets: ${{ steps.generate_targets.outputs.targets }}
|
||||
|
||||
steps:
|
||||
- name: Install prerequisites
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y jq
|
||||
|
||||
- name: Disable safe.directory check
|
||||
run: |
|
||||
git config --global --add safe.directory '*'
|
||||
|
||||
- name: Checkout QMK Firmware
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Generate build targets
|
||||
id: generate_targets
|
||||
run: |
|
||||
{ # Intentionally use `shuf` here so that we share manufacturers across all build groups -- some have a lot of ARM-based boards which inherently take longer
|
||||
counter=0
|
||||
echo -n '{'
|
||||
qmk find -km ${{ inputs.keymap }} 2>/dev/null | sort | uniq | shuf | xargs -L${{ inputs.slice_length }} | while IFS=$'\n' read target ; do
|
||||
if [ $counter -gt 0 ]; then
|
||||
echo -n ','
|
||||
fi
|
||||
counter=$((counter+1))
|
||||
printf "\"group %02d\":{" $counter
|
||||
echo -n '"targets":"'
|
||||
echo $target | tr ' ' '\n' | sort | uniq | xargs echo -n
|
||||
echo -n '"}'
|
||||
done
|
||||
echo -n '}'
|
||||
} | sed -e 's@\n@@g' > targets.json
|
||||
|
||||
# Output the target keys as a variable
|
||||
echo "targets=$(jq -c 'keys' targets.json)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Upload targets json
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: targets-${{ inputs.keymap }}
|
||||
path: targets.json
|
||||
|
||||
build_targets:
|
||||
name: "Compile ${{ matrix.target }} (${{ inputs.keymap }})"
|
||||
needs: generate_targets
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/qmk/qmk_cli
|
||||
continue-on-error: true
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
target: ${{ fromJson(needs.generate_targets.outputs.targets) }}
|
||||
|
||||
steps:
|
||||
- name: Install prerequisites
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y jq
|
||||
|
||||
- name: Disable safe.directory check
|
||||
run: |
|
||||
git config --global --add safe.directory '*'
|
||||
|
||||
- name: Checkout QMK Firmware
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Get target definitions
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: targets-${{ inputs.keymap }}
|
||||
path: .
|
||||
|
||||
- name: Deploy submodules
|
||||
run: |
|
||||
qmk git-submodule -f
|
||||
|
||||
- name: Dump targets
|
||||
run: |
|
||||
jq -r '.["${{ matrix.target }}"].targets' targets.json | tr ' ' '\n' | sort
|
||||
|
||||
- name: Build targets
|
||||
continue-on-error: true
|
||||
run: |
|
||||
export NCPUS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null)
|
||||
qmk mass-compile -t -j $NCPUS -e DUMP_CI_METADATA=yes $(jq -r '.["${{ matrix.target }}"].targets' targets.json) || touch .failed
|
||||
|
||||
- name: Upload binaries
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: firmware-${{ inputs.keymap }}-${{ matrix.target }}
|
||||
if-no-files-found: ignore
|
||||
path: |
|
||||
*.bin
|
||||
*.hex
|
||||
*.uf2
|
||||
.build/failed.*
|
||||
.failed
|
||||
|
||||
- name: Fail build if any group failed
|
||||
run: |
|
||||
# Exit with failure if the compilation stage failed
|
||||
[ ! -f .failed ] || exit 1
|
||||
|
||||
repack_firmware:
|
||||
if: always()
|
||||
name: "Repack artifacts"
|
||||
needs: build_targets
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout QMK Firmware
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download firmwares
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: firmware-${{ inputs.keymap }}-*
|
||||
path: .
|
||||
merge-multiple: true
|
||||
|
||||
- name: Upload all firmwares
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: firmware-${{ inputs.keymap }}
|
||||
if-no-files-found: ignore
|
||||
path: |
|
||||
*.bin
|
||||
*.hex
|
||||
*.uf2
|
||||
.build/failed.*
|
||||
.failed
|
||||
|
||||
- name: Generate output logs
|
||||
run: |
|
||||
# Generate the step summary markdown
|
||||
./util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
|
||||
# Truncate to a maximum of 1MB to deal with GitHub workflow limit
|
||||
truncate --size='<960K' $GITHUB_STEP_SUMMARY || true
|
||||
|
||||
- name: Delete temporary build artifacts
|
||||
uses: geekyeggo/delete-artifact@v4
|
||||
with:
|
||||
name: |
|
||||
firmware-${{ inputs.keymap }}-*
|
||||
targets-${{ inputs.keymap }}
|
||||
|
||||
- name: 'CI Discord Notification'
|
||||
if: always()
|
||||
working-directory: util/ci/
|
||||
env:
|
||||
DISCORD_WEBHOOK: ${{ secrets.CI_DISCORD_WEBHOOK }}
|
||||
run: |
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 ./discord-results.py --branch ${{ inputs.branch || github.ref_name }} --keymap ${{ inputs.keymap }} --url ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
Loading…
Reference in a new issue