From b8f968dc23942070055e43139728f742d4a42c2e Mon Sep 17 00:00:00 2001 From: Moritz 'e1mo' Fromm Date: Sat, 5 Aug 2023 13:04:52 +0200 Subject: [PATCH 1/2] Rework diff-hosts Now it'll detect wether there are changes to a hosts closure at all, lists build failures as such and is able to handle newly added or removed hosts. --- .github/scripts/diff-hosts.sh | 112 +++++++++++++++++++++++++++ .github/workflows/build-systems.yaml | 30 +------ 2 files changed, 116 insertions(+), 26 deletions(-) create mode 100755 .github/scripts/diff-hosts.sh diff --git a/.github/scripts/diff-hosts.sh b/.github/scripts/diff-hosts.sh new file mode 100755 index 0000000..8d756ab --- /dev/null +++ b/.github/scripts/diff-hosts.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# host_exists(flake, host) +# Checks if ${flake}#nixosConfigurations.$host +# Returns 0 if it does, 1 if it does not +function host_exists() { + local cmd="nix eval $1#nixosConfigurations --apply 'builtins.hasAttr \"$2\"'" + # echo "$cmd" + has_host=$(eval "$cmd") + if [[ $has_host == "true" ]]; then + return 0 + else + return 1 + fi +} + +# get_hosts(flake, available_hosts) +# Store a list of all available hosts in flake to $available_hosts +function get_hosts() { + local -n hosts=$2 + IFS=" " read -r -a hosts <<< "$(nix eval $1#nixosConfigurations --raw --apply 'x: builtins.concatStringsSep " " (builtins.attrNames x)')" +} + +set -o errexit +# When using to check local changes, the step summary is not needed +step_summary="${GITHUB_STEP_SUMMARY-/dev/null}" + +before_ref="${GITHUB_BASE_REF-main}" +before_ref="origin/${before_ref/#refs\/heads\//}" +if [[ $GITHUB_REF == "target/refs/main" ]]; then + # If triggered on main, compare with the previous commit + before_ref="$(git log HEAD~1 -1 --format=format:"%H")" +fi + +before_rev="$(git rev-parse "$before_ref")" +before_rev_abbr="$before_ref" +before_flake="git+file:.?ref=${before_rev}" + +after_rev="$(git rev-parse --verify HEAD)" +after_rev_abbr="$(git rev-parse --abbrev-ref HEAD)" +if [[ -z $(git status --short) ]]; then + # If the working tree is clean, we can use the latest commit hash directly + # and thus profit from even more reduced build times + after_flake="git+file:.?ref=${after_rev}" +else + # That way the script can be used to check local (non commited) changes + after_flake="." +fi + +before_hosts=() +get_hosts "$before_flake" before_hosts +after_hosts=() +get_hosts "$after_flake" after_hosts +all_hosts=( "${before_hosts[@]}" "${after_hosts[@]}" ) + +# Unite both arrays +# From https://stackoverflow.com/a/33153989 +declare -A _all_hosts +for k in "${all_hosts[@]}"; do _all_hosts["$k"]=1; done +all_hosts=("${!_all_hosts[@]}") + +echo -e "## Closure differences\n" >> "$step_summary" +dirty="" +if [[ -n $(git status --short) ]]; then + dirty="-DIRTY" +fi +echo -e "_Comparing prior $before_rev_abbr ([$before_rev]) with current $after_rev_abbr ([$after_rev]$dirty)_\n" | tee -a "$step_summary" +repo_url="${GITHUB_SERVER_URL}/$GITHUB_REPOSITORY" +{ + echo "[$before_rev]: $repo_url/commit/$before_rev"; + echo "[$after_rev]: $repo_url/commit/$after_rev"; +} >> "$step_summary" + + +for host in "${all_hosts[@]}"; do + host_exists "$before_flake" "$host" && a_exists=true || a_exists=false + host_exists "$after_flake" "$host" && b_exists=true || b_exists=false + if $a_exists && ! $b_exists; then + echo -e "**${host}** was removed\n" | tee -a "$step_summary" + elif ! $a_exists && $b_exists; then + echo -e "**${host}** was added\n" | tee -a "$step_summary" + else + echo "::group::Diff-closures for ${host}" + drv="nixosConfigurations.$host.config.system.build.toplevel" + diff_cmd="nix store diff-closures ${before_flake}#${drv} ${after_flake}#${drv}" + # Initially build host so that we don't spam the log + change_lines=$($diff_cmd 2>&1 | wc -l) + # Run once so that we have the colorful output in the github actions log + $diff_cmd && build_ok=true || build_ok=false + if [[ $change_lines -gt 0 ]] || [[ ! $build_ok ]]; then + if $build_ok; then + echo "ok" + echo "${host} changed" + echo -e "
\n${host} changed\n\n" >> "$step_summary" + else + echo "${host} failed to build" + echo -e "
\n${host} failed to build\n\n" >> "$step_summary" + fi + + { + echo '```'; + $diff_cmd --show-trace | sed -e 's/\x1b\[[0-9;]*m//g'; + echo '```'; + echo -e "\n
\n"; + } >> "$step_summary" 2>&1 + else + echo -e "**${host}** has not changed\n" | tee -a "$step_summary" + fi + + echo "::endgroup::" + fi +done diff --git a/.github/workflows/build-systems.yaml b/.github/workflows/build-systems.yaml index 1c04746..ff7f63d 100644 --- a/.github/workflows/build-systems.yaml +++ b/.github/workflows/build-systems.yaml @@ -26,6 +26,7 @@ jobs: echo Discovered the following system configs nix flake show . --json | jq ".nixosConfigurations|keys[]" -r - name: Build systems + id: build run: | echo "## Builds succeeded" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY @@ -43,29 +44,6 @@ jobs: echo "| ${host} | \`${out_path}\` |" >> $GITHUB_STEP_SUMMARY done - name: Diff closures - # Since this is only triggered by pushes to main, no need to compare to main for pushes - if: github.event_name != 'push' - run: | - # Compare to pull_request_target or, if this has no PR-Target, main - target_ref="${GITHUB_BASE_REF-main}" - target_ref="origin/${target_ref/#refs\/heads\//}" - if [[ $GITHUB_REF == "target/refs/main" ]]; then - # If triggered on main, compare with the previous commit - target_ref="$(git log HEAD~1 -1 --format=format:"%H")" - fi - echo -e "## Closure differences\n" >> $GITHUB_STEP_SUMMARY - echo "_Comparing current $(git rev-parse --abbrev-ref HEAD) ($(git rev-parse --verify HEAD)) to ${target_ref} ($(git rev-parse $target_ref))_" | tee -a $GITHUB_STEP_SUMMARY - for host in $(nix flake show . --json | jq ".nixosConfigurations|keys[]" -r); do - echo "::group::Diff-closures for ${host}" - drv="nixosConfigurations.$host.config.system.build.toplevel" - diff_cmd="nix store diff-closures git+file:.?ref=${target_ref}#${drv} .#${drv}" - # Get the nice and colorfull output for the logs, running twice won't (significantly) prolong the runtime - $diff_cmd - echo -e "
\nDiff for ${host}\n" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - # Strip color codes so that the summary stays readable - $diff_cmd | sed -e 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo -e "\n
" >> $GITHUB_STEP_SUMMARY - echo "::endgroup::" - done + # Still would like to see the changes for the non-failing hosts + if: ${{ success() }} || ${{ failure() }} + run: ./.github/scripts/diff-hosts.sh From 3ec3fb9416a936b0e37468663e4e6ca34fe0d7c6 Mon Sep 17 00:00:00 2001 From: Moritz 'e1mo' Fromm Date: Sat, 5 Aug 2023 13:49:10 +0200 Subject: [PATCH 2/2] Add error handling to ci build job --- .github/workflows/build-systems.yaml | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-systems.yaml b/.github/workflows/build-systems.yaml index ff7f63d..9ba73a1 100644 --- a/.github/workflows/build-systems.yaml +++ b/.github/workflows/build-systems.yaml @@ -28,21 +28,32 @@ jobs: - name: Build systems id: build run: | - echo "## Builds succeeded" >> $GITHUB_STEP_SUMMARY + echo "## Building" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "| Host | Out path |" >> $GITHUB_STEP_SUMMARY - echo "| ---- | -------- |" >> $GITHUB_STEP_SUMMARY + echo "| | Host | Out path |" >> $GITHUB_STEP_SUMMARY + echo "| --- | ---- | -------- |" >> $GITHUB_STEP_SUMMARY + any_failed=false for host in $(nix flake show . --json | jq ".nixosConfigurations|keys[]" -r); do echo "::group::Building ${host}" drv=".#nixosConfigurations.$host.config.system.build.toplevel" build_cmd="nix build ${drv}" - #cachix watch-exec chaos-jetzt-nixfiles -- $build_cmd - $build_cmd + failed=false + $build_cmd --show-trace || { failed=true; any_failed=true; } echo "::endgroup::" - out_path=$($build_cmd --print-out-paths) - echo -e "\x1b[32;1mSuccessfully built .#nixosConfigurations.${host}\x1b[0m" - echo "| ${host} | \`${out_path}\` |" >> $GITHUB_STEP_SUMMARY + if ! $failed; then + symbol="✅" + out_path=$($build_cmd --print-out-paths) + out_path="\`${out_path}\`" + echo -e "\x1b[32;1mSuccessfully built .#nixosConfigurations.${host}\x1b[0m" + else + symbol="❌" + out_path="_build failed_" + echo -e "\x1b[31;1mFailed to build .#nixosConfigurations.${host}\x1b[0m" + fi + echo "| $symbol | ${host} | ${out_path} |" >> $GITHUB_STEP_SUMMARY done + # Set return-code to 1 if any failed + ! $any_failed - name: Diff closures # Still would like to see the changes for the non-failing hosts if: ${{ success() }} || ${{ failure() }}