netdata_netdata/.github/workflows/build.yml

---
# CI code for building release artifacts.
name: Build
on:
  push: # Master branch checks only validate the build and generate artifacts for testing.
    branches:
      - master
  pull_request: null # PR checks only validate the build and generate artifacts for testing.
  workflow_dispatch: # Dispatch runs build and does limited validation, then pushes to the appropriate storage location.
    inputs:
      type:
        description: Build Type
        default: nightly
        required: true
      version:
        description: Version Tag
        default: nightly
        required: true
concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type.
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true
jobs:
  file-check: # Check what files changed if we’re being run in a PR or on a push.
    name: Check Modified Files
    runs-on: ubuntu-latest
    outputs:
      run: ${{ steps.check-run.outputs.run }}
      skip-go: ${{ steps.check-go.outputs.skip-go }}
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Check source files
        id: check-source-files
        uses: tj-actions/changed-files@v45
        with:
          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
          files: |
            **/*.c
            **/*.cc
            **/*.h
            **/*.hh
            **/*.in
            **/*.patch
            src/aclk/aclk-schemas/
            src/ml/dlib/
            src/fluent-bit/
            src/web/server/h2o/libh2o/            
          files_ignore: |
            netdata.spec.in
            **/*.md            
      - name: Check build files
        id: check-build-files
        uses: tj-actions/changed-files@v45
        with:
          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
          files: |
            **/*.cmake
            CMakeLists.txt
            netdata-installer.sh
            .github/data/distros.yml
            .github/workflows/build.yml
            .github/scripts/build-static.sh
            .github/scripts/get-static-cache-key.sh
            .github/scripts/gen-matrix-static.py
            .github/scripts/gen-matrix-build.py
            .github/scripts/run-updater-check.sh
            packaging/cmake/
            packaging/makeself/
            packaging/installer/
            packaging/windows/
            packaging/*.sh
            packaging/*.version
            packaging/*.checksums            
          files_ignore: |
            **/*.md
            packaging/repoconfig/            
      - name: List all changed files in pattern
        continue-on-error: true
        if: github.event_name != 'workflow_dispatch'
        env:
          CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
          CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
        run: |
          for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
            echo "$file was changed"
          done          
      - name: Check Run
        id: check-run
        run: |
          if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            echo 'run=true' >> "${GITHUB_OUTPUT}"
          else
            echo 'run=false' >> "${GITHUB_OUTPUT}"
          fi          
      - name: Check Go
        id: check-go
        run: |
          if [ '${{ github.event_name }}' == 'pull_request' ]; then
            if echo "${{ steps.check-source-files.outputs.other_changed_files }}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
              echo 'skip-go=' >> "${GITHUB_OUTPUT}"
            else
              echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
            fi
          else
            echo 'skip-go=' >> "${GITHUB_OUTPUT}"
          fi          

  build-dist: # Build the distribution tarball and store it as an artifact.
    name: Build Distribution Tarball
    runs-on: ubuntu-latest
    needs:
      - file-check
    outputs:
      distfile: ${{ steps.build.outputs.distfile }}
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Fix tags
        id: fix-tags
        if: github.event_name != 'push' && needs.file-check.outputs.run == 'true'
        run: |
          git fetch --tags --force          
      - name: Mark Stable
        id: channel
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true'
        run: |
          sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh          
      - name: Build
        id: build
        if: needs.file-check.outputs.run == 'true'
        run: |
          mkdir -p artifacts/
          tar --create --file "artifacts/netdata-$(git describe).tar.gz" \
              --sort=name --posix --auto-compress --exclude=artifacts/ --exclude=.git \
              --exclude=.gitignore --exclude=.gitattributes --exclude=.gitmodules \
              --transform "s/^\\.\\//netdata-$(git describe)\\//" --verbose .
          cd artifacts/
          echo "distfile=$(find . -name 'netdata-*.tar.gz')" >> "${GITHUB_OUTPUT}"          
      - name: Store
        id: store
        if: needs.file-check.outputs.run == 'true'
        uses: actions/upload-artifact@v4.6.1
        with:
          name: dist-tarball
          path: artifacts/*.tar.gz
          retention-days: 30
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Distribution tarball creation failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to create source tarball for distribution.
              Checkout: ${{ steps.checkout.outcome }}
              Fix Tags: ${{ steps.fix-tags.outcome }}
              Mark stable: ${{ steps.channel.outcome }}
              Build: ${{ steps.build.outcome }}
              Store: ${{ steps.store.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}          

  static-matrix: # Generate the static build matrix.
    name: Prepare Build Matrix
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
      - name: Prepare tools
        id: prepare
        run: |
          sudo apt-get update || true
          sudo apt-get install -y python3-ruamel.yaml          
      - name: Read build matrix
        id: set-matrix
        run: |
          matrix="$(.github/scripts/gen-matrix-static.py)"
          echo "Generated matrix: ${matrix}"
          echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"          
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Static build matrix preparation failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to prepare build matrix for build checks.
              Checkout: ${{ steps.checkout.outcome }}
              Prepare tools: ${{ steps.prepare.outcome }}
              Read build matrix: ${{ steps.set-matrix.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
          }}          

  build-static: # Build the static binary archives, and store them as artifacts.
    name: Build Static
    needs:
      - file-check
      - static-matrix
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.static-matrix.outputs.matrix) }}
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Fix tags
        id: fix-tags
        if: github.event_name != 'push' && needs.file-check.outputs.run == 'true'
        run: |
          git fetch --tags --force          
      - name: Mark Stable
        id: channel
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true'
        run: |
          sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh packaging/makeself/install-or-update.sh          
      - name: Get Cache Key
        if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true'
        id: cache-key
        run: .github/scripts/get-static-cache-key.sh ${{ matrix.arch }} "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache') }}"
      - name: Cache
        if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true'
        id: cache
        uses: actions/cache@v4
        with:
          path: artifacts/cache
          key: ${{ steps.cache-key.outputs.key }}
      - name: Set up QEMU
        id: qemu
        if: matrix.qemu && needs.file-check.outputs.run == 'true'
        run: |
          sudo apt-get update
          sudo apt-get upgrade -y
          sudo apt-get install -y qemu-user-static          
      - name: Build
        if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs.
        run: |
          export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
          [ "${{ matrix.qemu }}" == "true" ] || export SKIP_EMULATION=1
          .github/scripts/build-static.sh ${{ matrix.arch }}          
      - name: Build
        if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true'
        id: build
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 360
          max_attempts: 3
          command: |
            export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
            [ "${{ matrix.qemu }}" == "true" ] || export SKIP_EMULATION=1
            .github/scripts/build-static.sh ${{ matrix.arch }}            
      - name: Store
        id: store
        if: needs.file-check.outputs.run == 'true'
        uses: actions/upload-artifact@v4.6.1
        with:
          name: dist-static-${{ matrix.arch }}
          path: artifacts/*.gz.run
          retention-days: 30
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Static build failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to create static installer archive for ${{ matrix.arch }}.
              Checkout: ${{ steps.checkout.outcome }}
              Fix Tags: ${{ steps.fix-tags.outcome }}
              Mark stable: ${{ steps.channel.outcome }}
              Set up QEMU: ${{ steps.qemu.outcome }}
              Build: ${{ steps.build.outcome }}
              Store: ${{ steps.store.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}          

  windows-build: # Test building on Windows
    name: Test building on Windows
    runs-on: windows-latest
    needs:
      - file-check
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: Write-Output "SKIPPED"
      - name: Checkout
        uses: actions/checkout@v4
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        with:
          submodules: recursive
          lfs: true
      - name: Set Up Go
        id: golang
        if: needs.file-check.outputs.run == 'true'
        uses: actions/setup-go@v5
        with:
          go-version: "^1.23"
      - name: Set Up Dependencies
        id: deps
        if: needs.file-check.outputs.run == 'true'
        run: ./packaging/windows/install-dependencies.ps1
      - name: Build Netdata
        id: build
        if: needs.file-check.outputs.run == 'true'
        env:
          BUILD_DIR: ${{ github.workspace }}\build
        run: ./packaging/windows/build.ps1
      - name: Sign Agent Code
        id: sign-agent
        if: needs.file-check.outputs.run == 'true' && github.event_name != 'pull_request'
        uses: azure/trusted-signing-action@v0.5.1
        with:
          azure-tenant-id: ${{ secrets.CODE_SIGNING_TENNANT_ID }}
          azure-client-id: ${{ secrets.CODE_SIGNING_CLIENT_ID }}
          azure-client-secret: ${{ secrets.CODE_SIGNING_CLIENT_SECRET }}
          endpoint: "https://eus.codesigning.azure.net/"
          trusted-signing-account-name: Netdata
          certificate-profile-name: Netdata
          files-folder: ${{ github.workspace }}\build
          files-folder-filter: exe,dll
          files-folder-recurse: true
          file-digest: SHA256
          timestamp-rfc3161: "http://timestamp.acs.microsoft.com"
          timestamp-digest: SHA256
      - name: Package Netdata
        id: package
        if: needs.file-check.outputs.run == 'true'
        env:
          BUILD_DIR: ${{ github.workspace }}\build
        run: ./packaging/windows/package.ps1
      - name: Sign Installer
        id: sign-installer
        if: needs.file-check.outputs.run == 'true' && github.event_name != 'pull_request'
        uses: azure/trusted-signing-action@v0.5.1
        with:
          azure-tenant-id: ${{ secrets.CODE_SIGNING_TENNANT_ID }}
          azure-client-id: ${{ secrets.CODE_SIGNING_CLIENT_ID }}
          azure-client-secret: ${{ secrets.CODE_SIGNING_CLIENT_SECRET }}
          endpoint: "https://eus.codesigning.azure.net/"
          trusted-signing-account-name: Netdata
          certificate-profile-name: Netdata
          files-folder: ${{ github.workspace }}\packaging\windows
          files-folder-filter: msi
          file-digest: SHA256
          timestamp-rfc3161: "http://timestamp.acs.microsoft.com"
          timestamp-digest: SHA256
      - name: Upload Installer
        id: upload
        uses: actions/upload-artifact@v4.6.1
        with:
          name: windows-x86_64-installer
          path: packaging\windows\netdata*.msi
          retention-days: 30
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Windows build failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Windows build failed.
              Checkout: ${{ steps.checkout.outcome }}
              Set Up Dependencies: ${{ steps.deps.outcome }}
              Build Netdata: ${{ steps.build.outcome }}
              Sign Agent Code: ${{ steps.sign-agent.outcome }}
              Package Netdata: ${{ steps.package.outcome }}
              Sign Installer: ${{ steps.sign-installer.outcome }}
              Upload Installer: ${{ steps.upload.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}          

  prepare-upload: # Consolidate the artifacts for uploading or releasing.
    name: Prepare Artifacts
    runs-on: ubuntu-latest
    needs:
      - build-dist
      - build-static
      - windows-build
      - file-check
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
      - name: Prepare Environment
        id: prepare
        if: needs.file-check.outputs.run == 'true'
        run: mkdir -p artifacts
      - name: Retrieve Build Artifacts
        id: fetch-dist
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            pattern: dist-*
            path: dist-artifacts
            merge-multiple: true            
          attempt_limit: 3
          attempt_delay: 2000
      - name: Retrieve Windows Artifacts
        id: fetch-windows
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            pattern: windows-*-installer
            path: dist-artifacts
            merge-multiple: true            
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare Artifacts
        id: consolidate
        if: needs.file-check.outputs.run == 'true'
        working-directory: ./artifacts/
        run: |
          mv ../dist-artifacts/* . || exit 1
          ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1
          cp ../packaging/version ./latest-version.txt || exit 1
          sha256sum -b ./* > sha256sums.txt || exit 1
          cat sha256sums.txt          
      - name: Store Artifacts
        id: store
        if: needs.file-check.outputs.run == 'true'
        uses: actions/upload-artifact@v4.6.1
        with:
          name: final-artifacts
          path: artifacts/*
          retention-days: 30
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Failed to prepare release artifacts for upload:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to prepare release artifacts for upload.
              Checkout: ${{ steps.checkout.outcome }}
              Prepare environment: ${{ steps.prepare.outcome }}
              Fetch dist artifacts: ${{ steps.fetch-dist.outcome }}
              Fetch Windows installers: ${{ steps.fetch-windows.outcome }}
              Consolidate artifacts: ${{ steps.consolidate.outcome }}
              Store: ${{ steps.store.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}          

  artifact-verification-dist: # Verify the regular installer works with the consolidated artifacts.
    name: Test Consolidated Artifacts (Source)
    runs-on: ubuntu-latest
    needs:
      - prepare-upload
      - file-check
    services:
      apache: # This gets used to serve the dist tarball for the updater script.
        image: httpd:2.4
        ports:
          - 8080:80
        volumes:
          - ${{ github.workspace }}:/usr/local/apache2/htdocs/
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
      - name: Fetch artifacts
        id: fetch
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: artifacts            
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare artifacts directory
        id: prepare
        if: needs.file-check.outputs.run == 'true'
        run: |
          mkdir -p download/latest
          mv artifacts/* download/latest
          ls -al download/latest          
      - name: Verify that artifacts work with installer
        id: verify
        if: needs.file-check.outputs.run == 'true'
        env:
          NETDATA_TARBALL_BASEURL: http://localhost:8080/
        run: sh -x packaging/installer/kickstart.sh --build-only --dont-start-it --disable-telemetry --dont-wait
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Artifact verification for source tarball failed.'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Artifact verification for source tarball failed.
              Checkout: ${{ steps.checkout.outcome }}
              Fetch artifacts: ${{ steps.fetch.outcome }}
              Verify artifacts: ${{ steps.verify.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}          

  artifact-verification-static: # Verify the static installer works with the consolidated artifacts.
    name: Test Consolidated Artifacts (Static)
    runs-on: ubuntu-latest
    needs:
      - prepare-upload
      - file-check
    services:
      apache: # This gets used to serve the static archives.
        image: httpd:2.4
        ports:
          - 8080:80
        volumes:
          - ${{ github.workspace }}:/usr/local/apache2/htdocs/
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
      - name: Fetch artifacts
        id: fetch-artifacts
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: artifacts            
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare artifacts directory
        id: prepare
        if: needs.file-check.outputs.run == 'true'
        run: |
          mkdir -p download/latest
          mv artifacts/* download/latest
          ls -al download/latest          
      - name: Verify that artifacts work with installer
        id: verify
        if: needs.file-check.outputs.run == 'true'
        env:
          NETDATA_TARBALL_BASEURL: http://localhost:8080/
        run: sh -x packaging/installer/kickstart.sh --static-only --dont-start-it --disable-telemetry
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Artifact verification for static build failed.'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Artifact verification for static build failed.
              Checkout:  ${{ steps.checkout.outcome }}
              Fetch artifacts: ${{ steps.fetch-artifacts.outcome }}
              Verify artifacts: ${{ steps.verify.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}          

  artifact-verification-updater: # Test the generated dist archive using the updater code.
    name: Test Consolidated Artifacts (Updater)
    runs-on: ubuntu-latest
    needs:
      - prepare-upload
      - file-check
    services:
      apache: # This gets used to serve the dist tarball for the updater script.
        image: httpd:2.4
        ports:
          - 8080:80
        volumes:
          - ${{ github.workspace }}:/usr/local/apache2/htdocs/
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
      - name: Fetch artifacts
        id: fetch-artifacts
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: artifacts            
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare artifacts directory
        id: prepare
        if: needs.file-check.outputs.run == 'true'
        run: |
          mkdir -p download/latest
          mv artifacts/* download/latest
          ls -al download/latest          
      - name: Run Updater Check
        id: check
        if: needs.file-check.outputs.run == 'true'
        run: |
          docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host \
                 -v $PWD:/netdata -w /netdata \
                 ubuntu:latest /bin/sh -x /netdata/.github/scripts/run-updater-check.sh          
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Updater checks  failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Updater checks for failed.
              Checkout: ${{ steps.checkout.outcome }}
              Fetch artifacts: ${{ steps.fetch-artifacts.outcome }}
              Prepare artifact directory: ${{ steps.prepare.outcome }}
              Updater check: ${{ steps.check.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}          

  create-nightly: # Create a nightly build release in netdata/netdata-nightlies
    name: Create Nightly Release
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata'
    needs:
      - prepare-upload
      - artifact-verification-dist
      - artifact-verification-static
    steps:
      - name: Checkout Main Repo
        id: checkout-main
        uses: actions/checkout@v4
        with:
          path: main
      - name: Checkout Nightly Repo
        id: checkout-nightly
        uses: actions/checkout@v4
        with:
          repository: netdata/netdata-nightlies
          path: nightlies
          token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
      - name: Retrieve Artifacts
        id: fetch
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: final-artifacts            
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare version info
        id: version
        run: |
          # shellcheck disable=SC2129
          echo "version=$(cat main/packaging/version)" >> "${GITHUB_OUTPUT}"
          echo "commit=$(cd nightlies && git rev-parse HEAD)" >> "${GITHUB_OUTPUT}"
          echo "date=$(date +%F)" >> "${GITHUB_OUTPUT}"          
      - name: Create Release
        id: create-release
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: false
          artifactErrorsFailBuild: true
          artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/netdata-*.msi'
          owner: netdata
          repo: netdata-nightlies
          body: Netdata nightly build for ${{ steps.version.outputs.date }}.
          commit: ${{ steps.version.outputs.commit }}
          makeLatest: true
          tag: ${{ steps.version.outputs.version }}
          token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
      - name: Checkout netdata main Repo # Checkout back to netdata/netdata repo to the update latest packaged versions
        id: checkout-netdata
        uses: actions/checkout@v4
        with:
          token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
      - name: Init python environment for publish release metadata
        uses: actions/setup-python@v5
        id: init-python
        with:
          python-version: "3.12"
      - name: Setup python environment
        id: setup-python
        run: |
          pip install -r .github/scripts/modules/requirements.txt          
      - name: Check if the version is latest and published
        id: check-latest-version
        run: |
          python .github/scripts/check_latest_versions.py  ${{ steps.version.outputs.version }}          
      - name: SSH setup
        id: ssh-setup
        if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
        uses: shimataro/ssh-key-action@v2
        with:
          key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
          name: id_ecdsa
          known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
      - name: Sync release info to packages.netdata.cloud
        id: sync-releases
        continue-on-error: true
        if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
        run: |
          .github/scripts/upload-new-version-tags.sh packages.netdata.cloud          
      - name: Sync release info to packages2.netdata.cloud
        id: sync-releases2
        if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
        run: |
          .github/scripts/upload-new-version-tags.sh packages.netdata.cloud          
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Failed to draft release:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to create nightly release or attach artifacts.
              Checkout netdata/netdata: ${{ steps.checkout-main.outcome }}
              Checkout netdata/netdata-nightlies: ${{ steps.checkout-nightly.outcome }}
              Fetch artifacts: ${{ steps.fetch.outcome }}
              Prepare version info: ${{ steps.version.outcome }}
              Create release: ${{ steps.create-release.outcome }}
              Checkout back netdata/netdata: ${{ steps.checkout-netdata.outcome }}
              Init python environment: ${{ steps.init-python.outcome }}
              Setup python environment: ${{ steps.setup-python.outcome }}
              Check the nearly published release against the advertised: ${{ steps.check-latest-version.outcome }}
              Setup ssh: ${{ steps.ssh-setup.outcome }}
              Sync release info to packages.netdata.cloud: ${{ steps.sync-releases.outcome }}
              Sync release info to packages2.netdata.cloud: ${{ steps.sync-releases2.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && github.event_name == 'workflow_dispatch'
          }}          

  normalize-tag: # Fix the release tag if needed
    name: Normalize Release Tag
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release'
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
    steps:
      - name: Normalize Tag
        id: tag
        run: |
          if echo ${{ github.event.inputs.version }} | grep -qE '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$'; then
            echo "tag=v${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
          else
            echo "tag=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
          fi          

  upload-release: # Create the draft release and upload the build artifacts.
    name: Create Release Draft
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release' && github.repository == 'netdata/netdata'
    needs:
      - artifact-verification-dist
      - artifact-verification-static
      - normalize-tag
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
      - name: Retrieve Artifacts
        id: fetch
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: final-artifacts            
          attempt_limit: 3
          attempt_delay: 2000
      - name: Create Release
        id: create-release
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: false
          artifactErrorsFailBuild: true
          artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/netdata-*.msi'
          draft: true
          tag: ${{ needs.normalize-tag.outputs.tag }}
          token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Failed to draft release:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to create draft release or attach artifacts.
              Checkout: ${{ steps.checkout.outcome }}
              Fetch artifacts: ${{ steps.fetch.outcome }}
              Create draft release: ${{ steps.create-release.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && github.event_name == 'workflow_dispatch'
          }}          
      - name: Success Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'good'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Created agent draft release:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: "${{ github.repository }}: ${{ steps.create-release.outputs.html_url }}"
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            success()
            && github.event_name == 'workflow_dispatch'
          }}          

  # Remaining jobs are only used for CI checks, and not as part of the release process

  src-matrix: # Generate the shared build matrix for our Linux build tests.
    name: Prepare Build Matrix
    runs-on: ubuntu-latest
    if: github.event_name != 'workflow_dispatch'
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
      - name: Prepare tools
        id: prepare
        run: |
          sudo apt-get update || true
          sudo apt-get install -y python3-ruamel.yaml          
      - name: Read build matrix
        id: set-matrix
        run: |
          matrix="$(.github/scripts/gen-matrix-build.py)"
          echo "Generated matrix: ${matrix}"
          echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"          
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Build matrix preparation failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to prepare build matrix for build checks.
              Checkout: ${{ steps.checkout.outcome }}
              Prepare tools: ${{ steps.prepare.outcome }}
              Read build matrix: ${{ steps.set-matrix.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
          }}          

  source-build: # Test various source build arrangements.
    name: Test Source Build
    runs-on: ubuntu-latest
    if: github.event_name != 'workflow_dispatch'
    needs:
      - src-matrix
      - file-check
    strategy:
      fail-fast: false
      max-parallel: 8
      matrix: ${{ fromJson(needs.src-matrix.outputs.matrix) }}
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Setup Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
      - name: Build test environment
        id: build
        uses: Wandalen/wretry.action@v3
        with:
          action: docker/build-push-action@v6
          with: |
            push: false
            load: true
            file: .github/dockerfiles/Dockerfile.build_test
            build-args: |
              BASE=${{ matrix.distro }}
              PRE=${{ matrix.env_prep }}
              RMJSONC=${{ matrix.jsonc_removal }}
            tags: test:${{ matrix.artifact_key }}            
          attempt_limit: 3
          attempt_delay: 15000
      - name: netdata-installer on ${{ matrix.distro }}
        id: build-cloud
        if: needs.file-check.outputs.run == 'true'
        run: |
          docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --one-time-build ${{ needs.file-check.outputs.skip-go }}'          
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Build tests for ${{ matrix.distro }} failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Build tests for ${{ matrix.distro }} failed.
              Checkout: ${{ steps.checkout.outcome }}
              Set up Buildx: ${{ steps.buildx.outcome }}
              Build test environment: ${{ steps.build.outcome }}
              netdata-installer: ${{ steps.build-cloud.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}          

  macos-build: # Test building on macOS
    name: Test building on macOS
    runs-on: ${{ matrix.runner }}
    if: github.event_name != 'workflow_dispatch'
    needs:
      - file-check
    strategy:
      fail-fast: false
      max-parallel: 8
      matrix:
        include:
          - name: macos-13
            runner: macos-13
          - name: macos-14-M1
            runner: macos-14
          - name: macos-15-M1
            runner: macos-15
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - uses: actions/checkout@v4
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        with:
          submodules: recursive
      - name: Install latest bash
        id: install-bash
        if: needs.file-check.outputs.run == 'true'
        run: |
          brew install bash          
      - name: Install netdata dependencies
        id: install-nd-dep
        if: needs.file-check.outputs.run == 'true'
        run: |
          bash ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all          
      - name: Build from source
        id: build-source
        if: needs.file-check.outputs.run == 'true'
        run: |
          sudo bash ./netdata-installer.sh --install-no-prefix /usr/local/netdata --dont-wait --dont-start-it --one-time-build          
      - name: Test Agent start up
        id: test-agent
        if: needs.file-check.outputs.run == 'true'
        run: |
          /usr/local/netdata/usr/sbin/netdata -D > ./netdata.log 2>&1 &
          ./packaging/runtime-check.sh          
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Build & test from source macOS failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: macOS Build and test.
              Checkout: ${{ steps.checkout.outcome }}
              Setup runner: ${{ steps.install-bash.outcome }}
              Install netdata required packages: ${{ steps.install-nd-dep.outcome }}
              Build from source: ${{ steps.build-source.outcome }}
              Test Agent runtime: ${{ steps.test-agent.outcome }}              
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
          }}