healthcheck.yml

- name: Health check buildbot worker machines
  hosts: "{{ target_hosts | default('all') }}"
  become: true
  gather_facts: false
  tasks:
    - name: Run health checks
      shell: |
        errors=""
        warnings=""

        # Service checks
        systemctl is-active buildbot-worker@*.service >/dev/null 2>&1 || \
          errors="${errors}FAIL: buildbot worker service not running\n"
        systemctl is-active fail2ban >/dev/null 2>&1 || \
          errors="${errors}FAIL: fail2ban not active\n"
        systemctl is-active tuned >/dev/null 2>&1 || \
          warnings="${warnings}WARN: tuned not active\n"
        systemctl is-active dnf-automatic-install.timer >/dev/null 2>&1 || \
          systemctl is-active dnf-automatic.timer >/dev/null 2>&1 || \
          systemctl is-active dnf5-automatic.timer >/dev/null 2>&1 || \
          warnings="${warnings}WARN: no automatic updates timer active\n"
        systemctl is-active fstrim.timer >/dev/null 2>&1 || \
          warnings="${warnings}WARN: fstrim.timer not active\n"
        systemctl is-active chronyd >/dev/null 2>&1 || \
          warnings="${warnings}WARN: chronyd not active\n"

        # Security checks
        selinux_status=$(getenforce 2>/dev/null)
        if [ "$selinux_status" != "Enforcing" ] && [ -n "$selinux_status" ]; then
          errors="${errors}FAIL: SELinux is ${selinux_status}, expected Enforcing\n"
        fi
        sshd_effective=$(sshd -T 2>/dev/null)
        if [ -n "$sshd_effective" ]; then
          echo "$sshd_effective" | grep -q '^passwordauthentication no' || \
            errors="${errors}FAIL: PasswordAuthentication is not set to no\n"
          echo "$sshd_effective" | grep -q '^permitrootlogin no' || \
            errors="${errors}FAIL: PermitRootLogin is not set to no\n"
          echo "$sshd_effective" | grep -q '^kbdinteractiveauthentication no' || \
            warnings="${warnings}WARN: KbdInteractiveAuthentication not set to no\n"
          echo "$sshd_effective" | grep -q '^maxauthtries 3' || \
            warnings="${warnings}WARN: MaxAuthTries not set to 3\n"
          echo "$sshd_effective" | grep -q '^loglevel VERBOSE' || \
            warnings="${warnings}WARN: LogLevel not set to VERBOSE\n"
          echo "$sshd_effective" | grep -q '^logingracetime 30' || \
            warnings="${warnings}WARN: LoginGraceTime not set to 30\n"
          echo "$sshd_effective" | grep -q '^gssapiauthentication no' || \
            warnings="${warnings}WARN: GSSAPIAuthentication not set to no\n"
          echo "$sshd_effective" | grep -q '^allowusers ' || \
            errors="${errors}FAIL: AllowUsers not configured\n"
        else
          grep -q '^PasswordAuthentication no' /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf 2>/dev/null || \
            errors="${errors}FAIL: PasswordAuthentication is not set to no\n"
          grep -q '^PermitRootLogin no' /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf 2>/dev/null || \
            errors="${errors}FAIL: PermitRootLogin is not set to no\n"
          grep -q '^KbdInteractiveAuthentication no' /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf 2>/dev/null || \
            warnings="${warnings}WARN: KbdInteractiveAuthentication not set to no\n"
          grep -q '^MaxAuthTries 3' /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf 2>/dev/null || \
            warnings="${warnings}WARN: MaxAuthTries not set to 3\n"
          grep -q '^LogLevel VERBOSE' /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf 2>/dev/null || \
            warnings="${warnings}WARN: LogLevel not set to VERBOSE\n"
          grep -q '^LoginGraceTime 30' /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf 2>/dev/null || \
            warnings="${warnings}WARN: LoginGraceTime not set to 30\n"
          grep -q '^GSSAPIAuthentication no' /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf 2>/dev/null || \
            warnings="${warnings}WARN: GSSAPIAuthentication not set to no\n"
          grep -q '^AllowUsers' /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf 2>/dev/null || \
            errors="${errors}FAIL: AllowUsers not configured\n"
        fi

        default_zone=$(firewall-cmd --get-default-zone 2>/dev/null)
        if [ -n "$default_zone" ] && [ "$default_zone" != "drop" ] && [ "$default_zone" != "block" ]; then
          errors="${errors}FAIL: firewall default zone is ${default_zone}, expected drop or block\n"
        fi
        if [ -n "$default_zone" ]; then
          fw_services=$(firewall-cmd --list-services 2>/dev/null)
          for svc in $fw_services; do
            case "$svc" in
              ssh|dhcpv6-client) ;;
              *) warnings="${warnings}WARN: unexpected firewall service: ${svc}\n" ;;
            esac
          done
        fi

        avc_count=$(ausearch -m avc -ts recent 2>/dev/null | grep -c '^type=AVC' || true)
        if [ "$avc_count" -gt 0 ]; then
          warnings="${warnings}WARN: ${avc_count} recent SELinux AVC denials\n"
        fi

        # Resource checks
        root_usage=$(df / --output=pcent | tail -1 | tr -d ' %')
        if [ "$root_usage" -ge 90 ]; then
          errors="${errors}FAIL: root filesystem at ${root_usage}%\n"
        fi

        home_dev=$(df --output=source /home 2>/dev/null | tail -1)
        root_dev=$(df --output=source / | tail -1)
        if [ "$home_dev" != "$root_dev" ]; then
          home_usage=$(df /home --output=pcent | tail -1 | tr -d ' %')
          if [ "$home_usage" -ge 90 ]; then
            errors="${errors}FAIL: /home filesystem at ${home_usage}%\n"
          fi
        fi

        total_mem=$(awk '/MemTotal/ {print int($2/1024/1024)}' /proc/meminfo)
        total_swap=$(awk '/SwapTotal/ {print $2}' /proc/meminfo)
        if [ "$total_mem" -lt 8 ] && [ "$total_swap" -eq 0 ]; then
          warnings="${warnings}WARN: ${total_mem}GB RAM with no swap configured\n"
        fi

        zombie_count=$(ps -eo stat,args 2>/dev/null | grep -c '^Z.*buildbot-worker' || true)
        if [ "$zombie_count" -gt 0 ]; then
          errors="${errors}FAIL: ${zombie_count} zombie buildbot-worker processes\n"
        fi

        coredump_size=$(du -sm /var/lib/systemd/coredump/ 2>/dev/null | awk '{print $1}')
        if [ "${coredump_size:-0}" -ge 1024 ]; then
          warnings="${warnings}WARN: coredumps using ${coredump_size}MB\n"
        fi

        # System configuration checks
        systemd-analyze cat-config systemd/journald.conf 2>/dev/null | grep -q '^SystemMaxUse=500M' || \
          warnings="${warnings}WARN: journald SystemMaxUse not set to 500M\n"
        eff_limit=$(dnf --dump-main-config 2>/dev/null | awk '/^installonly_limit/{print $3}')
        if [ -z "$eff_limit" ]; then
          eff_limit=$(python3 -c 'import dnf; b = dnf.Base(); b.conf.read(); print(b.conf.installonly_limit)' 2>/dev/null)
        fi
        if [ -n "$eff_limit" ] && [ "$eff_limit" != "3" ]; then
          warnings="${warnings}WARN: installonly_limit is ${eff_limit}, expected 3\n"
        fi
        tuned_profile=$(tuned-adm active 2>/dev/null | awk -F': ' '{print $2}')
        if [ -n "$tuned_profile" ] && [ "$tuned_profile" != "throughput-performance" ]; then
          warnings="${warnings}WARN: tuned profile is ${tuned_profile}, expected throughput-performance\n"
        fi

        # Boot filesystem and kernel checks
        boot_usage=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
        if [ -n "$boot_usage" ]; then
          if [ "$boot_usage" -ge 90 ]; then
            errors="${errors}FAIL: /boot filesystem at ${boot_usage}%\n"
          elif [ "$boot_usage" -ge 75 ]; then
            warnings="${warnings}WARN: /boot filesystem at ${boot_usage}%\n"
          fi
        fi

        journal_usage=$(journalctl --disk-usage 2>/dev/null)
        if echo "$journal_usage" | grep -qE '[0-9]+\.?[0-9]*G'; then
          journal_size=$(echo "$journal_usage" | grep -oE '[0-9]+\.?[0-9]*G' | head -1)
          warnings="${warnings}WARN: journal using ${journal_size} on disk\n"
        fi

        rescue_count=$(ls /boot/vmlinuz-0-rescue-* 2>/dev/null | wc -l)
        if [ "$rescue_count" -gt 1 ]; then
          warnings="${warnings}WARN: ${rescue_count} rescue kernels in /boot (expected 1)\n"
        fi

        kernel_count=$(rpm -qa kernel-core 2>/dev/null | wc -l)
        if [ -n "$eff_limit" ] && [ "$kernel_count" -gt "$eff_limit" ]; then
          warnings="${warnings}WARN: ${kernel_count} kernels installed, installonly_limit is ${eff_limit}\n"
        fi

        # SystemTap checks for CPython test_dtrace
        os_id=""
        os_version_id=""
        if [ -r /etc/os-release ]; then
          . /etc/os-release
          os_id=${ID:-}
          os_version_id=${VERSION_ID%%.*}
        fi
        if ! { [ "$os_id" = "rhel" ] || [ "$os_id" = "centos" ]; } || [ "${os_version_id:-0}" -gt 8 ]; then
          rpm -q systemtap-client >/dev/null 2>&1 || \
            warnings="${warnings}WARN: systemtap-client not installed\n"
          rpm -q systemtap-runtime >/dev/null 2>&1 || \
            warnings="${warnings}WARN: systemtap-runtime not installed\n"
          rpm -q systemtap-devel >/dev/null 2>&1 || \
            warnings="${warnings}WARN: systemtap-devel not installed\n"
          rpm -q "kernel-devel-$(uname -r)" >/dev/null 2>&1 || \
            warnings="${warnings}WARN: matching kernel-devel for $(uname -r) not installed\n"
          command -v stap >/dev/null 2>&1 || \
            warnings="${warnings}WARN: stap command not available\n"
          command -v dtrace >/dev/null 2>&1 || \
            warnings="${warnings}WARN: dtrace command not available\n"
          id -nG buildbot-worker 2>/dev/null | grep -qw stapusr || \
            warnings="${warnings}WARN: buildbot-worker is not in stapusr\n"
          id -nG buildbot-worker 2>/dev/null | grep -qw stapdev || \
            warnings="${warnings}WARN: buildbot-worker is not in stapdev\n"
        fi

        # Buildbot-specific checks
        tac_file=$(find /var/lib/buildbot/worker /home/buildbot-worker -name buildbot.tac 2>/dev/null | head -1)
        if [ -n "$tac_file" ]; then
          grep -q '^delete_leftover_dirs = 1' "$tac_file" || \
            warnings="${warnings}WARN: delete_leftover_dirs not set to 1\n"
          grep -q '^keepalive = 60' "$tac_file" || \
            warnings="${warnings}WARN: keepalive not set to 60\n"

          log_dir=$(dirname "$tac_file")
          log_size=$(du -sm "$log_dir/twistd.log" 2>/dev/null | awk '{print $1}')
          if [ "${log_size:-0}" -ge 100 ]; then
            warnings="${warnings}WARN: twistd.log is ${log_size}MB\n"
          fi

          last_log=$(tail -5 "$log_dir/twistd.log" 2>/dev/null)
          echo "$last_log" | grep -q "Scheduling retry\|connection already shut down" && \
            errors="${errors}FAIL: worker disconnected from master\n"
        else
          errors="${errors}FAIL: no buildbot.tac found\n"
        fi

        # Summary
        hostname=$(hostname)
        if [ -z "$errors" ] && [ -z "$warnings" ]; then
          echo "${hostname}: ALL CHECKS PASSED"
        else
          printf "${hostname}:\n${errors}${warnings}"
        fi
      register: healthcheck
      changed_when: false

    - name: Show results
      debug:
        msg: "{{ healthcheck.stdout_lines }}"

    - name: Fail if any checks failed
      fail:
        msg: "Health check failures detected"
      when: "'FAIL' in healthcheck.stdout"