add a scsb connection checker on the recap_hosts

group_vars/checkmk/local_check_templates/scsb_connection_check.sh.j2

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+{{ ansible_managed | comment }}
+# States: 0=OK, 1=WARN, 2=CRIT, 3=UNKNOWN
+set -uo pipefail
+
+URL="{{ scsb_export_datadump_url | default('https://scsb.recaplib.org:9093/swagger-ui/index.html#/dataDump/exportDataDump') }}"
+TIMEOUT={{ scsb_export_datadump_timeout | default(10) }}
+REQUIRE_JSON={{ scsb_export_datadump_require_json | default(false) | ternary(1,0) }}
+WARN_MS={{ scsb_export_datadump_warn_ms | default(1500) }}
+CRIT_MS={{ scsb_export_datadump_crit_ms | default(5000) }}
+VERIFY_TLS={{ scsb_export_datadump_verify_tls | default(true) | ternary(1,0) }}
+SERVICE_NAME="{{ scsb_export_datadump_service_name | default('SCSB exportDataDump') }}"
+SECRETS_FILE="/etc/check_mk/scsb_export_datadump.env"
+
+# Load API key from root-only env file
+if [[ -r "$SECRETS_FILE" ]]; then
+  # shellcheck disable=SC1090
+  source "$SECRETS_FILE"
+fi
+SCSB_API_KEY="${SCSB_API_KEY:-}"
+
+tmpdir="$(mktemp -d)"
+body="$tmpdir/body"
+hdr="$tmpdir/headers"
+trap 'rm -rf "$tmpdir"' EXIT
+
+curl_opts=(-sS --max-time "$TIMEOUT" -o "$body" -D "$hdr" \
+  -H "Accept: application/json, */*;q=0.1" \
+  -H "User-Agent: checkmk-local-scsb/1.0")
+
+# TLS verification control
+if [ "$VERIFY_TLS" -eq 0 ]; then
+  curl_opts+=(-k)
+fi
+
+# Require API key
+if [[ -z "$SCSB_API_KEY" ]]; then
+  echo "2 $SERVICE_NAME | rta_ms=0;${WARN_MS};${CRIT_MS};0; Missing API token in $SECRETS_FILE (SCSB_API_KEY)"
+  exit 0
+fi
+curl_opts+=(-H "api_key: ${SCSB_API_KEY}")
+
+# Make request; capture status code & total time
+out="$(curl "${curl_opts[@]}" "$URL" -w '%{http_code} %{time_total}' 2>/dev/null || true)"
+code="${out%% *}"
+t_total="${out#* }"
+
+# Convert time->ms
+ms="$(awk -v t="$t_total" 'BEGIN{ if (t=="") t=0; printf("%.0f", t*1000) }')"
+ctype="$(awk -F': *' 'tolower($1)=="content-type"{ct=$2} END{print ct}' "$hdr" 2>/dev/null)"
+
+cmk_print() {
+  local state="$1"; shift
+  local msg="$*"
+  echo "$state $SERVICE_NAME | rta_ms=${ms};${WARN_MS};${CRIT_MS};0; $msg"
+}
+
+# If curl failed to give a code
+if ! [[ "$code" =~ ^[0-9]+$ ]]; then
+  cmk_print 3 "Unknown response; curl_output=$out"
+  exit 0
+fi
+
+# Non-200 → CRIT, include small snippet
+if [ "$code" -ne 200 ]; then
+  snippet="$(head -c 300 "$body" | tr '\n' ' ')"
+  cmk_print 2 "HTTP ${code}; ct=${ctype:-n/a}; ${snippet}"
+  exit 0
+fi
+
+# Optional JSON validation for real API endpoints
+if [ "$REQUIRE_JSON" -eq 1 ]; then
+  if ! command -v jq >/dev/null 2>&1; then
+    cmk_print 2 "jq not installed; cannot validate JSON; ct=${ctype:-n/a}"
+    exit 0
+  fi
+  if ! jq -e . < "$body" >/dev/null 2>&1; then
+    snippet="$(head -c 300 "$body" | tr '\n' ' ')"
+    cmk_print 2 "Invalid JSON; ct=${ctype:-n/a}; body=${snippet}"
+    exit 0
+  fi
+  base_msg="HTTP 200; JSON OK; ct=${ctype:-n/a}"
+else
+  base_msg="HTTP 200; ct=${ctype:-n/a}"
+fi
+
+state=0; msg="$base_msg; ${ms}ms"
+if [ "$ms" -ge "$CRIT_MS" ]; then
+  state=2; msg="$base_msg; slow response ${ms}ms"
+elif [ "$ms" -ge "$WARN_MS" ]; then
+  state=1; msg="$base_msg; high latency ${ms}ms"
+fi
+
+cmk_print "$state" "$msg"

playbooks/utils/checkmk_add_local_checks.yml

@@ -0,0 +1,121 @@
+---
+# To run the playbook:
+# confirm that the agent is installed already on your machine(s)
+# if it is not, run playbooks/utils/checkmk_agent.yml to install the CheckMK agent
+# rule_group environment variable specifies which rule group to apply to the hosts
+# by default the rails rule group will be applied
+# to run in with the another group, pass '-e rule_group=group_name'
+# by default this playbook runs in the staging environment
+# to run in production, pass '-e runtime_env=production'
+# For example `ansible-playbook playbooks/utils/checkmk_add_local_checks.yml --ask-vault-pass --limit=pdc-describe-staging2.princeton.edu`
+#  will run with the staging runtime environment and the rails rule group on pdc-describe-staging2
+
+- name: Install CheckMk local check scripts on host
+  hosts: "{{ runtime_env | default ('staging') }}"
+  remote_user: pulsys
+  become: true
+
+  vars_files:
+    - ../../group_vars/all/vault.yml
+    - ../../group_vars/checkmk/{{ runtime_env | default('staging') }}.yml
+    - ../../group_vars/checkmk/rule_{{ rule_group | default('rails') }}.yml
+    - ../../group_vars/checkmk/agent_common.yml
+    - ../../group_vars/checkmk/vault.yml
+
+  vars:
+    cmk_local_dir: /usr/lib/check_mk_agent/local
+    cmk_secret_dir: /etc/check_mk
+    secret_env_path: "{{ cmk_secret_dir }}/scsb_export_datadump.env"
+    in_recap_group: "{{ (group_names | select('search', 'recap_') | list | length) > 0 }}"
+
+  pre_tasks:
+      - name: stop playbook if you didn't use --limit
+        fail:
+          msg: "you must use -l or --limit"
+        when: ansible_limit is not defined
+        run_once: true
+      - name: set stripped agent hostname (only content before the first .) as fact
+        ansible.builtin.set_fact:
+          checkmk_agent_host_name: "{{ inventory_hostname | regex_search('^[^.]*') }}"
+
+  tasks:
+    - name: Fail early if token missing (only on recap_* hosts)
+      ansible.builtin.assert:
+        that:
+          - scsb_api_key is defined
+          - scsb_api_key | length > 0
+        fail_msg: "scsb_api_key is undefined/empty. Put it in group_vars/checkmk/vault.yml (vault)."
+      when: in_recap_group
+      no_log: true
+
+    - name: Ensure Checkmk dirs exist (recap_* only)
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+      loop:
+        - "{{ cmk_local_dir }}"
+        - "{{ cmk_secret_dir }}"
+      when: in_recap_group
+
+    - name: Install secret env file with API key (root-only) (recap_* only)
+      ansible.builtin.copy:
+        dest: "{{ secret_env_path }}"
+        owner: root
+        group: root
+        mode: "0600"
+        content: |
+          # SCSB local check secrets
+          SCSB_API_KEY={{ scsb_api_key }}
+      when: in_recap_group
+      no_log: true  # do not leak secret
+
+    - name: Install SCSB local check (shell) (recap_* only)
+      ansible.builtin.template:
+        src: "group_vars/checkmk/local_check_templates/scsb_export_datadump.sh.j2"
+        dest: "{{ cmk_local_dir }}/scsb_export_datadump"
+        owner: root
+        group: root
+        mode: "0755"
+      when: in_recap_group
+
+    - name: Ensure curl present (recap_* only)
+      ansible.builtin.package:
+        name: curl
+        state: present
+      when: in_recap_group
+
+    - name: Remove check and secret from non-recap hosts
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - "{{ cmk_local_dir }}/scsb_export_datadump"
+        - "{{ secret_env_path }}"
+      when: not in_recap_group
+
+    - name: write hostname using jinja2
+      ansible.builtin.template:
+        src: "../../group_vars/checkmk/local_check_templates/{{ item.template }}"
+        dest: "/usr/lib/check_mk_agent/local/{{ item.dest }}"
+        mode: "+x"
+      with_items: "{{ checkmk_local_scripts }}"
+
+    - name: "Add new local services on hosts."
+      checkmk.general.discovery:
+        server_url: "{{checkmk_agent_server_protocol}}://{{ checkmk_agent_server }}"
+        site: "{{ checkmk_agent_site }}"
+        automation_user: "{{ checkmk_agent_user }}"
+        automation_secret: "{{ vault_automation_secret }}"
+        hosts: "{{ checkmk_agent_host_name }}"
+        state: "new"
+
+    - name: "Start activation including foreign changes."
+      checkmk.general.activation:
+        server_url: "{{checkmk_agent_server_protocol}}://{{ checkmk_agent_server }}"
+        site: "{{ checkmk_agent_site }}"
+        automation_user: "{{ checkmk_agent_user }}"
+        automation_secret: "{{ vault_automation_secret }}"
+        force_foreign_changes: 'true'