#!/bin/bash set -u CONFIG="/etc/check_mk/prometheus.conf" OK=0 WARN=1 CRIT=2 UNKNOWN=3 if [[ ! -r "$CONFIG" ]]; then echo "${UNKNOWN} \"Prometheus\" - Configuration file ${CONFIG} missing or unreadable" exit 0 fi # shellcheck disable=SC1090 source "$CONFIG" PROMETHEUS_URL="${PROMETHEUS_URL:-http://127.0.0.1:9090}" PROMETHEUS_TOKEN="${PROMETHEUS_TOKEN:-}" TARGET_DOWN_STATE="${TARGET_DOWN_STATE:-2}" if ! command -v curl >/dev/null 2>&1; then echo "${UNKNOWN} \"Prometheus\" - curl is not installed" exit 0 fi if ! command -v jq >/dev/null 2>&1; then echo "${UNKNOWN} \"Prometheus\" - jq is not installed" exit 0 fi CURL_OPTS=( --silent --show-error --fail --connect-timeout 3 --max-time 8 ) AUTH_ARGS=() if [[ -n "$PROMETHEUS_TOKEN" ]]; then AUTH_ARGS=( --header "Authorization: Bearer ${PROMETHEUS_TOKEN}" ) fi # # --------------------------------------------------------------------------- # Prometheus health / version # --------------------------------------------------------------------------- # BUILD_RESPONSE="$( curl \ "${CURL_OPTS[@]}" \ "${AUTH_ARGS[@]}" \ "${PROMETHEUS_URL%/}/api/v1/status/buildinfo" \ 2>/dev/null )" CURL_RC=$? if [[ $CURL_RC -ne 0 || -z "$BUILD_RESPONSE" ]]; then echo "${CRIT} \"Prometheus\" - API unavailable at ${PROMETHEUS_URL}" exit 0 fi API_STATUS="$( printf '%s' "$BUILD_RESPONSE" | jq -r '.status // empty' 2>/dev/null )" VERSION="$( printf '%s' "$BUILD_RESPONSE" | jq -r '.data.version // empty' 2>/dev/null )" if [[ "$API_STATUS" != "success" || -z "$VERSION" ]]; then echo "${CRIT} \"Prometheus\" - API responded but build information is invalid" exit 0 fi echo "${OK} \"Prometheus\" - API reachable, version ${VERSION}" # # --------------------------------------------------------------------------- # Prometheus scrape targets # --------------------------------------------------------------------------- # TARGET_RESPONSE="$( curl \ "${CURL_OPTS[@]}" \ "${AUTH_ARGS[@]}" \ "${PROMETHEUS_URL%/}/api/v1/targets?state=active" \ 2>/dev/null )" CURL_RC=$? if [[ $CURL_RC -ne 0 || -z "$TARGET_RESPONSE" ]]; then echo "${UNKNOWN} \"Prometheus Targets\" - Unable to retrieve target status" exit 0 fi TARGET_API_STATUS="$( printf '%s' "$TARGET_RESPONSE" | jq -r '.status // empty' 2>/dev/null )" if [[ "$TARGET_API_STATUS" != "success" ]]; then echo "${UNKNOWN} \"Prometheus Targets\" - Prometheus returned an unsuccessful API response" exit 0 fi TOTAL="$( printf '%s' "$TARGET_RESPONSE" | jq '.data.activeTargets | length' 2>/dev/null )" UP="$( printf '%s' "$TARGET_RESPONSE" | jq '[.data.activeTargets[] | select(.health == "up")] | length' 2>/dev/null )" DOWN="$( printf '%s' "$TARGET_RESPONSE" | jq '[.data.activeTargets[] | select(.health != "up")] | length' 2>/dev/null )" if [[ -z "$TOTAL" || -z "$UP" || -z "$DOWN" ]]; then echo "${UNKNOWN} \"Prometheus Targets\" - Unable to parse target information" exit 0 fi PERFDATA="total=${TOTAL} up=${UP} down=${DOWN}" if [[ "$DOWN" -eq 0 ]]; then echo "${OK} \"Prometheus Targets\" ${PERFDATA} All ${TOTAL} active targets are UP" exit 0 fi # # Build readable DOWN target list. # # Prefer job + instance labels. Fall back to scrape URL where needed. # DOWN_LIST="$( printf '%s' "$TARGET_RESPONSE" | jq -r ' .data.activeTargets[] | select(.health != "up") | ( (.labels.job // "unknown-job") + "/" + (.labels.instance // .scrapeUrl // "unknown-target") + ( if (.lastError // "") != "" then " (" + .lastError + ")" else "" end ) ) ' 2>/dev/null | paste -sd '; ' - )" if [[ -z "$DOWN_LIST" ]]; then DOWN_LIST="unable to identify failed targets" fi echo "${TARGET_DOWN_STATE} \"Prometheus Targets\" ${PERFDATA} ${UP}/${TOTAL} targets UP, ${DOWN} DOWN: ${DOWN_LIST}" exit 0