checkmk_monintoring/CheckMK Agents/Linux/check_prometheus

#!/bin/bash

set -u

CONFIG="/etc/check_mk/prometheus.conf"

OK=0
WARN=1
CRIT=2
UNKNOWN=3

if [[ ! -r "$CONFIG" ]]; then
    echo "${UNKNOWN} \"Prometheus\" - Configuration file ${CONFIG} missing or unreadable"
    exit 0
fi

# shellcheck disable=SC1090
source "$CONFIG"

PROMETHEUS_URL="${PROMETHEUS_URL:-http://127.0.0.1:9090}"
PROMETHEUS_TOKEN="${PROMETHEUS_TOKEN:-}"
TARGET_DOWN_STATE="${TARGET_DOWN_STATE:-2}"

if ! command -v curl >/dev/null 2>&1; then
    echo "${UNKNOWN} \"Prometheus\" - curl is not installed"
    exit 0
fi

if ! command -v jq >/dev/null 2>&1; then
    echo "${UNKNOWN} \"Prometheus\" - jq is not installed"
    exit 0
fi

CURL_OPTS=(
    --silent
    --show-error
    --fail
    --connect-timeout 3
    --max-time 8
)

AUTH_ARGS=()

if [[ -n "$PROMETHEUS_TOKEN" ]]; then
    AUTH_ARGS=(
        --header "Authorization: Bearer ${PROMETHEUS_TOKEN}"
    )
fi

#
# ---------------------------------------------------------------------------
# Prometheus health / version
# ---------------------------------------------------------------------------
#

BUILD_RESPONSE="$(
    curl \
        "${CURL_OPTS[@]}" \
        "${AUTH_ARGS[@]}" \
        "${PROMETHEUS_URL%/}/api/v1/status/buildinfo" \
        2>/dev/null
)"

CURL_RC=$?

if [[ $CURL_RC -ne 0 || -z "$BUILD_RESPONSE" ]]; then
    echo "${CRIT} \"Prometheus\" - API unavailable at ${PROMETHEUS_URL}"
    exit 0
fi

API_STATUS="$(
    printf '%s' "$BUILD_RESPONSE" |
        jq -r '.status // empty' 2>/dev/null
)"

VERSION="$(
    printf '%s' "$BUILD_RESPONSE" |
        jq -r '.data.version // empty' 2>/dev/null
)"

if [[ "$API_STATUS" != "success" || -z "$VERSION" ]]; then
    echo "${CRIT} \"Prometheus\" - API responded but build information is invalid"
    exit 0
fi

echo "${OK} \"Prometheus\" - API reachable, version ${VERSION}"

#
# ---------------------------------------------------------------------------
# Prometheus scrape targets
# ---------------------------------------------------------------------------
#

TARGET_RESPONSE="$(
    curl \
        "${CURL_OPTS[@]}" \
        "${AUTH_ARGS[@]}" \
        "${PROMETHEUS_URL%/}/api/v1/targets?state=active" \
        2>/dev/null
)"

CURL_RC=$?

if [[ $CURL_RC -ne 0 || -z "$TARGET_RESPONSE" ]]; then
    echo "${UNKNOWN} \"Prometheus Targets\" - Unable to retrieve target status"
    exit 0
fi

TARGET_API_STATUS="$(
    printf '%s' "$TARGET_RESPONSE" |
        jq -r '.status // empty' 2>/dev/null
)"

if [[ "$TARGET_API_STATUS" != "success" ]]; then
    echo "${UNKNOWN} \"Prometheus Targets\" - Prometheus returned an unsuccessful API response"
    exit 0
fi

TOTAL="$(
    printf '%s' "$TARGET_RESPONSE" |
        jq '.data.activeTargets | length' 2>/dev/null
)"

UP="$(
    printf '%s' "$TARGET_RESPONSE" |
        jq '[.data.activeTargets[] | select(.health == "up")] | length' 2>/dev/null
)"

DOWN="$(
    printf '%s' "$TARGET_RESPONSE" |
        jq '[.data.activeTargets[] | select(.health != "up")] | length' 2>/dev/null
)"

if [[ -z "$TOTAL" || -z "$UP" || -z "$DOWN" ]]; then
    echo "${UNKNOWN} \"Prometheus Targets\" - Unable to parse target information"
    exit 0
fi

PERFDATA="total=${TOTAL} up=${UP} down=${DOWN}"

if [[ "$DOWN" -eq 0 ]]; then
    echo "${OK} \"Prometheus Targets\" ${PERFDATA} All ${TOTAL} active targets are UP"
    exit 0
fi

#
# Build readable DOWN target list.
#
# Prefer job + instance labels. Fall back to scrape URL where needed.
#
DOWN_LIST="$(
    printf '%s' "$TARGET_RESPONSE" |
        jq -r '
            .data.activeTargets[]
            | select(.health != "up")
            | (
                (.labels.job // "unknown-job")
                + "/"
                + (.labels.instance // .scrapeUrl // "unknown-target")
                + (
                    if (.lastError // "") != ""
                    then " (" + .lastError + ")"
                    else ""
                    end
                )
            )
        ' 2>/dev/null |
        paste -sd '; ' -
)"

if [[ -z "$DOWN_LIST" ]]; then
    DOWN_LIST="unable to identify failed targets"
fi

echo "${TARGET_DOWN_STATE} \"Prometheus Targets\" ${PERFDATA} ${UP}/${TOTAL} targets UP, ${DOWN} DOWN: ${DOWN_LIST}"

exit 0