checkmk_monintoring/CheckMK Agents/Linux/check_portainer

#!/bin/bash
# /usr/lib/check_mk_agent/local/300/check_portainer

set -u

CONFIG="/etc/check_mk/portainer.conf"

OK=0
WARN=1
CRIT=2
UNKNOWN=3

#
# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------
#

if [[ ! -r "$CONFIG" ]]; then
    printf '%s\n' "${UNKNOWN} \"Portainer\" - Configuration file ${CONFIG} missing or unreadable"
    exit 0
fi

# shellcheck disable=SC1090
source "$CONFIG"

PORTAINER_URL="${PORTAINER_URL:-https://127.0.0.1:9443}"
PORTAINER_TOKEN="${PORTAINER_TOKEN:-}"
PORTAINER_INSECURE="${PORTAINER_INSECURE:-yes}"
ALERT_STOPPED_CONTAINERS="${ALERT_STOPPED_CONTAINERS:-no}"
CHECK_UPDATES="${CHECK_UPDATES:-yes}"
PORTAINER_RELEASE_API="${PORTAINER_RELEASE_API:-https://api.github.com/repos/portainer/portainer/releases/latest}"

#
# ---------------------------------------------------------------------------
# Requirements
# ---------------------------------------------------------------------------
#

if ! command -v curl >/dev/null 2>&1; then
    printf '%s\n' "${UNKNOWN} \"Portainer\" - curl is not installed"
    exit 0
fi

if ! command -v jq >/dev/null 2>&1; then
    printf '%s\n' "${UNKNOWN} \"Portainer\" - jq is not installed"
    exit 0
fi

if ! command -v sort >/dev/null 2>&1; then
    printf '%s\n' "${UNKNOWN} \"Portainer\" - sort is not installed"
    exit 0
fi

if [[ -z "$PORTAINER_TOKEN" ]]; then
    printf '%s\n' "${UNKNOWN} \"Portainer\" - PORTAINER_TOKEN is not configured"
    exit 0
fi

#
# ---------------------------------------------------------------------------
# curl configuration
# ---------------------------------------------------------------------------
#

CURL_OPTS=(
    --silent
    --show-error
    --connect-timeout 3
    --max-time 10
)

if [[ "$PORTAINER_INSECURE" == "yes" ]]; then
    CURL_OPTS+=(--insecure)
fi

AUTH_HEADER="X-API-Key: ${PORTAINER_TOKEN}"

#
# ---------------------------------------------------------------------------
# Portainer health / version
# ---------------------------------------------------------------------------
#

STATUS_RESPONSE="$(
    curl \
        "${CURL_OPTS[@]}" \
        "${PORTAINER_URL%/}/api/status" \
        2>/dev/null
)"

CURL_RC=$?

if [[ $CURL_RC -ne 0 || -z "$STATUS_RESPONSE" ]]; then
    printf '%s\n' "${CRIT} \"Portainer\" - API unavailable at ${PORTAINER_URL}"
    exit 0
fi

if ! printf '%s' "$STATUS_RESPONSE" | jq -e . >/dev/null 2>&1; then
    printf '%s\n' "${UNKNOWN} \"Portainer\" - Status API returned invalid JSON"
    exit 0
fi

VERSION="$(
    printf '%s' "$STATUS_RESPONSE" |
        jq -r '.Version // .version // empty'
)"

if [[ -n "$VERSION" ]]; then
    printf '%s\n' "${OK} \"Portainer\" - API reachable, version ${VERSION}"
else
    printf '%s\n' "${OK} \"Portainer\" - API reachable"
fi

#
# ---------------------------------------------------------------------------
# Portainer update check
# ---------------------------------------------------------------------------
#

if [[ "$CHECK_UPDATES" == "yes" ]]; then

    if [[ -z "$VERSION" ]]; then
        printf '%s\n' \
            "${UNKNOWN} \"Portainer Update\" - Installed Portainer version could not be determined"

    else

        RELEASE_RESPONSE="$(
            curl \
                "${CURL_OPTS[@]}" \
                -H "Accept: application/vnd.github+json" \
                -H "User-Agent: checkmk-portainer-local-check" \
                "$PORTAINER_RELEASE_API" \
                2>/dev/null
        )"

        CURL_RC=$?

        if [[ $CURL_RC -ne 0 || -z "$RELEASE_RESPONSE" ]]; then
            printf '%s\n' \
                "${UNKNOWN} \"Portainer Update\" - Unable to retrieve latest release information"

        elif ! printf '%s' "$RELEASE_RESPONSE" | jq -e . >/dev/null 2>&1; then
            printf '%s\n' \
                "${UNKNOWN} \"Portainer Update\" - Release API returned invalid JSON"

        else

            LATEST_VERSION="$(
                printf '%s' "$RELEASE_RESPONSE" |
                    jq -r '.tag_name // empty'
            )"

            INSTALLED_VERSION="${VERSION#v}"
            LATEST_VERSION="${LATEST_VERSION#v}"

            if [[ -z "$LATEST_VERSION" ]]; then

                printf '%s\n' \
                    "${UNKNOWN} \"Portainer Update\" - Latest release version could not be determined"

            else

                #
                # Strip known suffixes before numeric comparison.
                #
                # Examples:
                #   2.36.0
                #   2.36.0-sts
                #   2.36.0-lts
                #

                INSTALLED_COMPARE="${INSTALLED_VERSION%%-*}"
                LATEST_COMPARE="${LATEST_VERSION%%-*}"

                if [[ ! "$INSTALLED_COMPARE" =~ ^[0-9]+([.][0-9]+)*$ ]]; then
                    printf '%s\n' \
                        "${UNKNOWN} \"Portainer Update\" - Unable to parse installed version ${INSTALLED_VERSION}"

                elif [[ ! "$LATEST_COMPARE" =~ ^[0-9]+([.][0-9]+)*$ ]]; then
                    printf '%s\n' \
                        "${UNKNOWN} \"Portainer Update\" - Unable to parse latest version ${LATEST_VERSION}"

                else

                    NEWEST="$(
                        printf '%s\n%s\n' \
                            "$INSTALLED_COMPARE" \
                            "$LATEST_COMPARE" |
                            sort -V |
                            tail -n 1
                    )"

                    if [[ "$INSTALLED_COMPARE" == "$LATEST_COMPARE" ]]; then

                        printf '%s\n' \
                            "${OK} \"Portainer Update\" - Current: installed ${INSTALLED_VERSION}, latest ${LATEST_VERSION}"

                    elif [[ "$NEWEST" == "$LATEST_COMPARE" ]]; then

                        printf '%s\n' \
                            "${WARN} \"Portainer Update\" - Update available: installed ${INSTALLED_VERSION}, latest ${LATEST_VERSION}"

                    else

                        printf '%s\n' \
                            "${OK} \"Portainer Update\" - Installed ${INSTALLED_VERSION} is newer than latest stable ${LATEST_VERSION}"

                    fi
                fi
            fi
        fi
    fi
fi

#
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
#

ENDPOINTS_RESPONSE="$(
    curl \
        "${CURL_OPTS[@]}" \
        -H "$AUTH_HEADER" \
        "${PORTAINER_URL%/}/api/endpoints" \
        2>/dev/null
)"

CURL_RC=$?

if [[ $CURL_RC -ne 0 || -z "$ENDPOINTS_RESPONSE" ]]; then
    printf '%s\n' "${CRIT} \"Portainer Endpoints\" - Unable to retrieve endpoints"
    exit 0
fi

if ! printf '%s' "$ENDPOINTS_RESPONSE" | jq -e 'type == "array"' >/dev/null 2>&1; then
    printf '%s\n' "${UNKNOWN} \"Portainer Endpoints\" - Endpoint API returned unexpected data"
    exit 0
fi

ENDPOINT_COUNT="$(
    printf '%s' "$ENDPOINTS_RESPONSE" |
        jq 'length'
)"

UP_ENDPOINTS=0
DOWN_ENDPOINTS=0

while IFS= read -r ENDPOINT; do

    ENDPOINT_ID="$(
        printf '%s' "$ENDPOINT" |
            jq -r '.Id // .ID // empty'
    )"

    ENDPOINT_NAME="$(
        printf '%s' "$ENDPOINT" |
            jq -r '.Name // "Unknown"'
    )"

    ENDPOINT_STATUS="$(
        printf '%s' "$ENDPOINT" |
            jq -r '.Status // 0'
    )"

    ENDPOINT_NAME="${ENDPOINT_NAME//\"/\'}"

    #
    # Portainer endpoint states:
    #   1 = UP
    #   2 = DOWN
    #

    if [[ "$ENDPOINT_STATUS" == "1" ]]; then

        UP_ENDPOINTS=$((UP_ENDPOINTS + 1))

        printf '%s\n' \
            "${OK} \"Portainer Endpoint ${ENDPOINT_NAME}\" - Endpoint is UP"

    else

        DOWN_ENDPOINTS=$((DOWN_ENDPOINTS + 1))

        printf '%s\n' \
            "${CRIT} \"Portainer Endpoint ${ENDPOINT_NAME}\" - Endpoint is DOWN, status ${ENDPOINT_STATUS}"

    fi

    #
    # -----------------------------------------------------------------------
    # Containers for this endpoint
    # -----------------------------------------------------------------------
    #

    if [[ -z "$ENDPOINT_ID" ]]; then
        printf '%s\n' \
            "${UNKNOWN} \"Docker Containers ${ENDPOINT_NAME}\" - Endpoint ID missing"
        continue
    fi

    CONTAINERS_RESPONSE="$(
        curl \
            "${CURL_OPTS[@]}" \
            -H "$AUTH_HEADER" \
            "${PORTAINER_URL%/}/api/endpoints/${ENDPOINT_ID}/docker/containers/json?all=1" \
            2>/dev/null
    )"

    CURL_RC=$?

    if [[ $CURL_RC -ne 0 || -z "$CONTAINERS_RESPONSE" ]]; then
        printf '%s\n' \
            "${UNKNOWN} \"Docker Containers ${ENDPOINT_NAME}\" - Unable to retrieve containers"
        continue
    fi

    if ! printf '%s' "$CONTAINERS_RESPONSE" | jq -e 'type == "array"' >/dev/null 2>&1; then
        printf '%s\n' \
            "${UNKNOWN} \"Docker Containers ${ENDPOINT_NAME}\" - Container API returned unexpected data"
        continue
    fi

    TOTAL_CONTAINERS="$(
        printf '%s' "$CONTAINERS_RESPONSE" |
            jq 'length'
    )"

    RUNNING_CONTAINERS="$(
        printf '%s' "$CONTAINERS_RESPONSE" |
            jq '[.[] | select(.State == "running")] | length'
    )"

    STOPPED_CONTAINERS="$(
        printf '%s' "$CONTAINERS_RESPONSE" |
            jq '[.[] | select(.State != "running")] | length'
    )"

    UNHEALTHY_CONTAINERS="$(
        printf '%s' "$CONTAINERS_RESPONSE" |
            jq '
                [
                    .[]
                    | select(
                        ((.Status // "") | test("\\(unhealthy\\)"))
                    )
                ]
                | length
            '
    )"

    PROBLEM_LIST="$(
        printf '%s' "$CONTAINERS_RESPONSE" |
            jq -r '
                .[]
                | select(
                    (.State != "running")
                    or
                    ((.Status // "") | test("\\(unhealthy\\)"))
                )
                | (
                    (
                        (.Names[0] // .Id // "unknown")
                        | sub("^/"; "")
                    )
                    + "="
                    + (.Status // .State // "unknown")
                )
            ' |
            paste -sd ';' -
    )"

    STATE=$OK

    if [[ "$UNHEALTHY_CONTAINERS" -gt 0 ]]; then
        STATE=$CRIT

    elif [[ "$ALERT_STOPPED_CONTAINERS" == "yes" && "$STOPPED_CONTAINERS" -gt 0 ]]; then
        STATE=$WARN
    fi

    PERFDATA="total=${TOTAL_CONTAINERS};;;0|running=${RUNNING_CONTAINERS};;;0|stopped=${STOPPED_CONTAINERS};;;0|unhealthy=${UNHEALTHY_CONTAINERS};;;0"

    DETAILS="${RUNNING_CONTAINERS}/${TOTAL_CONTAINERS} running"

    if [[ "$UNHEALTHY_CONTAINERS" -gt 0 ]]; then
        DETAILS="${DETAILS}, ${UNHEALTHY_CONTAINERS} unhealthy"
    fi

    if [[ "$STOPPED_CONTAINERS" -gt 0 ]]; then
        DETAILS="${DETAILS}, ${STOPPED_CONTAINERS} stopped"
    fi

    if [[ -n "$PROBLEM_LIST" ]]; then
        DETAILS="${DETAILS}: ${PROBLEM_LIST}"
    fi

    printf '%s\n' \
        "${STATE} \"Docker Containers ${ENDPOINT_NAME}\" ${PERFDATA} ${DETAILS}"

done < <(
    printf '%s' "$ENDPOINTS_RESPONSE" |
        jq -c '.[]'
)

#
# ---------------------------------------------------------------------------
# Endpoint summary
# ---------------------------------------------------------------------------
#

ENDPOINT_STATE=$OK

if [[ "$DOWN_ENDPOINTS" -gt 0 ]]; then
    ENDPOINT_STATE=$CRIT
fi

printf '%s\n' \
    "${ENDPOINT_STATE} \"Portainer Endpoints\" total=${ENDPOINT_COUNT};;;0|up=${UP_ENDPOINTS};;;0|down=${DOWN_ENDPOINTS};;;0 ${UP_ENDPOINTS}/${ENDPOINT_COUNT} endpoints UP"

exit 0