run_performance_kube.sh

#!/usr/bin/env bash
# Performance monitoring script for Flexo Kubernetes cluster
# Monitors pod status, CPU, memory usage, and triple counts from GraphDB
#
# Usage:
#   ./run_performance_kube.sh                    # Generate performance report
#   ./run_performance_kube.sh --verbose          # With verbose debug output
#
# Environment variables:
#   FLEXO_NAMESPACE - Kubernetes namespace (default: ts-flexo)
#   GRAPHDB_REPO_ID - GraphDB repository ID (default: openmbee)
#   GRAPHDB_PORT - GraphDB port (default: 7200)
#   KUBE_PERF_OUTPUT - Output directory (default: performance_results/kube)
#   KUBECTL_CMD - kubectl-compatible command/alias to use (default: kubectl)

# Parse arguments
VERBOSE=false
KUBECTL_CMD_OVERRIDE=""
while [[ $# -gt 0 ]]; do
    case $1 in
        --verbose|-v)
            VERBOSE=true
            shift
            ;;
        --kubectl|-k)
            shift
            if [[ $# -eq 0 ]]; then
                echo "Error: --kubectl requires a command value (e.g., --kubectl kdev)"
                exit 1
            fi
            KUBECTL_CMD_OVERRIDE="$1"
            shift
            ;;
        --help|-h)
            echo "Usage: $0 [--verbose] [--kubectl <command>]"
            echo ""
            echo "Options:"
            echo "  --verbose, -v       Show verbose output for debugging"
            echo "  --kubectl, -k       Use alternate kubectl command/alias (env: KUBECTL_CMD)"
            echo "  --help, -h          Show this help message"
            exit 0
            ;;
        *)
            echo "Unknown option: $1"
            echo "Use --help for usage information"
            exit 1
            ;;
    esac
done

NAMESPACE="${FLEXO_NAMESPACE:-ts-flexo}"
GRAPHDB_REPO_ID="${GRAPHDB_REPO_ID:-openmbee}"
GRAPHDB_PORT="${GRAPHDB_PORT:-7200}"
OUTPUT_DIR="${KUBE_PERF_OUTPUT:-performance_results/kube}"
KUBECTL_BIN="${KUBECTL_CMD_OVERRIDE:-${KUBECTL_CMD:-kubectl}}"

# Split the provided command into words (handles base command + default flags)
read -r -a KUBECTL_WORDS <<< "${KUBECTL_BIN}"
KUBECTL_EXECUTABLE="${KUBECTL_WORDS[0]}"

# Determine if the command is directly runnable; if not, fall back to login shell (helps aliases)
if command -v "${KUBECTL_EXECUTABLE}" >/dev/null 2>&1; then
    kubectl_cmd() {
        "${KUBECTL_WORDS[@]}" "$@"
    }
else
    kubectl_cmd() {
        local quoted_args=""
        local arg
        for arg in "$@"; do
            quoted_args+=" $(printf '%q' "$arg")"
        done
        bash -lc "${KUBECTL_BIN}${quoted_args}"
    }
fi

TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
OUTPUT_TIMESTAMP="${TIMESTAMP}"

# Create output directory
mkdir -p "${OUTPUT_DIR}"

get_next_test_number() {
    local dir="$1"
    local max_num=-1
    if [ -d "${dir}" ]; then
        local entry
        for entry in "${dir}"/test*; do
            [ -e "${entry}" ] || continue
            if [ -d "${entry}" ]; then
                local base
                base=$(basename "${entry}")
                local num="${base#test}"
                if [[ "${base}" == test* && "${num}" =~ ^[0-9]+$ ]]; then
                    if (( num > max_num )); then
                        max_num=${num}
                    fi
                fi
            fi
        done
    fi
    echo $((max_num + 1))
}

TEST_NUMBER=$(get_next_test_number "${OUTPUT_DIR}")
TEST_NAME="test${TEST_NUMBER}"
TEST_DIR="${OUTPUT_DIR}/${TEST_NAME}"
mkdir -p "${TEST_DIR}"

REPORT_FILE_JSON="${TEST_DIR}/kube_performance_${OUTPUT_TIMESTAMP}.json"
REPORT_FILE_TXT="${TEST_DIR}/kube_performance_${OUTPUT_TIMESTAMP}.txt"

# Function to get pod status
get_pod_status() {
    local pod_name=$1
    kubectl_cmd get pod "${pod_name}" -n "${NAMESPACE}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "NOT_FOUND"
}

# Function to get pod CPU and memory usage
get_pod_resources() {
    local pod_name=$1
    # Check if metrics server is available
    if ! kubectl_cmd top pod "${pod_name}" -n "${NAMESPACE}" --no-headers 2>/dev/null >/dev/null; then
        echo "N/A N/A"
        return 1
    fi
    kubectl_cmd top pod "${pod_name}" -n "${NAMESPACE}" --no-headers 2>/dev/null | awk '{print $2" "$3}' || echo "N/A N/A"
}

# Function to get pod CPU usage
get_pod_cpu() {
    local pod_name=$1
    local resources=$(get_pod_resources "${pod_name}" 2>/dev/null)
    if [ "${resources}" = "N/A N/A" ] || [ -z "${resources}" ]; then
        echo "N/A"
    else
        echo "${resources}" | awk '{print $1}'
    fi
}

# Function to get pod memory usage
get_pod_memory() {
    local pod_name=$1
    local resources=$(get_pod_resources "${pod_name}" 2>/dev/null)
    if [ "${resources}" = "N/A N/A" ] || [ -z "${resources}" ]; then
        echo "N/A"
    else
        echo "${resources}" | awk '{print $2}'
    fi
}

# Function to get pod memory request/limit
get_pod_memory_spec() {
    local pod_name=$1
    local resource_type=$2  # "requests" or "limits"
    kubectl_cmd get pod "${pod_name}" -n "${NAMESPACE}" -o jsonpath="{.spec.containers[0].resources.${resource_type}.memory}" 2>/dev/null || echo "N/A"
}

# Function to get pod CPU request/limit
get_pod_cpu_spec() {
    local pod_name=$1
    local resource_type=$2  # "requests" or "limits"
    kubectl_cmd get pod "${pod_name}" -n "${NAMESPACE}" -o jsonpath="{.spec.containers[0].resources.${resource_type}.cpu}" 2>/dev/null || echo "N/A"
}

# Function to list GraphDB repositories
list_graphdb_repositories() {
    local graphdb_pod=$(kubectl_cmd get pods -n "${NAMESPACE}" | grep graphdb | head -1 | awk '{print $1}')
    if [ -z "${graphdb_pod}" ]; then
        return 1
    fi
    local graphdb_port="${GRAPHDB_PORT}"
    kubectl_cmd exec "${graphdb_pod}" -n "${NAMESPACE}" -- \
        sh -c "curl -s 'http://localhost:${graphdb_port}/rest/repositories' 2>/dev/null" 2>/dev/null
}

# Function to auto-detect GraphDB repository ID
auto_detect_repo_id() {
    local repos_list=$(list_graphdb_repositories 2>/dev/null)
    if [ -n "${repos_list}" ]; then
        # Try to extract the first repository ID from JSON
        local repo_id=$(echo "${repos_list}" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4)
        if [ -n "${repo_id}" ]; then
            echo "${repo_id}"
            return 0
        fi
    fi
    return 1
}

# Function to query GraphDB for triple count
query_graphdb_triples() {
    local query_type=$1  # "all" or "named"
    local graphdb_pod=$(kubectl_cmd get pods -n "${NAMESPACE}" | grep graphdb | head -1 | awk '{print $1}')
    
    if [ -z "${graphdb_pod}" ]; then
        [ "${VERBOSE}" = true ] && echo "  [DEBUG] GraphDB pod not found" >&2
        echo "N/A"
        return 1
    fi
    
    local repo_id="${GRAPHDB_REPO_ID}"
    local graphdb_port="${GRAPHDB_PORT}"
    local result="N/A"
    
    [ "${VERBOSE}" = true ] && echo "  [DEBUG] Querying GraphDB pod: ${graphdb_pod}, repo: ${repo_id}, port: ${graphdb_port}" >&2
    
    # Method 1: Try GraphDB REST API size endpoint (most reliable)
    local size_response=$(kubectl_cmd exec "${graphdb_pod}" -n "${NAMESPACE}" -- \
        sh -c "curl -s 'http://localhost:${graphdb_port}/rest/repositories/${repo_id}/size' 2>&1" 2>/dev/null)
    
    [ "${VERBOSE}" = true ] && echo "  [DEBUG] Size endpoint response: ${size_response:0:200}" >&2
    
    if [ -n "${size_response}" ] && [ "${size_response}" != "N/A" ] && [ "${size_response}" != "" ] && ! echo "${size_response}" | grep -q "error\|Error\|404\|Not Found\|curl:"; then
        # Check if it's a JSON object with "total" field
        if echo "${size_response}" | grep -q '"total"'; then
            result=$(echo "${size_response}" | grep -o '"total":[0-9]*' | cut -d':' -f2)
            if [ -n "${result}" ] && echo "${result}" | grep -qE '^[0-9]+$'; then
                echo "${result}"
                return 0
            fi
        fi
        # Check if it's just a number
        if echo "${size_response}" | grep -qE '^[0-9]+$'; then
            echo "${size_response}"
            return 0
        fi
    fi
    
    # Method 2: Try SPARQL query via standard endpoint
    if [ "${query_type}" = "named" ]; then
        # Query for named graph triples - count all triples in named graphs
        local sparql_query="SELECT (COUNT(*) AS ?triples) WHERE { GRAPH ?g { ?s ?p ?o . } }"
    else
        # Query for all triples
        local sparql_query="SELECT (COUNT(*) AS ?triples) WHERE { ?s ?p ?o . }"
    fi
    local endpoint="http://localhost:${graphdb_port}/repositories/${repo_id}"
    
    [ "${VERBOSE}" = true ] && echo "  [DEBUG] Trying SPARQL endpoint: ${endpoint}" >&2
    
    # Try POST with proper URL encoding
    local response=$(kubectl_cmd exec "${graphdb_pod}" -n "${NAMESPACE}" -- \
        sh -c "curl -s -X POST '${endpoint}' \
        -H 'Content-Type: application/x-www-form-urlencoded' \
        -H 'Accept: application/sparql-results+json' \
        --data-urlencode 'query=${sparql_query}' 2>&1" 2>/dev/null)
    
    [ "${VERBOSE}" = true ] && echo "  [DEBUG] SPARQL POST response length: ${#response}" >&2
    
    if [ -n "${response}" ] && ! echo "${response}" | grep -q "curl:"; then
        # Try to extract value from SPARQL JSON response
        # Format: "triples": { "value": "264477" }
        if echo "${response}" | grep -q '"bindings"'; then
            # Look for "triples" key with "value" inside
            result=$(echo "${response}" | grep -o '"triples"[^}]*"value":"[^"]*"' | grep -o '"value":"[^"]*"' | cut -d'"' -f4)
            if [ -n "${result}" ] && [ "${result}" != "N/A" ] && echo "${result}" | grep -qE '^[0-9]+$'; then
                echo "${result}"
                return 0
            fi
            # Alternative: look for any "value" field in bindings
            result=$(echo "${response}" | grep -o '"value":"[^"]*"' | head -1 | cut -d'"' -f4)
            if [ -n "${result}" ] && [ "${result}" != "N/A" ] && echo "${result}" | grep -qE '^[0-9]+$'; then
                echo "${result}"
                return 0
            fi
        fi
        # Fallback: try to extract any "value" field
        if echo "${response}" | grep -q '"value"'; then
            result=$(echo "${response}" | grep -o '"value":"[^"]*"' | head -1 | cut -d'"' -f4)
            if [ -n "${result}" ] && [ "${result}" != "N/A" ] && echo "${result}" | grep -qE '^[0-9]+$'; then
                echo "${result}"
                return 0
            fi
        fi
    fi
    
    # Method 3: Try GET request
    response=$(kubectl_cmd exec "${graphdb_pod}" -n "${NAMESPACE}" -- \
        sh -c "curl -s -G '${endpoint}' \
        --data-urlencode 'query=${sparql_query}' \
        -H 'Accept: application/sparql-results+json' 2>&1" 2>/dev/null)
    
    if [ -n "${response}" ] && ! echo "${response}" | grep -q "curl:"; then
        if echo "${response}" | grep -q '"bindings"'; then
            result=$(echo "${response}" | grep -o '"triples"[^}]*"value":"[^"]*"' | grep -o '"value":"[^"]*"' | cut -d'"' -f4)
            if [ -n "${result}" ] && [ "${result}" != "N/A" ] && echo "${result}" | grep -qE '^[0-9]+$'; then
                echo "${result}"
                return 0
            fi
        fi
        if echo "${response}" | grep -q '"value"'; then
            result=$(echo "${response}" | grep -o '"value":"[^"]*"' | head -1 | cut -d'"' -f4)
            if [ -n "${result}" ] && [ "${result}" != "N/A" ] && echo "${result}" | grep -qE '^[0-9]+$'; then
                echo "${result}"
                return 0
            fi
        fi
    fi
    
    # Method 4: Try listing repositories to find the correct one
    [ "${VERBOSE}" = true ] && echo "  [DEBUG] Trying to list repositories..." >&2
    local repos_list=$(kubectl_cmd exec "${graphdb_pod}" -n "${NAMESPACE}" -- \
        sh -c "curl -s 'http://localhost:${graphdb_port}/rest/repositories' 2>&1" 2>/dev/null)
    
    [ "${VERBOSE}" = true ] && echo "  [DEBUG] Available repositories: ${repos_list:0:200}" >&2
    
    # Method 5: Try using GraphDB's internal query via workbench API
    response=$(kubectl_cmd exec "${graphdb_pod}" -n "${NAMESPACE}" -- \
        sh -c "curl -s -X POST 'http://localhost:${graphdb_port}/rest/repositories/${repo_id}/query' \
        -H 'Content-Type: application/sparql-query' \
        -H 'Accept: application/sparql-results+json' \
        -d '${sparql_query}' 2>&1" 2>/dev/null)
    
    if [ -n "${response}" ] && ! echo "${response}" | grep -q "curl:"; then
        if echo "${response}" | grep -q '"bindings"'; then
            result=$(echo "${response}" | grep -o '"triples"[^}]*"value":"[^"]*"' | grep -o '"value":"[^"]*"' | cut -d'"' -f4)
            if [ -n "${result}" ] && [ "${result}" != "N/A" ] && echo "${result}" | grep -qE '^[0-9]+$'; then
                echo "${result}"
                return 0
            fi
        fi
        if echo "${response}" | grep -q '"value"'; then
            result=$(echo "${response}" | grep -o '"value":"[^"]*"' | head -1 | cut -d'"' -f4)
            if [ -n "${result}" ] && [ "${result}" != "N/A" ] && echo "${result}" | grep -qE '^[0-9]+$'; then
                echo "${result}"
                return 0
            fi
        fi
    fi
    
    [ "${VERBOSE}" = true ] && echo "  [DEBUG] All GraphDB query methods failed" >&2
    echo "N/A"
    return 1
}

# Function to collect and display metrics
collect_and_display_metrics() {
    echo "============================================================"
    echo "Flexo Kubernetes Cluster Performance Report"
    echo "============================================================"
    echo "Namespace: ${NAMESPACE}"
    echo "Timestamp: $(date)"
    echo "Test Directory: ${TEST_DIR}"
    echo "Kubectl command: ${KUBECTL_BIN}"
    echo "JSON Report: ${REPORT_FILE_JSON}"
    echo "Text Report: ${REPORT_FILE_TXT}"
    echo "============================================================"
    echo ""
    
    # Verify namespace exists
    if ! kubectl_cmd get namespace "${NAMESPACE}" >/dev/null 2>&1; then
        echo "❌ Error: Namespace '${NAMESPACE}' not found!"
        echo "   Available namespaces:"
        kubectl_cmd get namespaces -o name 2>/dev/null | sed 's/namespace\///' | sed 's/^/     /' || echo "     (Unable to list namespaces)"
        return 1
    fi
    
    # Get all pods in namespace
    echo "Checking pod status..."
    PODS=$(kubectl_cmd get pods -n "${NAMESPACE}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
    
    if [ -z "${PODS}" ]; then
        echo "⚠️  No pods found in namespace ${NAMESPACE}"
        echo "   Checking if namespace exists..."
        if kubectl_cmd get namespace "${NAMESPACE}" >/dev/null 2>&1; then
            echo "   Namespace exists but has no pods"
        else
            echo "   Namespace does not exist!"
        fi
        return 1
    fi
    
    # Count pods
    POD_COUNT=$(echo "${PODS}" | wc -w | tr -d ' ')
    echo "Found ${POD_COUNT} pod(s)"
    
    # Collect node disk (ephemeral storage) information
    NODE_STORAGE_JSON="{}"
    NODE_STORAGE_SECTION=""
    NODE_STORAGE_VISUAL_SECTION=""
    NODE_STORAGE_AVAILABLE=false
    NODE_STORAGE_MESSAGE="  Node storage information unavailable (requires cluster-wide access)"
    NODE_STORAGE_JSON_FILE=""
    NODE_STORAGE_TABLE_FILE=""
    if NODE_INFO_JSON=$(kubectl_cmd get nodes -o json 2>/dev/null); then
        if [ -n "${NODE_INFO_JSON}" ]; then
            NODE_STORAGE_JSON_FILE=$(mktemp)
            if printf '%s' "${NODE_INFO_JSON}" | python3 - <<'PY' > "${NODE_STORAGE_JSON_FILE}" 2>/dev/null; then
import json
import sys
import re

def parse_bytes(value):
    if value in (None, '', 'N/A'):
        return None
    value = str(value)
    match = re.match(r'^([0-9.]+)([KMGTP]i)?$', value)
    if not match:
        try:
            return int(float(value))
        except Exception:
            return None
    number = float(match.group(1))
    unit = match.group(2)
    multipliers = {
        'Ki': 1024,
        'Mi': 1024 ** 2,
        'Gi': 1024 ** 3,
        'Ti': 1024 ** 4,
        'Pi': 1024 ** 5
    }
    if unit:
        number *= multipliers.get(unit, 1)
    return int(number)

def format_gib(value):
    if value is None:
        return "N/A"
    return f"{value / (1024 ** 3):.2f} Gi"

try:
    data = json.load(sys.stdin)
except Exception:
    sys.exit(1)

nodes = {}
total_capacity = 0
total_alloc = 0
capacity_seen = False
alloc_seen = False

for item in data.get("items", []):
    name = item.get("metadata", {}).get("name")
    if not name:
        continue
    cap_raw = item.get("status", {}).get("capacity", {}).get("ephemeral-storage")
    alloc_raw = item.get("status", {}).get("allocatable", {}).get("ephemeral-storage")
    cap_bytes = parse_bytes(cap_raw)
    alloc_bytes = parse_bytes(alloc_raw)
    nodes[name] = {
        "disk_capacity": format_gib(cap_bytes) if cap_bytes is not None else (cap_raw or "N/A"),
        "disk_allocatable": format_gib(alloc_bytes) if alloc_bytes is not None else (alloc_raw or "N/A"),
        "raw_capacity": cap_raw or "N/A",
        "raw_allocatable": alloc_raw or "N/A",
        "capacity_bytes": cap_bytes,
        "allocatable_bytes": alloc_bytes
    }
    if cap_bytes is not None:
        total_capacity += cap_bytes
        capacity_seen = True
    if alloc_bytes is not None:
        total_alloc += alloc_bytes
        alloc_seen = True

if not nodes:
    sys.exit(1)

totals = {
    "capacity_bytes": total_capacity if capacity_seen else None,
    "allocatable_bytes": total_alloc if alloc_seen else None,
    "capacity_pretty": format_gib(total_capacity if capacity_seen else None),
    "allocatable_pretty": format_gib(total_alloc if alloc_seen else None)
}

result = {"nodes": nodes, "totals": totals}
json.dump(result, sys.stdout, separators=(',', ':'))
PY
                NODE_STORAGE_JSON=$(cat "${NODE_STORAGE_JSON_FILE}")
                if [ -n "${NODE_STORAGE_JSON}" ] && [ "${NODE_STORAGE_JSON}" != "{}" ]; then
                    NODE_STORAGE_AVAILABLE=true
                    NODE_STORAGE_TABLE_FILE=$(mktemp)
                    if printf '%s' "${NODE_STORAGE_JSON}" | python3 - <<'PY' > "${NODE_STORAGE_TABLE_FILE}" 2>/dev/null; then
import json
import sys

try:
    data = json.load(sys.stdin)
except Exception:
    sys.exit(1)

nodes = data.get("nodes") or {}
if not nodes:
    sys.exit(1)

node_names = sorted(nodes.keys())
header = f"  {'NODE NAME':<30} {'DISK SIZE':<18} {'ALLOCATABLE':<18}"
line = "  " + "=" * 70
output_lines = [header, line]

for name in node_names:
    entry = nodes.get(name, {})
    disk = entry.get("disk_capacity", "N/A")
    alloc = entry.get("disk_allocatable", "N/A")
    output_lines.append(f"  {name:<30} {disk:<18} {alloc:<18}")

totals = data.get("totals") or {}
cap = totals.get("capacity_pretty", "N/A")
alloc_total = totals.get("allocatable_pretty", "N/A")
if cap != "N/A" or alloc_total != "N/A":
    output_lines.append("")
    output_lines.append(f"  Total Disk Size: {cap}")
    output_lines.append(f"  Total Allocatable: {alloc_total}")

sys.stdout.write("\n".join(output_lines))
PY
                        NODE_STORAGE_SECTION=$(cat "${NODE_STORAGE_TABLE_FILE}")
                    fi
                    if [ -z "${NODE_STORAGE_SECTION}" ]; then
                        set +e
                        NODE_STORAGE_SECTION=$(
                            printf '%s' "${NODE_STORAGE_JSON}" | python3 - <<'PY' 2>/dev/null
import json
import sys

try:
    data = json.loads(sys.stdin.read())
except Exception:
    sys.exit(1)

nodes = data.get("nodes") or {}
totals = data.get("totals") or {}

lines = []
if nodes:
    for name in sorted(nodes.keys()):
        entry = nodes.get(name, {})
        disk = entry.get("disk_capacity", "N/A")
        alloc = entry.get("disk_allocatable", "N/A")
        lines.append(f"{name}: disk={disk}, allocatable={alloc}")

cap = totals.get("capacity_pretty")
alloc_total = totals.get("allocatable_pretty")
if cap or alloc_total:
    lines.append(f"Cluster totals => capacity: {cap or 'N/A'}, allocatable: {alloc_total or 'N/A'}")

if lines:
    sys.stdout.write("\n".join(lines))
PY
                        )
                        PY_FALLBACK_STATUS=$?
                        set -e
                        if [ ${PY_FALLBACK_STATUS:-0} -ne 0 ]; then
                            NODE_STORAGE_SECTION=""
                        fi
                    fi
                    if [ -n "${NODE_STORAGE_JSON}" ]; then
                        set +e
                        NODE_STORAGE_VISUAL_SECTION=$(
                            printf '%s' "${NODE_STORAGE_JSON}" | python3 - <<'PY' 2>/dev/null
import json
import sys

BAR_WIDTH = 30

def parse_bytes(value):
    if value is None:
        return None
    try:
        return float(value)
    except Exception:
        return None

def format_gib(bytes_value):
    if bytes_value is None:
        return "N/A"
    return f"{bytes_value / (1024 ** 3):.2f} Gi"

try:
    data = json.loads(sys.stdin.read())
except Exception:
    sys.exit(1)

nodes = data.get("nodes") or {}
totals = data.get("totals") or {}

def build_bar(value, total):
    if value is None or total in (None, 0):
        return "[no data]"
    ratio = max(0.0, min(1.0, value / total))
    filled = int(round(ratio * BAR_WIDTH))
    bar = "#" * filled + "-" * (BAR_WIDTH - filled)
    return f"[{bar}] {ratio * 100:5.1f}%"

lines = []
for name in sorted(nodes.keys()):
    entry = nodes[name]
    cap_bytes = parse_bytes(entry.get("capacity_bytes"))
    alloc_bytes = parse_bytes(entry.get("allocatable_bytes"))
    used_bytes = None
    alloc_pct = None
    used_pct = None
    if cap_bytes is not None:
        if alloc_bytes is not None:
            alloc_pct = (alloc_bytes / cap_bytes) * 100
            used_bytes = cap_bytes - alloc_bytes
            used_pct = (used_bytes / cap_bytes) * 100
        else:
            used_bytes = None
    lines.append(f"{name}")
    lines.append(f"  Capacity:    {format_gib(cap_bytes)}")
    if alloc_bytes is not None:
        lines.append(f"  Allocatable: {format_gib(alloc_bytes)} ({alloc_pct:.1f}% of capacity)")
        lines.append(f"  Workload bar:   {build_bar(alloc_bytes, cap_bytes)}")
    else:
        lines.append("  Allocatable: N/A")
    if used_bytes is not None:
        lines.append(f"  Reserved/Other: {format_gib(used_bytes)} ({used_pct:.1f}% of capacity)")
        lines.append(f"  Reserved bar:   {build_bar(used_bytes, cap_bytes)}")
    lines.append("")

cap_total = parse_bytes(totals.get("capacity_bytes"))
alloc_total = parse_bytes(totals.get("allocatable_bytes"))
if cap_total or alloc_total:
    lines.append("Cluster totals")
    lines.append(f"  Capacity:    {format_gib(cap_total)}")
    if alloc_total is not None and cap_total:
        alloc_pct = (alloc_total / cap_total) * 100 if cap_total else None
        lines.append(f"  Allocatable: {format_gib(alloc_total)} ({alloc_pct:.1f}% of capacity)")
        lines.append(f"  Allocatable bar: {build_bar(alloc_total, cap_total)}")
        if cap_total is not None:
            reserved_total = cap_total - alloc_total
            reserved_pct = (reserved_total / cap_total) * 100 if cap_total else None
            lines.append(f"  Reserved/Other: {format_gib(reserved_total)} ({reserved_pct:.1f}% of capacity)")
            lines.append(f"  Reserved bar:   {build_bar(reserved_total, cap_total)}")

print("\n".join(line for line in lines if line.strip()))
PY
                        )
                        PY_VISUAL_STATUS=$?
                        set -e
                        if [ ${PY_VISUAL_STATUS:-0} -ne 0 ]; then
                            NODE_STORAGE_VISUAL_SECTION=""
                        fi
                    fi
                else
                    NODE_STORAGE_JSON="{}"
                fi
            fi
        fi
    fi
    if [ -n "${NODE_STORAGE_JSON_FILE}" ] && [ -f "${NODE_STORAGE_JSON_FILE}" ]; then
        rm -f "${NODE_STORAGE_JSON_FILE}"
    fi
    if [ -n "${NODE_STORAGE_TABLE_FILE}" ] && [ -f "${NODE_STORAGE_TABLE_FILE}" ]; then
        rm -f "${NODE_STORAGE_TABLE_FILE}"
    fi
    if [ -z "${NODE_STORAGE_JSON}" ]; then
        NODE_STORAGE_JSON="{}"
    fi
    
    # Collect data for each pod
    POD_NAMES=()
    POD_STATUSES=()
    POD_CPUS=()
    POD_MEMORIES=()
    POD_CPU_REQUESTS=()
    POD_CPU_LIMITS=()
    POD_MEMORY_REQUESTS=()
    POD_MEMORY_LIMITS=()
    
    for pod in ${PODS}; do
        echo "  Processing pod: ${pod}"
        POD_NAMES+=("${pod}")
        POD_STATUSES+=("$(get_pod_status "${pod}")")
        POD_CPUS+=("$(get_pod_cpu "${pod}")")
        POD_MEMORIES+=("$(get_pod_memory "${pod}")")
        POD_CPU_REQUESTS+=("$(get_pod_cpu_spec "${pod}" "requests")")
        POD_CPU_LIMITS+=("$(get_pod_cpu_spec "${pod}" "limits")")
        POD_MEMORY_REQUESTS+=("$(get_pod_memory_spec "${pod}" "requests")")
        POD_MEMORY_LIMITS+=("$(get_pod_memory_spec "${pod}" "limits")")
    done
    
    # Ensure we have at least one pod
    if [ ${#POD_NAMES[@]} -eq 0 ]; then
        echo "⚠️  No pod data collected!"
        return 1
    fi
    
    # Query GraphDB for triple counts
    echo ""
    echo "Querying GraphDB for triple counts..."
    
    # Auto-detect repository ID if not explicitly set
    if [ "${GRAPHDB_REPO_ID}" = "openmbee" ] || [ "${GRAPHDB_REPO_ID}" = "flexo" ]; then
        DETECTED_REPO=$(auto_detect_repo_id 2>/dev/null)
        if [ -n "${DETECTED_REPO}" ]; then
            if [ "${VERBOSE}" = true ]; then
                echo "  [DEBUG] Auto-detected repository ID: ${DETECTED_REPO}"
            fi
            GRAPHDB_REPO_ID="${DETECTED_REPO}"
        fi
    fi
    
    # First, try to list repositories if verbose or if query fails
    if [ "${VERBOSE}" = true ]; then
        echo "  [DEBUG] Using repository ID: ${GRAPHDB_REPO_ID}"
        REPOS_LIST=$(list_graphdb_repositories 2>/dev/null || echo "")
        if [ -n "${REPOS_LIST}" ]; then
            echo "  [DEBUG] Available repositories: ${REPOS_LIST:0:200}"
        fi
    fi
    
    if [ "${VERBOSE}" = true ]; then
        TRIPLE_COUNT_ALL=$(query_graphdb_triples "all" || echo "N/A")
        TRIPLE_COUNT_NAMED=$(query_graphdb_triples "named" || echo "N/A")
    else
        TRIPLE_COUNT_ALL=$(query_graphdb_triples "all" 2>/dev/null || echo "N/A")
        TRIPLE_COUNT_NAMED=$(query_graphdb_triples "named" 2>/dev/null || echo "N/A")
    fi
    
    # Display summary
    echo ""
    echo "============================================================"
    echo "Summary"
    echo "============================================================"
    echo ""
    
    # Check if metrics are available
    METRICS_AVAILABLE=true
    if ! kubectl_cmd top pod -n "${NAMESPACE}" --no-headers 2>/dev/null >/dev/null; then
        METRICS_AVAILABLE=false
        echo "⚠️  Metrics API not available - CPU/Memory usage will show N/A"
        echo "   Install metrics-server to enable resource usage monitoring"
        echo ""
    fi
    
    echo "Pod Status & Resources:"
    if [ "${METRICS_AVAILABLE}" = true ]; then
        printf "  %-40s %-10s %-12s %-12s %-20s %-20s\n" "POD NAME" "STATUS" "CPU" "MEMORY" "CPU REQ/LIMIT" "MEM REQ/LIMIT"
        echo "  $(printf '=%.0s' {1..130})"
        for i in "${!POD_NAMES[@]}"; do
            pod="${POD_NAMES[$i]}"
            cpu_req="${POD_CPU_REQUESTS[$i]:-N/A}"
            cpu_lim="${POD_CPU_LIMITS[$i]:-N/A}"
            mem_req="${POD_MEMORY_REQUESTS[$i]:-N/A}"
            mem_lim="${POD_MEMORY_LIMITS[$i]:-N/A}"
            printf "  %-40s %-10s %-12s %-12s %-20s %-20s\n" \
                "${pod}" \
                "${POD_STATUSES[$i]}" \
                "${POD_CPUS[$i]}" \
                "${POD_MEMORIES[$i]}" \
                "${cpu_req}/${cpu_lim}" \
                "${mem_req}/${mem_lim}"
        done
    else
        printf "  %-40s %-10s %-20s %-20s\n" "POD NAME" "STATUS" "CPU REQ/LIMIT" "MEM REQ/LIMIT"
        echo "  $(printf '=%.0s' {1..100})"
        for i in "${!POD_NAMES[@]}"; do
            pod="${POD_NAMES[$i]}"
            cpu_req="${POD_CPU_REQUESTS[$i]:-N/A}"
            cpu_lim="${POD_CPU_LIMITS[$i]:-N/A}"
            mem_req="${POD_MEMORY_REQUESTS[$i]:-N/A}"
            mem_lim="${POD_MEMORY_LIMITS[$i]:-N/A}"
            printf "  %-40s %-10s %-20s %-20s\n" \
                "${pod}" \
                "${POD_STATUSES[$i]}" \
                "${cpu_req}/${cpu_lim}" \
                "${mem_req}/${mem_lim}"
        done
    fi
    
    echo ""
    echo "============================================================"
    echo "Node Disk Capacity:"
    echo "============================================================"
    if [ "${NODE_STORAGE_AVAILABLE}" = true ] && [ -n "${NODE_STORAGE_SECTION}" ]; then
        printf "%s\n" "${NODE_STORAGE_SECTION}"
    else
        echo "${NODE_STORAGE_MESSAGE}"
    fi
    
    echo ""
    echo "Node Disk Allocation (Visual):"
    echo "============================================================"
    if [ "${NODE_STORAGE_AVAILABLE}" = true ] && [ -n "${NODE_STORAGE_VISUAL_SECTION}" ]; then
        printf "%s\n" "${NODE_STORAGE_VISUAL_SECTION}"
    else
        echo "${NODE_STORAGE_MESSAGE}"
    fi
    
    echo ""
    echo "============================================================"
    echo "Triple Counts (GraphDB):"
    echo "============================================================"
    echo "  All triples: ${TRIPLE_COUNT_ALL}"
    echo "  Named graph triples: ${TRIPLE_COUNT_NAMED}"
    if [ "${TRIPLE_COUNT_ALL}" = "N/A" ] || [ "${TRIPLE_COUNT_NAMED}" = "N/A" ]; then
        echo ""
        echo "  ⚠️  GraphDB query failed. To debug:"
        echo "     1. Check GraphDB pod: ${KUBECTL_BIN} get pods -n ${NAMESPACE} | grep graphdb"
        echo "     2. Verify repository ID: GRAPHDB_REPO_ID=${GRAPHDB_REPO_ID}"
        echo "     3. Check GraphDB port: GRAPHDB_PORT=${GRAPHDB_PORT}"
        echo "     4. Run with --verbose to see debug output"
        echo "     5. List repositories: ${KUBECTL_BIN} exec <graphdb-pod> -n ${NAMESPACE} -- curl http://localhost:${GRAPHDB_PORT}/rest/repositories"
        echo "     6. Test size endpoint: ${KUBECTL_BIN} exec <graphdb-pod> -n ${NAMESPACE} -- curl http://localhost:${GRAPHDB_PORT}/rest/repositories/${GRAPHDB_REPO_ID}/size"
    fi
    
    # Save reports
    echo ""
    echo "Generating reports..."
    
    # Create JSON structure
    JSON_PODS="{"
    for i in "${!POD_NAMES[@]}"; do
        pod="${POD_NAMES[$i]}"
        if [ "${JSON_PODS}" != "{" ]; then
            JSON_PODS="${JSON_PODS},"
        fi
        JSON_PODS="${JSON_PODS}\"${pod}\": {
      \"status\": \"${POD_STATUSES[$i]}\",
      \"cpu_usage\": \"${POD_CPUS[$i]}\",
      \"memory_usage\": \"${POD_MEMORIES[$i]}\",
      \"cpu_request\": \"${POD_CPU_REQUESTS[$i]}\",
      \"cpu_limit\": \"${POD_CPU_LIMITS[$i]}\",
      \"memory_request\": \"${POD_MEMORY_REQUESTS[$i]}\",
      \"memory_limit\": \"${POD_MEMORY_LIMITS[$i]}\"
    }"
    done
    JSON_PODS="${JSON_PODS}}"
    
    # Write JSON report
    cat > "${REPORT_FILE_JSON}" <<EOF
{
  "namespace": "${NAMESPACE}",
  "timestamp": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
  "pods": ${JSON_PODS},
  "node_storage": ${NODE_STORAGE_JSON},
  "triple_counts": {
    "all_triples": "${TRIPLE_COUNT_ALL}",
    "named_graph_triples": "${TRIPLE_COUNT_NAMED}"
  }
}
EOF
    
    # Write text report with table format
    {
        echo "============================================================"
        echo "Flexo Kubernetes Cluster Performance Report"
        echo "============================================================"
        echo "Namespace: ${NAMESPACE}"
        echo "Timestamp: $(date)"
        echo "Generated: $(date -u +"%Y-%m-%dT%H:%M:%SZ") UTC"
        echo "Kubectl command: ${KUBECTL_BIN}"
        echo ""
        echo "============================================================"
        echo "Pod Status & Resources"
        echo "============================================================"
        echo ""
        if [ "${METRICS_AVAILABLE}" = true ]; then
            printf "%-45s %-12s %-15s %-15s %-25s %-25s\n" "POD NAME" "STATUS" "CPU USAGE" "MEMORY USAGE" "CPU REQ/LIMIT" "MEM REQ/LIMIT"
            echo "$(printf '=%.0s' {1..150})"
            for i in "${!POD_NAMES[@]}"; do
                pod="${POD_NAMES[$i]}"
                cpu_req="${POD_CPU_REQUESTS[$i]:-N/A}"
                cpu_lim="${POD_CPU_LIMITS[$i]:-N/A}"
                mem_req="${POD_MEMORY_REQUESTS[$i]:-N/A}"
                mem_lim="${POD_MEMORY_LIMITS[$i]:-N/A}"
                printf "%-45s %-12s %-15s %-15s %-25s %-25s\n" \
                    "${pod}" \
                    "${POD_STATUSES[$i]}" \
                    "${POD_CPUS[$i]}" \
                    "${POD_MEMORIES[$i]}" \
                    "${cpu_req}/${cpu_lim}" \
                    "${mem_req}/${mem_lim}"
            done
        else
            printf "%-45s %-12s %-25s %-25s\n" "POD NAME" "STATUS" "CPU REQ/LIMIT" "MEM REQ/LIMIT"
            echo "$(printf '=%.0s' {1..120})"
            for i in "${!POD_NAMES[@]}"; do
                pod="${POD_NAMES[$i]}"
                cpu_req="${POD_CPU_REQUESTS[$i]:-N/A}"
                cpu_lim="${POD_CPU_LIMITS[$i]:-N/A}"
                mem_req="${POD_MEMORY_REQUESTS[$i]:-N/A}"
                mem_lim="${POD_MEMORY_LIMITS[$i]:-N/A}"
                printf "%-45s %-12s %-25s %-25s\n" \
                    "${pod}" \
                    "${POD_STATUSES[$i]}" \
                    "${cpu_req}/${cpu_lim}" \
                    "${mem_req}/${mem_lim}"
            done
        fi
        echo ""
        echo "============================================================"
        echo "Node Disk Capacity"
        echo "============================================================"
        if [ "${NODE_STORAGE_AVAILABLE}" = true ] && [ -n "${NODE_STORAGE_SECTION}" ]; then
            printf "%s\n" "${NODE_STORAGE_SECTION}"
        else
            echo "${NODE_STORAGE_MESSAGE}"
        fi
        echo ""
        echo "============================================================"
        echo "Node Disk Allocation (Visual)"
        echo "============================================================"
        if [ "${NODE_STORAGE_AVAILABLE}" = true ] && [ -n "${NODE_STORAGE_VISUAL_SECTION}" ]; then
            printf "%s\n" "${NODE_STORAGE_VISUAL_SECTION}"
        else
            echo "${NODE_STORAGE_MESSAGE}"
        fi
        echo ""
        echo "============================================================"
        echo "Triple Counts (GraphDB)"
        echo "============================================================"
        echo "Repository ID: ${GRAPHDB_REPO_ID}"
        echo "All triples: ${TRIPLE_COUNT_ALL}"
        echo "Named graph triples: ${TRIPLE_COUNT_NAMED}"
        if [ "${TRIPLE_COUNT_ALL}" = "N/A" ] || [ "${TRIPLE_COUNT_NAMED}" = "N/A" ]; then
            echo ""
        echo "⚠️  GraphDB query failed. To debug:"
        echo "   1. Check GraphDB pod: ${KUBECTL_BIN} get pods -n ${NAMESPACE} | grep graphdb"
        echo "   2. Verify repository ID: GRAPHDB_REPO_ID=${GRAPHDB_REPO_ID}"
        echo "   3. Check GraphDB port: GRAPHDB_PORT=${GRAPHDB_PORT}"
        echo "   4. Run with --verbose to see debug output"
        fi
        echo ""
        echo "============================================================"
        echo "Report Files"
        echo "============================================================"
        echo "JSON: ${REPORT_FILE_JSON}"
        echo "Text: ${REPORT_FILE_TXT}"
        echo "============================================================"
    } > "${REPORT_FILE_TXT}"
    
    echo "✓ JSON report saved to: ${REPORT_FILE_JSON}"
    echo "✓ Text report saved to: ${REPORT_FILE_TXT}"
    echo ""
    echo "============================================================"
}

# Main execution
set -e  # Enable error exit
collect_and_display_metrics