#!/usr/bin/env bash

set -u

SERVICE_NAME="connector_server"
LOG_DIR="/home/log/connector_server"
CHECK_DATE="$(date +%F)"
JOURNAL_LINES=1000
RUN_LOG_LINES=20000
ALARM_LOG_LINES=20000
IDLE_CONNECTION_SECONDS=600
IDLE_CONNECTION_MILLISECONDS=$((IDLE_CONNECTION_SECONDS * 1000))
GREP_COLOR_OPTION="--color=always"
HIGHLIGHT_PATTERN="restart|panic|error|fail|failed|invalid|critical|out of memory|oom[.]kill|err|warn|idle"

RUN_LOG="${LOG_DIR}/run.log"
ALARM_LOG="${LOG_DIR}/alarm.log"

print_section() {
    printf '\n===== %s =====\n' "$1"
}

print_missing() {
    printf '[skip] file is missing or unreadable: %s\n' "$1"
}

highlight_stream() {
    local extra_pattern="${1:-}"
    local pattern="$HIGHLIGHT_PATTERN"

    if [[ -n "$extra_pattern" ]]; then
        pattern="${pattern}|${extra_pattern}"
    fi

    grep $GREP_COLOR_OPTION -iE "${pattern}|$" || true
}

grep_one() {
    local pattern="$1"
    local file="$2"

    if [[ ! -r "$file" ]]; then
        print_missing "$file"
        return 0
    fi

    grep $GREP_COLOR_OPTION -F -- "$pattern" "$file" || true
}

grep_many() {
    local pattern="$1"
    shift
    local file
    local readable_files=()

    for file in "$@"; do
        [[ -r "$file" ]] && readable_files+=("$file")
    done

    if ((${#readable_files[@]} == 0)); then
        printf '[skip] no readable files matched the requested path\n'
        return 0
    fi

    grep $GREP_COLOR_OPTION -F -- "$pattern" "${readable_files[@]}" || true
}

filter_error_file() {
    local file="$1"

    if [[ ! -r "$file" ]]; then
        print_missing "$file"
        return 0
    fi

    grep -v "handshake error" "$file" \
        | grep -v runner \
        | grep -v promtail \
        | grep -vi yunxiao \
        | grep -iE "error|fail|failed|invalid|critical|panic" \
        | highlight_stream
}

filter_oom_files() {
    local file
    local readable_files=()

    for file in "$@"; do
        [[ -r "$file" ]] && readable_files+=("$file")
    done

    if ((${#readable_files[@]} == 0)); then
        printf '[skip] no readable files matched the requested path\n'
        return 0
    fi

    grep $GREP_COLOR_OPTION -iE "out of memory|oom[.]kill" \
        "${readable_files[@]}" || true
}

filter_kernel_warnings() {
    local file
    local readable_files=()

    for file in "$@"; do
        [[ -r "$file" ]] && readable_files+=("$file")
    done

    if ((${#readable_files[@]} == 0)); then
        printf '[skip] no readable files matched the requested path\n'
        return 0
    fi

    grep "kernel:" "${readable_files[@]}" \
        | grep -iE "err|warn" \
        | highlight_stream
}

filter_run_errors() {
    if [[ ! -r "$RUN_LOG" ]]; then
        print_missing "$RUN_LOG"
        return 0
    fi

    tail -n "$RUN_LOG_LINES" "$RUN_LOG" \
        | grep -iE "error|failed|invalid" \
        | grep -v tlvServerReq \
        | grep -v api \
        | grep -vi yunxiao \
        | highlight_stream
}

filter_alarm_errors() {
    if [[ ! -r "$ALARM_LOG" ]]; then
        print_missing "$ALARM_LOG"
        return 0
    fi

    tail -n "$ALARM_LOG_LINES" "$ALARM_LOG" \
        | grep -F -- "$CHECK_DATE" \
        | grep -vi yunxiao \
        | highlight_stream "$CHECK_DATE"
}

get_service_pid() {
    local pid=""
    local main_pid_line=""

    if command -v systemctl >/dev/null 2>&1; then
        pid="$(systemctl show -p MainPID --value "$SERVICE_NAME" 2>/dev/null || true)"
        if ! [[ "$pid" =~ ^[0-9]+$ ]] || ((pid <= 0)); then
            main_pid_line="$(systemctl show -p MainPID "$SERVICE_NAME" 2>/dev/null | head -1 || true)"
            main_pid_line="${main_pid_line#MainPID=}"
            pid="$main_pid_line"
        fi
    fi

    if [[ "$pid" =~ ^[0-9]+$ ]] && ((pid > 0)); then
        printf '%s\n' "$pid"
    fi
}

collect_service_pids() {
    {
        get_service_pid
        if command -v pgrep >/dev/null 2>&1; then
            pgrep -x "$SERVICE_NAME" 2>/dev/null || true
            pgrep -f "$SERVICE_NAME" 2>/dev/null || true
        fi
    } | awk '$1 ~ /^[0-9]+$/ && $1 > 0 { print $1 }' | sort -nu
}

collect_service_socket_inodes() {
    local pid
    local fd
    local target
    local inode

    while read -r pid; do
        [[ -n "$pid" ]] || continue
        [[ -d "/proc/$pid/fd" ]] || continue

        for fd in "/proc/$pid"/fd/*; do
            target="$(readlink "$fd" 2>/dev/null || true)"
            case "$target" in
                socket:\[*\])
                    inode="${target#socket:[}"
                    inode="${inode%]}"
                    printf '%s\n' "$inode"
                    ;;
            esac
        done
    done | awk '$1 ~ /^[0-9]+$/ && $1 > 0 { print $1 }' | sort -nu
}

service_runtime_info() {
    if ! command -v systemctl >/dev/null 2>&1; then
        printf '[skip] systemctl is not installed\n'
        return 0
    fi

    printf '$ systemctl is-active %s\n' "$SERVICE_NAME"
    systemctl is-active "$SERVICE_NAME" || true
    printf '$ systemctl show %s\n' "$SERVICE_NAME"
    systemctl show "$SERVICE_NAME" \
        -p MainPID \
        -p ActiveEnterTimestamp \
        -p ExecMainStartTimestamp \
        -p NRestarts \
        -p ExecMainStatus \
        -p Result || true
}

process_resource_info() {
    local pid
    pid="$(get_service_pid)"

    if [[ -z "$pid" ]]; then
        printf '[skip] could not determine MainPID for %s\n' "$SERVICE_NAME"
        return 0
    fi

    printf '$ ps -p %s -o pid,ppid,%%cpu,%%mem,rss,vsz,etime,cmd\n' "$pid"
    ps -p "$pid" -o pid,ppid,%cpu,%mem,rss,vsz,etime,cmd || true
    printf '$ open file descriptors for PID %s\n' "$pid"
    find "/proc/$pid/fd" -mindepth 1 -maxdepth 1 -type l 2>/dev/null | wc -l
}

system_resource_info() {
    printf '$ df -h /home /var\n'
    df -h /home /var 2>&1 || true
    printf '$ df -i /home /var\n'
    df -i /home /var 2>&1 || true
    printf '$ free -h\n'
    if command -v free >/dev/null 2>&1; then
        free -h || true
    else
        printf '[skip] free is not installed\n'
    fi
}

system_oom_info() {
    if ! command -v dmesg >/dev/null 2>&1; then
        printf '[skip] dmesg is not installed\n'
        return 0
    fi

    printf '$ dmesg -T: OOM and killed process records\n'
    dmesg -T 2>/dev/null \
        | grep -iE "out of memory|oom[.]kill|killed process" \
        | highlight_stream || true
}

service_journal_info() {
    if ! command -v journalctl >/dev/null 2>&1; then
        printf '[skip] journalctl is not installed\n'
        return 0
    fi

    printf '$ journalctl -u %s -n %s --no-pager\n' "$SERVICE_NAME" "$JOURNAL_LINES"
    journalctl -u "$SERVICE_NAME" -n "$JOURNAL_LINES" --no-pager 2>&1 \
        | grep -v "http: TLS handshake" | highlight_stream
}

normalize_ss_info() {
    awk '
        $1 ~ /^[A-Z][A-Z0-9_-]*$/ && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ {
            if (record != "") {
                print record
            }
            record = $0
            next
        }
        record != "" && ($0 ~ /lastsnd:/ || $0 ~ /lastrcv:/) {
            record = record " " $0
            next
        }
        END {
            if (record != "") {
                print record
            }
        }
    '
}

service_connection_info() {
    local pid
    local pids
    local socket_inodes
    local ss_output
    local connections
    local pid_connections
    local inode_connections
    local name_connections
    local idle_connections

    if ! command -v ss >/dev/null 2>&1; then
        printf '[skip] ss is not installed\n'
        return 0
    fi

    pid="$(get_service_pid)"
    pids="$(collect_service_pids)"
    socket_inodes="$(printf '%s\n' "$pids" | collect_service_socket_inodes)"
    ss_output="$(ss -Hantpie 2>/dev/null | normalize_ss_info || true)"

    pid_connections="$(
        while read -r pid; do
            [[ -n "$pid" ]] || continue
            printf '%s\n' "$ss_output" | grep -F "pid=$pid," || true
        done <<<"$pids"
    )"
    inode_connections="$(
        while read -r inode; do
            [[ -n "$inode" ]] || continue
            printf '%s\n' "$ss_output" | grep -E "(^|[[:space:]])ino:${inode}([[:space:]]|$)" || true
        done <<<"$socket_inodes"
    )"
    name_connections="$(printf '%s\n' "$ss_output" | grep -F "$SERVICE_NAME" || true)"
    connections="$(
        {
            printf '%s\n' "$pid_connections"
            printf '%s\n' "$inode_connections"
            printf '%s\n' "$name_connections"
        } | awk 'NF && !seen[$0]++'
    )"

    if [[ -z "$connections" ]]; then
        printf '[skip] no sockets associated with %s were found\n' "$SERVICE_NAME"
        printf '[diag] MainPID=%s\n' "${pid:-unknown}"
        if [[ -n "$pids" ]]; then
            printf '[diag] candidate PIDs: %s\n' "$(printf '%s\n' "$pids" | xargs echo)"
        else
            printf '[diag] candidate PIDs: none; pgrep did not find %s\n' "$SERVICE_NAME"
        fi
        if [[ -n "$socket_inodes" ]]; then
            printf '[diag] socket inodes from /proc: %s\n' "$(printf '%s\n' "$socket_inodes" | xargs echo)"
        else
            printf '[diag] socket inodes from /proc: none; run as root or check service PID\n'
        fi
        printf '[diag] raw socket lines containing service name: %s\n' \
            "$(printf '%s\n' "$ss_output" | grep -Fc "$SERVICE_NAME" || true)"
        return 0
    fi

    printf '$ ss -Hantpie: state summary for %s\n' "$SERVICE_NAME"
    printf '%s\n' "$connections" \
        | awk '
            { state[$1]++; total++ }
            END {
                printf "total=%d\n", total
                for (name in state) {
                    printf "%s=%d\n", name, state[name]
                }
            }
        ' \
        | sort

    printf '%s\n' "$connections" \
        | awk '$1 != "ESTAB" && $1 != "LISTEN" { print "[attention] " $0 }' \
        | highlight_stream "CLOSE-WAIT|SYN-SENT|SYN-RECV|FIN-WAIT|LAST-ACK|CLOSING|TIME-WAIT"

    printf '%s\n' "$connections" \
        | awk '$2 != 0 || $3 != 0 { print "[queue] " $0 }' \
        | highlight_stream "queue"

    printf '$ ss -Hantpi: ESTAB connections idle for at least %ss\n' \
        "$IDLE_CONNECTION_SECONDS"
    idle_connections="$(printf '%s\n' "$connections" \
        | awk -v threshold="$IDLE_CONNECTION_MILLISECONDS" '
            $1 == "ESTAB" {
                lastsnd = -1
                lastrcv = -1
                for (i = 1; i <= NF; i++) {
                    if ($i ~ /^lastsnd:/) {
                        split($i, value, ":")
                        lastsnd = value[2] + 0
                    }
                    if ($i ~ /^lastrcv:/) {
                        split($i, value, ":")
                        lastrcv = value[2] + 0
                    }
                }
                if (lastsnd >= threshold && lastrcv >= threshold) {
                    printf "[IDLE] lastsnd=%sms lastrcv=%sms %s -> %s\n", \
                        lastsnd, lastrcv, $4, $5
                }
            }
        ')"
    if [[ -n "$idle_connections" ]]; then
        printf '%s\n' "$idle_connections" | highlight_stream "IDLE"
    else
        printf '[ok] no ESTAB connections idle for at least %ss\n' \
            "$IDLE_CONNECTION_SECONDS"
    fi

    printf '$ ss -Hantpie: top ESTAB remote endpoints\n'
    printf '%s\n' "$connections" \
        | awk '$1 == "ESTAB" { print $5 }' \
        | sort \
        | uniq -c \
        | sort -nr \
        | head -20
}

run_journal_panic() {
    if ! command -v journalctl >/dev/null 2>&1; then
        printf '[skip] journalctl is not installed\n'
        return 0
    fi

    journalctl --no-pager -n "$JOURNAL_LINES" \
        | grep $GREP_COLOR_OPTION -F -- "panic" || true
}

run_journal_errors() {
    if ! command -v journalctl >/dev/null 2>&1; then
        printf '[skip] journalctl is not installed\n'
        return 0
    fi

    journalctl --no-pager -n "$JOURNAL_LINES" \
        | grep -iE "error|failed|invalid" \
        | grep -v promtail \
        | grep -v handshake \
        | grep -v runner \
        | grep -vi yunxiao \
        | grep -v "http: TLS handshake" \
        | highlight_stream
}

print_section "run.log: restart"
grep_one "restart" "$RUN_LOG"

shopt -s nullglob
MESSAGE_FILES=(/var/log/messages*)
shopt -u nullglob

print_section "messages*: panic"
grep_many "panic" "${MESSAGE_FILES[@]}"

print_section "syslog: panic"
grep_one "panic" /var/log/syslog

print_section "syslog.1: panic"
grep_one "panic" /var/log/syslog.1

print_section "messages*: out of memory / oom.kill"
filter_oom_files "${MESSAGE_FILES[@]}"

print_section "messages*: filtered errors"
for file in "${MESSAGE_FILES[@]}"; do
    filter_error_file "$file"
done
if ((${#MESSAGE_FILES[@]} == 0)); then
    printf '[skip] no /var/log/messages* files found\n'
fi

print_section "syslog: filtered errors"
filter_error_file /var/log/syslog

print_section "syslog.1: filtered errors"
filter_error_file /var/log/syslog.1

print_section "messages*: kernel err/warn"
filter_kernel_warnings "${MESSAGE_FILES[@]}"

print_section "journal: panic"
run_journal_panic

print_section "journal: filtered errors"
run_journal_errors

print_section "systemctl status: ${SERVICE_NAME}"
if command -v systemctl >/dev/null 2>&1; then
    systemctl status "$SERVICE_NAME" --no-pager --full 2>&1 | highlight_stream
else
    printf '[skip] systemctl is not installed\n'
fi

print_section "service runtime: ${SERVICE_NAME}"
service_runtime_info

print_section "process resources: ${SERVICE_NAME}"
process_resource_info

print_section "system resources: disk, inode, memory"
system_resource_info

print_section "system OOM records"
system_oom_info

print_section "service journal: ${SERVICE_NAME}"
service_journal_info

print_section "network connections: ${SERVICE_NAME}"
service_connection_info

print_section "run.log: recent errors"
filter_run_errors

print_section "alarm.log: ${CHECK_DATE}"
filter_alarm_errors