#!/usr/bin/env python3
"""Backup running configs from network devices via SSH."""

__version__ = "3.22.2"

import argparse
import configparser
import fnmatch
import getpass
import hashlib
import ipaddress
import html
import itertools
import json
import logging
import os
import re
import select
import shutil
import signal
import socket
import sqlite3
import struct
import subprocess
import sys
try:
    import termios  # POSIX TTY control — client-side raw mode for `connect`
    import tty
    import fcntl
except ImportError:
    termios = tty = fcntl = None
import tempfile
import threading
import time
try:
    import grp  # POSIX-only — used to diagnose "not in netops group" errors
except ImportError:
    grp = None
try:
    import pwd  # POSIX-only — used to resolve UID -> username for audit log
except ImportError:
    pwd = None
import logging.handlers
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta

import pexpect
if sys.platform == "win32":
    from pexpect.popen_spawn import PopenSpawn

# Base SSH options — no legacy algorithms; those are added only when needed
_SSH_BASE_OPTS = [
    "-o", "StrictHostKeyChecking=no",
    "-o", "UserKnownHostsFile=/dev/null",
]

# Prompt pattern for common network devices
# Handles trailing ANSI escape sequences (cursor positioning, etc.) including
# partial sequences that arrive mid-stream from old telnet full-screen switches.
# Pattern: hostname[#>%] + any mix of whitespace / ANSI sequences + end-of-buffer.
# `%` matches the Juniper FreeBSD shell prompt seen when logging in as root.
PROMPT_RE = r"[\w\-/\.()]+[#>%]\s*(?:\x1b[^\x1b\n]*)*\s*$"
_SHELL_PROMPT_RE = r"%\s*(?:\x1b[^\x1b\n]*)*\s*$"

# Tool availability (set by check_tools() at startup)
_HAS_SSH = False
_HAS_TELNET = False

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))


def _resolve_base_dir(env_var, script_dir_markers, fhs_default):
    """Resolve a base directory, preferring an in-place install.

    Priority, highest first:
      1. $<env_var> — explicit override (for testing or non-standard layouts).
      2. SCRIPT_DIR, if it already holds one of script_dir_markers. This is
         the back-compat path: existing run-in-place installs (e.g.
         /home/netops/netops on some sites) keep using files next to the
         script and are completely unaffected by packaging — even on a box
         that also happens to have the FHS directory.
      3. fhs_default, if that directory exists (a packaged install: the
         .deb's postinst creates /etc/netops and /var/lib/netops).
      4. SCRIPT_DIR — plain `git clone` / dev checkout fallback.
    """
    override = os.environ.get(env_var)
    if override:
        return os.path.abspath(os.path.expanduser(override))
    for marker in script_dir_markers:
        if os.path.exists(os.path.join(SCRIPT_DIR, marker)):
            return SCRIPT_DIR
    if os.path.isdir(fhs_default):
        return fhs_default
    return SCRIPT_DIR


# Config (read-only at runtime): netops.conf / legacy config.ini, secrets.conf.
# Packaged location: /etc/netops.  Override with NETOPS_CONFIG_DIR.
CONFIG_DIR = _resolve_base_dir(
    "NETOPS_CONFIG_DIR",
    ("netops.conf", "config.ini", "secrets.conf"),
    "/etc/netops",
)
# Mutable state: netops.db, configs/ (+ its git repo), device list.
# Packaged location: /var/lib/netops.  Override with NETOPS_STATE_DIR.
# Resolved independently of CONFIG_DIR so /etc vs /var/lib can diverge on a
# packaged box while staying unified on an in-place install.
STATE_DIR = _resolve_base_dir(
    "NETOPS_STATE_DIR",
    ("netops.db",),
    "/var/lib/netops",
)


def _resolve_log_dir():
    """Where netops.log / netops-debug.log live. A PACKAGED install gets a
    dedicated FHS log dir /var/log/netops (created by the postinst, covered by
    the AppArmor profile, rotated by /etc/logrotate.d/netops). A run-in-place /
    dev checkout keeps the log alongside the state so the tree stays
    self-contained. Override with NETOPS_LOG_DIR."""
    override = os.environ.get("NETOPS_LOG_DIR")
    if override:
        return os.path.abspath(os.path.expanduser(override))
    if STATE_DIR == "/var/lib/netops":
        return "/var/log/netops"
    return STATE_DIR


LOG_DIR = _resolve_log_dir()
CONFIGS_DIR = os.path.join(STATE_DIR, "configs")
LOG_FILE = os.path.join(LOG_DIR, "netops.log")
_CONFIG_FILE_NEW = os.path.join(CONFIG_DIR, "netops.conf")
_CONFIG_FILE_LEGACY = os.path.join(CONFIG_DIR, "config.ini")
# Prefer netops.conf for consistency with secrets.conf. Fall back to the
# legacy config.ini name so existing deployments keep working — a one-line
# rename completes the migration when the operator is ready. load_config()
# logs a deprecation note when the legacy filename is in use.
CONFIG_FILE = _CONFIG_FILE_NEW if os.path.exists(_CONFIG_FILE_NEW) else _CONFIG_FILE_LEGACY
DB_FILE = os.path.join(STATE_DIR, "netops.db")
SECRETS_FILE = os.path.join(CONFIG_DIR, "secrets.conf")

# secrets.conf is meant to be shareable with a dedicated `netops` group
# (mode 0640, group netops) so multiple operators can run netops — manually
# or as their login shell — without each keeping a private copy of the
# credentials. The packaged install sets this up in debian/postinst.
# Security rule: the owner may have any bits, the group may *read* (0o040)
# but not write or exec, and the world gets nothing. Forbidden mask is
# therefore group-write + group-exec + every world bit.
SECRET_GROUP = "netops"
_SECRET_MODE = 0o640
_SECRET_FORBIDDEN_MODE = 0o037

log = logging.getLogger("netops")


# ---------------------------------------------------------------------------
# Security audit log.
# Captures every operator action (console command, CLI invocation) with
# the Linux identity attached, so a compromised account's activity is
# traceable. Lands in syslog under the authpriv facility:
#   journalctl -t netops-audit
#   /var/log/auth.log
# rsyslog/syslog-ng can ship authpriv off-host for tamper-resistance.
# Falls back to a no-op handler when /dev/log isn't available (e.g.
# Windows, containers without syslog).
# ---------------------------------------------------------------------------

_audit = logging.getLogger("netops-audit")
_audit.propagate = False
_audit.setLevel(logging.INFO)
try:
    _audit_handler = logging.handlers.SysLogHandler(
        address="/dev/log",
        facility=logging.handlers.SysLogHandler.LOG_AUTHPRIV,
    )
    _audit_handler.setFormatter(
        logging.Formatter("netops-audit[%(process)d]: %(message)s"))
    _audit.addHandler(_audit_handler)
except (OSError, FileNotFoundError):
    # syslog socket unavailable — fall through to a NullHandler so
    # _audit_log() never raises. journalctl simply has no rows.
    _audit.addHandler(logging.NullHandler())


def _audit_identity():
    """Return the operator identity tuple for an audit-log entry.

    linux_user: real UID's username (sudo's invoking user when present
                via SUDO_USER, else the current real UID).
    euid_user:  effective UID's username (only included when it differs
                from linux_user — i.e. running under sudo).
    netops_admin: $NETOPS_ADMIN from the SSH authorized_keys forced-
                  command wrapper. The "claimed" identity at the SSH
                  layer; should usually match linux_user.
    ssh_from:   "<ip>:<port>" parsed from $SSH_CONNECTION. NULL when
                netops is invoked locally rather than over SSH.
    """
    real_uid = os.getuid() if hasattr(os, "getuid") else None
    eff_uid = os.geteuid() if hasattr(os, "geteuid") else None
    sudo_user = os.environ.get("SUDO_USER", "").strip()

    def _uid_to_name(uid):
        if uid is None or pwd is None:
            return str(uid) if uid is not None else "(unknown)"
        try:
            return pwd.getpwuid(uid).pw_name
        except KeyError:
            return str(uid)

    # When sudo'd, prefer $SUDO_USER over the real UID (which would be
    # 'root' inside the sudo session) — operator intent is to attribute
    # the action to the invoking human, not to root.
    linux_user = sudo_user or _uid_to_name(real_uid)
    euid_user = _uid_to_name(eff_uid)
    admin = os.environ.get("NETOPS_ADMIN", "").strip() or None
    ssh_conn = os.environ.get("SSH_CONNECTION", "").strip()
    ssh_from = None
    if ssh_conn:
        parts = ssh_conn.split()
        if len(parts) >= 2:
            ssh_from = f"{parts[0]}:{parts[1]}"
    return linux_user, euid_user, admin, ssh_from


def _audit_log(action, command=None, actor=None, **extra):
    """Record an audit-worthy event to syslog (authpriv).

    action:   short keyword (cli, console_start, console_cmd, console_exit,
              parse_error, etc.) — used for grep/journalctl filtering.
    command:  full command line as typed (or argv joined). Quoted with
              %r so embedded spaces / quotes don't blur the structure.
    actor:    overrides linux_user with a verified identity the *process* UID
              doesn't reflect. netopsd uses it so a brokered action is
              attributed to the SO_PEERCRED-verified operator, not to the
              `netops` service account the daemon runs as — otherwise
              `grep linux_user=alice` would miss every session Alice opened.
    **extra:  any additional k=v pairs to append (e.g. error=...).

    The line shape is space-separated k=v pairs (logfmt-style) so it
    parses cleanly in syslog tooling without ad-hoc regexes.
    """
    linux_user, euid_user, admin, ssh_from = _audit_identity()
    if actor:
        linux_user = actor
    parts = [f"action={action}", f"linux_user={linux_user}"]
    if euid_user and euid_user != linux_user:
        parts.append(f"euid_user={euid_user}")
    if admin and admin != linux_user:
        parts.append(f"netops_admin={admin}")
    if ssh_from:
        parts.append(f"ssh_from={ssh_from}")
    if command is not None:
        parts.append(f"cmd={command!r}")
    for k, v in extra.items():
        parts.append(f"{k}={v!r}" if isinstance(v, str) else f"{k}={v}")
    try:
        _audit.info(" ".join(parts))
    except Exception:
        # Audit MUST NEVER block a netops operation — fall through.
        pass


# ---------------------------------------------------------------------------
# Operational event log — instrumentation for 'digest health'.
# Per-SNMP-query rows would explode the DB (~15M rows/week/site), so we
# aggregate via a tick-local accumulator (_snmp_timing_*) and write one
# row per monitor-stp tick / per backup-device / per ssh-connect-attempt.
# ---------------------------------------------------------------------------

_snmp_timing_lock = threading.Lock()
_snmp_timing_state = {"times": [], "failed": 0, "enabled": False}


def _snmp_timing_start():
    """Reset the SNMP timing accumulator for the upcoming scope."""
    with _snmp_timing_lock:
        _snmp_timing_state["times"] = []
        _snmp_timing_state["failed"] = 0
        _snmp_timing_state["enabled"] = True


def _snmp_timing_record(duration_ms, success):
    """Called from _snmp_run. Cheap no-op when accumulation isn't enabled."""
    if not _snmp_timing_state["enabled"]:
        return
    with _snmp_timing_lock:
        _snmp_timing_state["times"].append(duration_ms)
        if not success:
            _snmp_timing_state["failed"] += 1


def _snmp_timing_stop():
    """Stop accumulating and return a small stats dict, or None if no data."""
    with _snmp_timing_lock:
        _snmp_timing_state["enabled"] = False
        times = _snmp_timing_state["times"]
        failed = _snmp_timing_state["failed"]
        _snmp_timing_state["times"] = []
        _snmp_timing_state["failed"] = 0
    if not times:
        return None
    return {
        "snmp_count": len(times),
        "snmp_failed": failed,
        "snmp_min_ms": min(times),
        "snmp_max_ms": max(times),
        "snmp_avg_ms": int(sum(times) / len(times)),
    }


def _record_op(op_type, *, started_at=None, duration_ms=0, success=True,
               ip=None, reason=None, extra=None, error=None):
    """Insert one row into op_events. Best-effort — failures here are
    swallowed so instrumentation doesn't break the operation that fired
    it."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    if started_at is None:
        started_at = now
    if isinstance(extra, dict):
        extra = " ".join(f"{k}={v}" for k, v in extra.items())
    try:
        conn = _db()
        conn.execute(
            "INSERT INTO op_events "
            "(op_type, started_at, duration_ms, success, ip, reason, extra, error) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
            (op_type, started_at, int(duration_ms), 1 if success else 0,
             ip, reason, extra, error))
        conn.commit()
        conn.close()
    except Exception as e:
        log.debug("op_events insert failed: %s", e)


# ---------------------------------------------------------------------------
# Database
# ---------------------------------------------------------------------------

def init_db():
    """Create the database and tables if they don't exist."""
    conn = sqlite3.connect(DB_FILE, timeout=30)
    # WAL lets a long-running writer (e.g. cron 'monitor stp') coexist with
    # concurrent readers/writers instead of blocking them on the whole DB.
    conn.execute("PRAGMA journal_mode=WAL")
    # Keep transient b-trees (a data-migration UPDATE over a large port_macs,
    # etc.) in RAM. init_db must set this like _db() does: SQLite otherwise
    # spills to /var/tmp, which the AppArmor profile denies, and the op dies
    # with SQLITE_CANTOPEN ("unable to open database file") mid-migration.
    conn.execute("PRAGMA temp_store=MEMORY")
    # 3.8.9 rename: mac_churn_* tables -> mac_flux_*. ALTER TABLE RENAME
    # preserves data; run BEFORE CREATE TABLE IF NOT EXISTS so an existing
    # pre-3.8.9 DB isn't left with both an empty mac_flux_alerts (created
    # below) AND the original mac_churn_alerts (data stranded). On a fresh
    # install neither table exists yet and these statements are no-ops.
    for old, new in (("mac_churn_alerts",    "mac_flux_alerts"),
                     ("mac_churn_whitelist", "mac_flux_whitelist")):
        has_old = conn.execute(
            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
            (old,)).fetchone()
        has_new = conn.execute(
            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
            (new,)).fetchone()
        if has_old and not has_new:
            conn.execute(f"ALTER TABLE {old} RENAME TO {new}")
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS devices (
            ip             TEXT PRIMARY KEY,
            base_mac       TEXT,
            hostname       TEXT,
            dns_name       TEXT,
            model          TEXT,
            serial         TEXT,
            firmware       TEXT,
            hardware_info  TEXT,
            proto          TEXT,
            username       TEXT,
            password_hash  TEXT,
            ssh_open       INTEGER DEFAULT 0,
            telnet_open    INTEGER DEFAULT 0,
            platform       TEXT,
            stp_enabled    INTEGER,
            stp_mode       TEXT,
            stp_last_check TEXT,
            snmp_enabled   INTEGER,
            snmp_proto     TEXT,
            snmp_community TEXT,
            snmp_v3_user   TEXT,
            snmp_last_ok   TEXT,
            snmp_last_check TEXT,
            snmp_diag      TEXT,
            chassis_type   TEXT,
            member_count   INTEGER,
            slot_count     INTEGER,
            link_up_count  INTEGER,
            status         TEXT NOT NULL,
            fail_reason    TEXT,
            duplicate_of   TEXT,
            role           TEXT NOT NULL DEFAULT 'switch',
            first_seen     TEXT NOT NULL,
            last_seen      TEXT NOT NULL
        );
        CREATE TABLE IF NOT EXISTS backups (
            id            INTEGER PRIMARY KEY AUTOINCREMENT,
            ip            TEXT NOT NULL,
            base_mac      TEXT,
            hostname      TEXT,
            filename      TEXT NOT NULL,
            config_hash   TEXT NOT NULL,
            changed       INTEGER NOT NULL DEFAULT 1,
            backed_up_at  TEXT NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_devices_mac ON devices(base_mac);
        CREATE INDEX IF NOT EXISTS idx_devices_status ON devices(status);
        CREATE INDEX IF NOT EXISTS idx_backups_ip ON backups(ip);
        CREATE INDEX IF NOT EXISTS idx_backups_mac ON backups(base_mac);

        CREATE TABLE IF NOT EXISTS stp_state (
            ip            TEXT NOT NULL,
            interface     TEXT NOT NULL,
            role          TEXT NOT NULL,
            state         TEXT NOT NULL,
            last_seen     TEXT NOT NULL,
            PRIMARY KEY (ip, interface)
        );
        CREATE TABLE IF NOT EXISTS stp_changes (
            id            INTEGER PRIMARY KEY AUTOINCREMENT,
            ip            TEXT NOT NULL,
            hostname      TEXT,
            interface     TEXT NOT NULL,
            old_role      TEXT,
            old_state     TEXT,
            new_role      TEXT NOT NULL,
            new_state     TEXT NOT NULL,
            changed_at    TEXT NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_stp_state_ip ON stp_state(ip);
        CREATE INDEX IF NOT EXISTS idx_stp_changes_ip ON stp_changes(ip);
        CREATE INDEX IF NOT EXISTS idx_stp_changes_time ON stp_changes(changed_at);

        CREATE TABLE IF NOT EXISTS stp_pending_changes (
            ip          TEXT NOT NULL,
            hostname    TEXT,
            interface   TEXT NOT NULL,
            old_role    TEXT, old_state TEXT,
            new_role    TEXT NOT NULL, new_state TEXT NOT NULL,
            first_seen  TEXT NOT NULL,
            PRIMARY KEY (ip, interface)
        );
        CREATE TABLE IF NOT EXISTS stp_root_state (
            ip                TEXT NOT NULL,
            instance          TEXT NOT NULL,   -- 'CIST' for now; future: MST1...
            root_priority     INTEGER,
            root_mac          TEXT,            -- canonical 'aa:bb:cc:dd:ee:ff'
            bridge_priority   INTEGER,
            bridge_mac        TEXT,
            is_root           INTEGER NOT NULL DEFAULT 0,
            tcn_count         INTEGER,
            last_tcn_seconds  INTEGER,
            updated_at        TEXT NOT NULL,
            PRIMARY KEY (ip, instance)
        );
        CREATE INDEX IF NOT EXISTS idx_stp_root_state_instance
            ON stp_root_state(instance, root_mac);
        CREATE TABLE IF NOT EXISTS stp_root_pending (
            ip             TEXT NOT NULL,
            instance       TEXT NOT NULL,
            old_priority   INTEGER, old_mac TEXT,
            new_priority   INTEGER, new_mac TEXT,
            first_seen     TEXT NOT NULL,
            PRIMARY KEY (ip, instance)
        );
        CREATE TABLE IF NOT EXISTS stp_root_changes (
            id            INTEGER PRIMARY KEY AUTOINCREMENT,
            ip            TEXT NOT NULL,
            hostname      TEXT,
            instance      TEXT NOT NULL,
            old_priority  INTEGER, old_mac TEXT,
            new_priority  INTEGER, new_mac TEXT,
            changed_at    TEXT NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_stp_root_changes_time
            ON stp_root_changes(changed_at);
        -- One row per (domain, instance) currently in root-bridge
        -- disagreement, holding a fingerprint of the contending root groups
        -- and their membership. Root disagreement is a standing config
        -- condition — it persists for days and its alert channel is the
        -- weekly 'digest stp' — so the per-minute 'monitor stp' tick logs the
        -- full breakdown only when this fingerprint changes, and collapses an
        -- unchanged pair to a single line carrying first_seen.
        CREATE TABLE IF NOT EXISTS stp_root_disagree_state (
            domain      TEXT NOT NULL,
            instance    TEXT NOT NULL,
            fingerprint TEXT NOT NULL,
            first_seen  TEXT NOT NULL,
            updated_at  TEXT NOT NULL,
            PRIMARY KEY (domain, instance)
        );
        CREATE TABLE IF NOT EXISTS stp_port_id_cache (
            -- Junos-only: maps the JUNIPER STP port-id (the index used by
            -- JUNIPER-MIMSTP-MIB) to the physical interface name. On VC
            -- chassis (multi-FPC EX-series), this index diverges from
            -- BRIDGE-MIB's dot1dBasePort, so the SNMP fast-path needs an
            -- authoritative mapping that doesn't rely on the BRIDGE-MIB
            -- chain. Populated by `discover` from
            -- `show spanning-tree interface | no-more` Port identifier
            -- column. Aligned-indexing devices populate it identically;
            -- the cache is consulted preferentially with BRIDGE-MIB
            -- fallback on miss.
            ip            TEXT NOT NULL,
            port_id       INTEGER NOT NULL,
            interface     TEXT NOT NULL,
            last_refresh  TEXT NOT NULL,
            PRIMARY KEY (ip, port_id)
        );
        CREATE TABLE IF NOT EXISTS device_members (
            ip          TEXT NOT NULL,   -- master device IP (matches devices.ip)
            member_id   INTEGER NOT NULL,-- 0/1/2 (Junos) or 1/2/3 (Aruba/HP)
            role        TEXT,            -- Master/Backup/Conductor/Standby/Member/Commander
            serial      TEXT,
            mac         TEXT,            -- canonical 'aa:bb:cc:dd:ee:ff'
            model       TEXT,
            status      TEXT,            -- vendor status word (Prsnt/Ready/Active/Up)
            updated_at  TEXT NOT NULL,
            PRIMARY KEY (ip, member_id)
        );
        CREATE INDEX IF NOT EXISTS idx_device_members_ip ON device_members(ip);
        CREATE INDEX IF NOT EXISTS idx_device_members_serial ON device_members(serial);
        CREATE TABLE IF NOT EXISTS port_state (
            ip          TEXT NOT NULL,
            interface   TEXT NOT NULL,
            link_up     INTEGER NOT NULL,
            last_seen   TEXT NOT NULL,
            PRIMARY KEY (ip, interface)
        );
        CREATE TABLE IF NOT EXISTS port_flaps (
            ip          TEXT NOT NULL,
            hostname    TEXT,
            interface   TEXT NOT NULL,
            flap_count  INTEGER NOT NULL DEFAULT 0,
            first_seen  TEXT NOT NULL,
            last_seen   TEXT NOT NULL,
            PRIMARY KEY (ip, interface)
        );
        CREATE INDEX IF NOT EXISTS idx_port_flaps_count ON port_flaps(flap_count);
        -- Append-only per-flap timeline (one row per detected link up-from-down
        -- transition). Unlike port_flaps (a counter the weekly clear-flap zeroes)
        -- this SURVIVES the reset, giving point-in-time forensics via
        -- 'show flap-history'. Pruned to [monitor] flap_history_days (default 30).
        CREATE TABLE IF NOT EXISTS flap_events (
            id          INTEGER PRIMARY KEY AUTOINCREMENT,
            ip          TEXT NOT NULL,
            hostname    TEXT,
            interface   TEXT NOT NULL,
            flapped_at  TEXT NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_flap_events_time ON flap_events(flapped_at);
        CREATE INDEX IF NOT EXISTS idx_flap_events_port ON flap_events(ip, interface);
        -- Append-only per-port error timeline. 'monitor topology' walks the
        -- IF-MIB error/discard counters + EtherLike FCS (CRC) counters every
        -- 30 min; each row records the DELTA when a cumulative counter rose
        -- since the last poll (so a row means "N new errors of this type in
        -- the last interval"). error_type is one of in_errors/out_errors/
        -- in_discards/out_discards/fcs_errors. Drives `show port-errors` and
        -- the errors column of `show mac`. Pruned to [monitor] port_error_days
        -- (default 365) so an intermittent fault stays visible for months.
        CREATE TABLE IF NOT EXISTS port_errors (
            id            INTEGER PRIMARY KEY AUTOINCREMENT,
            ip            TEXT NOT NULL,
            hostname      TEXT,
            interface     TEXT NOT NULL,
            error_type    TEXT NOT NULL,
            delta         INTEGER NOT NULL,
            counter_value INTEGER,
            at            TEXT NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_port_errors_time ON port_errors(at);
        CREATE INDEX IF NOT EXISTS idx_port_errors_port ON port_errors(ip, interface);
        -- Latest cumulative counter per (port, error_type). Internal scratch
        -- for delta computation only — NOT user-facing. One row per
        -- device×interface×error_type, updated in place each poll. A drop
        -- (counter < stored) means a device reboot / 32-bit wrap; we rebaseline
        -- silently rather than log a bogus negative delta.
        CREATE TABLE IF NOT EXISTS port_error_state (
            ip            TEXT NOT NULL,
            interface     TEXT NOT NULL,
            error_type    TEXT NOT NULL,
            counter_value INTEGER NOT NULL,
            updated_at    TEXT NOT NULL,
            PRIMARY KEY (ip, interface, error_type)
        );
        -- Per-port LLDP neighbor snapshot, populated by 'monitor topology'.
        -- One row per (src_ip, src_port) — only the most recent neighbor
        -- per port is kept (a port shouldn't have two simultaneous LLDP
        -- neighbors in practice; trunks aggregate at the LAG level). The
        -- graph walk uses neighbor_ip when the peer is in our devices
        -- table, falling back to chassis MAC / system name otherwise.
        CREATE TABLE IF NOT EXISTS topology_edges (
            src_ip            TEXT NOT NULL,
            src_port          TEXT NOT NULL,
            neighbor_chassis  TEXT,
            neighbor_sysname  TEXT,
            neighbor_port     TEXT,
            neighbor_port_desc TEXT,
            neighbor_ip       TEXT,
            neighbor_caps     TEXT,   -- LLDP system capabilities (csv: bridge,router,...)
            last_seen         TEXT NOT NULL,
            PRIMARY KEY (src_ip, src_port)
        );
        CREATE INDEX IF NOT EXISTS idx_topology_neighbor_ip ON topology_edges(neighbor_ip);
        CREATE INDEX IF NOT EXISTS idx_topology_src_ip ON topology_edges(src_ip);
        -- Operational event log — populated by instrumentation hooks
        -- (monitor stp ticks, backups, SSH attempts, LLDP scrape).
        -- 'digest health' aggregates these over the past 7 days. Rolled
        -- up per-tick rather than per-SNMP-query to keep the row count
        -- sane (~15M SNMP queries/week per site would blow up the DB).
        CREATE TABLE IF NOT EXISTS op_events (
            id          INTEGER PRIMARY KEY AUTOINCREMENT,
            op_type     TEXT NOT NULL,
            started_at  TEXT NOT NULL,
            duration_ms INTEGER NOT NULL,
            success     INTEGER NOT NULL,
            ip          TEXT,
            reason      TEXT,
            extra       TEXT,
            error       TEXT
        );
        CREATE INDEX IF NOT EXISTS idx_op_events_time ON op_events(started_at);
        CREATE INDEX IF NOT EXISTS idx_op_events_type_time ON op_events(op_type, started_at);
        -- Historical port<->MAC associations harvested from BRIDGE-MIB
        -- dot1dTpFdbTable (+ Q-BRIDGE dot1qTpFdbTable when VLAN-aware) on
        -- the 'monitor topology' SNMP pass (every 30 min). Lets STP-alert
        -- and flap-digest enrichment show the *last-known* MAC(s) on a port
        -- when the live FDB is empty (port flapping/down) — and feeds the
        -- non-trunk MAC-flux security signal. vlan defaults to 0 (= plain
        -- dot1d / VLAN-unaware) so the PK upsert stays deterministic.
        CREATE TABLE IF NOT EXISTS port_macs (
            ip          TEXT NOT NULL,
            interface   TEXT NOT NULL,
            mac         TEXT NOT NULL,        -- canonical 'aa:bb:cc:dd:ee:ff'
            vlan        INTEGER NOT NULL DEFAULT 0,
            oui_vendor  TEXT,
            first_seen  TEXT NOT NULL,
            last_seen   TEXT NOT NULL,
            times_seen  INTEGER NOT NULL DEFAULT 1,
            PRIMARY KEY (ip, interface, mac, vlan)
        );
        CREATE INDEX IF NOT EXISTS idx_port_macs_ip_if ON port_macs(ip, interface);
        CREATE INDEX IF NOT EXISTS idx_port_macs_last_seen ON port_macs(last_seen);
        -- 3.8.31 ID-loop: IP <-> MAC observations harvested from
        -- ipNetToMediaTable on the same 'monitor topology' SNMP pass.
        -- Closes the identification chain: a MAC from port_macs is joined
        -- here to find its IP, which dns_cache (below) resolves to a
        -- hostname. Index on mac is the hot lookup path used by
        -- _arp_lookup_for_mac during alert/digest enrichment.
        CREATE TABLE IF NOT EXISTS ip_arp (
            device_ip   TEXT NOT NULL,
            ifindex     INTEGER NOT NULL,
            ip          TEXT NOT NULL,
            mac         TEXT NOT NULL,        -- canonical 'aa:bb:cc:dd:ee:ff'
            first_seen  TEXT NOT NULL,
            last_seen   TEXT NOT NULL,
            PRIMARY KEY (device_ip, ifindex, ip, mac)
        );
        CREATE INDEX IF NOT EXISTS idx_ip_arp_mac        ON ip_arp(mac);
        CREATE INDEX IF NOT EXISTS idx_ip_arp_last_seen  ON ip_arp(last_seen);
        -- 3.8.31 reverse-DNS cache. Populated lazily by enrichment paths;
        -- NULL hostname = NXDOMAIN/timeout (cached so we don't retry every
        -- email). Separate prune window (1d) from the ip_arp/op_events
        -- retention because DNS PTR mappings are short-lived and cheap to
        -- re-resolve.
        CREATE TABLE IF NOT EXISTS dns_cache (
            ip            TEXT PRIMARY KEY,
            hostname      TEXT,
            looked_up_at  TEXT NOT NULL
        );
        -- Edge-trigger dedup for the non-trunk MAC-flux security notice.
        -- Mirrors the stp_pending / *_alerted_at preservation pattern: one
        -- email when a port first crosses mac_flux_threshold_24h, then
        -- suppressed until the episode clears (cleared_at set when the
        -- trailing-24h distinct-MAC count drops back under threshold).
        CREATE TABLE IF NOT EXISTS mac_flux_alerts (
            ip            TEXT NOT NULL,
            interface     TEXT NOT NULL,
            distinct_macs INTEGER NOT NULL,
            notified_at   TEXT NOT NULL,
            cleared_at    TEXT,
            PRIMARY KEY (ip, interface)
        );
        -- 3.8.8 flux-ignore list: operator-managed (via `netops ignore
        -- add|remove|list`) list of approved (ip, interface) ports that
        -- should never alert (e.g. an approved A/V switch downstream of
        -- an access port). Unioned with the 3.8.7 cfg key
        -- [monitor] mac_flux_excluded_ports at compute time. Lives in
        -- the DB rather than netops.conf so the netops user can manage
        -- it without sudo, with an audit trail (added_at, added_by).
        CREATE TABLE IF NOT EXISTS mac_flux_whitelist (
            ip          TEXT NOT NULL,
            interface   TEXT NOT NULL,
            reason      TEXT,
            added_at    TEXT NOT NULL,
            added_by    TEXT,
            PRIMARY KEY (ip, interface)
        );
        -- 3.8.10 port-flap silence/whitelist: operator-managed (via
        -- `netops ignore flap`) list of approved-or-noisy
        -- ports whose port_flaps rows should not page the operator
        -- via digest flap. Use case: a sleepy printer that link-flaps
        -- on every sleep cycle but isn't actually a network problem.
        -- Unlike mac_flux_whitelist, this table has an optional
        -- expires_at column so the operator can silence with a deadline
        -- ("until I order a new cable") rather than permanently. NULL
        -- expires_at = permanent. The reconciliation pass garbage-
        -- collects expired entries on each digest flap run.
        CREATE TABLE IF NOT EXISTS port_flap_whitelist (
            ip          TEXT NOT NULL,
            interface   TEXT NOT NULL,
            reason      TEXT,
            added_at    TEXT NOT NULL,
            added_by    TEXT,
            expires_at  TEXT,
            PRIMARY KEY (ip, interface)
        );
        -- Time-bounded alert-email mutes (`silence`). kind: 'ip' (one
        -- device, stored as its primary IP), 'name' (case-insensitive
        -- hostname substring), or 'all'. expires_at is NOT NULL by
        -- design — a silence is never permanent (30-day cap at the CLI);
        -- permanent per-port exclusions live in the `ignore` tables.
        CREATE TABLE IF NOT EXISTS alert_silences (
            id          INTEGER PRIMARY KEY AUTOINCREMENT,
            kind        TEXT NOT NULL,
            pattern     TEXT NOT NULL,
            reason      TEXT,
            created_at  TEXT NOT NULL,
            created_by  TEXT,
            expires_at  TEXT NOT NULL
        );
    """)
    # Migrate existing DBs: add columns if missing
    cols = [row[1] for row in conn.execute("PRAGMA table_info(devices)").fetchall()]
    if "firmware" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN firmware TEXT")
    if "hardware_info" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN hardware_info TEXT")
    if "password_hash" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN password_hash TEXT")
    if "ssh_open" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN ssh_open INTEGER DEFAULT 0")
    if "telnet_open" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN telnet_open INTEGER DEFAULT 0")
    if "stp_enabled" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN stp_enabled INTEGER")
    if "stp_last_check" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN stp_last_check TEXT")
    if "dns_name" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN dns_name TEXT")
    if "platform" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN platform TEXT")
    if "stp_mode" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN stp_mode TEXT")
    if "chassis_type" not in cols:
        # 'standalone' | 'stack' | 'chassis'. NULL until discovery re-runs.
        conn.execute("ALTER TABLE devices ADD COLUMN chassis_type TEXT")
    if "member_count" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN member_count INTEGER")
    if "slot_count" not in cols:
        # Mirrors member_count for every vendor we've probed; kept as its own
        # column so a future multi-blade chassis can diverge (e.g. HP 5400zl
        # with module slots per chassis member).
        conn.execute("ALTER TABLE devices ADD COLUMN slot_count INTEGER")
    if "link_up_count" not in cols:
        # Number of data-plane ports with link up, refreshed every monitor
        # stp poll. 0 means the switch is reachable via management only and
        # should be excluded from cross-switch consensus checks.
        conn.execute("ALTER TABLE devices ADD COLUMN link_up_count INTEGER")
    # SNMP columns: populated by 'discover'. snmp_enabled tri-state
    # (NULL=never probed, 1=verified working, 0=verified disabled/unreachable).
    # snmp_community / snmp_v3_user record the winning credential so future
    # polls don't have to walk the full list each time.
    if "snmp_enabled" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_enabled INTEGER")
    if "snmp_proto" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_proto TEXT")
    if "snmp_community" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_community TEXT")
    if "snmp_v3_user" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_v3_user TEXT")
    if "snmp_last_ok" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_last_ok TEXT")
    if "snmp_last_check" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_last_check TEXT")
    if "snmp_diag" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_diag TEXT")
    if "stp_disabled_since" not in cols:
        # Set to the timestamp when stp_enabled first transitioned 1->0
        # (or NULL/None -> 0). Cleared back to NULL on 0->1. Used by the
        # digest to require sustained-disabled state before listing, so a
        # transient empty SNMP poll (e.g. during a flap storm) doesn't
        # produce a false-positive in the next 8 AM digest.
        conn.execute("ALTER TABLE devices ADD COLUMN stp_disabled_since TEXT")
    if "snmp_consecutive_fails" not in cols:
        # Incremented on every 'monitor stp' poll that returns None (no STP
        # data via SNMP and, in auto mode, SSH fallback also failed). Reset
        # to 0 on any successful poll. Drives the unreachable-alert
        # hysteresis: alert only after N consecutive fails so a single
        # dropped UDP packet doesn't page anyone.
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_consecutive_fails INTEGER DEFAULT 0")
    if "unreachable_pending_since" not in cols:
        # 3.19.0 outage coalescing: stamped when the unreachable threshold
        # is crossed; the alert email is held until the outage stops
        # growing (or unreachable_coalesce_min passes) so one power event
        # produces ONE topology-grouped email.
        conn.execute(
            "ALTER TABLE devices ADD COLUMN unreachable_pending_since TEXT")
    if "unreachable_alerted_at" not in cols:
        # Timestamp the unreachable alert email went out. NULL = no current
        # outage alert. Set when consecutive-fails crosses the threshold;
        # cleared (and a recovery email sent) on the next successful poll.
        # Ensures one email per outage, not one per minute.
        conn.execute("ALTER TABLE devices ADD COLUMN unreachable_alerted_at TEXT")
    if "iface_terse" not in cols:
        # Set when the full Junos 'show interfaces | match' listing times
        # out: a large virtual chassis relays per-interface statistics
        # across members and can overrun (or abort at ~60s and truncate)
        # the listing no matter the timeout. The monitor then polls
        # 'show interfaces terse' for that device instead. DELIBERATELY
        # absent from upsert_device's preservation list: the nightly
        # backup's upsert resets it to 0, so the full listing (and its
        # richer last-flapped data) is re-tried once a day and the flag
        # re-learns. DDL only — auto-commits, no init_db commit needed.
        conn.execute(
            "ALTER TABLE devices ADD COLUMN iface_terse INTEGER DEFAULT 0")
    if "role" not in cols:
        # Functional role: 'switch' (default), 'firewall'. 3.8.28+. Auto-
        # derived from model during identification (Juniper `srx*` ->
        # firewall) and operator-settable via `netops mark firewall <ip>`.
        # Used to scope monitor-stp + digest-stp to switches only — STP
        # isn't meaningful on a pure-L3 SRX, and probing it just produces
        # 'Invalid input' noise. monitor-flap and backup paths stay
        # role-neutral; we still care if a firewall port is flapping.
        conn.execute("ALTER TABLE devices ADD COLUMN role TEXT "
                     "NOT NULL DEFAULT 'switch'")
    # Migrate from old password_idx column (drop data — idx is meaningless now)
    if "password_idx" in cols:
        conn.execute("UPDATE devices SET password_hash = NULL")

    # Phantom-evidence columns: per-port state-machine activity counter
    # (jnxMIMstCistEdgeDelayWhileExpiryCount). Captured on every Junos
    # SNMP poll so a confirmed STP transition can be evaluated against
    # whether the state machine actually ran (delta>0) or the MIB just
    # flickered (delta==0). Junos-only at the moment; NULL on other
    # platforms is fine.
    stp_state_cols = [r[1] for r in conn.execute(
        "PRAGMA table_info(stp_state)").fetchall()]
    if "edge_expiry" not in stp_state_cols:
        conn.execute("ALTER TABLE stp_state ADD COLUMN edge_expiry INTEGER")
    pending_cols = [r[1] for r in conn.execute(
        "PRAGMA table_info(stp_pending_changes)").fetchall()]
    if "edge_expiry_at_first_seen" not in pending_cols:
        conn.execute("ALTER TABLE stp_pending_changes "
                     "ADD COLUMN edge_expiry_at_first_seen INTEGER")
    if "tcn_at_first_seen" not in pending_cols:
        conn.execute("ALTER TABLE stp_pending_changes "
                     "ADD COLUMN tcn_at_first_seen INTEGER")
    # 3.8.44: LLDP neighbor system-capabilities (csv: bridge,router,wlan-
    # access-point,telephone,...) — lets the topology export type unmanaged
    # neighbors (AP / phone / switch / router) instead of generic ghosts.
    topo_cols = [r[1] for r in conn.execute(
        "PRAGMA table_info(topology_edges)").fetchall()]
    if "neighbor_caps" not in topo_cols:
        conn.execute("ALTER TABLE topology_edges ADD COLUMN neighbor_caps TEXT")

    # One-time data migrations, keyed on PRAGMA user_version (0 = never run).
    schema_ver = conn.execute("PRAGMA user_version").fetchone()[0]
    if schema_ver < 1:
        # v1: un-shift Junos EX/QFX FDB VLANs. Those platforms report
        # dot1qFdbId as vlan<<16 (e.g. 458752 = 7<<16 for VLAN 7), so old
        # port_macs rows carry a bogus >4094 VLAN. vlan is part of the PK,
        # so: (1) drop a shifted row that would collide with its correct
        # twin, (2) un-shift the rest in place, (3) drop anything still
        # unrecognizable. Idempotent + one-time.
        conn.execute(
            "DELETE FROM port_macs WHERE vlan>4094 AND (vlan & 65535)=0 "
            "AND EXISTS (SELECT 1 FROM port_macs c WHERE c.ip=port_macs.ip "
            "AND c.interface=port_macs.interface AND c.mac=port_macs.mac "
            "AND c.vlan=(port_macs.vlan >> 16))")
        conn.execute(
            "UPDATE port_macs SET vlan = vlan >> 16 "
            "WHERE vlan>4094 AND (vlan & 65535)=0 "
            "AND (vlan >> 16) BETWEEN 1 AND 4094")
        conn.execute("DELETE FROM port_macs WHERE vlan > 4094")
        # Commit the DML BEFORE bumping user_version — init_db() otherwise
        # never commits (the historical ALTER TABLEs persist via DDL
        # auto-commit, but this migration is pure DML with no DDL after it,
        # so without an explicit commit it rolls back on close and re-runs
        # forever). Persist the data first, then mark the migration done.
        conn.commit()
        conn.execute("PRAGMA user_version = 1")
        conn.commit()

    conn.close()


def _db():
    """Return a connection with row_factory set for dict-like access."""
    conn = sqlite3.connect(DB_FILE, timeout=30)
    conn.row_factory = sqlite3.Row
    # Keep SQLite's transient b-trees (large GROUP BY / ORDER BY sorts — e.g.
    # the per-day flap section of `digest health` over 100k+ flap_events on a
    # big fleet) in RAM. By default a spill goes to a temp file, and SQLite's
    # unix temp search prefers /var/tmp, which the AppArmor profile denies
    # (it allows /tmp, not /var/tmp) — so under the systemd timers a big sort
    # died with "unable to open database file" (SQLITE_CANTOPEN on the
    # /var/tmp/etilqs_* mknod). Working sets here are fleet-scale and bounded,
    # so in-memory temp is cheap and removes the filesystem dependency.
    conn.execute("PRAGMA temp_store=MEMORY")
    return conn


# Held flock file descriptors — a module-level dict keeps them alive for
# the process lifetime so the kernel-owned advisory lock stays in force.
# The kernel releases the lock automatically when the process exits, so
# there's no stale-lockfile problem.
_HELD_LOCKS = {}


def _release_advisory_lock(name):
    """Release a held advisory lock mid-process (chunked jobs). The kernel
    also releases on exit, so this is an optimization, not a correctness
    requirement."""
    fh = _HELD_LOCKS.pop(name, None)
    if fh is not None:
        try:
            fh.close()
        except OSError:
            pass


def _acquire_advisory_lock(name, wait=0):
    """Take an exclusive lock at STATE_DIR/.netops-<name>.lock.

    Returns True on success, False if the holder is still there at the
    deadline — the failure message names the current holder (pid + user
    + start time) so the operator knows what to wait for.

    `wait` = max seconds to block waiting for the lock (default 0 =
    non-blocking). When wait > 0, polls every 500 ms until acquired or
    the deadline expires; run_backup and run_topology use wait=90 so a
    collision with the per-minute monitor-stp tick is waited out rather
    than silently skipped.

    The lock lives in the state dir, not /tmp. The netops timers run as
    the `netops` user while ad-hoc runs may be root or a netops-group
    human; a /tmp lock file owned by one of them is unopenable by the
    others, because /tmp is sticky + world-writable and the kernel's
    fs.protected_regular hardening blocks the cross-user write-open even
    for root. STATE_DIR is netops-group-owned and not world-writable, so
    one group-writable lock file there is shareable by every netops
    runner. fcntl.lockf (POSIX record locks) is used; the kernel drops
    the lock when the holding process exits, so there is no stale-lock
    problem.
    """
    import fcntl as _fcntl
    lock_path = os.path.join(STATE_DIR, f".netops-{name}.lock")
    try:
        fh = open(lock_path, "a+")
    except OSError as e:
        log.warning("advisory lock %s open failed: %s — running without lock",
                    lock_path, e)
        return True
    # Group-writable so the netops user, root, and netops-group humans
    # can all share it (best-effort — only the creator can chmod it).
    try:
        os.fchmod(fh.fileno(), 0o664)
    except OSError:
        pass
    deadline = time.monotonic() + wait if wait > 0 else None
    while True:
        try:
            _fcntl.lockf(fh.fileno(), _fcntl.LOCK_EX | _fcntl.LOCK_NB)
            break  # acquired
        except (OSError, BlockingIOError):
            if deadline is None or time.monotonic() >= deadline:
                fh.seek(0)
                meta = (fh.read() or "").strip() or "(holder metadata unavailable)"
                fh.close()
                log.error("Another netops run holds the '%s' lock: %s", name, meta)
                log.error("Skipping this invocation. Wait for the current run, "
                          "or check %s.", lock_path)
                return False
            time.sleep(0.5)
    _HELD_LOCKS[name] = fh
    try:
        fh.seek(0); fh.truncate()
        admin = (os.environ.get("NETOPS_ADMIN")
                 or os.environ.get("USER") or "unknown")
        started = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        fh.write(f"pid={os.getpid()} user={admin} started={started} cmd={name}\n")
        fh.flush()
    except Exception:
        pass  # lock acquired; metadata write is best-effort
    return True


def upsert_device(ip, base_mac=None, hostname=None, model=None, serial=None,
                  firmware=None, hardware_info=None, proto=None, username=None,
                  password_hash=None, ssh_open=None, telnet_open=None,
                  status="active", fail_reason=None, duplicate_of=None,
                  dns_name=None, platform=None, role=None,
                  preserve_status=False):
    """Insert or update a device record.

    Preserves first_seen on update. When updating an existing device, None
    values keep the existing data (won't overwrite with NULL).

    preserve_status: if True and the device already exists, keep its current
    status (and fail_reason) instead of overwriting. Used by import flows
    that should not reverse a device's real operational state.
    """
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    existing = conn.execute("SELECT * FROM devices WHERE ip = ?", (ip,)).fetchone()
    first_seen = existing["first_seen"] if existing else now

    # Import-style flows: don't let the caller change a device's real state.
    if existing and preserve_status:
        status = existing["status"]
        fail_reason = existing["fail_reason"]

    # Columns populated by 'monitor stp', record_snmp_result, or other
    # background paths — NEVER by upsert_device callers. Because this uses
    # INSERT OR REPLACE (i.e. DELETE + INSERT), any column we don't list
    # would be reset to its DEFAULT on every device touch. Anything written
    # outside upsert_device MUST be read here and round-tripped, or it gets
    # silently zeroed. (Pain point: this broke per-event unreachable alerts —
    # nightly backup was wiping snmp_consecutive_fails / unreachable_alerted_at
    # at 01:30 and the threshold re-fired at 01:33 every morning.)
    stp_mode = None
    snmp_enabled = None
    snmp_proto = None
    snmp_community = None
    snmp_v3_user = None
    snmp_last_ok = None
    snmp_last_check = None
    snmp_diag = None
    stp_enabled = None
    stp_last_check = None
    stp_disabled_since = None
    link_up_count = None
    snmp_consecutive_fails = 0
    unreachable_alerted_at = None

    # Preserve existing values when caller passes None
    if existing:
        keys = existing.keys()
        base_mac = base_mac if base_mac is not None else existing["base_mac"]
        hostname = hostname if hostname is not None else existing["hostname"]
        model = model if model is not None else existing["model"]
        serial = serial if serial is not None else existing["serial"]
        firmware = firmware if firmware is not None else existing["firmware"]
        hardware_info = hardware_info if hardware_info is not None else existing["hardware_info"]
        proto = proto if proto is not None else existing["proto"]
        username = username if username is not None else existing["username"]
        password_hash = password_hash if password_hash is not None else existing["password_hash"]
        ssh_open = ssh_open if ssh_open is not None else existing["ssh_open"]
        telnet_open = telnet_open if telnet_open is not None else existing["telnet_open"]
        dns_name = dns_name if dns_name is not None else (existing["dns_name"] if "dns_name" in keys else None)
        platform = platform if platform is not None else (existing["platform"] if "platform" in keys else None)
        stp_mode = existing["stp_mode"] if "stp_mode" in keys else None
        snmp_enabled  = existing["snmp_enabled"]   if "snmp_enabled"   in keys else None
        snmp_proto    = existing["snmp_proto"]     if "snmp_proto"     in keys else None
        snmp_community = existing["snmp_community"] if "snmp_community" in keys else None
        snmp_v3_user  = existing["snmp_v3_user"]   if "snmp_v3_user"   in keys else None
        snmp_last_ok  = existing["snmp_last_ok"]   if "snmp_last_ok"   in keys else None
        snmp_last_check = existing["snmp_last_check"] if "snmp_last_check" in keys else None
        snmp_diag     = existing["snmp_diag"]      if "snmp_diag"      in keys else None
        stp_enabled   = existing["stp_enabled"]    if "stp_enabled"    in keys else None
        stp_last_check = existing["stp_last_check"] if "stp_last_check" in keys else None
        stp_disabled_since = existing["stp_disabled_since"] if "stp_disabled_since" in keys else None
        link_up_count = existing["link_up_count"]  if "link_up_count"  in keys else None
        snmp_consecutive_fails = (existing["snmp_consecutive_fails"]
                                  if "snmp_consecutive_fails" in keys else 0) or 0
        unreachable_alerted_at = (existing["unreachable_alerted_at"]
                                  if "unreachable_alerted_at" in keys else None)
        # role: caller wins (manual `mark firewall` etc.); otherwise
        # preserve what's there; otherwise derive from model. Never
        # silently demote a manual classification.
        if role is None:
            role = existing["role"] if "role" in keys else None

    # Default to 0 for new devices where ports weren't checked
    if ssh_open is None:
        ssh_open = 0
    if telnet_open is None:
        telnet_open = 0
    # Auto-derive role from model when caller didn't set one and there's
    # nothing on the existing row to preserve. Falls back to 'switch'.
    if role is None:
        role = derive_role(model)

    conn.execute("""
        INSERT OR REPLACE INTO devices
            (ip, base_mac, hostname, dns_name, model, serial, firmware, hardware_info,
             proto, username, password_hash, ssh_open, telnet_open, platform, stp_mode,
             snmp_enabled, snmp_proto, snmp_community, snmp_v3_user,
             snmp_last_ok, snmp_last_check, snmp_diag,
             stp_enabled, stp_last_check, stp_disabled_since, link_up_count,
             snmp_consecutive_fails, unreachable_alerted_at,
             status, fail_reason, duplicate_of, role, first_seen, last_seen)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                ?, ?, ?, ?, ?, ?, ?,
                ?, ?, ?, ?,
                ?, ?,
                ?, ?, ?, ?, ?, ?)
    """, (ip, base_mac, hostname, dns_name, model, serial, firmware, hardware_info,
          proto, username, password_hash, ssh_open, telnet_open, platform, stp_mode,
          snmp_enabled, snmp_proto, snmp_community, snmp_v3_user,
          snmp_last_ok, snmp_last_check, snmp_diag,
          stp_enabled, stp_last_check, stp_disabled_since, link_up_count,
          snmp_consecutive_fails, unreachable_alerted_at,
          status, fail_reason, duplicate_of, role, first_seen, now))
    conn.commit()
    conn.close()


def record_snmp_result(ip, *, enabled, proto=None, community=None,
                        v3_user=None, diag=None, ok_now=False):
    """Update SNMP-related columns on a device row. Called after a probe.

    enabled:   1 = working, 0 = verified disabled/unreachable, None = unknown
    proto:     'v2c' or 'v3' (winning protocol; None when no cred worked)
    community: winning v2c community string, or None
    v3_user:   winning v3 user name, or None
    diag:      free-form diagnostic message (cleared when probe succeeds)
    ok_now:    bump snmp_last_ok to now when True; always bump snmp_last_check
    """
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    if ok_now:
        conn.execute(
            "UPDATE devices SET snmp_enabled=?, snmp_proto=?, snmp_community=?,"
            " snmp_v3_user=?, snmp_diag=?, snmp_last_ok=?, snmp_last_check=?"
            " WHERE ip=?",
            (enabled, proto, community, v3_user, diag, now, now, ip))
    else:
        conn.execute(
            "UPDATE devices SET snmp_enabled=?, snmp_proto=?, snmp_community=?,"
            " snmp_v3_user=?, snmp_diag=?, snmp_last_check=?"
            " WHERE ip=?",
            (enabled, proto, community, v3_user, diag, now, ip))
    conn.commit()
    conn.close()


def reverse_dns(ip, timeout=2.0, _set_global_timeout=True):
    """Return the reverse DNS name for an IP, or empty string if unavailable.

    gethostbyaddr has no timeout parameter, so we bound it via the PROCESS-WIDE
    socket default timeout. Because that default is a global, a caller that fans
    reverse_dns out across a thread pool (resolve_dns_batch) must set it ONCE
    around the whole pool and pass _set_global_timeout=False here — otherwise the
    per-thread get/set/restore races and can leave the global stuck at `timeout`,
    which then silently breaks unrelated sockets in a long-lived process (it made
    the console `connect` client inherit a ~2s timeout and time out mid-login).
    """
    try:
        if _set_global_timeout:
            old = socket.getdefaulttimeout()
            socket.setdefaulttimeout(timeout)
        try:
            name, _, _ = socket.gethostbyaddr(ip)
            return name
        finally:
            if _set_global_timeout:
                socket.setdefaulttimeout(old)
    except (socket.herror, socket.gaierror, socket.timeout, OSError):
        return ""


def resolve_dns_batch(ips, threads=50, timeout=2.0):
    """Reverse-resolve a list of IPs in parallel. Returns {ip: dns_name_or_empty}."""
    results = {}
    if not ips:
        return results
    log.debug("Reverse DNS on %d IP(s) with %d threads", len(ips), threads)
    # Set the global socket default timeout ONCE around the whole pool (from this
    # one thread) and restore it in finally — instead of letting every worker
    # race on the global (which could leave it stuck at `timeout` and break
    # unrelated sockets later in a long-lived process). Workers pass
    # _set_global_timeout=False so they don't touch it.
    old = socket.getdefaulttimeout()
    socket.setdefaulttimeout(timeout)
    try:
        with ThreadPoolExecutor(max_workers=min(threads, max(1, len(ips)))) as pool:
            futures = {pool.submit(reverse_dns, ip, timeout, False): ip
                       for ip in ips}
            for future in as_completed(futures):
                ip = futures[future]
                results[ip] = future.result() or ""
    finally:
        socket.setdefaulttimeout(old)
    return results


def record_backup(ip, base_mac, hostname, filename, config_hash, changed):
    """Record a backup operation in the history."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    conn.execute("""
        INSERT INTO backups (ip, base_mac, hostname, filename, config_hash, changed, backed_up_at)
        VALUES (?, ?, ?, ?, ?, ?, ?)
    """, (ip, base_mac, hostname, filename, config_hash, int(changed), now))
    conn.commit()
    conn.close()


def _ip_sort_key(ip_str):
    """Return a sort key that orders IPs numerically (1.2.3.4 < 1.2.3.10).
    Falls back to the raw string for non-IP values so sort() never crashes.
    """
    try:
        return (0, int(ipaddress.ip_address(ip_str)))
    except (ValueError, TypeError):
        return (1, str(ip_str or ""))


def _sort_by_ip(rows, ip_field="ip"):
    """Return rows sorted numerically by their IP field."""
    return sorted(rows, key=lambda r: _ip_sort_key(r[ip_field]))


def get_devices(status=None):
    """Query devices, optionally filtered by status. Sorted numerically by IP."""
    conn = _db()
    if status:
        rows = conn.execute(
            "SELECT * FROM devices WHERE status = ?", (status,)
        ).fetchall()
        rows = _sort_by_ip(rows)
    else:
        # Sort by status first, then IP numerically within each status.
        rows = conn.execute("SELECT * FROM devices").fetchall()
        rows = sorted(rows, key=lambda r: (r["status"] or "", _ip_sort_key(r["ip"])))
    conn.close()
    return rows


def get_backups(ip=None, limit=None):
    """Query backup history, optionally filtered by IP."""
    conn = _db()
    sql = "SELECT * FROM backups"
    params = []
    if ip:
        sql += " WHERE ip = ?"
        params.append(ip)
    sql += " ORDER BY backed_up_at DESC"
    if limit:
        sql += " LIMIT ?"
        params.append(limit)
    rows = conn.execute(sql, params).fetchall()
    conn.close()
    return rows


def _device_identity(base_mac, serial):
    """Stable hardware identity for duplicate detection.

    Prefer the base MAC; fall back to the serial. L3 switches like the Ruckus
    ICX expose a serial but NO base MAC in their identity output, so a
    base-MAC-only key silently misses every secondary management IP / SVI of
    such a switch. Returns the identity string, or None when neither is known
    (can't dedup — keep the IP as its own device)."""
    bm = (base_mac or "").strip()
    if bm:
        return bm
    sn = (serial or "").strip()
    if sn:
        return sn
    return None


# Preference order when choosing which IP of a duplicate group is the
# "primary" (the one kept active for backup/monitoring). Lower = better.
_STATUS_PRIMARY_RANK = {"active": 0, "inactive": 1, "failed": 2,
                        "unknown": 3, "non-switch": 4, "duplicate": 5}


def _choose_primary(members):
    """Pick the primary row for an identity group: best operational status
    (active first), then a row not already flagged a duplicate, then the lowest
    IP. Deterministic and sticky — an already-active primary keeps winning, so
    the monitored IP doesn't churn."""
    def key(r):
        try:
            ipk = int(ipaddress.ip_address(r["ip"]))
        except ValueError:
            ipk = 0
        dup = 1 if (r["duplicate_of"] or "") else 0
        return (_STATUS_PRIMARY_RANK.get(r["status"], 9), dup, ipk)
    return sorted(members, key=key)[0]


def get_duplicates():
    """Devices that share a hardware identity (base MAC, else serial) with at
    least one other IP — the multiple management IPs / SVIs of one L3 switch.
    Grouped in Python so the serial fallback works (SQL can't express
    'base_mac else serial' as a single GROUP BY key)."""
    conn = _db()
    rows = conn.execute(
        "SELECT * FROM devices WHERE status != 'non-switch'").fetchall()
    conn.close()
    groups = {}
    for r in rows:
        ident = _device_identity(r["base_mac"], r["serial"])
        if ident:
            groups.setdefault(ident, []).append(r)
    out = []
    for ident in sorted(groups):
        members = groups[ident]
        if len(members) > 1:
            out.extend(_sort_by_ip(members))
    return out


def remove_device(ip):
    """Remove a device and its backup history from the database.

    Returns True if the device existed, False otherwise.
    """
    conn = _db()
    cursor = conn.execute("DELETE FROM devices WHERE ip = ?", (ip,))
    conn.execute("DELETE FROM backups WHERE ip = ?", (ip,))
    conn.commit()
    existed = cursor.rowcount > 0
    conn.close()
    return existed


def remove_devices_by_status(status):
    """Remove all devices with the given status. Returns count removed."""
    conn = _db()
    cursor = conn.execute("DELETE FROM devices WHERE status = ?", (status,))
    count = cursor.rowcount
    conn.commit()
    conn.close()
    return count


# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------

# Memory estimate per SSH session (pexpect + system ssh overhead)
_SSH_MEM_MB = 5
# Maximum fraction of system RAM to use for concurrent SSH sessions
_SSH_MEM_FRACTION = 0.25


def _max_open_files():
    """Return the OS soft limit on open file descriptors.

    Returns a practical integer even when the OS reports 'unlimited'.
    """
    try:
        import resource
        soft = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
        # macOS can report RLIM_INFINITY; fall back to kern.maxfilesperproc
        if soft > 100_000:
            try:
                import subprocess as _sp
                out = _sp.check_output(
                    ["sysctl", "-n", "kern.maxfilesperproc"],
                    stderr=_sp.DEVNULL, text=True,
                )
                return int(out.strip())
            except Exception:
                return 1024  # safe fallback
        return soft
    except (ImportError, AttributeError):
        # Windows: no resource module; query via ctypes
        try:
            import ctypes
            kernel32 = ctypes.windll.kernel32
            # _getmaxstdio returns CRT file handle limit (default 512)
            msvcrt = ctypes.cdll.msvcrt
            return msvcrt._getmaxstdio()
        except Exception:
            return 512  # safe fallback


def _total_memory_mb():
    """Return total system memory in MB."""
    # Linux / macOS
    try:
        pages = os.sysconf("SC_PHYS_PAGES")
        page_size = os.sysconf("SC_PAGE_SIZE")
        return (pages * page_size) // (1024 * 1024)
    except (ValueError, OSError, AttributeError):
        pass
    # Windows
    try:
        import ctypes

        class MEMORYSTATUSEX(ctypes.Structure):
            _fields_ = [
                ("dwLength", ctypes.c_ulong),
                ("dwMemoryLoad", ctypes.c_ulong),
                ("ullTotalPhys", ctypes.c_ulonglong),
                ("ullAvailPhys", ctypes.c_ulonglong),
                ("ullTotalPageFile", ctypes.c_ulonglong),
                ("ullAvailPageFile", ctypes.c_ulonglong),
                ("ullTotalVirtual", ctypes.c_ulonglong),
                ("ullAvailVirtual", ctypes.c_ulonglong),
                ("ullAvailExtendedVirtual", ctypes.c_ulonglong),
            ]

        mem = MEMORYSTATUSEX()
        mem.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
        ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(mem))
        return mem.ullTotalPhys // (1024 * 1024)
    except Exception:
        pass
    return 4096  # 4 GB conservative fallback


def _diagnose_config_access(path):
    """If the running user can't read `path` or traverse its directory,
    bail with a clear group-membership remedy. configparser.read silently
    ignores unreadable files — without this, a non-netops-group user
    gets an empty config and downstream "missing credentials" errors
    with no hint about the actual cause."""
    try:
        os.stat(path)
        return  # file exists and we can stat it
    except FileNotFoundError:
        return  # genuinely missing — load_config will use defaults
    except PermissionError:
        pass  # diagnose below
    except OSError:
        return  # other errors handled downstream
    user = getpass.getuser() or "(unknown)"
    in_group = False
    if grp is not None:
        try:
            g = grp.getgrnam(SECRET_GROUP)
            if user in g.gr_mem or os.getegid() == g.gr_gid:
                in_group = True
        except KeyError:
            pass
    log.error(
        "cannot read %s — permission denied. Your account ('%s') is "
        "%sa member of the '%s' group.",
        path, user, "" if in_group else "NOT ", SECRET_GROUP)
    if not in_group:
        log.error(
            "Ask your administrator for membership in the '%s' group.",
            SECRET_GROUP)
    else:
        log.error(
            "Ask your administrator to verify permissions on %s "
            "and %s.", os.path.dirname(path), path)
    sys.exit(2)


def load_config():
    """Read netops.conf (or legacy config.ini) and return a dict of
    settings with defaults."""
    cp = configparser.ConfigParser(interpolation=None)
    # Preserve case so [user_passwords] can distinguish e.g. 'admin' vs 'Admin'
    # (network devices often treat usernames as case-sensitive).
    cp.optionxform = str
    # configparser.read silently ignores files it can't open. That's a UX
    # trap when /etc/netops is 2750 root:netops and the running user isn't
    # in the netops group: the user gets an empty config + downstream
    # "missing credential" errors with no hint about the actual cause.
    # Probe explicitly so we can emit a clear group-membership remedy.
    _diagnose_config_access(CONFIG_FILE)
    cp.read(CONFIG_FILE, encoding="utf-8")
    # Nudge legacy filename users toward the new name for consistency
    # with secrets.conf. Soft warning; functionality unchanged.
    if CONFIG_FILE == _CONFIG_FILE_LEGACY and os.path.exists(_CONFIG_FILE_LEGACY):
        log.warning(
            "Using legacy config filename %s — rename to %s for consistency "
            "with secrets.conf. Both names are supported; the legacy fallback "
            "may be removed in a future release.",
            _CONFIG_FILE_LEGACY, _CONFIG_FILE_NEW)

    cfg = {
        "storage_mode": cp.get("general", "storage_mode", fallback="git"),
        "log_format": cp.get("general", "log_format", fallback="full"),
        "device_file": cp.get("general", "device_file", fallback="devices.txt"),
        "subnets": [
            s.strip()
            for s in cp.get("scan", "subnets", fallback="").split(",")
            if s.strip()
        ],
        "ports": [
            int(p.strip())
            for p in cp.get("scan", "ports", fallback="22,23").split(",")
            if p.strip()
        ],
        "timeout": cp.getfloat("scan", "timeout", fallback=1.0),
        "scan_threads": cp.getint("general", "scan_threads", fallback=200),
        "ssh_threads": cp.getint("general", "ssh_threads", fallback=20),
        "usernames": [
            u.strip()
            for u in cp.get("backup", "usernames", fallback="admin, root").split(",")
            if u.strip()
        ],
    }

    # --- Usernames: env var > secrets.conf > config.ini ---
    env_users = os.environ.get("NETOPS_USERNAMES", "").strip()
    if env_users:
        users = [u.strip() for u in env_users.split(",") if u.strip()]
        for u in users:
            _safe_credential_field(u, "NETOPS_USERNAMES entry")
        cfg["usernames"] = users

    # Read secrets.conf (same dir as script, cross-platform)
    secrets = configparser.ConfigParser(interpolation=None)
    secrets.optionxform = str
    if os.path.exists(SECRETS_FILE):
        secrets.read(SECRETS_FILE, encoding="utf-8")
        # Usernames from secrets.conf override config.ini (but env var wins)
        if not env_users:
            sec_users = secrets.get("credentials", "usernames", fallback="").strip()
            if sec_users:
                users = [u.strip() for u in sec_users.split(",") if u.strip()]
                for u in users:
                    _safe_credential_field(u, "secrets.conf [credentials] username")
                cfg["usernames"] = users

    # SECURITY: validate config.ini-sourced usernames too. Any value that
    # reaches _ssh_attempt must be safe regardless of where it originated.
    for u in cfg["usernames"]:
        _safe_credential_field(u, "netops.conf [backup] username")

    # Build passwords dict: {"default": "...", "paloalto_panos": "...", ...}
    passwords = {"default": cp.get("backup", "password", fallback="")}
    if cp.has_section("backup"):
        for key in cp.options("backup"):
            if key.startswith("password_"):
                device_type = key[len("password_"):]
                passwords[device_type] = cp.get("backup", key)
    cfg["passwords"] = passwords

    # --- Password list priority: env var > secrets.conf > config.ini ---
    env_pw = os.environ.get("NETOPS_PASSWORDS", "").strip()
    # Resolution order: env var → secrets.conf → config.ini.
    # secrets.conf is opt-in: defining the file does NOT mean opting out of
    # config.ini's [backup] passwords / password. Used to: if secrets.conf
    # existed but lacked a 'passwords' entry, password_list was set to []
    # and config.ini was ignored. That broke every call site that passed
    # password=None and relied on password_list (investigate_stp_ports,
    # monitor topology) while backup kept working because it also passes
    # the single password as a positional arg.
    def _from_config_ini():
        pw_list_raw = cp.get("backup", "passwords", fallback="")
        if pw_list_raw.strip():
            return [p.strip() for p in pw_list_raw.split(",") if p.strip()]
        return [passwords["default"]] if passwords["default"] else []

    if env_pw:
        cfg["password_list"] = [p.strip() for p in env_pw.split(",") if p.strip()]
        cfg["_pw_source"] = "env"
    elif os.path.exists(SECRETS_FILE):
        sec_pw = secrets.get("credentials", "passwords", fallback="").strip()
        if sec_pw:
            cfg["password_list"] = [p.strip() for p in sec_pw.split(",") if p.strip()]
            cfg["_pw_source"] = "secrets.conf"
        else:
            cfg["password_list"] = _from_config_ini()
            cfg["_pw_source"] = "config" if cfg["password_list"] else "none"
    else:
        cfg["password_list"] = _from_config_ini()
        cfg["_pw_source"] = "config" if cfg["password_list"] else "none"

    # Ensure default password is set for legacy callers
    if cfg["password_list"] and not passwords["default"]:
        passwords["default"] = cfg["password_list"][0]

    # --- Per-user password map (for LDAP/RADIUS-backed users and avoiding
    # lockouts from cross-product attempts). Merge config.ini then secrets.conf
    # per-key; secrets.conf wins. Values are lists of passwords for that user. ---
    user_passwords = {}
    if cp.has_section("user_passwords"):
        for user, pws in cp.items("user_passwords"):
            user_passwords[user] = [p.strip() for p in pws.split(",") if p.strip()]
    if os.path.exists(SECRETS_FILE) and secrets.has_section("user_passwords"):
        for user, pws in secrets.items("user_passwords"):
            user_passwords[user] = [p.strip() for p in pws.split(",") if p.strip()]
    cfg["user_passwords"] = user_passwords

    # --- Email config for alerts ---
    if cp.has_section("email"):
        cfg["smtp_server"] = cp.get("email", "smtp_server", fallback="")
        cfg["smtp_port"] = cp.get("email", "smtp_port", fallback="587")
        cfg["email_from"] = cp.get("email", "from", fallback="netops@localhost")
        cfg["email_to"] = cp.get("email", "to", fallback="")
        cfg["smtp_username"] = cp.get("email", "smtp_username", fallback="")
        cfg["smtp_password"] = cp.get("email", "smtp_password", fallback="")
        # Optional client logo for the HTML digest header. Per-site (each
        # client points at its own). Embedded CID (NOT base64 data-URI — Gmail/
        # Outlook strip those). Keep it in /etc/netops or /var/lib/netops so the
        # AppArmor profile can read it. Empty / missing file -> no logo.
        cfg["email_logo"] = cp.get("email", "logo", fallback="").strip()

    # Warn if config.ini contains passwords and has loose permissions
    if cfg["_pw_source"] == "config" and sys.platform != "win32":
        try:
            mode = os.stat(CONFIG_FILE).st_mode & 0o777
            if mode & _SECRET_FORBIDDEN_MODE:  # group-write or world access
                log.warning(
                    "WARNING: %s contains passwords and has permissions %04o — "
                    "consider moving passwords to secrets.conf or "
                    "'chmod 640 %s' (0640 root:%s is fine for a shared group)",
                    CONFIG_FILE, mode, CONFIG_FILE, SECRET_GROUP)
        except OSError:
            pass
    # Warn if secrets.conf has loose permissions
    if cfg["_pw_source"] == "secrets.conf" and sys.platform != "win32":
        try:
            mode = os.stat(SECRETS_FILE).st_mode & 0o777
            if mode & _SECRET_FORBIDDEN_MODE:
                log.warning(
                    "WARNING: %s has permissions %04o — group-write or "
                    "world access; consider 'chmod 640 %s' and 'chgrp %s %s'",
                    SECRETS_FILE, mode, SECRETS_FILE, SECRET_GROUP, SECRETS_FILE)
        except OSError:
            pass

    # --- Credentials must live in secrets.conf --------------------------
    # netops refuses to run if any credential material is present in
    # netops.conf (or legacy config.ini). ALL credentials — backup
    # passwords, per-user passwords, and the entire [snmp] / [snmp_v3:*]
    # configuration — belong in secrets.conf, which is 0640 root:netops so
    # a shared operator group can read it without keeping private copies.
    # This is unconditional (there is no opt-out): it guarantees an
    # incidental netops.conf edit can never expose a credential in a file
    # whose permissions are looser than secrets.conf's.
    violations = []
    # Probe the secrets file. We cannot use os.path.exists() — it returns
    # False on PermissionError too, which is exactly the case for a user
    # who isn't in the netops group: /etc/netops is 2750 root:netops, so
    # a non-group user can't stat through it, and the "missing file"
    # message they'd see is wrong and unhelpful. Distinguish:
    #   FileNotFoundError → genuinely missing
    #   PermissionError   → exists but the running user can't read the
    #                       parent dir or the file; diagnose group
    #                       membership and emit a remedy.
    secrets_state = "ok"
    try:
        secrets_st = os.stat(SECRETS_FILE)
    except FileNotFoundError:
        secrets_state = "missing"
    except PermissionError:
        secrets_state = "no-access"
        secrets_st = None
    if secrets_state == "missing":
        violations.append(
            f"{SECRETS_FILE} does not exist — all credentials live there. "
            f"Create it (0640 root:netops) with the credential sections.")
    elif secrets_state == "no-access":
        user = getpass.getuser() or "(unknown)"
        in_group = False
        if grp is not None:
            try:
                g = grp.getgrnam(SECRET_GROUP)
                if user in g.gr_mem or os.getegid() == g.gr_gid:
                    in_group = True
            except KeyError:
                pass
        if not in_group:
            violations.append(
                f"cannot read {SECRETS_FILE} — your account ('{user}') "
                f"is not a member of the '{SECRET_GROUP}' group. Ask "
                f"your administrator for membership in the "
                f"'{SECRET_GROUP}' group.")
        else:
            violations.append(
                f"cannot read {SECRETS_FILE} even though '{user}' is in "
                f"the '{SECRET_GROUP}' group. Ask your administrator "
                f"to verify permissions on {os.path.dirname(SECRETS_FILE)} "
                f"and {SECRETS_FILE}.")
    elif sys.platform != "win32" and secrets_st is not None:
        mode = secrets_st.st_mode & 0o777
        if mode & _SECRET_FORBIDDEN_MODE:
            violations.append(
                f"{SECRETS_FILE} has permissions {mode:04o} (must be "
                f"0640 or stricter — owner rw, optional group-read for "
                f"the {SECRET_GROUP} group; no group-write or world "
                f"access). Run 'chmod 640 {SECRETS_FILE} && chgrp "
                f"{SECRET_GROUP} {SECRETS_FILE}'.")
    config_cred_keys = []
    if cp.has_section("backup"):
        for key in cp.options("backup"):
            if (key in ("password", "passwords")
                    or key.startswith("password_")) and \
                    cp.get("backup", key, fallback="").strip():
                config_cred_keys.append(f"[backup] {key}")
    if cp.has_section("user_passwords") and cp.options("user_passwords"):
        config_cred_keys.append("[user_passwords] section")
    if cp.has_section("snmp"):
        config_cred_keys.append("[snmp] section")
    for section in cp.sections():
        if section.startswith("snmp_v3:"):
            config_cred_keys.append(f"[{section}] section")
    if config_cred_keys:
        violations.append(
            f"{CONFIG_FILE} contains credential material that must move to "
            f"{SECRETS_FILE}: {', '.join(config_cred_keys)}.")
    if violations:
        log.error("credential configuration error — all credentials must "
                  "live in %s, not %s:", SECRETS_FILE, CONFIG_FILE)
        for v in violations:
            log.error("  • %s", v)
        sys.exit(2)

    # Resolve device_file relative to the state dir if not absolute (it's an
    # operator-supplied input list and any failed_devices.txt is written
    # alongside it; both belong with the mutable state, not in /usr/bin).
    if not os.path.isabs(cfg["device_file"]):
        cfg["device_file"] = os.path.join(STATE_DIR, cfg["device_file"])

    # Cap threads to OS-safe limits (file descriptors + memory)
    fd_limit = _max_open_files()
    total_mem = _total_memory_mb()

    # Scan: limited by FDs only (lightweight TCP sockets, 1 FD each)
    scan_cap = max(fd_limit - 100, 10)  # reserve 100 FDs for logging, etc.
    scan_limit_reason = (
        f"OS fd limit {fd_limit} — 100 reserved for system = {scan_cap} max"
    )

    # Credential test / backup SSH pool: limited by FDs and memory
    # (whichever is lower). Each SSH session uses ~3 FDs and ~5 MB RAM.
    fd_ssh_cap = max((fd_limit - 100) // 3, 10)
    mem_ssh_cap = max(int(total_mem * _SSH_MEM_FRACTION / _SSH_MEM_MB), 10)
    if fd_ssh_cap <= mem_ssh_cap:
        ssh_cap = fd_ssh_cap
        ssh_limit_reason = (
            f"OS fd limit {fd_limit} — 100 reserved, "
            f"~3 fds per SSH session = {fd_ssh_cap} max"
        )
    else:
        ssh_cap = mem_ssh_cap
        usable_mb = int(total_mem * _SSH_MEM_FRACTION)
        ssh_limit_reason = (
            f"system memory {total_mem} MB — "
            f"25% usable ({usable_mb} MB) at ~{_SSH_MEM_MB} MB per SSH session "
            f"= {mem_ssh_cap} max"
        )

    raw_scan = cfg["scan_threads"]
    raw_ssh = cfg["ssh_threads"]
    cfg["scan_threads"] = min(raw_scan, scan_cap)
    cfg["ssh_threads"] = min(raw_ssh, ssh_cap)
    cfg["_thread_caps"] = {
        "fd_limit": fd_limit,
        "total_mem_mb": total_mem,
        "raw_scan": raw_scan,
        "raw_ssh": raw_ssh,
        "scan_cap": scan_cap,
        "scan_limit_reason": scan_limit_reason,
        "ssh_cap": ssh_cap,
        "ssh_limit_reason": ssh_limit_reason,
        "scan_capped": raw_scan > scan_cap,
        "ssh_capped": raw_ssh > ssh_cap,
    }

    # Site-wide expected STP mode. Compared case-insensitively against the
    # mode parsed from each device during 'monitor stp'; mismatches surface
    # in 'show spanning-tree mismatch' and the weekly 'digest stp' email.
    cfg["stp_expected_mode"] = cp.get(
        "monitor", "stp_expected_mode", fallback="mstp"
    ).strip().lower()

    # Threshold for 'digest backup' stale classification — devices whose
    # last successful backup is older than this are flagged in the email.
    cfg["backup_stale_days"] = cp.getint(
        "monitor", "backup_stale_days", fallback=7)

    # Non-trunk MAC-flux security threshold: a notice fires when a single
    # non-trunk port sees >= this many distinct MACs in a trailing 24h
    # window (rogue mini-switch / AP / MAC spoofing). 0 disables. Trunk /
    # uplink ports (LLDP switch<->switch links) are always excluded.
    # 3.8.9 rename: keys are now `mac_flux_*` (was `mac_churn_*` in
    # 3.8.0-3.8.8). _getint_legacy() reads the new name first; if that's
    # missing AND the old name is present, it reads the old name and emits
    # a one-time DEBUG hint. Existing netops.conf files with the old names
    # continue working unchanged; new installs use the new names.
    def _getint_legacy(new_key, old_key, fallback):
        if cp.has_option("monitor", new_key):
            return cp.getint("monitor", new_key)
        if cp.has_option("monitor", old_key):
            log.debug("[monitor] %s is the 3.8.9 rename of %s — both work, "
                      "but the new name is preferred", new_key, old_key)
            return cp.getint("monitor", old_key)
        return fallback
    def _get_legacy(new_key, old_key, fallback):
        if cp.has_option("monitor", new_key):
            return cp.get("monitor", new_key)
        if cp.has_option("monitor", old_key):
            log.debug("[monitor] %s is the 3.8.9 rename of %s — both work, "
                      "but the new name is preferred", new_key, old_key)
            return cp.get("monitor", old_key)
        return fallback
    cfg["mac_flux_threshold_24h"] = _getint_legacy(
        "mac_flux_threshold_24h", "mac_churn_threshold_24h", 5)
    # Retention (days) for the flap_events timeline behind `show flap-history`.
    # Pruned in the weekly health digest; independent of op_events (35d).
    try:
        cfg["flap_history_days"] = cp.getint("monitor", "flap_history_days")
    except (configparser.NoSectionError, configparser.NoOptionError, ValueError):
        cfg["flap_history_days"] = 30
    # Retention (days) for the FDB (port_macs) + ARP (ip_arp) location history
    # behind `show mac`. Default 365 — these are DEDUPLICATED tables (one row
    # per switch+port+MAC+VLAN, updated in place), so a year is cheap and gives
    # real device-movement forensics. Pruned separately from op_events.
    try:
        cfg["fdb_history_days"] = cp.getint("monitor", "fdb_history_days")
    except (configparser.NoSectionError, configparser.NoOptionError, ValueError):
        cfg["fdb_history_days"] = 365
    # Retention (days) for the port_errors timeline behind `show port-errors`
    # and the errors column of `show mac`. Default 365 — an intermittent fault
    # (the Hyper-V cluster case) needs to stay visible for months to correlate
    # against. Append-only but bounded (a row only when a counter rises), so a
    # year is cheap. Pruned in the weekly health digest.
    try:
        cfg["port_error_days"] = cp.getint("monitor", "port_error_days")
    except (configparser.NoSectionError, configparser.NoOptionError, ValueError):
        cfg["port_error_days"] = 365
    # Per-day flap threshold for the weekly health digest's flap section: a
    # port is listed only if it flapped at least this many times on its WORST
    # single day in the reporting window (computed from the flap_events
    # timeline). Keeps the consolidated digest focused on genuinely unstable
    # links instead of the long tail of one-off bounces. Default 50; lower to
    # widen, set 0 to list any port with a flap.
    try:
        cfg["digest_flap_min_per_day"] = cp.getint("monitor", "digest_flap_min_per_day")
    except (configparser.NoSectionError, configparser.NoOptionError, ValueError):
        cfg["digest_flap_min_per_day"] = 50
    # A non-trunk port with MORE than this many distinct MACs in 24h is
    # structurally an aggregation/uplink (not one rogue device) and is
    # excluded from the flux signal. 0 disables the ceiling check.
    cfg["mac_flux_uplink_ceiling"] = _getint_legacy(
        "mac_flux_uplink_ceiling", "mac_churn_uplink_ceiling", 24)
    # A real access port carries one data VLAN (+ maybe a voice VLAN). A
    # port whose MACs span more than this many VLANs is a trunk or a
    # virtualization host (VMware vSwitch uplink etc.) — excluded from the
    # flux signal so vMotion / many-VM ports don't false-alarm. 0 disables.
    cfg["mac_flux_max_vlans"] = _getint_legacy(
        "mac_flux_max_vlans", "mac_churn_max_vlans", 2)
    # 3.8.7 rotation filter — only MACs that look like brief passers-
    # through count toward the flux threshold. A MAC qualifies as
    # "rotating" when its times_seen <= max_times_seen AND its span
    # (last_seen - first_seen) < max_span_sec. Stable residents (busy
    # AV-room ports, multi-device desks) score 0 rotating MACs and drop
    # out cleanly. Set either to 0 to disable that part of the filter
    # and fall back to 3.8.6's "count all 24h distincts" semantics.
    cfg["mac_flux_rotation_max_times_seen"] = _getint_legacy(
        "mac_flux_rotation_max_times_seen",
        "mac_churn_rotation_max_times_seen", 3)
    cfg["mac_flux_rotation_max_span_sec"] = _getint_legacy(
        "mac_flux_rotation_max_span_sec",
        "mac_churn_rotation_max_span_sec", 7200)
    # 3.8.7 manual whitelist — approved (ip:interface) pairs that should
    # never alert (e.g. an approved A/V switch downstream of an access
    # port). Whitespace- or comma-separated list of ip:interface tokens.
    # Existing mac_flux_alerts rows for excluded ports clear on the
    # next 30-min tick because they drop out of the current flux list.
    raw = _get_legacy(
        "mac_flux_excluded_ports", "mac_churn_excluded_ports", "").strip()
    excluded = set()
    for tok in re.split(r"[\s,]+", raw):
        if ":" in tok:
            ip, _, iface = tok.partition(":")
            if ip and iface:
                excluded.add((ip.strip(), iface.strip()))
    cfg["mac_flux_excluded_ports"] = excluded

    # Per-event unreachable-alert hysteresis: 'monitor stp' emails after N
    # consecutive failed polls so a single dropped UDP packet doesn't page.
    # 3 polls at every-minute cadence = ~3 minutes of sustained failure.
    cfg["snmp_unreachable_polls"] = cp.getint(
        "monitor", "snmp_unreachable_polls", fallback=3)
    # After the unreachable threshold is crossed, hold the alert until no
    # NEW device has joined the outage for one poll (or this many minutes
    # pass) so one power event = one email, not a trickle of them.
    cfg["unreachable_coalesce_min"] = cp.getint(
        "monitor", "unreachable_coalesce_min", fallback=3)
    # Phase A slot broker: max concurrent SSH sessions the daemon grants
    # fleet-wide (0/absent = use ssh_threads).
    cfg["daemon_slot_cap"] = cp.getint("daemon", "slot_cap", fallback=0)

    # 'monitor stp' transport selection. 'auto' uses SNMP on devices with
    # snmp_enabled=1 (cached cred from discover) and falls back to SSH
    # otherwise — the load-reducing default. 'snmp' forces SNMP and skips
    # devices that haven't been probed yet; 'ssh' keeps the legacy path
    # everywhere.
    cfg["stp_poll_mode"] = cp.get(
        "monitor", "stp_mode", fallback="auto").strip().lower()
    if cfg["stp_poll_mode"] not in ("auto", "snmp", "ssh"):
        log.warning("[monitor] stp_mode=%r invalid; using 'auto'",
                    cfg["stp_poll_mode"])
        cfg["stp_poll_mode"] = "auto"

    # Optional STP domain mapping — for multi-site networks where switches
    # in different L2 domains (separate sites, IPsec-tunneled offices, etc.)
    # legitimately have different root bridges. Keys are IP prefixes, values
    # are domain names. Longest-prefix match wins. Empty map = single
    # implicit domain (backward-compatible default for single-site setups).
    cfg["stp_domains"] = {}
    if cp.has_section("stp_domains"):
        for prefix_str, domain in cp.items("stp_domains"):
            try:
                net = ipaddress.ip_network(prefix_str.strip(), strict=False)
            except ValueError:
                log.warning("[stp_domains] invalid prefix %r — skipping",
                            prefix_str)
                continue
            cfg["stp_domains"][net] = domain.strip()

    # Optional remote git URL for pushing committed backups. Supports any
    # form git itself does: github.com SSH or HTTPS, on-prem ssh:// URL,
    # bare repo on a network share, etc. Authentication (keys, PAT, credential
    # helper) is the user's responsibility — netops doesn't store secrets.
    # When empty, backups stay local-only.
    cfg["git_remote_url"] = cp.get("backup", "git_remote_url", fallback="").strip()
    cfg["git_branch"] = cp.get("backup", "git_branch", fallback="main").strip() or "main"

    # --- SNMP credentials (optional) ---
    # Accept multiple v2c communities and multiple v3 users. The probe tries
    # the cached winner first (per-device); list-walk happens only during
    # discover. The entire [snmp] / [snmp_v3:*] config lives in secrets.conf.
    cfg["snmp"] = _load_snmp_config(secrets)

    return cfg


def _load_snmp_config(secrets):
    """Parse [snmp] + [snmp_v3:USER] sections from secrets.conf.

    SNMP credentials (v2c communities, v3 passphrases) are credentials, so
    the entire [snmp] / [snmp_v3:*] configuration lives only in secrets.conf
    — netops.conf is rejected at startup if it contains those sections.

    Returns a dict with keys:
      communities    list of v2c community strings
      v3_users       list of dicts (name, auth_proto, auth_pass, priv_proto,
                     priv_pass, security_level)
      timeout        per-request UDP timeout, seconds
      retries        retry count on no-response
      enabled        True if at least one credential is configured
    """
    out = {"communities": [], "v3_users": [], "timeout": 5, "retries": 1,
           "enabled": False}
    if not secrets:
        return out

    if secrets.has_section("snmp"):
        # 'communities' (plural, comma-list) takes precedence; fall back to
        # singular 'community' for the single-string case.
        raw = secrets.get("snmp", "communities", fallback="").strip()
        if raw:
            out["communities"] = [c.strip() for c in raw.split(",") if c.strip()]
        else:
            single = secrets.get("snmp", "community", fallback="").strip()
            if single:
                out["communities"] = [single]
        out["timeout"] = secrets.getint("snmp", "timeout", fallback=5)
        out["retries"] = secrets.getint("snmp", "retries", fallback=1)

    # v3 users: one [snmp_v3:USERNAME] section per user.
    user_keys = ("auth_proto", "auth_pass", "priv_proto", "priv_pass",
                 "security_level")
    for section in sorted(s for s in secrets.sections()
                          if s.startswith("snmp_v3:")):
        username = section.split(":", 1)[1].strip()
        if not username:
            continue
        record = {"name": username}
        for key in user_keys:
            record[key] = secrets.get(section, key, fallback="").strip()
        # security_level defaults to authPriv when auth+priv are present,
        # authNoPriv when only auth, noAuthNoPriv otherwise.
        if not record["security_level"]:
            if record["auth_pass"] and record["priv_pass"]:
                record["security_level"] = "authPriv"
            elif record["auth_pass"]:
                record["security_level"] = "authNoPriv"
            else:
                record["security_level"] = "noAuthNoPriv"
        out["v3_users"].append(record)

    out["enabled"] = bool(out["communities"]) or bool(out["v3_users"])
    return out


# ---------------------------------------------------------------------------
# Logging
# ---------------------------------------------------------------------------

def _open_log_handler(path, mode, level):
    """Build a file log handler, or return None if the path isn't writable.

    The log dir is group-writable (2770 netops:netops) and the log file is kept
    0664 so ANY netops-group operator — not just the service user — can append,
    so a human's `show`/console run is logged like the timers' runs. If the file
    still isn't writable (e.g. an odd pre-existing mode we can't fix because we
    don't own it), we skip file logging and keep console output rather than crash.
    """
    try:
        handler = logging.FileHandler(path, mode=mode, encoding="utf-8")
    except OSError as exc:
        print(f"netops: file logging disabled ({path}: {exc.strerror}); "
              "logging to console only.", file=sys.stderr)
        return None
    # Keep it group-writable so the next netops-group operator can append too.
    # Best-effort: only the file's owner can chmod — a group member appending to
    # an already-0664 file needs no chmod, so a failure here is harmless.
    try:
        os.chmod(path, 0o664)
    except OSError:
        pass
    handler.setLevel(level)
    handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
    return handler


def setup_logging(debug=False, debug_file=None, log_format="full", log_mode="append"):
    """Configure logging to stdout, netops.log, and optionally a separate debug file.

    debug=True       → DEBUG on console; netops.log also gets DEBUG.
    debug_file=PATH  → additionally write DEBUG to PATH (keeps netops.log as-is).
                       PATH is separate from netops.log so routine runs stay clean.
    log_mode="truncate" → open log files in write mode (fresh each run).
                          Default "append" keeps history.
    Both debug flags can be combined.
    """
    console_level = logging.DEBUG if debug else logging.INFO
    main_file_level = logging.DEBUG if debug else logging.INFO
    file_mode = "w" if log_mode == "truncate" else "a"

    if debug or log_format == "full":
        console_fmt = "%(asctime)s %(levelname)s %(message)s"
    else:
        console_fmt = "%(message)s"

    console_handler = logging.StreamHandler(sys.stdout)
    console_handler.setLevel(console_level)
    console_handler.setFormatter(logging.Formatter(console_fmt))

    handlers = [console_handler]

    # netops.log — normal log (INFO, or DEBUG when --debug on console).
    # Skipped (with a stderr note) when the caller can't write it, e.g. a
    # netops-group operator running a read-only command — console still works.
    file_handler = _open_log_handler(LOG_FILE, file_mode, main_file_level)
    if file_handler is not None:
        handlers.insert(0, file_handler)

    # Optional dedicated DEBUG file — separate from netops.log.
    if debug_file:
        debug_handler = _open_log_handler(debug_file, file_mode, logging.DEBUG)
        if debug_handler is not None:
            handlers.append(debug_handler)

    root = logging.getLogger()
    root.setLevel(logging.DEBUG)  # handlers filter per-destination
    root.handlers = handlers

    # Suppress noisy third-party loggers unless debug is going to the console.
    if not debug:
        logging.getLogger("pexpect").setLevel(logging.WARNING)


# ---------------------------------------------------------------------------
# Connection layer — pexpect + system ssh/telnet
# ---------------------------------------------------------------------------

def check_tools():
    """Check for required external tools (ssh, telnet) and set availability flags.

    ssh is required. telnet is optional but needed for devices without SSH.
    Prints install instructions if tools are missing.
    """
    global _HAS_SSH, _HAS_TELNET

    _HAS_SSH = shutil.which("ssh") is not None
    _HAS_TELNET = shutil.which("telnet") is not None

    if not _HAS_SSH:
        log.error("OpenSSH client not found. SSH is required.")
        if sys.platform == "darwin":
            log.error("  Install: brew install openssh")
        elif sys.platform == "win32":
            log.error("  Install: Settings > Apps > Optional Features > OpenSSH Client")
            log.error("  Or: Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0")
        else:
            log.error("  Install: sudo apt install openssh-client  (Debian/Ubuntu)")
            log.error("           sudo dnf install openssh-clients  (RHEL/Fedora)")
        sys.exit(1)

    if not _HAS_TELNET:
        log.warning("telnet not found — telnet fallback disabled (SSH-only mode).")
        if sys.platform == "darwin":
            log.warning("  Install: brew install telnet")
        elif sys.platform == "win32":
            log.warning("  Install: dism /online /Enable-Feature /FeatureName:TelnetClient")
            log.warning("  Or: Enable-WindowsOptionalFeature -Online -FeatureName TelnetClient")
        else:
            log.warning("  Install: sudo apt install telnet  (Debian/Ubuntu)")
            log.warning("           sudo dnf install telnet  (RHEL/Fedora)")


# ---------------------------------------------------------------------------
# Credential-field validators
#
# Every field that gets interpolated into an ssh/telnet argv MUST pass
# _safe_credential_field() before it reaches the spawn layer. The check
# closes the SSH ProxyCommand injection where a username beginning with
# '-' (or containing shell-quoted ssh option syntax) is parsed by ssh as
# an option flag and executed via /bin/sh before authentication.
# ---------------------------------------------------------------------------

_CRED_FIELD_RE = re.compile(r"^[A-Za-z0-9._+\-/@]+$")


def _safe_credential_field(value, field_name):
    """Reject credential fields that could be confused with ssh option flags.

    A username starting with '-' is interpretable by ssh as the start of an
    option (see -oProxyCommand= injection). Reject rather than escape;
    legitimate device usernames never start with '-' or contain shell
    metacharacters in this fleet.
    """
    if not isinstance(value, str) or not value:
        raise ValueError(f"invalid {field_name}: empty or non-string")
    if value.startswith("-"):
        raise ValueError(
            f"invalid {field_name}: leading '-' is forbidden "
            "(would be parsed by ssh as an option flag)")
    if not _CRED_FIELD_RE.match(value):
        raise ValueError(
            f"invalid {field_name}: contains characters outside "
            "[A-Za-z0-9._+-/@] — refusing to pass to ssh/telnet")


def _safe_ip(value):
    """Reject anything that isn't a valid IPv4/IPv6 literal.

    Defends against ip=`-oProxyCommand=...` style smuggling at the host
    position. Devices in the DB should always pass this; reject early if not.
    """
    try:
        ipaddress.ip_address(value)
    except (ValueError, TypeError) as e:
        raise ValueError(f"invalid IP: {value!r}: {e}")


# Per-thread login context. The netopsd broker sets `plain` so a `connect`
# session is a RAW passthrough — netops sends NO login-setup commands (no
# paging-disable, no `enable`); the operator lands exactly where a manual SSH
# would, and elevating/paging is theirs to decide. Automated logins (backup,
# monitor, discover) leave it unset and get the normal setup, since they need
# privileged mode + disabled paging to scrape full configs. Thread-local so
# concurrent broker sessions and the automated logins never interfere; the
# login runs synchronously in the same thread that sets this.
_login_ctx = threading.local()


def _login_plain():
    return getattr(_login_ctx, "plain", False)


def _spawn(cmd, timeout=10):
    """Spawn a child process using PTY (Unix) or PopenSpawn (Windows).

    `cmd` may be a list (argv form — preferred, NO shell/shlex involved)
    or a string (legacy form — pexpect re-tokenizes via shlex, which is
    UNSAFE for any cmd that interpolates user-controlled fields). All
    new call sites should pass a list.

    codec_errors="replace" keeps the UTF-8 decoder from raising on raw bytes
    that arrive mid-stream (e.g. telnet IAC negotiation, 0xFB / 0xFF) before
    the child wraps them into text.
    """
    if sys.platform == "win32":
        return PopenSpawn(cmd, timeout=timeout, encoding="utf-8", codec_errors="replace")
    if isinstance(cmd, list):
        # argv form — pexpect calls execvp directly, no shlex re-parse.
        child = pexpect.spawn(cmd[0], args=cmd[1:], timeout=timeout,
                              encoding="utf-8", codec_errors="replace",
                              maxread=200000)
    else:
        child = pexpect.spawn(cmd, timeout=timeout, encoding="utf-8",
                              codec_errors="replace", maxread=200000)
    # Use a wide terminal so the remote device's CLI doesn't wrap long
    # commands across lines (Junos echoes '...' continuations on wrap,
    # which slip past send_command's single-line echo stripper).
    try:
        child.setwinsize(40, 300)
    except Exception:
        pass  # non-pty or platform without winsize support — harmless
    return child


def _create_askpass(password):
    """Create a temporary SSH_ASKPASS script for non-interactive password auth (Windows)."""
    fd, path = tempfile.mkstemp(suffix=".sh")
    with os.fdopen(fd, "w") as f:
        f.write(f"#!/bin/sh\necho '{password}'\n")
    os.chmod(path, 0o700)
    return path


# Algorithms we filter out even when a switch advertises them, because they
# would fail the openssh-client command-line parse or are universally
# considered too weak. Other legacy algorithms (group1-sha1, ssh-rsa,
# 3des-cbc, hmac-md5, etc.) are added on-demand when a specific device's
# negotiation error tells us they're needed — never offered upfront.
_REJECT_ALGOS = {
    "ssh-dss",  # removed in OpenSSH 10; '+' prefix can't reintroduce it
    "des",      # 56-bit cipher; OpenSSH refuses regardless
    "none",     # explicitly-disabled cipher
}


def _parse_algo_offer(output):
    """Parse SSH algorithm negotiation failures and return retry -o flags.

    OpenSSH prints messages like:
      Unable to negotiate ... no matching key exchange method found.
        Their offer: diffie-hellman-group14-sha1,diffie-hellman-group1-sha1
      Unable to negotiate ... no matching host key type found.
        Their offer: ssh-rsa,ssh-dss
      Unable to negotiate ... no matching MAC found.
        Their offer: hmac-sha1,hmac-md5

    The "Their offer:" may appear on the same line or a continuation line.

    All five negotiable layers are covered (kex, host-key, cipher, MAC,
    pubkey accepted). Anything not matched falls back to KexAlgorithms +
    HostKeyAlgorithms as a safety net.
    """
    extra_opts = []
    # Find all "Their offer: <algos>" occurrences anywhere in the output
    for m in re.finditer(r"Their offer:\s*(\S+)", output):
        raw_algos = m.group(1)
        # Filter out unsupported algorithms (see _REJECT_ALGOS for why)
        algos = ",".join(a for a in raw_algos.split(",") if a not in _REJECT_ALGOS)
        if not algos:
            continue
        # Look at text before this match to determine the algorithm category
        preceding = output[:m.start()]
        category = preceding.rsplit("no matching", 1)[-1] if "no matching" in preceding else ""
        if "key exchange" in category:
            extra_opts += ["-o", f"KexAlgorithms=+{algos}"]
        elif "host key" in category:
            extra_opts += ["-o", f"HostKeyAlgorithms=+{algos}"]
            extra_opts += ["-o", f"PubkeyAcceptedAlgorithms=+{algos}"]
        elif "cipher" in category:
            extra_opts += ["-o", f"Ciphers=+{algos}"]
        elif "MAC" in category or "mac" in category:
            extra_opts += ["-o", f"MACs=+{algos}"]
        else:
            extra_opts += ["-o", f"KexAlgorithms=+{algos}"]
            extra_opts += ["-o", f"HostKeyAlgorithms=+{algos}"]
    return extra_opts


def _ssh_attempt(ip, username, password, timeout, extra_opts=None):
    """Single SSH connection attempt. Returns child on success, None on failure.

    On algorithm negotiation failure, returns ('negotiate', output) so the
    caller can retry with appropriate options.
    """
    # SECURITY: never interpolate an unvalidated field into the ssh argv.
    # In THIS per-device connect path a rejected value means "can't connect
    # to this device" — return None (the normal failure sentinel) so the
    # caller fails just this one device instead of aborting an entire fleet
    # poll. The hard rejection of operator-supplied values happens earlier
    # at the CLI (apply_overrides) and config (load_config) boundaries, so
    # the RCE surface stays closed; here we must stay non-fatal because the
    # username can be a per-device DB sentinel like "(nouser)".
    try:
        _safe_credential_field(username, "username")
        _safe_ip(ip)
    except ValueError as e:
        log.debug("  %s: skipping ssh — %s", ip, e)
        return None

    opts = list(_SSH_BASE_OPTS)
    if extra_opts:
        opts += extra_opts
    # argv form — no shell, no shlex re-parse. ssh sees each list element
    # as exactly one argument regardless of internal whitespace or quotes.
    argv = ["ssh"] + opts + ["-o", f"ConnectTimeout={timeout}",
                             f"{username}@{ip}"]
    log.debug("  SSH argv: %s", argv)

    askpass = None
    child = None
    try:
        if sys.platform == "win32":
            askpass = _create_askpass(password)
            env = os.environ.copy()
            env["SSH_ASKPASS"] = askpass
            env["SSH_ASKPASS_REQUIRE"] = "prefer"
            env["DISPLAY"] = ":0"
            child = PopenSpawn(argv, timeout=timeout, encoding="utf-8",
                               codec_errors="replace", env=env)
        else:
            child = pexpect.spawn(argv[0], args=argv[1:], timeout=timeout,
                                  encoding="utf-8", codec_errors="replace",
                                  maxread=200000)
            # Wide terminal so long device CLI commands don't wrap.
            try:
                child.setwinsize(40, 300)
            except Exception:
                pass

        # Wait for password prompt, prompt, or failure
        idx = child.expect([
            r"[Pp]assword\s*:",          # 0 — password prompt
            PROMPT_RE,                    # 1 — already at command prompt (key auth)
            r"Permission denied",         # 2 — auth failed
            pexpect.TIMEOUT,              # 3
            pexpect.EOF,                  # 4
        ])

        if idx == 0:
            child.sendline(password)
            idx2 = child.expect([
                PROMPT_RE,                # 0 — command prompt (success)
                r"[Pp]ress any key",      # 1 — HP ProCurve post-login banner
                r"[Tt]erminal type\?",    # 2 — old JUNOS terminal type prompt
                r"[Pp]assword\s*:",       # 3 — wrong password, prompted again
                r"Permission denied",     # 4
                pexpect.TIMEOUT,          # 5
                pexpect.EOF,              # 6
            ])
            if idx2 == 1:
                # HP ProCurve "Press any key to continue" banner
                child.sendline("")
                child.expect(PROMPT_RE, timeout=timeout)
            elif idx2 == 2:
                # Old JUNOS "Terminal type? [vt100]" — accept default
                child.sendline("")
                child.expect(PROMPT_RE, timeout=timeout)
            elif idx2 != 0:
                child.close()
                return None
        elif idx == 1:
            pass  # Already at prompt
        elif idx == 4:
            # EOF — check for algorithm negotiation failure
            output = child.before or ""
            if "no matching" in output or "Bad SSH2" in output:
                child.close()
                return ("negotiate", output)
            child.close()
            return None
        else:
            child.close()
            return None

        # Detect platform from prompt and disable paging
        prompt_text = child.after or ""

        # Juniper root login lands in the FreeBSD shell (% prompt); enter JunOS CLI.
        if re.search(_SHELL_PROMPT_RE, prompt_text):
            child.sendline("cli")
            child.expect(PROMPT_RE, timeout=5)
            prompt_text = child.after or ""

        # Broker (`connect`): raw passthrough — send NO login-setup commands
        # (no paging-disable, no `enable`); the operator lands exactly where a
        # manual SSH would. Automated logins fall through and get the setup.
        if _login_plain():
            return child

        # Ruckus FastIron's `SSH@`/`telnet@` literal prefix lives JUST BEFORE
        # what PROMPT_RE captures (the regex's `[\w\-/\.]+` stops at `@`),
        # so child.after has only the suffix `hostname[#>]`. Concatenate the
        # tail of child.before so the FastIron prefix is visible to the check.
        _post_login_setup(child, password, prompt_text)

        return child

    except (pexpect.ExceptionPexpect, OSError) as e:
        log.debug("  SSH failed for %s@%s: %s", username, ip, e)
        if child:
            try:
                child.close()
            except Exception:
                pass
        return None
    finally:
        if askpass:
            try:
                os.unlink(askpass)
            except Exception:
                pass


def _post_login_setup(child, password, prompt_text=None):
    """Platform-aware session setup after ANY login (SSH or telnet):
    disable paging, and on ProCurve/Cisco enter privileged mode.

    Was inline in the SSH path only — the telnet fallback had its own
    pre-platform-era block that sent ProCurve's `no page` to everything
    (a no-op on FastIron, leaving the pager ACTIVE) and ran `enable`
    (which corrupts FastIron sessions). A 04:15 telnet-fallback capture
    full of '--More--' pager fragments emailed as a config-change alert
    was the result.
    """
    if prompt_text is None:
        prompt_text = child.after or ""
    prompt_tail = ((child.before or "")[-40:] + prompt_text)

    if re.search(r"(?:SSH|telnet)@[\w\-/\.()]+[#>]", prompt_tail):
        # Ruckus FastIron (ICX) — disable paging; never run `enable`
        # (privileged-mode entry on FastIron prompts for User Name + Password,
        # which corrupts the session if we just send the login password)
        child.sendline("skip-page-display")
        child.expect(PROMPT_RE, timeout=5)
    elif "@" in prompt_tail:
        # Juniper (user@host> prompt). Check prompt_tail, NOT prompt_text:
        # PROMPT_RE captures only the part AFTER the '@' (it stops at '@'),
        # so the '@' lives in child.before. Without this, Junos fell into
        # the ProCurve/Cisco branch below and got sent 'no page' /
        # 'terminal length 0' / 'enable', all of which it rejects.
        child.sendline("set cli screen-length 0")
        child.expect(PROMPT_RE, timeout=5)
    else:
        # HP ProCurve / Cisco IOS — disable paging. Both forms are sent
        # because the prompt alone can't distinguish the two; whichever
        # the device doesn't recognize returns an "Invalid input" line
        # that's harmlessly absorbed by the next PROMPT_RE expect.
        child.sendline("no page")
        child.expect(PROMPT_RE, timeout=5)
        child.sendline("terminal length 0")
        child.expect(PROMPT_RE, timeout=5)

        # Check if we're in unprivileged mode (> instead of #). (The broker
        # never reaches here — it returned a raw session above.)
        if ">" in child.after:
            child.sendline("enable")
            idx = child.expect([
                r"[Pp]assword\s*:",   # 0 — enable password
                PROMPT_RE,            # 1 — already enabled
                pexpect.TIMEOUT,      # 2
            ], timeout=5)
            if idx == 0:
                child.sendline(password)
                child.expect(PROMPT_RE, timeout=5)


def ssh_connect(ip, username, password, timeout=10):
    """Connect to a device via SSH, modern-first with on-demand legacy fallback.

    First attempt uses openssh-client's built-in defaults — modern crypto
    only. If a specific switch fails kex/host-key/cipher/MAC negotiation,
    the device's error message is parsed and ONLY the algorithms that
    device advertises get added to a retry. This keeps modern devices
    talking modern crypto and only opts the legacy algorithms back in for
    the legacy switches that need them.

    No lockout risk in the retry loop: a kex/cipher/MAC failure happens
    before the password prompt, so it doesn't count as a failed login on
    Juniper's progressive-delay logic.
    """
    extra_opts = []
    while True:
        result = _ssh_attempt(ip, username, password, timeout, extra_opts=extra_opts or None)

        if not isinstance(result, tuple):
            return result  # child or None

        # Algorithm negotiation failed — parse and accumulate needed options.
        output = result[1]
        new_opts = _parse_algo_offer(output)
        if not new_opts:
            return None  # can't parse what's needed — give up
        extra_opts += new_opts
        log.debug("  SSH algo negotiation, retrying with: %s", extra_opts)


def telnet_connect(ip, username, password, timeout=10):
    """Connect to a device via telnet using the system telnet command.

    Handles HP ProCurve "Press any key" banner, optional username prompt,
    and password-only login. Returns a pexpect child on success, or None.
    """
    # SECURITY: validate ip before it lands in argv. argv-list spawn means
    # ip can't smuggle a -option; non-fatal per-device (return None) so a
    # corrupt DB ip fails just this device, not the whole run.
    try:
        _safe_ip(ip)
    except ValueError as e:
        log.debug("  %s: skipping telnet — %s", ip, e)
        return None
    argv = ["telnet", ip]
    log.debug("  telnet argv: %s", argv)

    try:
        child = _spawn(argv, timeout=timeout)

        # First expect: banner, username, password, or prompt
        idx = child.expect([
            r"[Pp]ress any key",          # 0 — HP ProCurve banner
            r"[Uu]ser\s*[Nn]ame\s*:",     # 1 — username prompt
            r"[Pp]assword\s*:",            # 2 — password prompt (no username)
            PROMPT_RE,                     # 3 — already at command prompt
            r"[Cc]onnection refused",      # 4
            pexpect.TIMEOUT,              # 5
            pexpect.EOF,                  # 6
        ])

        if idx == 0:
            # Press any key
            child.sendline("")
            idx = child.expect([
                r"[Uu]ser\s*[Nn]ame\s*:",  # 0 — username prompt
                r"[Pp]assword\s*:",         # 1 — password prompt
                PROMPT_RE,                  # 2 — already at prompt
                pexpect.TIMEOUT,            # 3
            ], timeout=timeout)

        if idx == 1:
            # Username prompt
            child.sendline(username)
            idx = child.expect([
                r"[Pp]assword\s*:",   # 0
                pexpect.TIMEOUT,      # 1
            ], timeout=timeout)
            if idx != 0:
                child.close()
                return None

        if idx == 2:
            # Password prompt (no username asked)
            child.sendline(password)
        elif idx == 0:
            # After username, now at password prompt
            child.sendline(password)
        elif idx == 3:
            # Already at command prompt
            pass
        else:
            child.close()
            return None

        # Wait for command prompt after login
        if idx != 3:
            idx = child.expect([
                PROMPT_RE,                # 0 — success
                r"[Pp]assword\s*:",       # 1 — wrong password
                r"[Ll]ogin incorrect",    # 2
                pexpect.TIMEOUT,          # 3
                pexpect.EOF,              # 4
            ])
            if idx != 0:
                child.close()
                return None

        # Juniper root login lands in the FreeBSD shell (% prompt); enter JunOS CLI.
        if re.search(_SHELL_PROMPT_RE, child.after or ""):
            child.sendline("cli")
            child.expect(PROMPT_RE, timeout=5)

        # Broker (`connect`): raw passthrough — skip paging-disable + enable.
        # (The Junos shell->CLI transition above is kept so the operator lands
        # in the CLI, not the FreeBSD shell.)
        if _login_plain():
            return child

        # Same platform-aware setup as the SSH path (FastIron gets
        # skip-page-display and NO `enable`; Junos gets screen-length 0).
        _post_login_setup(child, password)

        return child

    except (pexpect.ExceptionPexpect, OSError) as e:
        log.debug("  telnet failed for %s: %s", ip, e)
        try:
            child.close()
        except Exception:
            pass
        return None


def classify_fail_reason(ssh_open, telnet_open):
    """Return a specific failure reason based on observed port state.

    - Both ports closed/filtered → unreachable
    - One or both ports open but auth failed → credentials rejected
    """
    if not ssh_open and not telnet_open:
        return "unreachable (no SSH/telnet port open)"
    protos = []
    if ssh_open:
        protos.append("SSH")
    if telnet_open:
        protos.append("telnet")
    return f"credentials rejected ({'/'.join(protos)} reachable)"


# --- Slot-broker client (jobs → netopsd) -----------------------------------
# Maps a connect reason to a broker priority class. Anything unmapped is
# treated as backup-tier.
_SLOT_CLASS_FOR_REASON = {
    "monitor_stp_fallback": "tick",
    "monitor_topology_fallback": "tick",
    "investigate_stp": "tick",
    "monitor_config": "config",
    "backup": "backup",
    "discover": "bulk",
    "test": "bulk",
    "retest": "bulk",
}
_SLOT_DAEMON_STATE = {"available": None}   # None = not probed yet


class _DeviceSlot:
    """A held session slot. Releasing = telling the daemon (best-effort)
    and closing the socket — the daemon reaps on EOF regardless, so a
    crashed process can never leak a slot."""
    def __init__(self, sock):
        self._sock = sock

    def release(self):
        try:
            _send_frame(self._sock, {"op": "release"})
        except Exception:
            pass
        try:
            self._sock.close()
        except Exception:
            pass


def _slot_daemon_available():
    """One cached probe per process: is netopsd up with the slot broker?"""
    if _SLOT_DAEMON_STATE["available"] is None:
        try:
            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            sock.settimeout(2)
            sock.connect(NETOPSD_SOCK)
            _send_frame(sock, {"op": "ping"})
            resp = _recv_frame(sock)
            sock.close()
            _SLOT_DAEMON_STATE["available"] = bool(
                isinstance(resp, dict) and resp.get("ok")
                and resp.get("slots"))
        except Exception:
            _SLOT_DAEMON_STATE["available"] = False
        if _SLOT_DAEMON_STATE["available"]:
            log.debug("slot broker: netopsd available — per-session "
                      "admission control active (flock skipped)")
    return _SLOT_DAEMON_STATE["available"]


def _acquire_device_slot(ip, reason, timeout=180):
    """A slot for one device session, or None (daemon down / in-daemon
    direct path handled separately / timeout — timeout logs)."""
    if _IN_DAEMON:
        return None                     # daemon acquires directly
    if not _slot_daemon_available():
        return None
    cls = _SLOT_CLASS_FOR_REASON.get(reason or "", "backup")
    try:
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        sock.settimeout(timeout + 15)
        sock.connect(NETOPSD_SOCK)
        _send_frame(sock, {"op": "slot", "ip": ip, "cls": cls,
                           "job": reason or "", "timeout": timeout})
        resp = _recv_frame(sock)
        if isinstance(resp, dict) and resp.get("ok"):
            return _DeviceSlot(sock)
        sock.close()
        if isinstance(resp, dict) and resp.get("reason") == "slot-timeout":
            log.warning("%s: slot wait timed out (%ds, class %s)",
                        ip, timeout, cls)
        return None
    except Exception as e:
        log.debug("%s: slot acquire failed (%s) — continuing without", ip, e)
        return None


def _ssh_job_gate(wait=90, verb=""):
    """Job-level SSH serialization. When the slot broker is up, jobs need
    no global lock — per-device slots + the cap govern everything, and
    monitor ticks interleave with long sweeps. Otherwise fall back to the
    legacy whole-job flock."""
    if _slot_daemon_available():
        return True
    return _acquire_advisory_lock("ssh", wait=wait)


def connect_device(ip, usernames, password, timeout=10, password_list=None,
                   known_username=None, known_password_hash=None,
                   user_passwords=None, reason=None):
    """Try SSH then telnet to connect to a device.

    reason: optional tag for op_events instrumentation — describes WHY
    this connection is being made ('backup', 'investigate_stp',
    'monitor_topology_fallback', 'monitor_stp_fallback', 'discover',
    etc.). Used by 'digest health' to group SSH activity by purpose.

    If password_list is provided, tries each password with each username.
    Otherwise uses the single password.

    If user_passwords is provided (dict of {user: [pw, ...]}), any user present
    in that mapping is restricted to ONLY those passwords — the bulk
    password_list is not tried for that user. Intended for LDAP/RADIUS-backed
    accounts to avoid lockouts from random-password guesses.

    Iteration is user-major: each user's passwords are fully exhausted before
    moving to the next user.

    If known_username/known_password_hash are set (from a previous successful
    connection stored in the DB) AND the matching plaintext is still in the
    current cred config, the function trusts that cached pair: it tries the
    pair once via SSH and then telnet, and if both fail it returns None
    rather than falling through to the multi-cred grid below.

    Trusting the cache is deliberate. Without it, a transient rate-limit
    refusal on a single management interface (common on ProCurve when a
    multi-IP chassis is hit in parallel) would trigger up to len(usernames)
    × len(passwords) extra SSH sessions per device per discovery, dragging
    the device deeper into rate-limit territory and amplifying the false
    negative across the rest of the run. Genuine credential rotation
    requires a manual re-test ('netops test ssh retest' after updating creds);
    that's a planned operation rather than something discovery needs to
    auto-detect on every run.

    The combo grid below only runs for devices with no cached cred at
    all (initial discovery), or where the cached hash references a
    password no longer present in the current config (we have nothing
    to "trust").

    Returns (child, proto, username, pw_hash, ssh_open, telnet_open)
    or (None, None, None, None, False, False).
    pw_hash is the SHA-256 hex digest of the working password.
    """
    # Filter Nones at the entrypoint so every downstream call site (telnet
    # password-only loop, ssh combo sweep, hash-encode lookup) sees a
    # clean list. An incomplete config entry would otherwise propagate a
    # None into ssh_connect / telnet_connect and crash deep in pexpect.
    pw_list = [p for p in (password_list if password_list else [password]) if p]
    user_passwords = {u: [p for p in (pws or []) if p]
                      for u, pws in (user_passwords or {}).items()}
    has_ssh = bool(check_ports(ip, [22], timeout=2))
    has_telnet = _HAS_TELNET and bool(check_ports(ip, [23], timeout=2))

    # Phase A: hold a per-device session slot from the broker for the
    # lifetime of this connection (attached to the child; released in
    # disconnect / on failure). None when the daemon is down (legacy
    # flock governs) or when running inside the daemon itself.
    _slot = _acquire_device_slot(ip, reason)

    # op_events recording — wrap the whole attempt so the digest can see
    # per-reason latency + success rate. The helper records once and
    # returns the result tuple unchanged so call sites stay tidy.
    _cd_t0 = time.monotonic()
    _cd_started_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    def _finalize(result, error=None):
        child, proto, _user, _hash, _so, _to = result
        if _slot is not None:
            if child is not None:
                child._netops_slot = _slot
            else:
                _slot.release()
        _record_op("ssh_connect" if not (has_telnet and not has_ssh)
                    else "telnet_connect",
                   started_at=_cd_started_at,
                   duration_ms=int((time.monotonic() - _cd_t0) * 1000),
                   success=(child is not None),
                   ip=ip, reason=reason, error=error)
        return result

    def pws_for(user):
        """Passwords to try for a given user.

        Mapped users get ONLY their listed passwords (safe for lockout-prone
        accounts). Unmapped users fall back to the bulk list. Always
        returns a None-filtered list — incomplete config (e.g. a
        placeholder entry in [user_passwords]) would otherwise propagate
        Nones into ssh_connect / hash-encode call sites and crash with
        'NoneType' object has no attribute 'encode' (observed on one site's
        Aruba-CX fleet, which blocked all SSH-based investigation).
        """
        if user in user_passwords:
            return [p for p in user_passwords[user] if p]
        return [p for p in pw_list if p]

    # --- Try known-good credentials first (no lockout risk) ---
    if known_username is not None and known_password_hash is not None:
        # Hash may match either a user-specific or bulk-list password.
        candidates = list(pws_for(known_username)) + [p for p in pw_list if p]
        known_pw = None
        for pw in candidates:
            if hashlib.sha256(pw.encode()).hexdigest() == known_password_hash:
                known_pw = pw
                break
        if known_pw is not None:
            if has_ssh:
                log.debug("  trying known-good SSH %s on %s", known_username, ip)
                child = ssh_connect(ip, known_username, known_pw, timeout=timeout)
                if child is not None:
                    return _finalize((child, "SSH", known_username, known_password_hash, has_ssh, has_telnet))
            if has_telnet:
                log.debug("  trying known-good telnet %s on %s", known_username, ip)
                child = telnet_connect(ip, known_username, known_pw, timeout=timeout)
                if child is not None:
                    return _finalize((child, "telnet", known_username or "(no user)", known_password_hash, has_ssh, has_telnet))
            # Cached cred refused on every reachable protocol. Bail without
            # walking the combo grid — see docstring: trusting the cached
            # pair eliminates the rate-limit cascade that hit one core switch's
            # 84.1 management interface during a parallel discovery.
            log.debug("  cached cred rejected on %s — not grinding combos", ip)
            return _finalize((None, None, None, None, has_ssh, has_telnet),
                             error="cached cred rejected")

    # --- Try user/password combos (user-major: exhaust user1 before user2) ---
    if has_ssh:
        for username in usernames:
            for pw in pws_for(username):
                log.debug("  trying SSH %s on %s", username, ip)
                child = ssh_connect(ip, username, pw, timeout=timeout)
                if child is not None:
                    pw_hash = hashlib.sha256(pw.encode()).hexdigest()
                    return _finalize((child, "SSH", username, pw_hash, has_ssh, has_telnet))

    if has_telnet:
        # Password-only devices (e.g. HP ProCurve) — try each bulk password
        # with no username first.
        for pw in pw_list:
            log.debug("  trying telnet (no user) on %s", ip)
            child = telnet_connect(ip, "", pw, timeout=timeout)
            if child is not None:
                pw_hash = hashlib.sha256(pw.encode()).hexdigest()
                return _finalize((child, "telnet", "(no user)", pw_hash, has_ssh, has_telnet))
        for username in usernames:
            for pw in pws_for(username):
                log.debug("  trying telnet %s on %s", username, ip)
                child = telnet_connect(ip, username, pw, timeout=timeout)
                if child is not None:
                    pw_hash = hashlib.sha256(pw.encode()).hexdigest()
                    return _finalize((child, "telnet", username, pw_hash, has_ssh, has_telnet))

    return _finalize((None, None, None, None, has_ssh, has_telnet),
                     error="all credentials exhausted")


_ANSI_RE = re.compile(
    r"\x1b"           # ESC
    r"(?:"
    r"\[[\d;?]*[a-zA-Z~]"   # CSI sequences: ESC [ params letter  (includes \x1b[?25h)
    r"|[()][0-9A-Za-z]"     # Character set selection: ESC ( B, ESC ) 0, etc.
    r"|[>=<]"               # Keypad mode: ESC > (numeric), ESC = (application)
    r"|[\x20-\x2f]+[\x30-\x7e]"  # ESC intermediate+ final (e.g. ESC # 8)
    r"|[A-Z]"               # Simple ESC-letter (cursor save/restore, etc.)
    r")"
)


_ANSI_TRUNCATED_RE = re.compile(r"\x1b\[[\d;?]*")


def _strip_ansi(text):
    """Remove ANSI escape sequences from text.

    Second pass catches truncated CSI sequences (ESC [ params with no
    terminator) — ProCurve sometimes emits 'ESC[24;' and bails, which
    the main regex can't match since it requires a final letter. Without
    this pass, the visible '[24;' fragment leaks into emails.
    """
    text = _ANSI_RE.sub("", text)
    return _ANSI_TRUNCATED_RE.sub("", text)


_PAGER_RE = (
    r"---\(more[^)]*\)---"
    r"|-- MORE --(?:, next page: Space, next line: Enter, quit: Control-C)?"
    r"|---more---"
    r"|(?:^|[\r\n])\s*--More--"
    r"(?:, next page: Space, next line: Return key, quit: Control-c)?"
)

# Pager residue that leaks into output when the continuation text arrives
# in a LATER read than the '--More--' the expect matched on (telnet-
# fallback ICX sessions with paging active glued these fragments between
# config lines — and emailed them as a config-change alert). The
# replacement newline restores the line break the pager redraw consumed.
_PAGER_FRAGMENT_RE = re.compile(
    r",? ?(?:--\s?More\s?--|-- MORE --)?"
    r"(?:, )?next page: Space, next line: (?:Return key|Enter), "
    r"quit: Control-[cC] *")

# Trailing prompt fragment: PROMPT_RE matches at 'HOSTNAME#', leaving the
# FastIron 'SSH@'/'telnet@' prefix in child.before — it ended up as the
# last line of every FastIron capture.
_PROMPT_FRAGMENT_RE = re.compile(r"\n(?:SSH|telnet)@\s*$")


def send_command(child, command, timeout=30):
    """Send a command and return the output (stripped of echo, prompt, and ANSI codes).

    Handles pager prompts (---(more)---, -- MORE --, etc.) by sending space
    to continue until the full output is received.
    """
    child.sendline(command)
    chunks = []
    while True:
        idx = child.expect([PROMPT_RE, _PAGER_RE], timeout=timeout)
        chunks.append(child.before)
        if idx == 0:
            # Prompt matched — we have all the output
            break
        else:
            # Pager prompt — paging was NOT disabled for this session (the
            # platform's pager-disable command was rejected or unsupported).
            # Flag the child: FastIron's line-redraw pager corrupts captured
            # output (merged lines, dropped chars), so backup_device refuses
            # any config captured over a paged session. Send space to drain
            # the session cleanly regardless.
            try:
                child._netops_paged = True
            except Exception:
                pass
            child.send(" ")
    output = _strip_ansi("".join(chunks))
    output = _PAGER_FRAGMENT_RE.sub("\n", output).replace("\x08", "")
    # Strip the command echo (first line)
    lines = output.splitlines()
    if lines and command in lines[0]:
        lines = lines[1:]
    out = "\n".join(lines).strip()
    return _PROMPT_FRAGMENT_RE.sub("", out)


def get_hostname(child):
    """Extract hostname from the device prompt.

    Handles both HP ProCurve ('hostname#') and Juniper ('user@hostname>').
    """
    # Same retry pattern as _detect_platform — one transient prompt
    # stall shouldn't blow away identification for a real switch.
    for attempt in (1, 2):
        try:
            child.sendline("")
            child.expect(PROMPT_RE, timeout=15)
            break
        except pexpect.TIMEOUT:
            if attempt == 2:
                raise
            log.debug("get_hostname: prompt timed out, retrying once")
    # Combine before+after so split ANSI sequences are intact for stripping
    raw = (child.before or "") + (child.after or "")
    clean = _strip_ansi(raw)
    m = re.search(r"([\w\-/\.()]+)[#>]\s*$", clean)
    if not m:
        return None
    name = m.group(1)
    # Juniper prompt: user@hostname — strip the username prefix
    if "@" in clean:
        at_match = re.search(r"@([\w\-/\.()]+)[#>]\s*$", clean)
        if at_match:
            name = at_match.group(1)
    return name


def disconnect(child):
    """Disconnect from a device (and release its broker slot, if any)."""
    try:
        child.sendline("exit")
        child.close()
    except Exception:
        pass
    slot = getattr(child, "_netops_slot", None)
    if slot is not None:
        slot.release()
        child._netops_slot = None


def _is_cisco_banner(text):
    """True if `show version` output is Cisco IOS/IOS-XE OR NX-OS (Nexus).

    Detection strings, and what emits them:
      - "Cisco IOS Software"           — IOS / IOS-XE (Catalyst)
      - "Cisco Systems, Inc."          — older IOS copyright line
      - "Cisco Nexus Operating System" — NX-OS banner (Nexus)
      - "NX-OS"                         — NX-OS, belt-and-suspenders

    NX-OS is folded into the 'cisco-ios' platform: show running-config,
    show spanning-tree, and 'terminal length 0' behave the same, and a
    Nexus lands at a '#' prompt so the IOS-only `enable` step is skipped.
    Modern NX-OS says "Cisco and/or its affiliates" (NOT "Cisco Systems,
    Inc.") and has no "Cisco IOS Software" line — which is why the pre-NX-OS
    check misclassified Nexus gear as non-switch.
    """
    return ("Cisco IOS Software" in text
            or "Cisco Systems, Inc." in text
            or "Cisco Nexus Operating System" in text
            or "NX-OS" in text)


def get_device_id(output):
    """Extract unique device identifiers from device output.

    Accepts output from 'show system' (HP) or 'show version' + 'show chassis hardware' (Juniper).
    Returns dict with base_mac, serial, firmware, model (any may be absent).
    """
    info = {}

    # --- HP ProCurve / ArubaOS-Switch ---
    m = re.search(r"Base MAC Addr\s*:\s*(\S+)", output)
    if m:
        info["base_mac"] = m.group(1)
    m = re.search(r"Serial Number\s*:\s*(\S+)", output)
    if m:
        info["serial"] = m.group(1)
    m = re.search(r"Firmware revision\s*:\s*(\S+)", output)
    if m:
        info["firmware"] = m.group(1)
    if "firmware" not in info:
        m = re.search(r"Software revision\s*:\s*(\S+)", output)
        if m:
            info["firmware"] = m.group(1)

    # HP/Aruba model from 'show version' first line:
    #   "HP J9729A 2920-48G-PoE+ Switch"
    #   "Aruba JL321A 2930M-24G-PoE+ Switch"
    #   "HP J8697A E5406-44G-PoE+/4G-SFP v2 zl Switch with Premium Software"
    if "model" not in info:
        m = re.search(r"^((?:HP|Aruba)\s+\S+\s+.+?)(?:\s+[Ss]witch.*)?\s*$",
                      output, re.MULTILINE)
        if m:
            info["model"] = m.group(1).strip()
    # Aruba 2920/2930 stack model lives in `show stacking` member rows:
    #   "  1  b8d4e7-0c2cc0     Aruba JL322A 2930M-48G-PoE+ Switch    255 Commander"
    #   "  1  8030e0-eb7d80     Aruba R0M67A 2930M-40G-8SR-PoE-Cla... 255 Standby"
    # Anchor on the trailing "<priority> <status>" columns rather than
    # the "Switch" keyword — newer Aruba SKUs (R0M67A) don't include
    # "Switch" in the model column, and the device truncates long names
    # to fit the column with a literal "..." marker. The regex tolerates
    # both forms; the trailing "..." (if any) is dropped from group 2.
    if "model" not in info:
        m = re.search(
            r"^\s*\d+\s+[\da-fA-F:\-]+\s+(?:HP|Aruba)\s+(\S+)\s+"
            r"(\S+?)(?:\.\.\.|\s+Switch)?\s+\d{1,3}\s+"
            r"(?:Commander|Standby|Member)\b",
            output, re.MULTILINE)
        if m:
            info["model"] = f"Aruba {m.group(1)} {m.group(2)}"
    # Standalone Aruba 2920/2930F/2930M model lives in `show modules`
    # ("Chassis: 2930M-48G-PoE+  JL322A         Serial Number: SG...")
    # — show stacking returns nothing on a standalone, so this is the
    # path that covers non-stacked switches. Anchor on "Serial Number:"
    # rather than the part-number prefix so it also catches newer
    # Aruba SKUs that don't start with J (e.g. R0M67A).
    if "model" not in info:
        m = re.search(
            r"^\s*Chassis:\s+(\S+)\s+(\S+)\s+Serial Number:",
            output, re.MULTILINE)
        if m:
            info["model"] = f"Aruba {m.group(2)} {m.group(1)}"

    # --- ArubaOS-CX ---
    if "base_mac" not in info:
        m = re.search(r"Base MAC Address\s*:\s*(\S+)", output)
        if m:
            info["base_mac"] = m.group(1)
    if "serial" not in info:
        m = re.search(r"Chassis Serial Nbr\s*:\s*(\S+)", output)
        if m:
            info["serial"] = m.group(1)
    if "firmware" not in info:
        m = re.search(r"ArubaOS-CX Version\s*:\s*(\S+)", output)
        if m:
            info["firmware"] = m.group(1)
    if "model" not in info:
        m = re.search(r"Product Name\s*:\s*(.+)", output)
        if m:
            info["model"] = m.group(1).strip()

    # --- Juniper JunOS ---
    # Junos version: "Junos: 23.4R2-S6.9" or "JUNOS Base OS Software Suite [12.3R12-S21]"
    if "firmware" not in info:
        m = re.search(r"Junos:\s*(\S+)", output)
        if m:
            info["firmware"] = m.group(1)
    if "firmware" not in info:
        m = re.search(r"JUNOS Base OS Software Suite \[([^\]]+)\]", output)
        if m:
            info["firmware"] = m.group(1)
    # Juniper model: "Model: srx345-dual-ac" or "Model: ex3300-48t"
    if "model" not in info:
        m = re.search(r"^Model:\s*(\S+)", output, re.MULTILINE)
        if m:
            info["model"] = m.group(1)
    # Juniper serial from show chassis hardware: first alphanumeric 6+ chars on Chassis line
    # (Part numbers have hyphens like 750-034247, serials don't: GA0214410099, DS3817AF0171)
    if "serial" not in info:
        m = re.search(r"^Chassis\b[^\n]*?\b([A-Z][A-Z0-9]{5,})\b", output, re.MULTILINE)
        if m:
            info["serial"] = m.group(1)
    # Juniper MAC from show chassis mac-addresses:
    #   older: "Public base address     28:c0:da:35:10:00"
    #   newer: "Base address    ec:38:73:5f:7a:5e"
    if "base_mac" not in info:
        m = re.search(r"(?:Public\s+)?[Bb]ase\s+address\s+([\da-fA-F:]+)", output)
        if m:
            info["base_mac"] = m.group(1).replace(":", "")

    # --- Cisco IOS (Catalyst / classic IOS) ---
    # show version banner:
    #   "Cisco IOS Software, C3560 Software (C3560-IPBASEK9-M), Version 12.2(55)SE1, RELEASE SOFTWARE (fc1)"
    #   "Model number                    : WS-C3560G-24PS-S"   (preferred SKU)
    #   "cisco WS-C3560G-24PS (PowerPC405) processor (...)"    (fallback model line)
    #   "System serial number            : FOC1002Z8S9"
    #   "Processor board ID FOC1002Z8S9"                       (fallback serial)
    #   "Base ethernet MAC Address       : 00:16:C7:DA:30:80"
    if "firmware" not in info:
        m = re.search(r"Cisco IOS Software.*?Version\s+(\S+?),", output)
        if m:
            info["firmware"] = m.group(1)
    # Labels are case-insensitive: classic IOS prints "Model number" /
    # "System serial number" / "Processor board ID" (lowercase), but IOS-XE
    # (Catalyst 9K) and NX-OS use "Model Number" / "System Serial Number" /
    # "Processor Board ID" (capitalised). One regex, IGNORECASE, covers both.
    if "model" not in info:
        m = re.search(r"^\s*Model number\s*:\s*(\S+)", output,
                      re.MULTILINE | re.IGNORECASE)
        if m:
            info["model"] = m.group(1)
    if "model" not in info:
        m = re.search(r"^\s*cisco\s+(\S+)\s+\(.*processor", output, re.MULTILINE)
        if m:
            info["model"] = m.group(1)
    if "serial" not in info:
        m = re.search(r"^\s*System serial number\s*:\s*(\S+)", output,
                      re.MULTILINE | re.IGNORECASE)
        if m:
            info["serial"] = m.group(1)
    if "serial" not in info:
        m = re.search(r"^\s*Processor board ID\s+(\S+)", output,
                      re.MULTILINE | re.IGNORECASE)
        if m:
            info["serial"] = m.group(1)
    if "base_mac" not in info:
        m = re.search(r"Base ethernet MAC Address\s*:\s*([\da-fA-F:]+)", output)
        if m:
            info["base_mac"] = m.group(1).replace(":", "").lower()

    # --- Cisco NX-OS (Nexus) ---
    # show version differs from IOS:
    #   "  NXOS: version 10.2(7) [Maintenance Release]"  (older: "system: version X")
    #   "  cisco Nexus9000 C93180YC-EX chassis"          (IOS says "... (...) processor")
    #   "  Processor Board ID FDO21122NJP"               (caught by the IGNORECASE serial regex above)
    if "firmware" not in info:
        m = re.search(r"(?:NXOS|system):\s*version\s+(\S+)", output)
        if m:
            info["firmware"] = m.group(1)
    if "model" not in info:
        m = re.search(r"^\s*cisco\s+(.+?)\s+[Cc]hassis\b", output, re.MULTILINE)
        if m:
            info["model"] = m.group(1).strip()

    # --- Ruckus FastIron (ICX) ---
    # show version banner format:
    #   "  HW: Stackable ICX7750-48F"   or   "  HW: ICX7450-48-HPOE"
    #   "        SW: Version 08.0.80fT201"
    #   "      Serial  #:CRH3303P005"     (first match wins — UNIT 1)
    if "model" not in info:
        m = re.search(r"^\s*HW:\s*(?:Stackable\s+)?(ICX\S+)", output, re.MULTILINE)
        if m:
            info["model"] = m.group(1)
    if "firmware" not in info:
        m = re.search(r"^\s*SW:\s*Version\s+(\S+)", output, re.MULTILINE)
        if m:
            info["firmware"] = m.group(1)
    if "serial" not in info:
        m = re.search(r"^\s*Serial\s*#\s*:\s*(\S+)", output, re.MULTILINE)
        if m:
            info["serial"] = m.group(1)

    return info


def looks_like_switch(platform, dev_id):
    """Positive identification of a managed network switch.

    Uses STRUCTURED command output, not free-text banner content, since
    login banners and welcome messages are operator-editable. Two signals:

    1. Platform was confidently detected from prompt format or vendor-
       specific command behavior (junos / fastiron / cisco-ios / aruba-cx).
    2. `get_device_id` extracted labeled fields from `show system` /
       `show version` output. The labels themselves ("Base MAC Addr",
       "Serial Number", "Software revision", "HW: ICX...", etc.) are
       hard-coded in firmware output formatters, NOT in customizable
       MOTD/banner text. A Linux server with a "Welcome to ProCurve"
       MOTD won't extract any structured fields; a switch with a
       stripped MOTD still returns the same `show system` field layout.

    A device counts as a switch if EITHER:
      - The runtime platform is in {junos, fastiron, cisco-ios, aruba-cx}.
      - get_device_id extracted firmware AND (serial OR base_mac).
        Both are needed — firmware alone could appear in a server's
        uptime banner; the pairing with a hardware identifier confirms
        a switch's structured CLI.
    """
    if platform in ("junos", "fastiron", "cisco-ios", "aruba-cx"):
        return True
    if not dev_id:
        return False
    has_firmware = bool(dev_id.get("firmware"))
    has_hwid = bool(dev_id.get("serial") or dev_id.get("base_mac"))
    return has_firmware and has_hwid


class NotASwitchError(Exception):
    """Raised by backup_device when the connected target isn't a switch
    (server BMC, Linux server, etc). run_backup catches this and records
    status='non-switch' instead of treating it as a backup failure."""


def derive_role(model):
    """Map a model string to its functional role.

    Returns 'firewall' for known firewall model patterns, otherwise
    'switch' (the safe default for everything netops auto-classifies
    via the existing identification path). The intent is NOT vendor
    detection — platform handles that. This is purely about whether
    STP / L2 monitoring applies to the device.

      Juniper SRX (incl. vSRX)        -> firewall
      anything else / unknown / empty -> switch  (caller may then
                                                  override via
                                                  `netops mark <role>`)
    """
    if not model:
        return "switch"
    m = model.strip().lower()
    # Juniper SRX / vSRX: 'srx345-dual-ac', 'srx210he2', 'vsrx', etc.
    if re.match(r"^v?srx[\d\-]", m) or m == "srx" or m == "vsrx":
        return "firewall"
    return "switch"


# ---------------------------------------------------------------------------
# STP state parsers
# ---------------------------------------------------------------------------

def parse_stp_junos(output):
    """Parse Juniper 'show spanning-tree interface' output.

    Returns list of dicts: [{interface, role, state}, ...].
    Only includes ports with a meaningful STP role (skips header/blank lines).
    """
    results = []
    for line in output.splitlines():
        # ge-0/0/0  128:490  128:490  32768.xxxx  200000  FWD  DESG
        m = re.match(
            r"(\S+)\s+"           # interface
            r"\d+:\d+\s+"        # port ID
            r"\d+:\d+\s+"        # designated port ID
            r"\S+\s+"            # designated bridge ID
            r"\d+\s+"            # cost
            r"(\S+)\s+"          # state (FWD/BLK)
            r"(\S+)",            # role (DESG/ROOT/ALT/DIS)
            line)
        if m:
            results.append({
                "interface": m.group(1),
                "state": m.group(2),
                "role": m.group(3),
            })
    return results


def parse_junos_stp_port_id_table(output):
    """Extract (port_id, interface) pairs from 'show spanning-tree interface'
    output for the JUNIPER STP-port-id cache.

    The same table parse_stp_junos uses, but we capture the port-id half of
    the second column ('128:490' -> 490). The port-id is the index used by
    JUNIPER-MIMSTP-MIB; on VC switches this diverges from BRIDGE-MIB's
    dot1dBasePort. Strips Junos '.0' logical-unit suffix so the interface
    name matches what the SNMP collector emits.

    Returns list of (port_id_int, interface_str) tuples.
    """
    results = []
    for line in output.splitlines():
        m = re.match(
            r"(\S+)\s+"           # interface
            r"\d+:(\d+)\s+"      # port ID — capture the port-id half
            r"\d+:\d+\s+"        # designated port ID
            r"\S+\s+"            # designated bridge ID
            r"\d+\s+"            # cost
            r"\S+\s+"            # state
            r"\S+",              # role
            line)
        if m:
            ifname = m.group(1)
            port_id = int(m.group(2))
            if "." in ifname:
                ifname = ifname.split(".", 1)[0]
            results.append((port_id, ifname))
    return results


def _upsert_junos_stp_port_id_cache(ip, port_ids):
    """Replace stp_port_id_cache rows for `ip` with the given mapping.

    `port_ids` is a list of (port_id, interface) tuples produced by
    parse_junos_stp_port_id_table. Empty input is a no-op (don't blow away
    a previous cache when the parse came back empty for a transient reason).
    """
    if not port_ids:
        return
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    try:
        # Replace strategy so a port that disappeared since the last
        # discover doesn't linger as a stale entry.
        conn.execute("DELETE FROM stp_port_id_cache WHERE ip = ?", (ip,))
        conn.executemany(
            "INSERT INTO stp_port_id_cache (ip, port_id, interface, last_refresh)"
            " VALUES (?, ?, ?, ?)",
            [(ip, pid, name, now) for pid, name in port_ids]
        )
        conn.commit()
    finally:
        conn.close()


def parse_stp_procurve(output):
    """Parse HP ProCurve 'show spanning-tree' output (MSTP format).

    Returns list of dicts: [{interface, role, state}, ...].
    ProCurve doesn't have an explicit Role column — we infer from State.
    """
    results = []
    for line in output.splitlines():
        # 1/1   10GbE-T  | 4000  128  Forwarding  | 8030e0-eb5d0a  2  Yes
        # Trk1  Trk      | 2000  128  Forwarding  | ...
        m = re.match(
            r"\s*(\S+)\s+"       # port (1/1, Trk1, etc.)
            r"\S+\s+\|\s+"      # type + separator
            r"\S+\s+"           # cost (number or Auto)
            r"\d+\s+"           # priority
            r"(\w+)",           # state (Forwarding/Blocking/Disabled/Learning)
            line)
        if m:
            port = m.group(1)
            state = m.group(2)
            if state == "Disabled":
                role = "Disabled"
            elif state == "Blocking":
                role = "Alternate"
            elif state == "Forwarding":
                role = "Designated"
            else:
                role = state
            results.append({
                "interface": port,
                "state": state,
                "role": role,
            })
    return results


def parse_stp_aruba_cx(output):
    """Parse ArubaOS-CX 'show spanning-tree' output.

    Returns list of dicts: [{interface, role, state}, ...].
    CX has explicit Role and State columns.
    """
    results = []
    for line in output.splitlines():
        # 1/1/6   Designated  Forwarding  2000  128  P2P  ...
        m = re.match(
            r"(\S+)\s+"          # port (1/1/6)
            r"(\S+)\s+"         # role (Designated/Alternate/Root/Disabled)
            r"(\S+)\s+"         # state (Forwarding/Blocking/Down)
            r"\d+\s+"           # cost
            r"\d+\s+"           # priority
            r"\S+",             # type (P2P/Shr)
            line)
        if m:
            results.append({
                "interface": m.group(1),
                "role": m.group(2),
                "state": m.group(3),
            })
    return results


def parse_stp_cisco(output):
    """Parse Cisco IOS 'show spanning-tree' (PVST / Rapid-PVST / MST).

    Cisco runs a spanning tree PER VLAN (or per MST instance), so each
    interface appears once per VLAN section. We collapse to one (role, state)
    per interface — keeping the MOST-FORWARDING entry — so a port that forwards
    in any VLAN reads Forwarding (per-VLAN load-balance blocking isn't a fleet
    STP event), while a port blocked in EVERY VLAN reads Blocking. IOS column
    abbreviations are normalized to the full-word vocabulary the procurve /
    aruba-cx parsers and is_stp_blocking() use. Returns [{interface, role,
    state}, ...].
    """
    ROLE = {"Desg": "Designated", "Root": "Root", "Altn": "Alternate",
            "Back": "Backup", "Mstr": "Master", "Boun": "Boundary"}
    STS = {"FWD": "Forwarding", "BLK": "Blocking", "LRN": "Learning",
           "LIS": "Listening", "DIS": "Disabled", "BKN": "Broken"}
    STATE_RANK = {"Forwarding": 5, "Learning": 4, "Listening": 3,
                  "Blocking": 2, "Broken": 1, "Disabled": 0}
    best = {}  # interface -> (rank, role, state)
    for line in output.splitlines():
        # Gi0/25   Desg FWD 4   128.25  P2p
        m = re.match(r"^\s*(\S+)\s+(Desg|Root|Altn|Back|Mstr|Boun)\s+"
                     r"(FWD|BLK|LRN|LIS|DIS|BKN)\b", line)
        if not m:
            continue
        role = ROLE.get(m.group(2), m.group(2))
        state = STS.get(m.group(3), m.group(3))
        rank = STATE_RANK.get(state, -1)
        intf = m.group(1)
        if intf not in best or rank > best[intf][0]:
            best[intf] = (rank, role, state)
    return [{"interface": i, "role": r, "state": s}
            for i, (rank, r, s) in best.items()]


def parse_stp_mode_cisco(output):
    """STP mode from Cisco 'show spanning-tree' ('... protocol rstp') or
    'show spanning-tree summary' ('Switch is in rapid-pvst mode')."""
    m = re.search(r"protocol\s+(rstp|mstp|mst|rapid-pvst|pvst|ieee)", output, re.I)
    if not m:
        m = re.search(r"in\s+(rapid-pvst|pvst|mst)\s+mode", output, re.I)
    if not m:
        return None
    p = m.group(1).lower()
    if p in ("rstp", "rapid-pvst"):
        return "rstp"
    if p in ("mstp", "mst"):
        return "mstp"
    return "stp"  # ieee / pvst


def parse_ifaces_cisco(output):
    """Cisco 'show interfaces status' -> {interface: link_up_bool}. The Port
    column ('Gi0/2') matches the names in 'show spanning-tree'. 'connected' is
    up; notconnect / disabled / err-disabled / inactive are down."""
    ifaces = {}
    for line in output.splitlines():
        m = re.match(r"^\s*(\S+)\s+.*?\b(connected|notconnect|disabled|"
                     r"err-disabled|inactive|monitoring|faulty|sfpAbsent)\b", line)
        if not m:
            continue
        ifaces[m.group(1)] = (m.group(2) == "connected")
    return ifaces


def parse_stp_root_cisco(output):
    """Per-instance root-bridge info from Cisco 'show spanning-tree'.

    MODE-AGNOSTIC by design — the same code covers what some sites run today and
    where it's headed:
      - PVST / Rapid-PVST: one section per VLAN  ('VLAN0220', 'VLAN0221', ...)
      - MSTP:              one section per MST instance ('MST0' = CIST, 'MST1')
      - plain 802.1D:      a single 'VLAN0001' section
    Each section's Root ID / Bridge ID block becomes a root_entry keyed by the
    section name, so the existing per-(ip, instance) root-change tracking keeps
    working through a future PVST->MSTP migration. tcn_count/last_tcn need
    'show spanning-tree detail' (not collected here) so they stay None.
    Returns [{instance, root_priority, root_mac, bridge_priority, bridge_mac,
    tcn_count, last_tcn_seconds}, ...].
    """
    entries = []
    cur = None
    part = None  # 'root' | 'bridge' — which ID block an indented 'Address' belongs to
    for line in output.splitlines():
        sec = re.match(r"^(VLAN\d+|MST\d+)\s*$", line.strip())
        if sec:
            if cur and cur.get("root_mac"):
                entries.append(cur)
            cur = {"instance": sec.group(1), "root_priority": None,
                   "root_mac": None, "bridge_priority": None, "bridge_mac": None,
                   "tcn_count": None, "last_tcn_seconds": None}
            part = None
            continue
        if cur is None:
            continue
        if "Root ID" in line:
            part = "root"
            m = re.search(r"Priority\s+(\d+)", line)
            if m:
                cur["root_priority"] = int(m.group(1))
            continue
        if "Bridge ID" in line:
            part = "bridge"
            m = re.search(r"Priority\s+(\d+)", line)
            if m:
                cur["bridge_priority"] = int(m.group(1))
            continue
        m = re.search(r"Address\s+([0-9A-Fa-f][0-9A-Fa-f.:-]+)", line)
        if m:
            mac = _normalize_stp_mac(m.group(1))
            if part == "root" and not cur["root_mac"]:
                cur["root_mac"] = mac
            elif part == "bridge" and not cur["bridge_mac"]:
                cur["bridge_mac"] = mac
            continue
    if cur and cur.get("root_mac"):
        entries.append(cur)
    return entries


def is_stp_blocking(port_info):
    """Return True if this port represents a real STP block (not just link-down)."""
    role = port_info.get("role", "")
    state = port_info.get("state", "")
    # Juniper: BLK + ALT = real block; BLK + DIS = no link
    if state == "BLK" and role == "ALT":
        return True
    # ProCurve: Blocking state = real block
    if state == "Blocking":
        return True
    # ArubaOS-CX: Alternate + Blocking = real block
    if role == "Alternate" and state == "Blocking":
        return True
    return False


def _is_stp_offline_state(role, state):
    """A port's STP state is 'offline' when it's Disabled/Down/missing (GONE)."""
    if (role, state) == ("GONE", "GONE"):
        return True
    return role in ("DIS", "Disabled") and state in ("BLK", "Disabled", "Down")


def _is_stp_transient_state(role, state):
    """Learning/Listening are convergence steps in the STP state machine, not
    steady states. A port caught in LRN/LIS is mid-transition; using it as an
    alert reference produces phantom events like 'DESG/LRN -> GONE/GONE' when
    the port later disappears before settling."""
    return state in ("LRN", "Learning", "LIS", "Listening")


def should_alert_stp_change(old, new):
    """Return True if a change from old (role, state) to new (role, state) is
    alertable (topology-relevant), as opposed to a link-flap artifact.

    Rules:
      - either side is transient (LRN/LIS): silent — wait for the port to settle.
      - old -> down/gone: alert only if the port was previously blocking
        (a suppressed redundant path going down matters; a forwarding port
        going down is a link event, not STP).
      - down/gone -> new: silent (link coming back up is a flap recovery).
      - Any other role/state transition: alert.
    """
    if _is_stp_transient_state(*old) or _is_stp_transient_state(*new):
        return False
    new_off = _is_stp_offline_state(*new)
    old_off = _is_stp_offline_state(*old)
    if new_off:
        return is_stp_blocking({"role": old[0], "state": old[1]})
    if old_off:
        return False
    return True


# Junos data-plane Ethernet prefixes — the only interfaces we count for
# link-state, flap tracking, and in-service heuristics. Excludes management
# (me/bme/vme/fxp/em) and internal virtual (dsc/gre/ipip/lo0/lsi/etc.)
# interfaces that always report 'Physical link is Up' but carry no user
# traffic.
_JUNOS_DATAPLANE_PREFIXES = ("ge-", "xe-", "et-", "fe-", "ae")


def _is_junos_dataplane(name):
    return name.startswith(_JUNOS_DATAPLANE_PREFIXES)


def parse_ifaces_junos_terse(output):
    """Parse 'show interfaces terse' physical rows. Returns
    {physical_iface: link_up_bool} restricted to data-plane ports.

    Columns: 'Interface  Admin  Link  [Proto ...]'. Logical units (name
    contains '.') are skipped; a port is up only when admin AND link are
    both 'up'. This is the cheap fallback for the full filtered
    'show interfaces' listing: terse skips per-interface statistics, so
    it returns in ~1s even on a virtual chassis where the stats relay
    across members makes the full listing time out — at the cost of the
    per-port 'Last flapped' data (inferred-flap detection pauses; flaps
    observed across two polls are still counted).
    """
    results = {}
    for line in output.splitlines():
        m = re.match(r"^(\S+)\s+(up|down)\s+(up|down)\b", line)
        if not m:
            continue
        iface = m.group(1)
        if "." in iface or not _is_junos_dataplane(iface):
            continue
        results[iface] = (m.group(2) == "up" and m.group(3) == "up")
    return results


def parse_ifaces_junos(output):
    """Parse filtered 'show interfaces | match "^Physical|Last flapped"'.
    Returns {physical_iface: link_up_bool} restricted to data-plane ports.

    Header form: 'Physical interface: NAME, Enabled, Physical link is Up'.
    Admin-disabled interfaces show 'Administratively down' instead of
    'Enabled'. A port is up only when admin is 'Enabled' AND link is 'Up'.
    """
    results = {}
    for line in output.splitlines():
        m = re.match(
            r"^Physical interface:\s*(\S+?),\s+([^,]+?),\s+"
            r"Physical link is\s+(Up|Down)",
            line)
        if not m:
            continue
        iface = m.group(1)
        if not _is_junos_dataplane(iface):
            continue
        admin_up = "enabled" in m.group(2).lower()
        link_up = (m.group(3) == "Up")
        results[iface] = admin_up and link_up
    return results


def parse_ifaces_aruba_cx(output):
    """Parse full 'show interface' output. Returns {iface: link_up_bool}.

    Uses the 'Interface NAME is (up|down|disabled)' line as link state.
    Skips vlan/lag/loopback pseudo-interfaces.
    """
    results = {}
    for line in output.splitlines():
        m = re.match(r"^Interface\s+(\S+)\s+is\s+(up|down|disabled)", line)
        if not m:
            continue
        iface = m.group(1)
        if iface.startswith(("vlan", "lag", "loopback", "Port")):
            continue
        results[iface] = (m.group(2) == "up")
    return results


def parse_ifaces_procurve(output):
    """Parse HP ProCurve 'show interfaces brief' output. Returns {iface: link_up_bool}.

    A port is up only when Enabled=Yes AND Status=Up.
    """
    results = {}
    for line in output.splitlines():
        #   1/1          10GbE-T    | No        Yes     Up     5000FDx    MDI  off  0
        m = re.match(
            r"^\s+(\S+)\s+\S+\s+\|\s+\S+\s+(Yes|No)\s+(Up|Down)\b", line)
        if not m:
            continue
        results[m.group(1)] = (m.group(2) == "Yes" and m.group(3) == "Up")
    return results


# ---------------------------------------------------------------------------
# SNMP — credential probing and config inspection
# ---------------------------------------------------------------------------
#
# We shell out to net-snmp (snmpget) rather than pulling in pysnmp:
# 1. Matches the working POC and the existing test-server toolchain.
# 2. No new Python dependency. pysnmp v6 is asyncio-only and would force a
#    refactor of the existing ThreadPoolExecutor-based polling paths.
# 3. Easier to debug — operators can reproduce a failure with the same
#    snmpget command we ran.
#
# sysUpTime.0 (1.3.6.1.2.1.1.3.0) is the canary OID: universally supported,
# returns immediately, and a successful response confirms (a) SNMP is enabled,
# (b) our credentials are accepted, and (c) any ACL on the device permits us.

_SNMP_SYSUPTIME_OID = "1.3.6.1.2.1.1.3.0"


def _snmp_run(args, timeout):
    """Run an snmpget/snmpwalk command. Returns (rc, stdout, stderr).
    Records timing into the per-tick accumulator if one is active."""
    t0 = time.monotonic()
    try:
        proc = subprocess.run(args, capture_output=True, text=True,
                              timeout=timeout)
        rc, out, err = proc.returncode, proc.stdout or "", proc.stderr or ""
    except subprocess.TimeoutExpired:
        rc, out, err = -1, "", "subprocess timeout"
    except FileNotFoundError:
        rc, out, err = -1, "", "snmpget not found (install net-snmp / snmp tools)"
    except Exception as e:
        rc, out, err = -1, "", f"snmpget exec error: {e}"
    duration_ms = int((time.monotonic() - t0) * 1000)
    _snmp_timing_record(duration_ms, rc == 0 and "Timeout" not in (out + err))
    return rc, out, err


def _snmp_v2c_args(community, ip, oid, timeout, retries):
    return ["snmpget", "-v2c", "-c", community,
            "-t", str(timeout), "-r", str(retries),
            "-On", ip, oid]


def _snmp_v3_args(user_record, ip, oid, timeout, retries):
    """Build snmpget args for a v3 user record from cfg['snmp']['v3_users']."""
    args = ["snmpget", "-v3",
            "-l", user_record.get("security_level") or "noAuthNoPriv",
            "-u", user_record["name"],
            "-t", str(timeout), "-r", str(retries),
            "-On"]
    if user_record.get("auth_pass"):
        args += ["-a", user_record.get("auth_proto") or "SHA",
                 "-A", user_record["auth_pass"]]
    if user_record.get("priv_pass"):
        args += ["-x", user_record.get("priv_proto") or "AES",
                 "-X", user_record["priv_pass"]]
    args += [ip, oid]
    return args


def _snmp_response_ok(rc, stdout, stderr):
    """Did snmpget return a usable value? net-snmp's exit code isn't enough —
    timeouts and auth failures can still be rc=0 with an error in stdout."""
    if rc != 0:
        return False
    s = stdout.strip()
    if not s:
        return False
    bad = ("Timeout: No Response", "No Such Object", "No Such Instance",
           "authenticationFailure", "Unknown user name",
           "Authentication failure", "wrongDigest", "decryptionError")
    if any(b in s for b in bad):
        return False
    if any(b in (stderr or "") for b in bad):
        return False
    return True


def snmp_probe(ip, snmp_cfg, cached_proto=None, cached_cred=None,
               full_walk=False):
    """Probe an IP for SNMP reachability.

    Strategy (each step short-circuits on success):
      1. If a cached credential is supplied, try it. On success, return.
      2. If the cached cred failed AND full_walk=False, return failure
         without re-trying — this keeps routine polls to a single UDP
         roundtrip, even when the cached cred has drifted. Discovery
         re-walks and refreshes the cache.
      3. If no cached cred OR full_walk=True, iterate communities then
         v3 users, stopping at the first that responds. Used by 'discover'
         and by first-time polls on never-probed devices.

    Returns a dict:
      {ok: bool, proto: 'v2c'|'v3'|None, community: str|None,
       v3_user: str|None, error: str}

    The probe costs one UDP roundtrip per attempt — bounded by snmp_cfg
    timeout * (cached_attempt + len(communities) + len(v3_users)) when
    full_walk=True. With defaults (5s timeout, 1 retry, 2 communities, 1 v3
    user) that's ~25s worst-case per dead device. Don't call full_walk=True
    inside the per-minute monitor loop.
    """
    timeout = snmp_cfg.get("timeout", 5)
    retries = snmp_cfg.get("retries", 1)
    # net-snmp's per-attempt timeout × (retries+1) is the wall-clock budget;
    # add a small subprocess fence so a hung snmpget can't block forever.
    sub_timeout = (timeout * (retries + 1)) + 5

    result = {"ok": False, "proto": None, "community": None,
              "v3_user": None, "error": ""}

    def try_v2c(community):
        args = _snmp_v2c_args(community, ip, _SNMP_SYSUPTIME_OID,
                              timeout, retries)
        rc, out, err = _snmp_run(args, sub_timeout)
        if _snmp_response_ok(rc, out, err):
            return True, ""
        return False, (out.strip() or err.strip() or "no response")

    def try_v3(user_record):
        args = _snmp_v3_args(user_record, ip, _SNMP_SYSUPTIME_OID,
                             timeout, retries)
        rc, out, err = _snmp_run(args, sub_timeout)
        if _snmp_response_ok(rc, out, err):
            return True, ""
        return False, (out.strip() or err.strip() or "no response")

    # ---- 1. Cached credential first ----
    if cached_proto == "v2c" and cached_cred:
        ok, err = try_v2c(cached_cred)
        if ok:
            result.update({"ok": True, "proto": "v2c", "community": cached_cred})
            return result
        if not full_walk:
            result["error"] = err
            return result
    elif cached_proto == "v3" and cached_cred:
        match = next((u for u in snmp_cfg.get("v3_users", [])
                      if u["name"] == cached_cred), None)
        if match:
            ok, err = try_v3(match)
            if ok:
                result.update({"ok": True, "proto": "v3", "v3_user": cached_cred})
                return result
            if not full_walk:
                result["error"] = err
                return result

    # ---- 2. Full credential walk (discovery only) ----
    last_err = ""
    for community in snmp_cfg.get("communities", []):
        if cached_proto == "v2c" and cached_cred == community:
            continue  # already tried above
        ok, err = try_v2c(community)
        if ok:
            result.update({"ok": True, "proto": "v2c", "community": community})
            return result
        last_err = err
    for user_record in snmp_cfg.get("v3_users", []):
        if cached_proto == "v3" and cached_cred == user_record["name"]:
            continue
        ok, err = try_v3(user_record)
        if ok:
            result.update({"ok": True, "proto": "v3",
                           "v3_user": user_record["name"]})
            return result
        last_err = err

    result["error"] = last_err or "no credentials configured"
    return result


def _parse_snmp_config_junos(output):
    """Parse 'show configuration snmp | display set' output.

    Junos has two community forms:
      set snmp community NAME                          (read-only by default)
      set snmp community NAME authorization read-write
      set snmp community NAME clients PREFIX           (per-community ACL)
    The bare-name form has no trailing token, so the regex must accept
    end-of-line — earlier versions required \\s+ after the name and missed
    the read-only case.
    """
    communities, v3_users, acl_refs = [], [], []
    for line in output.splitlines():
        line = line.strip()
        m = re.match(r"set snmp community\s+(\S+?)(?:\s+|$)", line)
        if m and m.group(1) not in communities:
            communities.append(m.group(1))
        m = re.match(r"set snmp community\s+\S+\s+clients\s+(\S+)", line)
        if m:
            acl_refs.append(("clients-list", m.group(1)))
        m = re.match(r"set snmp v3 usm local-engine user\s+(\S+)", line)
        if m and m.group(1) not in v3_users:
            v3_users.append(m.group(1))
        m = re.match(r"set snmp client-list\s+(\S+)\s+(\S+)", line)
        if m:
            acl_refs.append((f"client-list:{m.group(1)}", m.group(2)))
    return communities, v3_users, acl_refs


def _parse_snmp_config_aruba_cx(output):
    """Parse the combined output of 'show snmp community' + 'show snmpv3 users'
    (and optionally 'show running-config') for Aruba-CX.

    Aruba-CX presents each in a fixed-width table:
      'show snmp community':
        Community            Access-level ACL Name      ACL Type
        --------             ------------ ----------    --------
        public               read-only    none          none
        netops_ro            read-only    siem-allow    extended

      'show snmpv3 users':
        User       AuthMode PrivMode Status   Context
        --------   -------- -------- -------- -------
        monitor    sha      aes      enabled

    Per-community ACLs are the de-facto SNMP source-IP filter on Aruba-CX
    when ACL Name != 'none'. 'snmp-server vrf <name>' bindings also matter
    (probe must come from a host reachable on that VRF).
    """
    communities, v3_users, acl_refs = [], [], []
    in_comm_table, in_user_table = False, False
    for line in output.splitlines():
        s = line.rstrip()
        if (re.match(r"^Community\b", s.lstrip())
                and "Access-level" in s and "ACL" in s):
            in_comm_table, in_user_table = True, False
            continue
        if (re.match(r"^User\b", s.lstrip())
                and "AuthMode" in s and "PrivMode" in s):
            in_user_table, in_comm_table = True, False
            continue
        if not s.strip():
            in_comm_table = in_user_table = False
            continue
        if s.lstrip().startswith("---"):
            continue
        if in_comm_table:
            tok = s.split()
            if len(tok) >= 1 and tok[0] not in communities:
                communities.append(tok[0])
            # Tabular ACL Name is column 3 (0-indexed 2). 'none' means no ACL.
            if len(tok) >= 3 and tok[2].lower() != "none":
                acl_refs.append(("acl", f"{tok[2]} (community {tok[0]})"))
            continue
        if in_user_table:
            tok = s.split()
            if tok and tok[0] not in v3_users:
                v3_users.append(tok[0])
            continue

    # Running-config form (no quotes on Aruba-CX, single-line).
    # We also pick up ACL entries that gate SNMP — both the per-community
    # 'access-list ipv4 NAME' bindings and inline 'permit/deny ... eq snmp'
    # entries that operators use as datapath filters. The latter aren't a
    # 'configured ACL Name' on the community itself, but they DO block
    # probes silently, so surface them.
    for line in output.splitlines():
        s = line.strip()
        m = re.match(r"snmp-server community\s+(\S+)", s)
        if m and m.group(1) not in communities:
            communities.append(m.group(1))
        m = re.match(r"snmpv3 user\s+(\S+)", s)
        if m and m.group(1) not in v3_users:
            v3_users.append(m.group(1))
        m = re.match(r"snmp-server vrf\s+(\S+)", s)
        if m:
            ref = ("vrf", m.group(1))
            if ref not in acl_refs:
                acl_refs.append(ref)
        # ACL entries referencing SNMP. Examples:
        #   80 permit udp 192.107.39.0/24 any eq snmp
        #   90 deny tcp any any eq snmp
        # We don't track the parent ACL name (would require multi-line
        # context); just recording that the device has SNMP-gating ACL
        # entries is enough to give the operator a pointer.
        m = re.match(
            r"\d+\s+(permit|deny)\s+\S+\s+(\S+)\s+\S+(?:\s+\S+)*\s+eq\s+snmp",
            s)
        if m:
            verb, src = m.group(1), m.group(2)
            ref = ("acl-entry", f"{verb} {src} eq snmp")
            if ref not in acl_refs:
                acl_refs.append(ref)
    return communities, v3_users, acl_refs


def _parse_snmp_config_procurve(output):
    """Parse ProCurve 'show snmp-server' / 'show running-config' / 'show ip
    authorized-managers' for SNMP info.

    ProCurve splits this across multiple outputs:
      - 'show snmp-server' is human-formatted (table of communities + v3
        users + trap receivers + source-IP policy). The default 'public'
        community appears here even when not explicitly in running-config.
      - 'show running-config | include snmp' shows non-default
        'snmp-server community "NAME" ...' lines plus 'access-method snmp'
        toggles. Quotes are optional but typical on newer ProCurves.
      - 'show ip authorized-managers' (and the equivalent
        'ip authorized-managers' lines in running-config) gate SNMP at the
        IP level — this is the de-facto ACL mechanism on ProCurve. When
        present and 'access-method snmp' is enabled, SNMP probes from
        outside that prefix will fail with no diagnostic.
    """
    communities, v3_users, acl_refs = [], [], []
    snmp_gated = False  # True if access-method snmp / authorized-managers active

    # ---- Tabular form from 'show snmp-server' ----
    # Community-name column is 32 chars; first whitespace-delimited token is
    # the name. We bail out of the table when we hit the next header or a
    # blank line so trap-receiver / excluded-MIB sections don't pollute.
    in_comm_table, in_user_table = False, False
    for line in output.splitlines():
        s = line.rstrip()
        if "Community Name" in s and "MIB View" in s:
            in_comm_table, in_user_table = True, False
            continue
        if "User Name" in s and "Auth Protocol" in s:
            in_user_table, in_comm_table = True, False
            continue
        if not s.strip():
            in_comm_table = in_user_table = False
            continue
        if s.lstrip().startswith("---"):
            continue
        if in_comm_table:
            tok = s.split()
            if tok and tok[0] not in communities:
                communities.append(tok[0])
        elif in_user_table:
            tok = s.split()
            if tok and tok[0] not in v3_users:
                v3_users.append(tok[0])

    # ---- Config-form lines ----
    for line in output.splitlines():
        s = line.strip()
        # Newer ProCurves quote the name; older versions don't.
        m = re.match(r"snmp-server community\s+\"?([^\"\s]+)\"?\s*(.*)$", s)
        if m:
            name = m.group(1)
            if name not in communities:
                communities.append(name)
            tail = m.group(2)
            tm = re.search(r"restricted-acl\s+(\S+)", tail)
            if tm:
                acl_refs.append(("restricted-acl", tm.group(1)))
            continue
        m = re.match(r"snmpv3 user\s+\"?([^\"\s]+)\"?", s)
        if m and m.group(1) not in v3_users:
            v3_users.append(m.group(1))
            continue
        # 'ip authorized-managers IP MASK access {manager|operator}' is the
        # real SNMP source-IP filter on ProCurve. Capture both forms — the
        # one-liner 'access manager' suffix and the multi-line block with
        # 'access-method snmp' as a child.
        m = re.match(
            r"ip authorized-managers\s+(\S+)\s+(\S+)(?:\s+access\s+(\S+))?",
            s)
        if m:
            ip_addr, mask = m.group(1), m.group(2)
            role = m.group(3) or "manager"
            acl_refs.append(("authorized-managers",
                             f"{ip_addr}/{mask} ({role})"))
            continue
        if re.match(r"access-method\s+snmp", s):
            snmp_gated = True

    # ---- Multi-line 'show ip authorized-managers' parser ----
    #   Address : 199.7.244.64
    #   Mask    : 255.255.255.192
    #   Access  : Manager
    #   Access Method : snmp        <-- this is the SNMP-relevant entry
    # An 'Access Method : all' entry also gates SNMP (along with everything
    # else); 'web' / 'ssh' / 'telnet' / 'tftp' do not.
    cur = {}
    snmp_relevant_methods = ("snmp", "all")
    for line in output.splitlines():
        s = line.strip()
        m = re.match(r"Address\s*:\s*(\S+)", s)
        if m:
            cur = {"addr": m.group(1)}
            continue
        m = re.match(r"Mask\s*:\s*(\S+)", s)
        if m and "addr" in cur:
            cur["mask"] = m.group(1)
            continue
        m = re.match(r"Access\s*:\s*(\S+)", s)
        if m and "addr" in cur:
            cur["role"] = m.group(1)
            continue
        m = re.match(r"Access Method\s*:\s*(\S+)", s)
        if m and "addr" in cur:
            method = m.group(1).lower()
            if method in snmp_relevant_methods:
                role = cur.get("role", "Manager")
                ref = f"{cur['addr']}/{cur.get('mask','?')} ({role}, {method})"
                # Avoid duplicates — same prefix may appear from running-config
                # and from 'show ip authorized-managers' both.
                if ("authorized-managers", ref) not in acl_refs:
                    acl_refs.append(("authorized-managers", ref))
            cur = {}
            continue

    if snmp_gated:
        # If only 'access-method snmp' was found without an explicit
        # authorized-managers block in this output, still note the gate so
        # operators see why the probe might be ACL-blocked.
        if not any(k == "authorized-managers" for k, _ in acl_refs):
            acl_refs.append(("authorized-managers", "configured (see device)"))
    return communities, v3_users, acl_refs


def inspect_snmp_via_ssh(child, platform):
    """Pull the SNMP-related running-config from a device and parse it.

    Used as a fallback diagnostic when snmp_probe fails — answers whether
    SNMP is even configured on the device, and if so, what credentials and
    ACLs the device expects (so the operator can see the drift).

    Returns a dict with: communities, v3_users, acl_refs, excerpt, error.
    'excerpt' is a trimmed slice of the raw output suitable for the digest
    email; never longer than ~30 lines.
    """
    out = {"communities": [], "v3_users": [], "acl_refs": [],
           "excerpt": "", "error": ""}
    try:
        if platform == "junos":
            raw = send_command(
                child,
                "show configuration snmp | display set | no-more",
                timeout=15)
            out["communities"], out["v3_users"], out["acl_refs"] = \
                _parse_snmp_config_junos(raw)
        elif platform == "aruba-cx":
            # 'show snmp-server' / running-config sectioning don't exist on
            # current Aruba-CX firmware ('Invalid input: snmp-server'). The
            # community/v3 tables come from dedicated 'show' commands; the
            # vrf bindings (which gate where SNMP listens) come from
            # running-config.
            raw_a = send_command(child, "show snmp community", timeout=15)
            raw_b = send_command(child, "show snmpv3 users", timeout=15)
            try:
                raw_c = send_command(child, "show running-config", timeout=30)
            except Exception:
                raw_c = ""
            # Trim running-config to snmp-related lines so the parser doesn't
            # spend cycles on irrelevant config. Join sections with a blank
            # line so the table-state machine resets between them — without
            # the gap, running-config 'snmp-server vrf default' lines bleed
            # into the v3-user table state and produce false-positive users.
            rc_lines = [ln for ln in raw_c.splitlines()
                        if "snmp" in ln.lower()]
            raw = raw_a + "\n\n" + raw_b + "\n\n" + "\n".join(rc_lines)
            out["communities"], out["v3_users"], out["acl_refs"] = \
                _parse_snmp_config_aruba_cx(raw)
        else:  # procurve
            raw_a = send_command(child, "show snmp-server", timeout=15)
            raw_b = send_command(child,
                                 "show running-config | include snmp",
                                 timeout=15)
            # 'ip authorized-managers' is the actual SNMP ACL mechanism
            # on ProCurve. The 'include snmp' grep above usually catches
            # it via the 'access-method snmp' child line, but querying
            # directly removes that dependency on indented-context behavior
            # which differs between K-series and older firmware.
            raw_c = send_command(child, "show ip authorized-managers",
                                 timeout=10)
            # Blank-line separator forces the table-state machine to reset
            # between the human-formatted 'show snmp-server' output and the
            # config-form lines that follow.
            raw = raw_a + "\n\n" + raw_b + "\n\n" + raw_c
            out["communities"], out["v3_users"], out["acl_refs"] = \
                _parse_snmp_config_procurve(raw)
    except Exception as e:
        out["error"] = f"ssh inspect failed: {e}"
        return out

    # Trim excerpt: keep lines that mention 'snmp' (case-insensitive) so the
    # diag email isn't bloated with full running-config.
    interesting = [ln for ln in raw.splitlines()
                   if "snmp" in ln.lower()][:30]
    out["excerpt"] = "\n".join(interesting)
    return out


def diff_snmp_config(device_info, snmp_cfg):
    """Compare what the device has configured against what we have configured.

    Returns a short human-readable reason string suitable for snmp_diag.
    Examples:
      'no SNMP configured on device'
      'cred drift: device has [old_ro], we have [new_ro]'
      'acl/clients-list restricts access (refs: clients=10.0.0.0/8)'
    """
    dev_comms = set(device_info.get("communities") or [])
    dev_users = set(device_info.get("v3_users") or [])
    our_comms = set(snmp_cfg.get("communities") or [])
    our_users = set(u["name"] for u in snmp_cfg.get("v3_users") or [])
    acl_refs = device_info.get("acl_refs") or []

    if not dev_comms and not dev_users:
        return "no SNMP configured on device"
    overlap_v2 = dev_comms & our_comms
    overlap_v3 = dev_users & our_users
    if not overlap_v2 and not overlap_v3:
        return (f"cred drift: device={sorted(dev_comms | dev_users)} "
                f"ours={sorted(our_comms | our_users)}")
    # We have a credential match but probe still failed → ACL or transport block.
    if acl_refs:
        ref_str = ", ".join(f"{k}={v}" for k, v in acl_refs[:5])
        return f"creds match but probe failed; acl/restrictions: {ref_str}"
    return "creds match but probe failed (network/firewall block?)"


# ---------------------------------------------------------------------------
# SNMP-based STP collection (replaces SSH polls for monitor stp on devices
# where snmp_enabled=1)
# ---------------------------------------------------------------------------
#
# Standards used (work on Aruba-CX and ProCurve):
#   1.3.6.1.2.1.1.3.0          sysUpTime (centiseconds since boot)
#   1.3.6.1.2.1.2.2.1.8.<idx>   ifOperStatus (1=up, 2=down)
#   1.3.6.1.2.1.2.2.1.9.<idx>   ifLastChange (centiseconds at last state change)
#   1.3.6.1.2.1.17.1.4.1.2.<bp> dot1dBasePortIfIndex (bridge-port -> ifIndex)
#   1.3.6.1.2.1.17.2.4.0        dot1dStpTopChanges (TCN counter, Counter32)
#   1.3.6.1.2.1.17.2.5.0        dot1dStpDesignatedRoot (BridgeId, 8 bytes)
#   1.3.6.1.2.1.17.1.1.0        dot1dBaseBridgeAddress (own MAC, 6 bytes)
#   1.3.6.1.2.1.17.2.7.0        dot1dStpTimeSinceTopologyChange (TimeTicks)
#   1.3.6.1.2.1.17.2.15.1.3.<bp> dot1dStpPortState (1=disabled,2=blocking,
#                                3=listening,4=learning,5=forwarding,6=broken)
#   1.3.6.1.2.1.17.2.16.0       dot1dStpVersion (0=stp,2=rstp,3=mstp) — RSTP-MIB
#   1.3.6.1.2.1.17.6.1.4.1.5.<bp> dot1dStpExtPortRole (1=root,2=designated,
#                                  3=alternate,4=backup,5=disabled) — RSTP-MIB
#   1.3.6.1.2.1.31.1.1.1.1.<idx> ifName (e.g. "1/1/1", "ge-0/0/0.0")
#
# Junos vendor MIB (JUNIPER-MIMSTP-MIB at 1.3.6.1.4.1.2636.3.46.1):
#   1.3.6.1.4.1.2636.3.46.1.1.6.1.12.<bp>  jnxMIMstCistPortState
#                                          (1=disabled,2=discarding,3=blocking,
#                                           4=learning,5=forwarding)
#   1.3.6.1.4.1.2636.3.46.1.1.6.1.35.<bp>  jnxMIMstCistCurrentPortRole
#                                          (0=disabled,1=alternate,2=backup,
#                                           3=root,4=designated)
#   1.3.6.1.4.1.2636.3.46.1.1.3.1.8.<vc>   jnxMIMstCistRoot (BridgeId hex)
#
# The Junos MSTP table is indexed by dot1dBasePort numbers (verified empirically
# on STABLES-BDF and MAD-RACK4-0; see reference_junos_snmp_stp memory entry).

# OID constants — keep grouped for readability and so the cron-time hot path
# isn't building strings repeatedly.
_OID_SYSUPTIME            = "1.3.6.1.2.1.1.3.0"
_OID_IF_OPER_STATUS       = "1.3.6.1.2.1.2.2.1.8"
_OID_IF_LAST_CHANGE       = "1.3.6.1.2.1.2.2.1.9"
_OID_IF_NAME              = "1.3.6.1.2.1.31.1.1.1.1"
_OID_DOT1D_BASE_PORT_IFIDX = "1.3.6.1.2.1.17.1.4.1.2"
# Per-interface error / discard counters (IF-MIB ifTable), indexed by ifIndex.
# Cumulative Counter32 — diffed against the prior poll's snapshot to get a
# per-interval delta (see _record_port_errors). dot3StatsFCSErrors is the
# EtherLike-MIB CRC/FCS counter (also ifIndex-indexed): the classic
# bad-cable / failing-SFP / duplex-mismatch signal.
_OID_IF_IN_DISCARDS   = "1.3.6.1.2.1.2.2.1.13"
_OID_IF_IN_ERRORS     = "1.3.6.1.2.1.2.2.1.14"
_OID_IF_OUT_DISCARDS  = "1.3.6.1.2.1.2.2.1.19"
_OID_IF_OUT_ERRORS    = "1.3.6.1.2.1.2.2.1.20"
_OID_DOT3_FCS_ERRORS  = "1.3.6.1.2.1.10.7.2.1.3"
# (error_type label -> OID). Stored verbatim in port_errors.error_type.
_IFACE_ERROR_OIDS = (
    ("in_errors",    _OID_IF_IN_ERRORS),
    ("out_errors",   _OID_IF_OUT_ERRORS),
    ("in_discards",  _OID_IF_IN_DISCARDS),
    ("out_discards", _OID_IF_OUT_DISCARDS),
    ("fcs_errors",   _OID_DOT3_FCS_ERRORS),
)
# Forwarding DB (learned MAC -> bridge port). dot1d index = 6 MAC octets;
# Q-BRIDGE index = <vlan>.<6 MAC octets> (VLAN-aware). Value = dot1dBasePort.
_OID_DOT1D_TPFDB_PORT     = "1.3.6.1.2.1.17.4.3.1.2"
_OID_DOT1Q_TPFDB_PORT     = "1.3.6.1.2.1.17.7.1.2.2.1.2"
# ARP / neighbor caches — L3 device IP<->MAC mappings. Walked on every
# device during 'monitor topology'; pure L2 access switches return empty
# fast. Lets us turn a MAC harvested from port_macs into an IP, which the
# dns_cache layer then turns into a hostname — closing the ID loop in
# alert emails.
#
# ipNetToPhysicalTable (RFC 4293, dual-stack) is tried first because
# modern Aruba CX / recent Cisco / Juniper only expose this one. Index =
# <ifIndex>.<addrType>.<addrLen>.<addr_octets>; we restrict to IPv4
# (addrType=1, addrLen=4). Value = MAC octets.
#
# ipNetToMediaTable (RFC 1213, IPv4-only legacy) is the fallback for
# older devices (Juniper EX 4200/4300, ProCurve) that still ship it but
# don't expose the modern table. Index = <ifIndex>.<a.b.c.d>.
_OID_IP_NET_TO_PHYSICAL_PHYS = "1.3.6.1.2.1.4.35.1.4"
_OID_IP_NET_TO_MEDIA_PHYS    = "1.3.6.1.2.1.4.22.1.2"
_OID_DOT1D_BRIDGE_ADDR    = "1.3.6.1.2.1.17.1.1.0"
_OID_DOT1D_STP_PRIORITY   = "1.3.6.1.2.1.17.2.2.0"
_OID_DOT1D_STP_VERSION    = "1.3.6.1.2.1.17.2.16.0"
_OID_DOT1D_STP_TOPCHANGES = "1.3.6.1.2.1.17.2.4.0"
_OID_DOT1D_STP_ROOT_COST  = "1.3.6.1.2.1.17.2.6.0"
_OID_DOT1D_STP_ROOT_PORT  = "1.3.6.1.2.1.17.2.7.0"
_OID_DOT1D_STP_TIME_SINCE = "1.3.6.1.2.1.17.2.3.0"
_OID_DOT1D_STP_DESIG_ROOT = "1.3.6.1.2.1.17.2.5.0"
_OID_DOT1D_STP_PORT_STATE = "1.3.6.1.2.1.17.2.15.1.3"
_OID_DOT1D_STP_PORT_ENABLE = "1.3.6.1.2.1.17.2.15.1.4"  # 1=enabled, 2=disabled
_OID_DOT1D_STP_PORT_DESIG = "1.3.6.1.2.1.17.2.15.1.8"  # dot1dStpPortDesignatedBridge
_OID_DOT1D_STP_EXT_ROLE   = "1.3.6.1.2.1.17.6.1.4.1.5"  # RSTP-MIB; rarely populated
_OID_JNX_MSTP_PORT_STATE  = "1.3.6.1.4.1.2636.3.46.1.1.6.1.12"
_OID_JNX_MSTP_PORT_ROLE   = "1.3.6.1.4.1.2636.3.46.1.1.6.1.35"
_OID_JNX_MSTP_ROOT        = "1.3.6.1.4.1.2636.3.46.1.1.3.1.8"
# State-machine activity counters used for phantom-event evidence
# gathering (verified empirically against CLI 'show spanning-tree
# interface ... detail' and 'show spanning-tree bridge'):
#   1.3.6.1.4.1.2636.3.46.1.1.6.1.14.<bp>  jnxMIMstCistEdgeDelayWhileExpiryCount
#                                          (port-level state-machine timer
#                                           expiries — incrementing means the
#                                           STP state machine actually ran)
#   1.3.6.1.4.1.2636.3.46.1.1.3.1.35.0     jnxMIMstCistTopChanges
#                                          (CIST topology-change count for the
#                                           VC master — incrementing means a
#                                           TC was processed by this bridge)
_OID_JNX_MSTP_EDGE_EXPIRY = "1.3.6.1.4.1.2636.3.46.1.1.6.1.14"
_OID_JNX_MSTP_TC_COUNT    = "1.3.6.1.4.1.2636.3.46.1.1.3.1.35.0"


class SnmpCollectError(Exception):
    """Raised when SNMP STP collection can't complete (timeout, no creds,
    parse failure). The poll_one path catches this to fall back or fail."""


def _resolve_snmp_cred(device_row, snmp_cfg):
    """Look up the cached cred for a device and return a callable that
    runs snmpget/snmpbulkwalk for it. Raises SnmpCollectError when the
    device has no cached cred or its cred isn't loaded in cfg."""
    keys = device_row.keys() if hasattr(device_row, "keys") else []
    proto = device_row["snmp_proto"] if "snmp_proto" in keys else None
    if proto == "v2c":
        community = device_row["snmp_community"] if "snmp_community" in keys else None
        if not community:
            raise SnmpCollectError("v2c cached but community missing")
        return ("v2c", community, None)
    elif proto == "v3":
        user_name = device_row["snmp_v3_user"] if "snmp_v3_user" in keys else None
        if not user_name:
            raise SnmpCollectError("v3 cached but user missing")
        v3 = next((u for u in snmp_cfg.get("v3_users") or []
                   if u["name"] == user_name), None)
        if not v3:
            raise SnmpCollectError(
                f"v3 user '{user_name}' cached but not in [snmp_v3:*] config")
        return ("v3", None, v3)
    raise SnmpCollectError("device not yet snmp-probed")


def _snmp_args_for(cred, ip, oid, op, timeout, retries, hex_octets=False):
    """Build snmpget or snmpbulkwalk argv for the given cred tuple.
    op = 'get' for snmpget, 'walk' for snmpbulkwalk.

    hex_octets=True adds -Ox so OCTET STRINGs render as 'Hex-STRING: AA BB
    ...' rather than net-snmp's content-guessed STRING. Essential for
    binary columns like an LLDP port-ID that carries a raw MAC: without
    it, net-snmp emits the raw bytes and the subprocess pipe UTF-8-decodes
    them into U+FFFD replacement chars, destroying the value. Text values
    (an interface-name port-ID) come back as hex too but decode cleanly
    back to text downstream, so it is safe to force on the ID columns."""
    proto, community, v3 = cred
    binary = "snmpget" if op == "get" else "snmpbulkwalk"
    _ox = ["-Ox"] if hex_octets else []
    if proto == "v2c":
        args = [binary, "-v2c", "-c", community,
                "-t", str(timeout), "-r", str(retries), "-On"] + _ox
        if op == "walk":
            args += ["-Cr50"]  # bulk repeats
        args += [ip, oid]
        return args
    # v3
    args = [binary, "-v3",
            "-l", v3.get("security_level") or "noAuthNoPriv",
            "-u", v3["name"],
            "-t", str(timeout), "-r", str(retries), "-On"] + _ox
    if v3.get("auth_pass"):
        args += ["-a", v3.get("auth_proto") or "SHA",
                 "-A", v3["auth_pass"]]
    if v3.get("priv_pass"):
        args += ["-x", v3.get("priv_proto") or "AES",
                 "-X", v3["priv_pass"]]
    if op == "walk":
        args += ["-Cr50"]
    args += [ip, oid]
    return args


# Output line shapes from net-snmp -On format:
#   .1.3.6.1.2.1.2.2.1.8.514 = INTEGER: 1
#   .1.3.6.1.2.1.2.2.1.8.514 = INTEGER: up(1)
#   .1.3.6.1.2.1.31.1.1.1.1.514 = STRING: ge-0/0/0.0
#   .1.3.6.1.2.1.17.2.5.0 = Hex-STRING: 00 00 02 00 00 11 12 11
#   .1.3.6.1.2.1.1.3.0 = Timeticks: (123456) 0:20:34.56
_SNMP_LINE_RE = re.compile(
    r"^\.([\d\.]+)\s*=\s*(\w[\w-]*):\s*(.*?)\s*$")


def _parse_snmp_value(type_name, raw_value):
    """Convert a net-snmp '-On' value string into a Python value, given the
    declared SNMP type. Returns None if unparseable."""
    type_name = type_name.lower()
    raw = raw_value.strip().rstrip('"').lstrip('"')
    if type_name in ("integer", "counter32", "counter64", "gauge32",
                     "unsigned32"):
        # Could be 'INTEGER: 5' or 'INTEGER: forwarding(5)' — extract trailing int
        m = re.search(r"\(?(\-?\d+)\)?\s*$", raw)
        if m:
            return int(m.group(1))
        return None
    if type_name == "timeticks":
        # 'Timeticks: (12345) ...' — extract centisecs
        m = re.search(r"\((\d+)\)", raw_value)
        return int(m.group(1)) if m else None
    if type_name == "hex-string":
        # '00 11 22 33 44 55' → bytes
        try:
            return bytes(int(b, 16) for b in raw.split())
        except ValueError:
            return None
    if type_name == "string":
        # Already a string
        return raw
    if type_name == "oid":
        return raw
    return raw  # unknown type — return raw text


def _snmp_walk_dict(cred, ip, base_oid, timeout=5, retries=1, op_timeout=15,
                    hex_octets=False):
    """Walk a column OID and return {trailing_index: value}.

    trailing_index is everything after base_oid (typically a single integer
    for ifTable / dot1dStpPortTable). Strips the literal base_oid prefix
    plus the leading dot.
    """
    args = _snmp_args_for(cred, ip, base_oid, "walk", timeout, retries,
                          hex_octets=hex_octets)
    rc, out, err = _snmp_run(args, op_timeout)
    if rc != 0 or not out.strip():
        raise SnmpCollectError(
            f"snmpwalk {base_oid} on {ip} failed: {(err or out).strip()[:120]}")
    if any(b in (out + err) for b in
           ("Authentication failure", "Unknown user name",
            "Timeout: No Response", "wrongDigest", "decryptionError")):
        raise SnmpCollectError(
            f"snmpwalk {base_oid} on {ip}: {(out or err).strip().splitlines()[0][:120]}")

    prefix = "." + base_oid + "."
    result = {}
    for line in out.splitlines():
        m = _SNMP_LINE_RE.match(line)
        if not m:
            continue
        full_oid, type_name, raw_value = m.groups()
        full_oid = "." + full_oid
        if not full_oid.startswith(prefix):
            continue
        idx_part = full_oid[len(prefix):]
        # Single-column tables have one trailing integer; we keep it as int.
        # Multi-element indexes (rare here) stay as the dot-string.
        if idx_part.isdigit():
            key = int(idx_part)
        else:
            key = idx_part
        val = _parse_snmp_value(type_name, raw_value)
        if val is not None:
            result[key] = val
    return result


def _snmp_get_value(cred, ip, oid, timeout=5, retries=1, op_timeout=10):
    """Get a single OID value. Returns the parsed value (int/bytes/string)
    or raises SnmpCollectError."""
    args = _snmp_args_for(cred, ip, oid, "get", timeout, retries)
    rc, out, err = _snmp_run(args, op_timeout)
    if rc != 0 or not _snmp_response_ok(rc, out, err):
        raise SnmpCollectError(
            f"snmpget {oid} on {ip} failed: {(out or err).strip()[:120]}")
    m = _SNMP_LINE_RE.match(out.strip().splitlines()[0])
    if not m:
        raise SnmpCollectError(f"snmpget {oid} on {ip}: unparseable: {out[:80]}")
    _, type_name, raw_value = m.groups()
    val = _parse_snmp_value(type_name, raw_value)
    if val is None:
        raise SnmpCollectError(f"snmpget {oid} on {ip}: parse-empty")
    return val


# ---- State / role translators per platform vocabulary ----
# Existing SSH parsers use platform-specific strings; SNMP collectors must
# emit the same so the downstream is_stp_blocking / should_alert_stp_change
# logic doesn't need any changes.

_DOT1D_STATE_TO_ARUBA = {
    1: "Disabled", 2: "Blocking", 3: "Listening",
    4: "Learning", 5: "Forwarding", 6: "Broken",
}
_DOT1D_STATE_TO_PROCURVE = _DOT1D_STATE_TO_ARUBA  # same vocabulary

_RSTP_ROLE_TO_ARUBA = {
    1: "Root", 2: "Designated", 3: "Alternate",
    4: "Backup", 5: "Disabled",
}
_RSTP_ROLE_TO_PROCURVE = {
    1: "Root", 2: "Designated", 3: "Alternate",
    4: "Backup", 5: "Disabled",
}

_JUNOS_STATE_INT_TO_STR = {
    1: "DIS", 2: "BLK", 3: "BLK", 4: "LRN", 5: "FWD",
}
_JUNOS_ROLE_INT_TO_STR = {
    0: "DIS", 1: "ALT", 2: "BACKUP", 3: "ROOT", 4: "DESG",
}

_DOT1D_STP_VERSION_TO_MODE = {0: "stp", 2: "rstp", 3: "mstp"}


def _infer_role_dot1d(bp, state_int, enable_int, root_port_bp, role_strings):
    """Derive STP role from BRIDGE-MIB fields.

    The RSTP-MIB role extension (dot1dStpExtPortRole) isn't populated on
    Aruba-CX or most ProCurves. We use dot1dStpRootPort directly: it
    identifies *which specific bridge-port* is the root port for this
    device. That's authoritative — the device's own STP machine names it.

      - root_port_bp == 0       -> this device IS the root; every active
                                   port is Designated for its segment.
      - bp == root_port_bp      -> this is the root port (Root in
                                   forwarding; Alternate if blocking
                                   during convergence — unusual).
      - any other port          -> Designated (forwarding) or Alternate
                                   (blocking).

    Avoids the VSX / cluster MAC gotcha entirely: the device tells us
    which port is the root port without us guessing from MACs.

    role_strings: (designated, root, alternate, disabled) — vendor vocab.
    """
    designated, root, alternate, disabled = role_strings
    # dot1dStpPortEnable=2 means STP is administratively disabled on this
    # port. SSH parsers report these as Role=Disabled regardless of state;
    # honour that vocabulary so cross-poll diffs don't churn.
    if state_int == 1 or enable_int == 2:
        return disabled
    if root_port_bp and bp == root_port_bp:
        # This port is named by the device as its root port.
        if state_int == 5:
            return root
        return alternate
    # Either we're root (root_port_bp == 0) or this is a non-root port.
    if state_int == 5:
        return designated
    if state_int in (2, 3):
        return alternate
    return alternate


def _bytes_to_mac(b):
    """Convert a 6-byte address into 'aa:bb:cc:dd:ee:ff' canonical form.
    Matches _normalize_stp_mac output so cross-switch consensus works."""
    if b is None or len(b) < 6:
        return None
    return ":".join(f"{x:02x}" for x in b[:6])


def _bridge_id_to_priority_mac(b):
    """A BridgeId is 2 bytes priority + 6 bytes MAC. Returns (priority, mac)."""
    if b is None or len(b) < 8:
        return None, None
    priority = (b[0] << 8) | b[1]
    return priority, _bytes_to_mac(b[2:8])


def _snmp_collect_common_iface(cred, ip):
    """Walk the standard ifTable + dot1dBasePortIfIndex once. Returned dicts
    are reused by the per-platform STP collector so we don't re-walk these
    on every call. Returns (sysup_centisec, base_to_ifidx, ifidx_to_name,
    ifidx_to_oper, ifidx_to_lastchg)."""
    sysup = _snmp_get_value(cred, ip, _OID_SYSUPTIME)
    base_to_ifidx = _snmp_walk_dict(cred, ip, _OID_DOT1D_BASE_PORT_IFIDX)
    ifidx_to_name = _snmp_walk_dict(cred, ip, _OID_IF_NAME)
    ifidx_to_oper = _snmp_walk_dict(cred, ip, _OID_IF_OPER_STATUS)
    ifidx_to_lastchg = _snmp_walk_dict(cred, ip, _OID_IF_LAST_CHANGE)
    return sysup, base_to_ifidx, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg


def _build_ifaces_and_flaps(sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg,
                             strip_unit_suffix=False):
    """Convert SNMP ifTable walks into the {iface: link_up_bool} +
    {iface: seconds_since_flap} dicts the existing pipeline expects.

    strip_unit_suffix=True drops the '.0' Junos logical-unit suffix from
    ifNames, matching what parse_ifaces_junos produces."""
    ifaces = {}
    last_flapped = {}
    for idx, oper in ifidx_to_oper.items():
        name = ifidx_to_name.get(idx)
        if not name:
            continue
        if strip_unit_suffix and "." in name:
            name = name.split(".", 1)[0]
        is_up = (oper == 1)
        ifaces[name] = is_up
        # Last-flap age only meaningful when the port is currently up — for
        # down ports the existing pipeline doesn't use the flap timestamp.
        lc = ifidx_to_lastchg.get(idx)
        if is_up and lc is not None:
            age_centi = max(0, sysup - lc)
            last_flapped[name] = age_centi // 100
    return ifaces, last_flapped


def _build_root_entries_standard(cred, ip, bridge_addr_bytes=None):
    """Build a single CIST root-entry dict from standard BRIDGE-MIB + RSTP-MIB
    fields. Used by Aruba-CX and ProCurve."""
    try:
        root_id = _snmp_get_value(cred, ip, _OID_DOT1D_STP_DESIG_ROOT)
    except SnmpCollectError:
        return []
    if bridge_addr_bytes is None:
        try:
            bridge_addr_bytes = _snmp_get_value(cred, ip, _OID_DOT1D_BRIDGE_ADDR)
        except SnmpCollectError:
            bridge_addr_bytes = None
    try:
        tcn = _snmp_get_value(cred, ip, _OID_DOT1D_STP_TOPCHANGES)
    except SnmpCollectError:
        tcn = None
    try:
        last_tcn_ticks = _snmp_get_value(cred, ip, _OID_DOT1D_STP_TIME_SINCE)
    except SnmpCollectError:
        last_tcn_ticks = None
    # dot1dStpPriority (this device's bridge priority for the CIST). Pulling
    # it explicitly closes the parity gap with the SSH path, which scraped
    # the priority from 'show spanning-tree bridge' / equivalent output.
    try:
        bridge_priority = _snmp_get_value(cred, ip, _OID_DOT1D_STP_PRIORITY)
    except SnmpCollectError:
        bridge_priority = None

    root_priority, root_mac = _bridge_id_to_priority_mac(root_id)
    bridge_mac = _bytes_to_mac(bridge_addr_bytes) if bridge_addr_bytes else None
    return [{
        "instance": "CIST",
        "root_priority": root_priority,
        "root_mac": root_mac,
        "bridge_priority": bridge_priority,
        "bridge_mac": bridge_mac,
        "tcn_count": tcn,
        "last_tcn_seconds": (last_tcn_ticks // 100) if last_tcn_ticks is not None else None,
    }]


def _snmp_collect_stp_aruba_cx(cred, ip):
    """SNMP-based STP poll for Aruba-CX. Returns (ports, ifaces, stp_mode,
    root_entries, last_flapped) — same shape as the SSH poll path.

    Aruba-CX doesn't populate the RSTP-MIB role extension. We use
    dot1dStpRootCost (0 → we are root) plus dot1dStpDesignatedRoot
    (the root bridge ID) to derive role. This avoids the VSX/cluster
    pitfall where dot1dBaseBridgeAddress reports the chassis MAC but the
    STP bridge ID uses a different virtual cluster MAC."""
    sysup, base_to_ifidx, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg = \
        _snmp_collect_common_iface(cred, ip)
    state_by_bp = _snmp_walk_dict(cred, ip, _OID_DOT1D_STP_PORT_STATE)
    try:
        enable_by_bp = _snmp_walk_dict(cred, ip, _OID_DOT1D_STP_PORT_ENABLE)
    except SnmpCollectError:
        enable_by_bp = {}
    try:
        own_mac_bytes = _snmp_get_value(cred, ip, _OID_DOT1D_BRIDGE_ADDR)
    except SnmpCollectError:
        own_mac_bytes = None
    try:
        root_port_bp = _snmp_get_value(cred, ip, _OID_DOT1D_STP_ROOT_PORT)
    except SnmpCollectError:
        root_port_bp = 0  # treat as 'we are root' if unreadable

    if not state_by_bp:
        ifaces, last_flapped = _build_ifaces_and_flaps(
            sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg)
        return [], ifaces, None, [], last_flapped

    role_strings = ("Designated", "Root", "Alternate", "Disabled")
    ports = []
    for bp, state_int in state_by_bp.items():
        ifidx = base_to_ifidx.get(bp)
        name = ifidx_to_name.get(ifidx) if ifidx else None
        if not name:
            continue
        # When the link is down, Aruba-CX 'show spanning-tree' renders the
        # port as Role=Disabled / State=Down. Mirror BOTH overrides so the
        # downstream offline-detection (_is_stp_offline_state) recognises
        # the result; otherwise transient combos like 'Disabled/Forwarding'
        # or 'Disabled/Blocking' (briefly emitted while the STP machine
        # catches up to the link drop) bypass the offline check, get
        # confirmed across two polls, and fire false-positive alerts that
        # bury the real Alternate/Blocking events you actually want to see.
        oper = ifidx_to_oper.get(ifidx) if ifidx else None
        if oper is not None and oper != 1:
            role_str = "Disabled"
            state_str = "Down"
        else:
            role_str = _infer_role_dot1d(bp, state_int,
                                          enable_by_bp.get(bp, 1),
                                          root_port_bp, role_strings)
            state_str = _DOT1D_STATE_TO_ARUBA.get(state_int, "Unknown")
        ports.append({
            "interface": name,
            "state": state_str,
            "role":  role_str,
        })

    stp_mode = None
    try:
        ver = _snmp_get_value(cred, ip, _OID_DOT1D_STP_VERSION)
        stp_mode = _DOT1D_STP_VERSION_TO_MODE.get(ver)
    except SnmpCollectError:
        pass

    ifaces, last_flapped = _build_ifaces_and_flaps(
        sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg)
    root_entries = _build_root_entries_standard(cred, ip,
                                                 bridge_addr_bytes=own_mac_bytes)
    return ports, ifaces, stp_mode, root_entries, last_flapped


def _snmp_collect_stp_procurve(cred, ip):
    """SNMP-based STP poll for HP ProCurve. Same standard MIBs as Aruba-CX.

    The SSH parser_stp_procurve infers role from state alone (no Role column
    in 'show spanning-tree' output). The SNMP path can do better with
    dot1dStpPortDesignatedBridge — distinguishes Designated vs Root/Alternate
    properly. We map back to ProCurve's vocabulary so downstream comparisons
    against the existing stp_state rows still work after the first transition
    poll."""
    sysup, base_to_ifidx, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg = \
        _snmp_collect_common_iface(cred, ip)
    state_by_bp = _snmp_walk_dict(cred, ip, _OID_DOT1D_STP_PORT_STATE)
    try:
        enable_by_bp = _snmp_walk_dict(cred, ip, _OID_DOT1D_STP_PORT_ENABLE)
    except SnmpCollectError:
        enable_by_bp = {}
    try:
        own_mac_bytes = _snmp_get_value(cred, ip, _OID_DOT1D_BRIDGE_ADDR)
    except SnmpCollectError:
        own_mac_bytes = None
    try:
        root_port_bp = _snmp_get_value(cred, ip, _OID_DOT1D_STP_ROOT_PORT)
    except SnmpCollectError:
        root_port_bp = 0

    if not state_by_bp:
        ifaces, last_flapped = _build_ifaces_and_flaps(
            sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg)
        return [], ifaces, None, [], last_flapped

    # ProCurve SSH parser only emits Designated/Alternate/Disabled — never
    # Root. Collapse Root → Designated to keep cross-poll diffs byte-stable
    # against existing stp_state rows.
    role_strings = ("Designated", "Designated", "Alternate", "Disabled")
    ports = []
    for bp, state_int in state_by_bp.items():
        ifidx = base_to_ifidx.get(bp)
        name = ifidx_to_name.get(ifidx) if ifidx else None
        if not name:
            continue
        # When the link is down, ProCurve 'show spanning-tree' shows
        # Role=Disabled / State=Down. Mirror BOTH overrides so transient
        # Disabled/Forwarding or Disabled/Blocking pseudo-states (briefly
        # emitted while STP catches up to the link drop) don't slip past
        # _is_stp_offline_state and fire false-positive alerts.
        oper = ifidx_to_oper.get(ifidx) if ifidx else None
        if oper is not None and oper != 1:
            role_str = "Disabled"
            state_str = "Down"
        else:
            role_str = _infer_role_dot1d(bp, state_int,
                                          enable_by_bp.get(bp, 1),
                                          root_port_bp, role_strings)
            state_str = _DOT1D_STATE_TO_PROCURVE.get(state_int, "Unknown")
        ports.append({"interface": name, "state": state_str, "role": role_str})

    stp_mode = None
    try:
        ver = _snmp_get_value(cred, ip, _OID_DOT1D_STP_VERSION)
        stp_mode = _DOT1D_STP_VERSION_TO_MODE.get(ver)
    except SnmpCollectError:
        pass

    ifaces, last_flapped = _build_ifaces_and_flaps(
        sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg)
    root_entries = _build_root_entries_standard(cred, ip,
                                                 bridge_addr_bytes=own_mac_bytes)
    return ports, ifaces, stp_mode, root_entries, last_flapped


def _snmp_collect_stp_junos(cred, ip):
    """SNMP-based STP poll for Junos. Uses JUNIPER-MIMSTP-MIB for state/role
    (indexed by dot1dBasePort) and standard ifTable for interface names.

    Junos MSTP port-IDs are dot1dBasePort numbers — see
    reference_junos_snmp_stp memory entry for empirical validation."""
    sysup, base_to_ifidx, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg = \
        _snmp_collect_common_iface(cred, ip)
    state_by_bp = _snmp_walk_dict(cred, ip, _OID_JNX_MSTP_PORT_STATE)
    try:
        role_by_bp = _snmp_walk_dict(cred, ip, _OID_JNX_MSTP_PORT_ROLE)
    except SnmpCollectError:
        role_by_bp = {}
    # Authoritative JUNIPER STP port-id -> interface map (Junos VC switches
    # have diverged JUNIPER vs BRIDGE-MIB indexing — the cache is populated
    # at discover time from CLI 'show spanning-tree interface'). Cache miss
    # falls back to the BRIDGE-MIB chain below; that fallback works on the
    # 76% of Junos devices where the two indexings happen to align.
    conn = _db()
    port_id_cache = {r["port_id"]: r["interface"] for r in conn.execute(
        "SELECT port_id, interface FROM stp_port_id_cache WHERE ip = ?",
        (ip,)).fetchall()}
    conn.close()
    # Per-port state-machine activity counter — used by the poll loop to
    # tell whether a state change matches a real state-machine event or
    # just an SNMP-cache flicker. None on parse/timeout failure (treated
    # as 'unknown' downstream, never counted as an event).
    try:
        edge_expiry_by_bp = _snmp_walk_dict(cred, ip, _OID_JNX_MSTP_EDGE_EXPIRY)
    except SnmpCollectError:
        edge_expiry_by_bp = {}
    # Bridge-wide CIST topology-change counter; complements per-port
    # edge_expiry — incrementing means the bridge processed a TC, which
    # legitimately drives port-level state transitions.
    try:
        tcn_count_value = _snmp_get_value(cred, ip, _OID_JNX_MSTP_TC_COUNT)
    except SnmpCollectError:
        tcn_count_value = None

    if not state_by_bp:
        ifaces, last_flapped = _build_ifaces_and_flaps(
            sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg,
            strip_unit_suffix=True)
        return [], ifaces, None, [], last_flapped

    # Reverse-map physical-iface names to ifIndex so the link-down override
    # works regardless of which path resolved the name (cache or fallback).
    # Filtered to physical entries (no '.N' suffix) because ifOperStatus is
    # most reliable on the physical interface row, not the logical unit.
    name_to_ifidx = {n: i for i, n in ifidx_to_name.items() if "." not in n}

    ports = []
    for bp, state_int in state_by_bp.items():
        # Prefer the CLI-populated cache (authoritative). Falls through to
        # the BRIDGE-MIB chain only on cache miss — this can mislabel ports
        # on partial-overlap-indexing devices, but on first-deploy the
        # cache is empty for everyone and we want to keep working until the
        # next discover populates it.
        name = port_id_cache.get(bp)
        if name is None:
            ifidx_fallback = base_to_ifidx.get(bp)
            name = ifidx_to_name.get(ifidx_fallback) if ifidx_fallback else None
            if name and "." in name:
                # Strip Junos '.0' logical-unit suffix to match parse_stp_junos.
                name = name.split(".", 1)[0]
        if not name:
            continue
        # Link-down override: when ifOperStatus is down, emit DIS/BLK so the
        # offline-detector recognises the result. Without this, a flap
        # transient (link drops, jnxMIMstCistCurrentPortRole momentarily
        # still reads 'designated' before the role machine catches up)
        # produces alert noise like 'DESG/FWD -> DESG/BLK' that buries the
        # real Alternate/Blocking topology events.
        ifidx = name_to_ifidx.get(name)
        oper = ifidx_to_oper.get(ifidx) if ifidx else None
        if oper is not None and oper != 1:
            role_str = "DIS"
            state_str = "BLK"
        else:
            role_int = role_by_bp.get(bp, 0)
            role_str = _JUNOS_ROLE_INT_TO_STR.get(role_int, "DIS")
            state_str = _JUNOS_STATE_INT_TO_STR.get(state_int, "DIS")
        ports.append({
            "interface": name,
            "state": state_str,
            "role":  role_str,
            "edge_expiry": edge_expiry_by_bp.get(bp),
        })

    # Junos root from jnxMIMstCistRoot. The table is indexed by VC-member;
    # member 0 is the master and gives the CIST root for the chassis.
    root_entries = []
    try:
        root_walk = _snmp_walk_dict(cred, ip, _OID_JNX_MSTP_ROOT)
        if root_walk:
            # Take the lowest-indexed entry (master member) as authoritative.
            first_idx = sorted(root_walk.keys())[0]
            root_id = root_walk[first_idx]
            root_priority, root_mac = _bridge_id_to_priority_mac(root_id)
            try:
                bridge_addr_bytes = _snmp_get_value(cred, ip,
                                                    _OID_DOT1D_BRIDGE_ADDR)
            except SnmpCollectError:
                bridge_addr_bytes = None
            # Junos populates standard dot1dStpPriority for the CIST too, so
            # the priority value matches what Aruba/ProCurve report.
            try:
                bridge_priority = _snmp_get_value(cred, ip,
                                                   _OID_DOT1D_STP_PRIORITY)
            except SnmpCollectError:
                bridge_priority = None
            root_entries = [{
                "instance": "CIST",
                "root_priority": root_priority,
                "root_mac": root_mac,
                "bridge_priority": bridge_priority,
                "bridge_mac": _bytes_to_mac(bridge_addr_bytes)
                              if bridge_addr_bytes else None,
                # tcn_count populated for phantom-evidence cross-check; if
                # this advances between two polls a TC was processed and
                # state-machine activity is expected. last_tcn_seconds is
                # left None because Junos exposes that as Timeticks at a
                # different OID and the alert path doesn't use it yet.
                "tcn_count": tcn_count_value,
                "last_tcn_seconds": None,
            }]
    except SnmpCollectError:
        pass

    # MSTP being populated implies stp_mode = 'mstp' on Junos. Other modes use
    # different OID trees we're not walking here, so we'd return None in that
    # case (preserving the last SSH-derived value).
    stp_mode = "mstp"

    ifaces, last_flapped = _build_ifaces_and_flaps(
        sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg,
        strip_unit_suffix=True)
    return ports, ifaces, stp_mode, root_entries, last_flapped


def snmp_collect_stp(ip, cred, platform):
    """Dispatch SNMP STP collection by platform. Returns the same tuple as the
    SSH poll_one path: (ports, ifaces, stp_mode, root_entries, last_flapped).
    Raises SnmpCollectError on hard failure (caller handles fallback)."""
    if platform == "junos":
        return _snmp_collect_stp_junos(cred, ip)
    if platform == "aruba-cx":
        return _snmp_collect_stp_aruba_cx(cred, ip)
    if platform == "procurve":
        return _snmp_collect_stp_procurve(cred, ip)
    raise SnmpCollectError(f"unknown platform {platform!r}")


def parse_stp_mode_junos(output):
    """Extract STP mode from 'show spanning-tree bridge' on Junos.

    Returns a lowercased mode string ('mstp'/'rstp'/'vstp') or None if
    the output doesn't contain a recognizable 'Enabled protocol' line.
    """
    m = re.search(r"Enabled protocol\s*:\s*(\S+)", output)
    return m.group(1).strip().lower() if m else None


def parse_stp_mode_aruba_cx(output):
    """Extract STP mode from 'show spanning-tree' on ArubaOS-CX.

    Header line looks like: 'Spanning tree status      : Enabled Protocol: MSTP'
    """
    m = re.search(r"Protocol:\s*(\S+)", output)
    return m.group(1).strip().lower() if m else None


def parse_stp_mode_procurve(output):
    """Extract STP mode from 'show spanning-tree' on HP ProCurve.

    Line looks like: 'Force Version : MSTP-operation'. The '-operation'
    suffix is stripped so procurve, aruba-cx, and junos all normalize to
    the same mode tokens ('mstp'/'rstp'/etc.).
    """
    # ArubaOS-Switch (2930M etc.) reports 'Mode : RPVST' / 'MSTP'; classic
    # ProCurve reports 'Force Version : MSTP-operation'. Both normalize to the
    # shared tokens, so a migration shows as the per-switch mode flipping
    # rpvst -> mstp.
    m = re.search(r"(?im)^\s*Mode\s*:\s*(RPVST|MSTP|RSTP|PVST|STP)\b", output)
    if m:
        return m.group(1).lower()
    m = re.search(r"Force Version\s*:\s*(\S+)", output)
    if not m:
        return None
    return m.group(1).strip().lower().split("-")[0]


# ---------------------------------------------------------------------------
# STP root-bridge parsers
#
# Return list[dict] (one per STP instance). The first MVP only tracks the
# CIST/MST0 instance — that's what carries the cross-region root identity
# and is what every switch in a correctly-configured network agrees on.
# Adding per-MSTI tracking later is additive (more instances in the list).
#
# Canonical MAC form used across all parsers: 'aa:bb:cc:dd:ee:ff' lowercase,
# so outputs from Junos ('xx:xx:...'), ProCurve ('aabbcc-ddeeff'), and
# ArubaOS-CX ('xx:xx:...') compare byte-for-byte.
# ---------------------------------------------------------------------------

def _normalize_stp_mac(s):
    """Normalize an STP bridge MAC to 'aa:bb:cc:dd:ee:ff' lowercase.

    Accepts colon-separated ('02:00:00:11:12:11'), ProCurve two-halves
    ('020000-111211'), and Cisco dotted ('0200.0011.1211'). Returns None
    if the input doesn't contain 12 hex digits.
    """
    if not s:
        return None
    hexchars = re.sub(r"[^0-9a-fA-F]", "", s)
    if len(hexchars) != 12:
        return None
    return ":".join(hexchars[i:i+2] for i in range(0, 12, 2)).lower()


def parse_stp_root_junos(output):
    """Parse CIST root-bridge info from 'show spanning-tree bridge'.

    Expected lines under 'STP bridge parameters for CIST':
        Root ID                           : 0.02:00:00:11:12:11
        Bridge ID                         : 32768.28:c0:da:44:c5:01
        Number of topology changes        : 92
        Time since last topology change   : 263520 seconds
    """
    def _prio_mac(s):
        # '0.02:00:00:11:12:11' -> (0, '02:00:00:11:12:11')
        m = re.match(r"\s*(\d+)\.([0-9a-fA-F:]+)", s)
        if not m:
            return (None, None)
        return (int(m.group(1)), _normalize_stp_mac(m.group(2)))

    entry = {"instance": "CIST"}
    m = re.search(r"^\s*Root ID\s*:\s*(\S+)", output, re.MULTILINE)
    if m:
        entry["root_priority"], entry["root_mac"] = _prio_mac(m.group(1))
    m = re.search(r"^\s*Bridge ID\s*:\s*(\S+)", output, re.MULTILINE)
    if m:
        entry["bridge_priority"], entry["bridge_mac"] = _prio_mac(m.group(1))
    m = re.search(r"Number of topology changes\s*:\s*(\d+)", output)
    if m:
        entry["tcn_count"] = int(m.group(1))
    m = re.search(r"Time since last topology change\s*:\s*(\d+)", output)
    if m:
        entry["last_tcn_seconds"] = int(m.group(1))
    if entry.get("root_mac") is None:
        return []
    return [entry]


def parse_stp_root_procurve(output):
    """Parse CIST root-bridge info from ProCurve 'show spanning-tree'.

    'CST Root' fields are the universal CIST root (compare across switches).
    'Switch MAC Address'/'Switch Priority' are this bridge. ProCurve's
    'Time Since Last Change' is imprecise ('3 days') so we don't record it.

    ArubaOS-Switch RPVST (Aruba 2930M etc.) instead prints a per-VLAN root
    table ('VLAN ID | Root Mac Address | Root Priority | Root Path-Cost | Root
    Port'); we emit one entry per VLAN (instance 'VLANx') so per-(ip,instance)
    root tracking shows root placement per VLAN through a PVST->MSTP migration.
    """
    if (re.search(r"(?im)^\s*Mode\s*:\s*RPVST\b", output)
            or re.search(r"(?im)^\s*VLAN\s+Root Mac\b", output)):
        sw = re.search(r"Switch MAC Address\s*:\s*(\S+)", output)
        bridge_mac = _normalize_stp_mac(sw.group(1)) if sw else None
        entries = []
        for line in output.splitlines():
            # '  1     8c85c1-6ac80a   32,768     0   This switch is root  2'
            m = re.match(r"^\s*(\d+)\s+(\S+)\s+([\d,]+)\s+\d+\b", line)
            if not m:
                continue
            root_mac = _normalize_stp_mac(m.group(2))
            if not root_mac:
                continue
            root_pri = int(m.group(3).replace(",", ""))
            entries.append({
                "instance": "VLAN%s" % m.group(1),
                "root_mac": root_mac, "root_priority": root_pri,
                "bridge_mac": bridge_mac,
                "bridge_priority": root_pri if root_mac == bridge_mac else None,
                "tcn_count": None, "last_tcn_seconds": None,
            })
        return entries

    entry = {"instance": "CIST"}
    m = re.search(r"CST Root MAC Address\s*:\s*(\S+)", output)
    if m:
        entry["root_mac"] = _normalize_stp_mac(m.group(1))
    m = re.search(r"CST Root Priority\s*:\s*(\d+)", output)
    if m:
        entry["root_priority"] = int(m.group(1))
    m = re.search(r"Switch MAC Address\s*:\s*(\S+)", output)
    if m:
        entry["bridge_mac"] = _normalize_stp_mac(m.group(1))
    m = re.search(r"Switch Priority\s*:\s*(\d+)", output)
    if m:
        entry["bridge_priority"] = int(m.group(1))
    m = re.search(r"Topology Change Count\s*:\s*(\d+)", output)
    if m:
        entry["tcn_count"] = int(m.group(1))
    if entry.get("root_mac") is None:
        return []
    return [entry]


def parse_stp_root_aruba_cx(output):
    """Parse CIST root-bridge info from ArubaOS-CX 'show spanning-tree'.

    Expected block (first MAC-Address line after each 'Priority' header):
        MST0
          Root ID    Priority   : 0
                     MAC-Address: 02:00:00:11:12:11
          ...
          Bridge ID  Priority  : 32768
                     MAC-Address: 00:02:00:00:00:01
        Number of topology changes    : 335
        Last topology change occurred : 263567 seconds ago
    """
    entry = {"instance": "CIST"}
    # Split at MST0 so we don't accidentally pick up MST1 fields.
    section = output
    m = re.search(r"^MST0\b", output, re.MULTILINE)
    if m:
        end = re.search(r"^MST[1-9]\d*\b", output[m.end():], re.MULTILINE)
        section = output[m.start():m.start() + (end.start() if end else len(output))]

    m = re.search(
        r"Root ID\s*Priority\s*:\s*(\d+)\s*\n\s*MAC-Address\s*:\s*(\S+)", section)
    if m:
        entry["root_priority"] = int(m.group(1))
        entry["root_mac"] = _normalize_stp_mac(m.group(2))
    m = re.search(
        r"Bridge ID\s*Priority\s*:\s*(\d+)\s*\n\s*MAC-Address\s*:\s*(\S+)", section)
    if m:
        entry["bridge_priority"] = int(m.group(1))
        entry["bridge_mac"] = _normalize_stp_mac(m.group(2))
    m = re.search(r"Number of topology changes\s*:\s*(\d+)", section)
    if m:
        entry["tcn_count"] = int(m.group(1))
    m = re.search(r"Last topology change occurred\s*:\s*(\d+)\s*seconds", section)
    if m:
        entry["last_tcn_seconds"] = int(m.group(1))
    if entry.get("root_mac") is None:
        return []
    return [entry]


# ---------------------------------------------------------------------------
# Hardware / stacking / chassis parsers
#
# Each parser returns a list of member dicts:
#   {member_id: int, role: str|None, serial: str|None,
#    mac: str|None (canonical), model: str|None, status: str|None}
#
# 'standalone' devices still return a single-element list so the caller
# can uniformly upsert into device_members.
# ---------------------------------------------------------------------------

def parse_junos_vc(output):
    """Parse 'show virtual-chassis'. Member rows look like:
        0 (FPC 0)  Prsnt    BP0211020542 ex4200-48t 128  Master*      ...
        1 (FPC 1)  Prsnt    BP0211020034 ex4200-48t 128  Backup       ...
    Standalone devices still report one member (role 'Master*').
    Junos does not expose per-member MAC here — left None.
    """
    members = []
    line_re = re.compile(
        r"^\s*(\d+)\s+\(FPC\s+\d+\)\s+"   # member id
        r"(\S+)\s+"                        # status (Prsnt, NotPrsnt, etc.)
        r"(\S+)\s+"                        # serial
        r"(\S+)\s+"                        # model
        r"\d+\s+"                          # mstr priority
        r"(\w+)\*?")                       # role (Master*/Backup/Linecard)
    for line in output.splitlines():
        m = line_re.match(line)
        if not m:
            continue
        members.append({
            "member_id": int(m.group(1)),
            "status":    m.group(2),
            "serial":    m.group(3),
            "model":     m.group(4),
            "role":      m.group(5),
            "mac":       None,
        })
    return members


def parse_aruba_cx_vsf(vsf_output, module_output):
    """Parse ArubaOS-CX VSF stack. Combines 'show vsf' (gives member_id,
    MAC, model token, role) with 'show module' (gives per-member serial).
    """
    members = {}
    # 'show vsf' member table:
    #   1   9c:37:08:2d:a1:00   JL659A         Conductor
    vsf_re = re.compile(
        r"^\s*(\d+)\s+([0-9a-fA-F:]{17})\s+(\S+)\s+(\S+)")
    for line in vsf_output.splitlines():
        m = vsf_re.match(line)
        if not m:
            continue
        mid = int(m.group(1))
        members[mid] = {
            "member_id": mid,
            "mac":       _normalize_stp_mac(m.group(2)),
            "model":     m.group(3),
            "role":      m.group(4),
            "serial":    None,
            "status":    None,
        }
    # 'show module' line modules — 'name' col is '<member>/<slot>':
    #   1/1  JL659A  6300M 48SR5 CL6 PoE 4SFP56 Swch        SG47LMQ0WC Active (local)
    # Multi-word description forces us to anchor on the status keyword at
    # the end (Active/Ready/Standby/Down) to locate the serial before it.
    mod_re = re.compile(
        r"^\s*(\d+)/\d+\s+"
        r"\S+\s+"
        r".+?\s+"
        r"([A-Z0-9]{8,})\s+"                          # serial (e.g. SG47LMQ0WC)
        r"(Active|Ready|Standby|Down)(?:\s+\(.*?\))?\s*$")
    # Only read the 'Line Modules' section — Management Modules usually
    # have matching serials but we want the switch serial specifically.
    in_lines = False
    for line in module_output.splitlines():
        if re.match(r"\s*Line Modules", line):
            in_lines = True; continue
        if not in_lines:
            continue
        m = mod_re.match(line)
        if not m:
            continue
        mid = int(m.group(1))
        if mid in members and members[mid].get("serial") is None:
            members[mid]["serial"] = m.group(2)
            members[mid]["status"] = m.group(3)
    return [members[mid] for mid in sorted(members)]


def parse_aruba_cx_standalone(system_output, module_output):
    """Single-member record for a non-VSF ArubaOS-CX device. Uses
    'show system' (chassis serial + base MAC + product name).
    """
    serial = None; mac = None; model = None
    m = re.search(r"Chassis Serial Nbr\s*:\s*(\S+)", system_output)
    if m: serial = m.group(1)
    m = re.search(r"Base MAC Address\s*:\s*(\S+)", system_output)
    if m: mac = _normalize_stp_mac(m.group(1))
    m = re.search(r"Product Name\s*:\s*(\S.+?)\s*$", system_output, re.MULTILINE)
    if m: model = m.group(1).strip()
    if serial is None and mac is None:
        return []
    return [{
        "member_id": 1, "role": None,
        "serial": serial, "mac": mac, "model": model,
        "status": "Active",
    }]


def parse_procurve_stack(stacking_output, system_output):
    """Parse a ProCurve/Aruba 29xx/38xx stack. 'show stacking' gives
    member_id, MAC, (truncated) model, priority, status. 'show system'
    per-member sections add serial numbers.
    """
    members = {}
    # show stacking row:
    #   1  8030e0-eb7d80     Aruba R0M67A 2930M-...           255 Standby
    stk_re = re.compile(
        r"^\s+(\d+)\s+"                # member id
        r"([0-9a-fA-F-]{13})\s+"       # MAC (ProCurve 'aabbcc-ddeeff')
        r"(.+?)\s+"                    # model (greedy-stopped by priority)
        r"(\d+)\s+"                    # priority
        r"(\S+)\s*$")                  # status
    for line in stacking_output.splitlines():
        m = stk_re.match(line)
        if not m:
            continue
        mid = int(m.group(1))
        members[mid] = {
            "member_id": mid,
            "mac":       _normalize_stp_mac(m.group(2)),
            "model":     m.group(3).strip().rstrip("."),
            "role":      m.group(5),    # Commander/Standby/Member
            "status":    m.group(5),
            "serial":    None,
        }
    # 'show system' per-member blocks:
    #   Member :1
    #     MAC Addr           : 8030e0-eb7d80
    #     Serial Number      : SG8BKJS0V1
    current_id = None
    for line in system_output.splitlines():
        m = re.match(r"\s*Member\s*:(\d+)", line)
        if m:
            current_id = int(m.group(1)); continue
        if current_id is None:
            continue
        m = re.search(r"Serial Number\s*:\s*(\S+)", line)
        if m and current_id in members:
            members[current_id]["serial"] = m.group(1)
    return [members[mid] for mid in sorted(members)]


# Junos 'Last flapped : TS (X ago)' relative forms seen in the wild:
#   (00:00:41 ago)     — HH:MM:SS
#   (11:06:53 ago)     — HH:MM:SS
#   (1d 02:48 ago)     — Nd HH:MM  (no SS)
#   (3d 08:11 ago)     — Nd HH:MM
#   (5w2d 11:32 ago)   — NwNd HH:MM
#   (36w1d 22:12 ago)  — NwNd HH:MM
#   (216w1d 02:12 ago) — NwNd HH:MM
# The seconds portion is optional (present only on short durations).
_JUNOS_LAST_FLAPPED_RE = re.compile(
    r"Last flapped\s*:.*?\(\s*"
    r"(?:(\d+)w)?\s*(?:(\d+)d)?\s*"
    r"(\d+):(\d+)(?::(\d+))?\s+ago\s*\)"
)


def parse_last_flapped_junos(iface_output):
    """Extract seconds-since-last-link-flap from Junos 'show interfaces <port>'.

    Returns an int (seconds) or None if the field is absent / 'Never' /
    unparseable. Handles all seen relative-time formats, including long
    durations that lack the seconds field ('3d 08:11 ago').
    """
    m = _JUNOS_LAST_FLAPPED_RE.search(iface_output)
    if not m:
        return None
    w = int(m.group(1) or 0)
    d = int(m.group(2) or 0)
    h, mi = int(m.group(3)), int(m.group(4))
    s = int(m.group(5) or 0)
    return ((w * 7 + d) * 24 + h) * 3600 + mi * 60 + s


def parse_junos_iface_flaps(output):
    """Parse filtered 'show interfaces | match "^Physical|Last flapped"' and
    return {physical_iface: seconds_since_flap_or_None}, restricted to
    data-plane ports. Entries marked 'Never' become None. Used every poll
    to detect sub-poll flaps via gap-vs-timestamp comparison.
    """
    results = {}
    current = None
    for line in output.splitlines():
        m = re.match(r"^Physical interface:\s*(\S+?),", line)
        if m:
            name = m.group(1)
            if not _is_junos_dataplane(name):
                current = None
                continue
            current = name
            results[current] = None
            continue
        if current is None:
            continue
        if re.match(r"^\s*Last flapped\s*:\s*Never", line):
            results[current] = None
            current = None
            continue
        m = _JUNOS_LAST_FLAPPED_RE.search(line)
        if m:
            w = int(m.group(1) or 0)
            d = int(m.group(2) or 0)
            h, mi = int(m.group(3)), int(m.group(4))
            s = int(m.group(5) or 0)
            results[current] = ((w * 7 + d) * 24 + h) * 3600 + mi * 60 + s
            current = None
    return results


_ARUBA_UP_FOR_UNIT = {
    "second": 1, "seconds": 1,
    "minute": 60, "minutes": 60,
    "hour": 3600, "hours": 3600,
    "day": 86400, "days": 86400,
    "week": 7 * 86400, "weeks": 7 * 86400,
    "month": 30 * 86400, "months": 30 * 86400,
    "year": 365 * 86400, "years": 365 * 86400,
}


def _aruba_up_for_seconds(n, unit):
    return n * _ARUBA_UP_FOR_UNIT.get(unit.lower(), 365 * 86400)


def parse_last_flapped_aruba_cx(iface_output):
    """Extract seconds-since-last-link-up from ArubaOS-CX 'show interface'.

    Parses 'Link state: up for N <unit>'. Returns None if the port is
    currently down or the format doesn't match.
    """
    m = re.search(r"Link state:\s*up for\s+(\d+)\s+(\w+)", iface_output)
    if not m:
        return None
    return _aruba_up_for_seconds(int(m.group(1)), m.group(2))


def parse_aruba_cx_iface_flaps(output):
    """Parse full 'show interface' output on ArubaOS-CX and return
    {physical_iface: seconds_since_flap_or_None}. 'down' ports stay None
    (we don't know when they went down from this field). Used to detect
    sub-poll flaps every poll.
    """
    results = {}
    current = None
    for line in output.splitlines():
        m = re.match(r"^Interface\s+(\S+)\s+is\s+(up|down|disabled)", line)
        if m:
            current = m.group(1)
            results[current] = None
            continue
        if current is None:
            continue
        m = re.match(r"\s*Link state:\s*(up|down)\s+for\s+(\d+)\s+(\w+)", line)
        if m and m.group(1) == "up":
            results[current] = _aruba_up_for_seconds(int(m.group(2)), m.group(3))
            current = None
    return results


def parse_procurve_standalone(system_output):
    """Single-member record for a non-stacked ProCurve. 'show system'
    on a standalone has top-level MAC + Serial (no 'Member :N' sections).
    """
    serial = None; mac = None
    m = re.search(r"Base MAC Addr\s*:\s*(\S+)", system_output)
    if m: mac = _normalize_stp_mac(m.group(1))
    m = re.search(r"Serial Number\s*:\s*(\S+)", system_output)
    if m: serial = m.group(1)
    if serial is None and mac is None:
        return []
    return [{
        "member_id": 1, "role": None,
        "serial": serial, "mac": mac, "model": None,
        "status": "Up",
    }]


def collect_hardware(child, platform):
    """Run vendor-specific hardware/stacking commands and return
    (chassis_type, members). chassis_type is 'standalone' or 'stack'.
    Returns (None, []) on error or unsupported platform — callers should
    treat that as 'leave existing rows alone'.
    """
    try:
        if platform == "junos":
            out = send_command(child, "show virtual-chassis | no-more", timeout=20)
            members = parse_junos_vc(out)
            return ("stack" if len(members) > 1 else "standalone"), members
        if platform == "aruba-cx":
            vsf_out = send_command(child, "show vsf", timeout=15)
            mod_out = send_command(child, "show module", timeout=15)
            if "Invalid input" in vsf_out or not vsf_out.strip():
                sys_out = send_command(child, "show system", timeout=15)
                return "standalone", parse_aruba_cx_standalone(sys_out, mod_out)
            members = parse_aruba_cx_vsf(vsf_out, mod_out)
            return ("stack" if len(members) > 1 else "standalone"), members
        if platform == "procurve":
            stk_out = send_command(child, "show stacking", timeout=15)
            sys_out = send_command(child, "show system", timeout=15)
            members = parse_procurve_stack(stk_out, sys_out)
            if not members:
                return "standalone", parse_procurve_standalone(sys_out)
            return ("stack" if len(members) > 1 else "standalone"), members
    except Exception as e:
        log.debug("collect_hardware: %s — failed: %s", platform, e)
    return None, []


def _upsert_device_members(ip, chassis_type, members):
    """Replace device_members rows for this IP and set chassis_type +
    member_count + slot_count on the master devices row. Called from
    discover/retest once hardware has been collected.
    """
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    conn.execute("DELETE FROM device_members WHERE ip = ?", (ip,))
    for mm in members:
        conn.execute("""
            INSERT INTO device_members
                (ip, member_id, role, serial, mac, model, status, updated_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (ip, mm["member_id"], mm.get("role"), mm.get("serial"),
              mm.get("mac"), mm.get("model"), mm.get("status"), now))
    count = len(members) if members else None
    conn.execute(
        "UPDATE devices SET chassis_type = ?, member_count = ?, slot_count = ? "
        "WHERE ip = ?",
        (chassis_type, count, count, ip))
    conn.commit()
    conn.close()


# ---------------------------------------------------------------------------
# Subnet scanner (merged from scan_switches.py)
# ---------------------------------------------------------------------------

def check_ports(ip_str, ports, timeout):
    """Return a list of open TCP ports on the host (subset of ports).

    DEBUG logs one line per *responsive* host (open ports, or explicit refusal).
    Silent timeouts are intentionally not logged — on a /16 that would be 65k
    noise lines. Seeing "refused" vs nothing at all distinguishes "host alive
    but no service" from "nothing there / filtered".
    """
    open_ports = []
    refused = []
    for port in ports:
        try:
            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
                s.settimeout(timeout)
                s.connect((ip_str, port))
            open_ports.append(port)
        except socket.timeout:
            continue
        except ConnectionRefusedError:
            refused.append(port)
        except OSError:
            continue
    if open_ports:
        log.debug("scan %s: open %s", ip_str, ",".join(str(p) for p in open_ports))
    elif refused:
        log.debug("scan %s: host alive, refused %s",
                  ip_str, ",".join(str(p) for p in refused))
    return open_ports


def scan_subnet(subnet, ports, timeout, threads, on_responder=None):
    """Return a dict of {ip: [open_ports]} for hosts with any open ports in the subnet.

    on_responder, if given, is called as on_responder(ip, open_ports) the
    instant each responder is found — used to persist responders to the DB
    in flight, so a crash or SSH-drop mid-/8 doesn't lose hours of scan
    work. The callback runs on the main thread (the as_completed loop), so
    it's serial; wrap any DB work in its own try/except.
    """
    found = {}
    network = ipaddress.ip_network(subnet, strict=False)
    total = network.num_addresses - 2 if network.prefixlen < 31 else network.num_addresses
    done = 0

    # Process in batches to avoid creating millions of futures at once
    # (a /8 has 16M+ hosts — submitting all at once exhausts memory)
    batch_size = threads * 4
    host_iter = (str(ip) for ip in network.hosts())

    with ThreadPoolExecutor(max_workers=threads) as pool:
        while True:
            batch = list(itertools.islice(host_iter, batch_size))
            if not batch:
                break
            futures = {pool.submit(check_ports, ip, ports, timeout): ip
                       for ip in batch}
            for future in as_completed(futures):
                done += 1
                ip = futures[future]
                pct = done * 100 // total
                sys.stdout.write(f"\rScanning {subnet}... ({done}/{total}) {pct}%  ")
                sys.stdout.flush()
                open_ports = future.result()
                if open_ports:
                    found[ip] = open_ports
                    if on_responder is not None:
                        try:
                            on_responder(ip, open_ports)
                        except Exception as e:
                            log.debug("scan_subnet on_responder(%s) failed: %s",
                                      ip, e)

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()
    return found


def write_devices_file(filepath, ips, identity=None):
    """Write discovered IPs to the device list file, preserving any commented lines.

    identity is an optional dict {ip: "label (mac)"} for inline comments.
    """
    comments = []
    if os.path.exists(filepath):
        with open(filepath, encoding="utf-8") as f:
            for line in f:
                if line.strip().startswith("#"):
                    comments.append(line)

    with open(filepath, "w", encoding="utf-8") as f:
        if comments:
            for c in comments:
                f.write(c)
        else:
            f.write("# One device IP per line\n")
        for ip in ips:
            if identity and ip in identity:
                f.write(f"{ip}  # {identity[ip]}\n")
            else:
                f.write(ip + "\n")


def write_failed_file(filepath, failures):
    """Write failed devices to a file with reasons.

    failures is a list of (ip, reason) tuples.
    """
    with open(filepath, "w", encoding="utf-8") as f:
        f.write("# Devices that failed credential check — re-test with -r after fixing\n")
        for ip, reason in failures:
            f.write(f"{ip}  # {reason}\n")


def write_telnet_report(filepath, telnet_only_ips, telnet_and_ssh_ips):
    """Write a report of devices with telnet exposure for security review."""
    if not telnet_only_ips and not telnet_and_ssh_ips:
        # Clean up stale report if no telnet devices found
        if os.path.exists(filepath):
            os.remove(filepath)
        return
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(f"# Telnet security report — updated {datetime.now():%Y-%m-%d %H:%M}\n")
        f.write("#\n")
        total = len(telnet_only_ips) + len(telnet_and_ssh_ips)
        f.write(f"# {total} device(s) with telnet exposure\n\n")
        if telnet_only_ips:
            f.write("# TELNET ONLY — enable SSH on these devices\n")
            for ip in telnet_only_ips:
                f.write(f"{ip}\n")
            f.write("\n")
        if telnet_and_ssh_ips:
            f.write("# SSH + TELNET — consider disabling telnet\n")
            for ip in telnet_and_ssh_ips:
                f.write(f"{ip}\n")


def prune_telnet_report(filepath, ssh_passed_ips):
    """Remove IPs from the telnet report that now connect via SSH."""
    if not ssh_passed_ips or not os.path.exists(filepath):
        return
    remove = set(ssh_passed_ips)
    telnet_only = []
    telnet_and_ssh = []
    current_section = None
    with open(filepath, encoding="utf-8") as f:
        for line in f:
            stripped = line.strip()
            if "TELNET ONLY" in stripped:
                current_section = "only"
            elif "SSH + TELNET" in stripped:
                current_section = "both"
            elif stripped and not stripped.startswith("#"):
                ip = stripped.split()[0]
                if ip not in remove:
                    if current_section == "only":
                        telnet_only.append(ip)
                    elif current_section == "both":
                        telnet_and_ssh.append(ip)
    removed = len(remove) - len(remove - set(telnet_only) - set(telnet_and_ssh))
    write_telnet_report(filepath, telnet_only, telnet_and_ssh)
    if removed:
        log.info("Removed %d device(s) from telnet report (now using SSH)", removed)


def run_scan(cfg, dry_run=False):
    """Scan all configured subnets and update the device list file. Returns the list of found IPs.

    When dry_run is True, found IPs are printed but the file is not written.
    """
    # Collect results: {ip: [open_ports]}
    all_results = {}
    port_names = "/".join(str(p) for p in cfg["ports"])

    def _persist_responder(ip, ports):
        """Upsert each responder the instant scan_subnet finds it, so a
        crash mid-/8 doesn't lose hours of work. status='active' matches
        the final-pass behavior; the final pass still runs to add DNS."""
        proto = "SSH" if 22 in ports else ("telnet" if 23 in ports else None)
        try:
            upsert_device(ip, proto=proto,
                          ssh_open=1 if 22 in ports else 0,
                          telnet_open=1 if 23 in ports else 0,
                          status="active")
        except Exception as e:
            log.debug("scan persistence: %s upsert failed: %s", ip, e)

    for subnet in cfg["subnets"]:
        log.info("Scanning %s for open ports (%s)...", subnet, port_names)
        results = scan_subnet(subnet, cfg["ports"], cfg["timeout"],
                              cfg["scan_threads"],
                              on_responder=(None if dry_run else _persist_responder))
        log.info("Found %d host(s) in %s", len(results), subnet)
        for ip in sorted(results, key=ipaddress.ip_address):
            log.debug("  %s (ports: %s)", ip, ", ".join(str(p) for p in results[ip]))
        all_results.update(results)

    # De-duplicate and sort
    all_found = sorted(all_results.keys(), key=ipaddress.ip_address)

    # Warn about telnet
    telnet_only = []
    telnet_and_ssh = []
    for ip in all_found:
        ports = all_results[ip]
        has_ssh = 22 in ports
        has_telnet = 23 in ports
        if has_telnet and has_ssh:
            telnet_and_ssh.append(ip)
        elif has_telnet and not has_ssh:
            telnet_only.append(ip)

    # Compare against existing device list
    device_file = cfg["device_file"]
    if os.path.exists(device_file):
        existing = set(parse_devices(device_file))
    else:
        existing = set()
    scanned = set(all_found)

    added = sorted(scanned - existing, key=ipaddress.ip_address)
    removed = sorted(existing - scanned, key=ipaddress.ip_address)
    unchanged = len(scanned & existing)

    if added or removed:
        if added:
            log.info("New devices (%d):", len(added))
            for ip in added:
                log.info("  + %s", ip)
        if removed:
            log.info("Removed devices (%d):", len(removed))
            for ip in removed:
                log.info("  - %s", ip)
        if unchanged:
            log.info("Unchanged: %d device(s)", unchanged)
    elif scanned:
        log.info("Device list unchanged (%d device(s))", len(scanned))

    if all_found:
        if dry_run:
            log.info("Dry run — found %d device(s) total", len(all_found))
        else:
            log.info("Resolving reverse DNS for %d device(s)...", len(all_found))
            dns_map = resolve_dns_batch(all_found)
            for ip in all_found:
                ports = all_results[ip]
                proto = "SSH" if 22 in ports else ("telnet" if 23 in ports else None)
                upsert_device(ip, proto=proto,
                              ssh_open=1 if 22 in ports else 0,
                              telnet_open=1 if 23 in ports else 0,
                              dns_name=dns_map.get(ip, ""),
                              status="active")
            log.info("Updated database with %d scanned device(s)", len(all_found))
    else:
        log.info("No hosts found across all subnets.")

    # --- Post-scan warnings ---
    if telnet_only or telnet_and_ssh:
        log.info("--- Telnet Warnings ---")
        if telnet_only:
            log.warning("Telnet only — enable SSH on these devices:")
            for ip in telnet_only:
                log.warning("  ! %s", ip)
        if telnet_and_ssh:
            log.warning("SSH + Telnet — consider disabling telnet:")
            for ip in telnet_and_ssh:
                log.warning("  ! %s", ip)

    return all_found


def run_discover(cfg, dry_run=False, email=False):
    """Scan subnets, test credentials, and store results in the database.

    When dry_run is True, results are displayed but the database is not updated.
    When email is True, a diff of devices that appeared / became unreachable /
    had hardware/firmware/hostname changes since the pre-run state is emailed.
    Intended for cron-scheduled weekly runs; manual invocations omit the flag
    so they don't trigger alerts.
    """
    # No ssh lock for the SCAN phase — it is pure TCP port probing and
    # starves nothing (a /8 sweep can run for hours). The lock (legacy
    # mode) or broker slots (daemon mode) come into play only in the
    # credential phase below, and legacy mode takes the lock in CHUNKS so
    # the per-minute monitor ticks interleave with a long sweep.
    # Snapshot every device BEFORE discovery modifies them so we can compute
    # a clean diff when run with --email. We include all statuses so status
    # transitions (active -> failed / duplicate) surface in the digest with
    # their real end-state, not just "disappeared".
    pre_snapshot = {}
    if email and not dry_run:
        pre_snapshot = {r["ip"]: {
            "hostname": r["hostname"] or "",
            "model":    r["model"] or "",
            "serial":   r["serial"] or "",
            "firmware": r["firmware"] or "",
            "status":   r["status"],
            "snmp_enabled": r["snmp_enabled"] if "snmp_enabled" in r.keys() else None,
            "snmp_diag":    r["snmp_diag"]    if "snmp_diag"    in r.keys() else None,
        } for r in get_devices()}

    # --- Scan phase ---
    all_results = {}
    port_names = "/".join(str(p) for p in cfg["ports"])

    def _persist_responder(ip, ports):
        """Persist each responder the instant scan_subnet finds it, so a
        long /8 scan or a credential-phase crash doesn't lose hours of
        work. preserve_status keeps any prior real state (e.g. 'active')
        intact on already-known devices; brand-new responders get
        status='scanned' — neutral, excluded from monitor-stp's
        WHERE status='active' — until the final write resolves them."""
        proto = "SSH" if 22 in ports else ("telnet" if 23 in ports else None)
        try:
            upsert_device(ip, proto=proto,
                          ssh_open=1 if 22 in ports else 0,
                          telnet_open=1 if 23 in ports else 0,
                          status="scanned", preserve_status=True)
        except Exception as e:
            log.debug("scan persistence: %s upsert failed: %s", ip, e)

    for subnet in cfg["subnets"]:
        log.info("Scanning %s for open ports (%s)...", subnet, port_names)
        results = scan_subnet(subnet, cfg["ports"], cfg["timeout"],
                              cfg["scan_threads"],
                              on_responder=(None if dry_run else _persist_responder))
        log.info("Found %d host(s) in %s", len(results), subnet)
        for ip in sorted(results, key=ipaddress.ip_address):
            log.debug("  %s (ports: %s)", ip, ", ".join(str(p) for p in results[ip]))
        all_results.update(results)

    all_found = sorted(all_results.keys(), key=ipaddress.ip_address)

    if not all_found:
        log.info("No hosts found across all subnets.")
        return

    # Separate SSH-capable from telnet-only
    ssh_hosts = []
    telnet_only = []
    telnet_and_ssh = []
    for ip in all_found:
        ports = all_results[ip]
        has_ssh = 22 in ports
        has_telnet = 23 in ports
        if has_ssh:
            ssh_hosts.append(ip)
            if has_telnet:
                telnet_and_ssh.append(ip)
        elif has_telnet:
            telnet_only.append(ip)

    log.info("Found %d host(s) total — %d with SSH, %d telnet-only",
             len(all_found), len(ssh_hosts), len(telnet_only))

    # --- Credential test phase ---
    # Test all hosts: SSH-capable via SSH (+ telnet fallback), telnet-only via telnet
    test_hosts = ssh_hosts + telnet_only
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        return

    passed = []
    failed = []

    snmp_cfg = cfg.get("snmp") or {}

    def test_one(ip):
        log.debug("test_one: %s — starting credential test", ip)
        # Look up known-good credentials from DB to avoid lockouts. Also pull
        # cached SNMP creds so the SNMP probe can fast-path the cred we last
        # saw working on this device.
        conn = _db()
        row = conn.execute(
            "SELECT username, password_hash, snmp_proto, snmp_community, snmp_v3_user"
            " FROM devices WHERE ip = ? AND status = 'active'",
            (ip,)).fetchone()
        conn.close()
        known_user = row["username"] if row else None
        known_pw_hash = row["password_hash"] if row else None
        cached_snmp_proto = row["snmp_proto"] if row else None
        cached_snmp_cred = None
        if row:
            if cached_snmp_proto == "v2c":
                cached_snmp_cred = row["snmp_community"]
            elif cached_snmp_proto == "v3":
                cached_snmp_cred = row["snmp_v3_user"]
        if known_user:
            log.debug("test_one: %s — trying known-good user=%s first", ip, known_user)

        child, proto, username, pw_hash, ssh_open, telnet_open = connect_device(
            ip, cfg["usernames"], passwords["default"],
            password_list=cfg.get("password_list"),
            known_username=known_user, known_password_hash=known_pw_hash,
            user_passwords=cfg.get("user_passwords"),
            reason="discover",
        )
        if child is not None:
            log.debug("test_one: %s — connected via %s as %s", ip, proto, username)
            try:
                platform = _detect_platform(child)
                hostname = get_hostname(child)
            except Exception as e:
                # Session damaged (e.g. Ruckus FastIron `enable` reauth
                # left it in User Name:/Password: state that doesn't
                # match PROMPT_RE). Credentials WORKED — return PASS
                # with empty identification rather than crash the whole
                # discover run on one misbehaving device.
                log.debug("test_one: %s — platform/hostname detect failed: %s",
                          ip, e)
                platform = None
                hostname = ""
            base_mac = ""
            serial = ""
            firmware = ""
            model = ""
            hardware_info = ""
            id_output = ""
            dev_id = {}
            snmp_result = {"ok": False, "proto": None, "community": None,
                           "v3_user": None, "diag": ""}
            try:
                if platform == "junos":
                    ver = send_command(child, "show version | no-more", timeout=15)
                    hw = send_command(child, "show chassis hardware | no-more", timeout=15)
                    mac_out = send_command(child, "show chassis mac-addresses | no-more", timeout=15)
                    id_output = ver + "\n" + hw + "\n" + mac_out
                    hardware_info = hw
                elif platform == "fastiron":
                    # Ruckus multi-unit stacks emit many --More-- pages
                    # in show version; each pager round-trip needs its
                    # own timeout window. Generous timeout avoids the
                    # transient-non-switch bucketing seen on one switch.
                    ver_out = send_command(child, "show version", timeout=30)
                    id_output = ver_out
                    hardware_info = ver_out
                else:
                    sys_out = send_command(child, "show system", timeout=10)
                    ver_out = send_command(child, "show version", timeout=10)
                    id_output = sys_out + "\n" + ver_out
                    # Cisco IOS lands in the procurve bucket by prompt
                    # alone (`hostname#`/`hostname>` is identical). Re-tag
                    # post-banner so downstream platform-keyed code (STP,
                    # FDB, future features) can dispatch correctly.
                    if _is_cisco_banner(ver_out):
                        platform = "cisco-ios"
                    else:
                        # Aruba 2920/2930F/2930M model isn't in `show
                        # system` or `show version`. Two probe commands
                        # cover both topologies:
                        #   show stacking — STACK model row:
                        #     "1  mac  Aruba JL322A 2930M-48G-PoE+
                        #      Switch  255 Commander"
                        #   show modules — STANDALONE chassis row:
                        #     "Chassis: 2930M-48G-PoE+  JL322A  Serial
                        #      Number: SG..."
                        # Either matches a regex in get_device_id; on
                        # non-applicable devices both return "Invalid
                        # input" or empty output and the regexes don't
                        # match — harmless.
                        try:
                            stack_out = send_command(child, "show stacking",
                                                     timeout=10)
                            id_output = id_output + "\n" + stack_out
                        except Exception:
                            pass
                        try:
                            mod_out = send_command(child, "show modules",
                                                   timeout=10)
                            id_output = id_output + "\n" + mod_out
                        except Exception:
                            pass
                    hardware_info = sys_out
                dev_id = get_device_id(id_output)
                base_mac = dev_id.get("base_mac", "")
                serial = dev_id.get("serial", "")
                firmware = dev_id.get("firmware", "")
                model = dev_id.get("model", "")
                if not model and platform not in ("junos", "fastiron"):
                    for sys_line in id_output.splitlines():
                        sys_line = sys_line.strip()
                        lower = sys_line.lower()
                        # Skip header lines that name fields rather
                        # than values; show-modules expansion-card
                        # rows that would otherwise win the keyword
                        # race; and show-stacking member rows (we
                        # have a structured regex for those — if it
                        # missed, grabbing the raw row would store a
                        # truncated/columnated mess as the model).
                        if "system name" in lower or "module" in lower:
                            continue
                        if re.search(r"\s+\d{1,3}\s+"
                                     r"(?:Commander|Standby|Member)\s*$",
                                     sys_line):
                            continue
                        if any(kw in lower for kw in
                               ["procurve", "aruba", "cisco", "juniper",
                                "palo", "switch", "router", "firewall"]):
                            model = sys_line.strip("; ")
                            break
                chassis_type, members = collect_hardware(child, platform)
            except Exception as e:
                log.debug("test_one: %s — identification failed: %s", ip, e)
                chassis_type, members = None, []

            # Junos STP port-id -> interface mapping cache. Captured here
            # while the SSH session is open so we don't pay a second connect
            # at monitor time. On VC switches the JUNIPER STP port-id can
            # diverge from BRIDGE-MIB's dot1dBasePort; the SNMP fast-path
            # consults stp_port_id_cache as authoritative.
            junos_stp_port_ids = []
            if platform == "junos":
                try:
                    stp_out = send_command(child,
                                           "show spanning-tree interface | no-more",
                                           timeout=15)
                    junos_stp_port_ids = parse_junos_stp_port_id_table(stp_out)
                    log.debug("test_one: %s — captured %d STP port-id entries",
                              ip, len(junos_stp_port_ids))
                except Exception as e:
                    log.debug("test_one: %s — STP port-id capture failed: %s",
                              ip, e)
            # SNMP probe + (on failure) SSH-side config inspection. Done
            # while the SSH session is still open so we don't pay a second
            # connect to diagnose. Skipped entirely when [snmp] is empty.
            if snmp_cfg.get("enabled"):
                probe = snmp_probe(ip, snmp_cfg,
                                   cached_proto=cached_snmp_proto,
                                   cached_cred=cached_snmp_cred,
                                   full_walk=True)
                snmp_result.update({
                    "ok": probe["ok"],
                    "proto": probe.get("proto"),
                    "community": probe.get("community"),
                    "v3_user": probe.get("v3_user"),
                })
                if not probe["ok"] and platform:
                    try:
                        dev_snmp = inspect_snmp_via_ssh(child, platform)
                        snmp_result["diag"] = diff_snmp_config(dev_snmp, snmp_cfg)
                    except Exception as e:
                        snmp_result["diag"] = f"snmp probe failed; ssh inspect error: {e}"
                elif not probe["ok"]:
                    snmp_result["diag"] = (probe.get("error")
                                           or "snmp probe failed")
                log.debug("test_one: %s — snmp ok=%s proto=%s diag=%s",
                          ip, probe["ok"], probe.get("proto"),
                          snmp_result["diag"])
            disconnect(child)
            log.debug("test_one: %s — detected hostname=%s model=%s firmware=%s mac=%s serial=%s chassis=%s members=%d",
                      ip, hostname or "?", model or "?", firmware or "?",
                      base_mac or "?", serial or "?",
                      chassis_type or "?", len(members))
            label = hostname or ip
            if model:
                label = f"{label} ({model})"
            mac_info = f", mac={base_mac}" if base_mac else ""
            is_switch = looks_like_switch(platform, dev_id)
            info = {"hostname": hostname, "base_mac": base_mac, "serial": serial,
                    "firmware": firmware, "model": model,
                    "hardware_info": hardware_info, "proto": proto,
                    "username": username, "password_hash": pw_hash,
                    "ssh_open": ssh_open, "telnet_open": telnet_open,
                    "chassis_type": chassis_type, "members": members,
                    "snmp": snmp_result,
                    "junos_stp_port_ids": junos_stp_port_ids,
                    # Persist ONLY a confident platform (cisco-ios covers Nexus
                    # NX-OS too); leave the procurve/aruba-cx bucket as None so
                    # the monitor pass classifies it via _classify_platform.
                    # Without this a Nexus discovered active sits at the
                    # monitor's 'procurve' placeholder and, because the monitor
                    # treats procurve as final, never gets re-tagged cisco-ios —
                    # so STP monitoring runs the wrong (ProCurve) parser.
                    "platform": (platform if platform in
                                 ("junos", "fastiron", "cisco-ios") else None),
                    "is_switch": is_switch}
            if not is_switch:
                info["fail_reason"] = ("not a switch (no recognized vendor "
                                       "banner in show version)")
                log.debug("test_one: %s — NON-SWITCH (credentials work but "
                          "no switch banner)", ip)
                return (ip, f"NON-SWITCH proto={proto}, user={username}, "
                            f"host={label} — excluded from backup",
                        True, info)
            log.debug("test_one: %s — PASS", ip)
            return (ip, f"PASS proto={proto}, user={username}"
                        f"{mac_info}, host={label}", True, info)
        else:
            reason = classify_fail_reason(ssh_open, telnet_open)
            log.debug("test_one: %s — FAIL (%s)", ip, reason)
            return (ip, f"FAIL — {reason}", False,
                    {"hostname": None, "base_mac": "", "serial": "",
                     "firmware": "", "model": "", "hardware_info": "",
                     "proto": None, "username": None, "password_hash": None,
                     "ssh_open": ssh_open, "telnet_open": telnet_open,
                     "chassis_type": None, "members": [],
                     "snmp": None,
                     "fail_reason": reason})

    if test_hosts:
        test_results = []
        total = len(test_hosts)
        done = 0
        # Daemon mode: broker slots govern each session — no whole-job
        # lock. Legacy flock mode: take the lock per CHUNK with short
        # release gaps so monitor ticks run in between instead of
        # starving for the whole sweep.
        _legacy_gate = not _slot_daemon_available()
        _CRED_CHUNK = 25
        for _chunk_at in range(0, total, _CRED_CHUNK):
            chunk = test_hosts[_chunk_at:_chunk_at + _CRED_CHUNK]
            if _legacy_gate:
                if not _acquire_advisory_lock("ssh", wait=120):
                    _record_op("discover_run",
                               started_at=datetime.now().strftime(
                                   "%Y-%m-%d %H:%M:%S"),
                               duration_ms=0, success=False,
                               reason="ssh-lock-held")
                    log.warning("discover: ssh lock unavailable mid-run — "
                                "%d host(s) left untested this week",
                                total - _chunk_at)
                    break
            with ThreadPoolExecutor(max_workers=cfg["ssh_threads"]) as pool:
                futures = {pool.submit(test_one, ip): ip for ip in chunk}
                for future in as_completed(futures):
                    done += 1
                    pct = done * 100 // total
                    sys.stdout.write(f"\rTesting credentials... ({done}/{total}) {pct}%  ")
                    sys.stdout.flush()
                    ip_for_future = futures[future]
                    try:
                        result = future.result()
                    except Exception as e:
                        log.error("test_one(%s) raised unhandled %s: %s",
                                  ip_for_future, type(e).__name__, e)
                        result = (ip_for_future,
                                  f"FAIL — internal error: {type(e).__name__}",
                                  False,
                                  {"hostname": None, "base_mac": "", "serial": "",
                                   "firmware": "", "model": "", "hardware_info": "",
                                   "proto": None, "username": None, "password_hash": None,
                                   "ssh_open": None, "telnet_open": None,
                                   "chassis_type": None, "members": [],
                                   "snmp": None,
                                   "fail_reason": f"internal error: {type(e).__name__}"})
                    test_results.append(result)
            if _legacy_gate:
                _release_advisory_lock("ssh")
                time.sleep(2)

        sys.stdout.write("\r" + " " * 120 + "\r")
        sys.stdout.flush()

        test_results.sort(key=lambda r: ipaddress.ip_address(r[0]))

        # Group passing IPs by Base MAC for dedup
        mac_groups = {}  # mac -> [(ip, info), ...]
        info_map = {}    # ip -> info
        non_switches = []
        for ip, result, ok, info in test_results:
            print(f"  {ip}: {result}")
            info_map[ip] = info
            if ok:
                # Non-switches go in their own bucket — not added to passed/
                # mac_groups so they bypass the active-fleet dedup pipeline.
                if not info.get("is_switch", True):
                    non_switches.append(ip)
                    continue
                ident = _device_identity(info.get("base_mac"), info.get("serial"))
                if ident:
                    mac_groups.setdefault(ident, []).append(ip)
                else:
                    passed.append(ip)  # no MAC/serial — can't dedup, keep it
            else:
                failed.append((ip, info.get("fail_reason")
                               or "all connection attempts failed"))

        # Dedup: keep first IP per MAC (sorted), skip duplicates
        dedup_skipped = []
        for mac, ips_for_mac in mac_groups.items():
            passed.append(ips_for_mac[0])
            if len(ips_for_mac) > 1:
                for dup_ip in ips_for_mac[1:]:
                    dedup_skipped.append((dup_ip, ips_for_mac[0], mac))
                    failed.append((dup_ip, f"duplicate of {ips_for_mac[0]} ({mac})"))

        if dedup_skipped:
            log.info("--- Deduplication ---")
            grouped = {}
            for dup_ip, kept_ip, mac in dedup_skipped:
                grouped.setdefault((kept_ip, mac), []).append(dup_ip)
            for (kept_ip, mac), dup_ips in grouped.items():
                log.info("  %s (%s): keeping %s, skipping %s",
                         mac, kept_ip, kept_ip, ", ".join(dup_ips))

    # --- Write to database ---
    if not dry_run and test_hosts:
        # Reverse DNS for every host we touched (passed + failed) so unreachable
        # devices still get names for identification.
        all_touched = list(set([ip for ip, _ in failed] + list(passed)))
        log.info("Resolving reverse DNS for %d device(s)...", len(all_touched))
        dns_map = resolve_dns_batch(all_touched)

        for ip in passed:
            info = info_map[ip]
            upsert_device(ip, base_mac=info["base_mac"], hostname=info["hostname"],
                          dns_name=dns_map.get(ip, ""),
                          platform=info.get("platform"),
                          model=info.get("model"), serial=info.get("serial"),
                          firmware=info.get("firmware"),
                          hardware_info=info.get("hardware_info"),
                          proto=info["proto"], username=info["username"],
                          password_hash=info.get("password_hash"),
                          ssh_open=info.get("ssh_open"),
                          telnet_open=info.get("telnet_open"), status="active")
            if info.get("members"):
                _upsert_device_members(ip, info.get("chassis_type"), info["members"])
            if info.get("junos_stp_port_ids"):
                _upsert_junos_stp_port_id_cache(ip, info["junos_stp_port_ids"])
            snmp_info = info.get("snmp")
            if snmp_info is not None and snmp_cfg.get("enabled"):
                record_snmp_result(
                    ip,
                    enabled=1 if snmp_info["ok"] else 0,
                    proto=snmp_info.get("proto"),
                    community=snmp_info.get("community"),
                    v3_user=snmp_info.get("v3_user"),
                    diag=snmp_info.get("diag") or None,
                    ok_now=bool(snmp_info["ok"]),
                )
        for ip in non_switches:
            info = info_map[ip]
            upsert_device(ip, base_mac=info.get("base_mac"),
                          hostname=info.get("hostname"),
                          dns_name=dns_map.get(ip, ""),
                          model=info.get("model"), serial=info.get("serial"),
                          firmware=info.get("firmware"),
                          hardware_info=info.get("hardware_info"),
                          proto=info.get("proto"), username=info.get("username"),
                          password_hash=info.get("password_hash"),
                          ssh_open=info.get("ssh_open"),
                          telnet_open=info.get("telnet_open"),
                          status="non-switch",
                          fail_reason=info.get("fail_reason"))
        # Look up prior status for credential-failed IPs. If the device was
        # ever active before, treat the new miss as a regression ("failed").
        # If it was never active (or absent from DB), it's a first-time
        # discovery miss — could be a Linux box, SAN BMC, VMware ESXi,
        # printer, etc. that happened to have port 22 open. Classify as
        # "unknown" rather than "failed" to keep the failed bucket
        # meaningful (only true regressions).
        fail_ips = [ip for ip, _ in failed]
        prior_status_map = {}
        if fail_ips:
            cn = _db()
            placeholder = ",".join("?" * len(fail_ips))
            for r in cn.execute(
                    f"SELECT ip, status FROM devices WHERE ip IN ({placeholder})",
                    fail_ips):
                prior_status_map[r["ip"]] = r["status"]
            cn.close()
        n_failed = 0
        n_unknown = 0
        n_duplicate = 0
        for ip, reason in failed:
            info = info_map.get(ip, {})
            if "duplicate of" in reason:
                kept_ip = reason.split("duplicate of ")[1].split(" (")[0]
                upsert_device(ip, base_mac=info.get("base_mac"),
                              hostname=info.get("hostname"),
                              dns_name=dns_map.get(ip, ""),
                              model=info.get("model"), serial=info.get("serial"),
                              firmware=info.get("firmware"),
                              hardware_info=info.get("hardware_info"),
                              proto=info.get("proto"), username=info.get("username"),
                              password_hash=info.get("password_hash"),
                              ssh_open=info.get("ssh_open"),
                              telnet_open=info.get("telnet_open"),
                              status="duplicate", duplicate_of=kept_ip)
                n_duplicate += 1
                continue
            if prior_status_map.get(ip) == "active":
                new_status = "failed"
                n_failed += 1
            else:
                new_status = "unknown"
                n_unknown += 1
            upsert_device(ip, dns_name=dns_map.get(ip, ""),
                          status=new_status, fail_reason=reason)
        log.info("Updated database: %d active, %d non-switch, "
                 "%d failed (regression), %d unknown (new, no auth), "
                 "%d duplicate",
                 len(passed), len(non_switches),
                 n_failed, n_unknown, n_duplicate)

    # --- Post-discover warnings ---
    # Warn about telnet-only devices that passed (they work but should get SSH)
    telnet_passed = [ip for ip in telnet_only if ip in passed]
    if telnet_passed or telnet_and_ssh:
        log.info("--- Telnet Warnings ---")
        if telnet_passed:
            log.warning("Telnet only — enable SSH on these devices:")
            for ip in telnet_passed:
                log.warning("  ! %s", ip)
        if telnet_and_ssh:
            log.warning("SSH + Telnet — consider disabling telnet:")
            for ip in telnet_and_ssh:
                log.warning("  ! %s", ip)

    # --- Summary ---
    log.info("--- Discovery Summary ---")
    log.info("Passed:      %d", len(passed))
    log.info("Failed:      %d", len(failed))

    # Fold duplicate reconciliation into the weekly discover: secondary switch
    # IPs (L3 SVIs) get reclassified to 'duplicate' + reachability-checked on
    # the same cadence, and grouping on this run's freshly-refreshed serials
    # re-verifies each duplicate still belongs to its switch. Fail-soft so a
    # reconcile hiccup never breaks discovery.
    if not dry_run:
        try:
            reconcile_duplicates(cfg)
        except Exception as e:
            log.warning("Duplicate reconciliation skipped: %s", e)

    # --- Diff email (cron-only; --email flag gates this) ---
    if email and not dry_run:
        _send_discovery_digest(cfg, pre_snapshot)


def _send_discovery_digest(cfg, pre_snapshot):
    """Compare the post-discovery DB state against pre_snapshot and email
    a diff. Always sends when invoked (no-change weeks still report 'clean'
    so the recipient knows the weekly run completed)."""
    if not cfg.get("smtp_server"):
        log.debug("No [email] config — skipping discovery digest email")
        return

    post = {r["ip"]: {
        "hostname": r["hostname"] or "",
        "model":    r["model"] or "",
        "serial":   r["serial"] or "",
        "firmware": r["firmware"] or "",
        "status":   r["status"],
        "snmp_enabled": r["snmp_enabled"] if "snmp_enabled" in r.keys() else None,
        "snmp_diag":    r["snmp_diag"]    if "snmp_diag"    in r.keys() else None,
    } for r in get_devices()}

    def was_active(info):
        return info and info.get("status") == "active"

    new_devices = []      # appeared (new IP or non-active -> active)
    gone_devices = []     # was active, no longer active
    hostname_changes = [] # (ip, old_hostname, new_hostname)
    hardware_changes = [] # (ip, hostname, old_model, new_model, old_serial, new_serial)
    firmware_changes = [] # (ip, hostname, old_firmware, new_firmware)
    snmp_broken_now = []  # active devices where SNMP is currently broken
    snmp_newly_broken = [] # SNMP transitioned from working/unknown to broken
    snmp_newly_fixed  = [] # SNMP transitioned from broken to working

    for ip, after in post.items():
        before = pre_snapshot.get(ip)
        if after["status"] == "active" and not was_active(before):
            new_devices.append((ip, after["hostname"] or ip, after["model"], after["serial"]))
            continue
        if before and was_active(before) and after["status"] != "active":
            gone_devices.append((ip, before["hostname"] or ip, after["status"]))
            continue
        if before and was_active(before) and after["status"] == "active":
            if before["hostname"] != after["hostname"]:
                hostname_changes.append((ip, before["hostname"] or "(none)",
                                         after["hostname"] or "(none)"))
            if (before["model"] != after["model"]
                    or before["serial"] != after["serial"]):
                hardware_changes.append((ip, after["hostname"] or ip,
                                         before["model"], after["model"],
                                         before["serial"], after["serial"]))
            if before["firmware"] != after["firmware"]:
                firmware_changes.append((ip, after["hostname"] or ip,
                                         before["firmware"] or "(none)",
                                         after["firmware"] or "(none)"))
        # SNMP status — only meaningful for active devices.
        if after["status"] == "active" and after["snmp_enabled"] == 0:
            snmp_broken_now.append((ip, after["hostname"] or ip,
                                    after["snmp_diag"] or "snmp probe failed"))
            if before and before.get("snmp_enabled") in (1, None):
                snmp_newly_broken.append(ip)
        elif (after["status"] == "active" and after["snmp_enabled"] == 1
              and before and before.get("snmp_enabled") == 0):
            snmp_newly_fixed.append((ip, after["hostname"] or ip))
    # IPs in pre_snapshot that are no longer in post at all (status cleared) —
    # rare; counts as 'gone' for symmetry.
    for ip, before in pre_snapshot.items():
        if ip not in post and was_active(before):
            gone_devices.append((ip, before["hostname"] or ip, "(removed)"))

    subject_bits = []
    if new_devices:      subject_bits.append(f"{len(new_devices)} new")
    if gone_devices:     subject_bits.append(f"{len(gone_devices)} unreachable")
    if hostname_changes: subject_bits.append(f"{len(hostname_changes)} hostname")
    if hardware_changes: subject_bits.append(f"{len(hardware_changes)} hardware")
    if firmware_changes: subject_bits.append(f"{len(firmware_changes)} firmware")
    if snmp_broken_now:  subject_bits.append(f"{len(snmp_broken_now)} snmp")
    subject = (f"[netops] Discovery digest — {', '.join(subject_bits)}"
               if subject_bits else "[netops] Discovery digest — no changes")

    pre_active  = sum(1 for v in pre_snapshot.values() if v.get("status") == "active")
    post_active = sum(1 for v in post.values()          if v.get("status") == "active")
    lines = [f"Discovery digest at {datetime.now():%Y-%m-%d %H:%M}",
             f"Pre-run: {pre_active} active of {len(pre_snapshot)} total.  "
             f"Post-run: {post_active} active of {len(post)} total.", ""]
    if new_devices:
        lines.append(f"--- New devices ({len(new_devices)}) ---")
        for ip, name, model, serial in sorted(new_devices):
            lines.append(f"  {name} ({ip})  [{model or '?'}]  serial={serial or '?'}")
        lines.append("")
    if gone_devices:
        lines.append(f"--- Now unreachable ({len(gone_devices)}) ---")
        for ip, name, status in sorted(gone_devices):
            lines.append(f"  {name} ({ip})  status={status}")
        lines.append("")
    if hostname_changes:
        lines.append(f"--- Hostname changes ({len(hostname_changes)}) ---")
        for ip, old, new in sorted(hostname_changes):
            lines.append(f"  {ip}  {old} -> {new}")
        lines.append("")
    if hardware_changes:
        lines.append(f"--- Hardware changes ({len(hardware_changes)}) ---")
        for ip, name, om, nm, os_, ns in sorted(hardware_changes):
            lines.append(f"  {name} ({ip})")
            if om != nm:
                lines.append(f"    model:  {om or '(none)'} -> {nm or '(none)'}")
            if os_ != ns:
                lines.append(f"    serial: {os_ or '(none)'} -> {ns or '(none)'}")
        lines.append("")
    if firmware_changes:
        lines.append(f"--- Firmware changes ({len(firmware_changes)}) ---")
        for ip, name, of, nf in sorted(firmware_changes):
            lines.append(f"  {name} ({ip})  {of} -> {nf}")
        lines.append("")
    if snmp_broken_now:
        # Group by reason for compactness — the diag string is the dimension
        # operators want to triage on (acl drift vs cred drift vs no-snmp).
        by_reason = {}
        for ip, name, diag in snmp_broken_now:
            by_reason.setdefault(diag, []).append((ip, name))
        lines.append(f"--- SNMP issues ({len(snmp_broken_now)}) ---")
        if snmp_newly_broken:
            lines.append(f"  ({len(snmp_newly_broken)} newly broken since last run)")
        if snmp_newly_fixed:
            lines.append(f"  ({len(snmp_newly_fixed)} newly fixed)")
            for ip, name in sorted(snmp_newly_fixed):
                lines.append(f"    + {name} ({ip})  now reachable")
        for reason in sorted(by_reason):
            entries = sorted(by_reason[reason])
            lines.append(f"  {reason}  ({len(entries)})")
            for ip, name in entries:
                lines.append(f"    {name} ({ip})")
        lines.append("")
    if not subject_bits:
        lines.append("No changes detected since the previous discovery run.")

    log.info("--- Discovery Digest ---")
    log.info("New:          %d", len(new_devices))
    log.info("Unreachable:  %d", len(gone_devices))
    log.info("Hostname:     %d", len(hostname_changes))
    log.info("Hardware:     %d", len(hardware_changes))
    log.info("Firmware:     %d", len(firmware_changes))
    log.info("SNMP broken:  %d (%d newly)", len(snmp_broken_now), len(snmp_newly_broken))
    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("Discovery digest email %s", detail)
    else:
        log.error("Discovery digest email failed: %s", detail)


def run_retest(cfg):
    """Re-test failed devices from the database and promote passing ones."""
    if not _ssh_job_gate(wait=90):
        print("test ssh retest: another SSH-heavy job holds the lock "
              "(waited 90s) — try again in a minute.")
        return
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        return

    # Read failed + unknown + inactive from DB; fall back to failed_devices.txt
    db_failed = (get_devices(status="failed")
                 + get_devices(status="unknown")
                 + get_devices(status="inactive"))
    if db_failed:
        ips = [r["ip"] for r in db_failed]
    else:
        device_file = cfg["device_file"]
        failed_file = os.path.join(os.path.dirname(device_file), "failed_devices.txt")
        if os.path.exists(failed_file):
            ips = parse_failed_file(failed_file)
            if ips:
                log.info("No failed devices in DB — reading from %s", failed_file)
        else:
            ips = []

    if not ips:
        log.info("No failed devices to re-test.")
        return

    log.info("Re-testing %d failed device(s)...", len(ips))

    passed = []
    still_failed = []

    def test_one(ip):
        child, proto, username, pw_hash, ssh_open, telnet_open = connect_device(
            ip, cfg["usernames"], passwords["default"],
            password_list=cfg.get("password_list"),
            user_passwords=cfg.get("user_passwords"),
        )
        if child is not None:
            try:
                platform = _detect_platform(child)
                hostname = get_hostname(child)
            except Exception as e:
                # Session damaged (e.g. Ruckus FastIron `enable` reauth
                # left it in User Name:/Password: state). Credentials
                # WORKED — return PASS with empty identification rather
                # than crash the whole retest run.
                log.debug("test_one: %s — platform/hostname detect failed: %s",
                          ip, e)
                platform = None
                hostname = ""
            base_mac = ""
            serial = ""
            firmware = ""
            model = ""
            hardware_info = ""
            id_output = ""
            dev_id = {}
            try:
                if platform == "junos":
                    ver = send_command(child, "show version | no-more", timeout=15)
                    hw = send_command(child, "show chassis hardware | no-more", timeout=15)
                    mac_out = send_command(child, "show chassis mac-addresses | no-more", timeout=15)
                    id_output = ver + "\n" + hw + "\n" + mac_out
                    hardware_info = hw
                elif platform == "fastiron":
                    # Ruckus multi-unit stacks emit many --More-- pages
                    # in show version; each pager round-trip needs its
                    # own timeout window. Generous timeout avoids the
                    # transient-non-switch bucketing seen on one switch.
                    ver_out = send_command(child, "show version", timeout=30)
                    id_output = ver_out
                    hardware_info = ver_out
                else:
                    sys_out = send_command(child, "show system", timeout=10)
                    ver_out = send_command(child, "show version", timeout=10)
                    id_output = sys_out + "\n" + ver_out
                    # Cisco IOS lands in the procurve bucket by prompt
                    # alone (`hostname#`/`hostname>` is identical). Re-tag
                    # post-banner so downstream platform-keyed code (STP,
                    # FDB, future features) can dispatch correctly.
                    if _is_cisco_banner(ver_out):
                        platform = "cisco-ios"
                    else:
                        # Aruba 2920/2930F/2930M model isn't in `show
                        # system` or `show version`. Two probe commands
                        # cover both topologies:
                        #   show stacking — STACK model row:
                        #     "1  mac  Aruba JL322A 2930M-48G-PoE+
                        #      Switch  255 Commander"
                        #   show modules — STANDALONE chassis row:
                        #     "Chassis: 2930M-48G-PoE+  JL322A  Serial
                        #      Number: SG..."
                        # Either matches a regex in get_device_id; on
                        # non-applicable devices both return "Invalid
                        # input" or empty output and the regexes don't
                        # match — harmless.
                        try:
                            stack_out = send_command(child, "show stacking",
                                                     timeout=10)
                            id_output = id_output + "\n" + stack_out
                        except Exception:
                            pass
                        try:
                            mod_out = send_command(child, "show modules",
                                                   timeout=10)
                            id_output = id_output + "\n" + mod_out
                        except Exception:
                            pass
                    hardware_info = sys_out
                dev_id = get_device_id(id_output)
                base_mac = dev_id.get("base_mac", "")
                serial = dev_id.get("serial", "")
                firmware = dev_id.get("firmware", "")
                model = dev_id.get("model", "")
                if not model and platform not in ("junos", "fastiron"):
                    for sys_line in id_output.splitlines():
                        sys_line = sys_line.strip()
                        lower = sys_line.lower()
                        # Skip header lines that name fields rather
                        # than values; show-modules expansion-card
                        # rows that would otherwise win the keyword
                        # race; and show-stacking member rows (we
                        # have a structured regex for those — if it
                        # missed, grabbing the raw row would store a
                        # truncated/columnated mess as the model).
                        if "system name" in lower or "module" in lower:
                            continue
                        if re.search(r"\s+\d{1,3}\s+"
                                     r"(?:Commander|Standby|Member)\s*$",
                                     sys_line):
                            continue
                        if any(kw in lower for kw in
                               ["procurve", "aruba", "cisco", "juniper",
                                "palo", "switch", "router", "firewall"]):
                            model = sys_line.strip("; ")
                            break
                chassis_type, members = collect_hardware(child, platform)
            except Exception:
                chassis_type, members = None, []
            disconnect(child)
            is_switch = looks_like_switch(platform, dev_id)
            label = hostname or ip
            if model:
                label = f"{label} ({model})"
            mac_info = f", mac={base_mac}" if base_mac else ""
            info = {"hostname": hostname, "base_mac": base_mac, "serial": serial,
                    "firmware": firmware, "model": model,
                    "hardware_info": hardware_info, "proto": proto,
                    "username": username, "password_hash": pw_hash,
                    "ssh_open": ssh_open, "telnet_open": telnet_open,
                    "chassis_type": chassis_type, "members": members,
                    "is_switch": is_switch}
            if not is_switch:
                info["fail_reason"] = ("not a switch (no recognized vendor "
                                       "banner in show version)")
                return (ip, f"NON-SWITCH proto={proto}, user={username}, "
                            f"host={label} — excluded from backup",
                        True, info)
            return (ip, f"PASS proto={proto}, user={username}"
                        f"{mac_info}, host={label}", True, info)
        else:
            reason = classify_fail_reason(ssh_open, telnet_open)
            return (ip, f"FAIL — {reason}", False,
                    {"hostname": None, "base_mac": "", "serial": "",
                     "firmware": "", "model": "", "hardware_info": "",
                     "proto": None, "username": None, "password_hash": None,
                     "ssh_open": ssh_open, "telnet_open": telnet_open,
                     "chassis_type": None, "members": [],
                     "fail_reason": reason})

    results = []
    total = len(ips)
    done = 0
    with ThreadPoolExecutor(max_workers=cfg["ssh_threads"]) as pool:
        futures = {pool.submit(test_one, ip): ip for ip in ips}
        for future in as_completed(futures):
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rRe-testing credentials... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()
            ip_for_future = futures[future]
            try:
                result = future.result()
            except Exception as e:
                log.error("test_one(%s) raised unhandled %s: %s",
                          ip_for_future, type(e).__name__, e)
                result = (ip_for_future,
                          f"FAIL — internal error: {type(e).__name__}",
                          False,
                          {"hostname": None, "base_mac": "", "serial": "",
                           "firmware": "", "model": "", "hardware_info": "",
                           "proto": None, "username": None, "password_hash": None,
                           "ssh_open": None, "telnet_open": None,
                           "chassis_type": None, "members": [],
                           "fail_reason": f"internal error: {type(e).__name__}"})
            results.append(result)

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    results.sort(key=lambda r: ipaddress.ip_address(r[0]))

    info_map = {}
    mac_groups = {}
    non_switches = []
    for ip, result, ok, info in results:
        print(f"  {ip}: {result}")
        info_map[ip] = info
        if ok:
            if not info.get("is_switch", True):
                non_switches.append(ip)
                continue
            ident = _device_identity(info.get("base_mac"), info.get("serial"))
            if ident:
                mac_groups.setdefault(ident, []).append(ip)
            else:
                passed.append(ip)
        else:
            still_failed.append((ip, info.get("fail_reason")
                                 or "all connection attempts failed"))

    # Dedup: keep first IP per MAC, skip duplicates
    dedup_skipped = []
    for mac, ips_for_mac in mac_groups.items():
        passed.append(ips_for_mac[0])
        if len(ips_for_mac) > 1:
            for dup_ip in ips_for_mac[1:]:
                dedup_skipped.append((dup_ip, ips_for_mac[0], mac))
                still_failed.append((dup_ip, f"duplicate of {ips_for_mac[0]} ({mac})"))

    if dedup_skipped:
        log.info("--- Deduplication ---")
        grouped = {}
        for dup_ip, kept_ip, mac in dedup_skipped:
            grouped.setdefault((kept_ip, mac), []).append(dup_ip)
        for (kept_ip, mac), dup_ips in grouped.items():
            log.info("  %s (%s): keeping %s, skipping %s",
                     mac, kept_ip, kept_ip, ", ".join(dup_ips))

    # --- Write to database ---
    for ip in passed:
        info = info_map[ip]
        upsert_device(ip, base_mac=info["base_mac"], hostname=info["hostname"],
                      platform=info.get("platform"),
                      model=info.get("model"), serial=info.get("serial"),
                      firmware=info.get("firmware"),
                      hardware_info=info.get("hardware_info"),
                      proto=info["proto"], username=info["username"],
                      password_hash=info.get("password_hash"),
                          ssh_open=info.get("ssh_open"),
                          telnet_open=info.get("telnet_open"), status="active")
        if info.get("members"):
            _upsert_device_members(ip, info.get("chassis_type"), info["members"])
    for ip in non_switches:
        info = info_map[ip]
        upsert_device(ip, base_mac=info.get("base_mac"),
                      hostname=info.get("hostname"),
                      model=info.get("model"), serial=info.get("serial"),
                      firmware=info.get("firmware"),
                      hardware_info=info.get("hardware_info"),
                      proto=info.get("proto"), username=info.get("username"),
                      password_hash=info.get("password_hash"),
                      ssh_open=info.get("ssh_open"),
                      telnet_open=info.get("telnet_open"),
                      status="non-switch",
                      fail_reason=info.get("fail_reason"))
    # Look up prior statuses so retest preserves the failed-vs-unknown
    # distinction. A device that came in as "unknown" stays "unknown"
    # if creds still don't work — don't promote it to "failed".
    still_fail_ips = [ip for ip, _ in still_failed]
    prior_status_map = {}
    if still_fail_ips:
        cn = _db()
        placeholder = ",".join("?" * len(still_fail_ips))
        for r in cn.execute(
                f"SELECT ip, status FROM devices WHERE ip IN ({placeholder})",
                still_fail_ips):
            prior_status_map[r["ip"]] = r["status"]
        cn.close()
    for ip, reason in still_failed:
        info = info_map.get(ip, {})
        if "duplicate of" in reason:
            kept_ip = reason.split("duplicate of ")[1].split(" (")[0]
            upsert_device(ip, base_mac=info.get("base_mac"),
                          hostname=info.get("hostname"),
                          model=info.get("model"), serial=info.get("serial"),
                          firmware=info.get("firmware"),
                          hardware_info=info.get("hardware_info"),
                          proto=info.get("proto"), username=info.get("username"),
                          password_hash=info.get("password_hash"),
                          status="duplicate", duplicate_of=kept_ip)
        else:
            # Preserve "unknown" if the row was already unknown; otherwise
            # this is a regression from a prior active state.
            new_status = ("unknown"
                          if prior_status_map.get(ip) == "unknown"
                          else "failed")
            upsert_device(ip, status=new_status, fail_reason=reason)

    log.info("--- Retest Summary ---")
    log.info("Promoted:      %d", len(passed))
    log.info("Non-switch:    %d", len(non_switches))
    log.info("Still failing: %d", len(still_failed))


def run_snmp_test(cfg, retest=False):
    """Re-probe SNMP credentials on devices already in the DB.

    Unlike 'discover', this does NO subnet scan and NO SSH login — it just
    fires the SNMP probe (cached winner first, then the full community/v3
    list via full_walk) at each target and refreshes the snmp_* columns.
    Fast enough to run ad hoc right after you've fixed an SNMP config on a
    switch, without waiting for the weekly discover.

      test snmp          probe every active device
      test snmp retest   probe only the 'no-snmp' set (snmp_enabled != 1) —
                         the devices SNMP currently can't poll

    Never opens SSH, so it doesn't contend for the ssh advisory lock and can
    run alongside a backup/monitor tick.
    """
    snmp_cfg = cfg.get("snmp") or {}
    if not snmp_cfg.get("enabled"):
        log.error("No SNMP credentials configured. Add a [snmp] community "
                  "or an [snmp_v3:USER] section to secrets.conf.")
        return

    conn = _db()
    base_sql = ("SELECT ip, hostname, snmp_enabled, snmp_proto, snmp_community,"
                " snmp_v3_user FROM devices WHERE status = 'active'")
    if retest:
        # The 'no-snmp' set: never tested (NULL) or tested/failed (0).
        base_sql += " AND (snmp_enabled IS NULL OR snmp_enabled != 1)"
    rows = conn.execute(base_sql).fetchall()
    conn.close()

    rows = _sort_by_ip(rows)
    if not rows:
        if retest:
            log.info("No active devices with broken/untested SNMP — nothing "
                     "to re-test. ('show no-snmp' lists this set.)")
        else:
            log.info("No active devices to test. Run 'discover' first.")
        return

    total = len(rows)
    scope = "broken/untested" if retest else "active"
    log.info("Probing SNMP on %d %s device(s) (no subnet scan)...", total, scope)

    def _probe_one(row):
        ip = row["ip"]
        cached_proto = row["snmp_proto"]
        cached_cred = None
        if cached_proto == "v2c":
            cached_cred = row["snmp_community"]
        elif cached_proto == "v3":
            cached_cred = row["snmp_v3_user"]
        probe = snmp_probe(ip, snmp_cfg, cached_proto=cached_proto,
                           cached_cred=cached_cred, full_walk=True)
        return (ip, probe)

    results = {}
    done = 0
    threads = min(20, total)
    with ThreadPoolExecutor(max_workers=max(1, threads)) as pool:
        futures = {pool.submit(_probe_one, r): r["ip"] for r in rows}
        for future in as_completed(futures):
            ip, probe = future.result()
            results[ip] = probe
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rProbing SNMP... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()
    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    ok_now = 0
    newly_fixed = []   # (ip, hostname) — was broken/untested, now works
    still_broken = []  # (ip, hostname, diag)
    for r in rows:
        ip = r["ip"]
        probe = results.get(ip) or {"ok": False, "error": "no result"}
        prior_enabled = r["snmp_enabled"]
        if probe["ok"]:
            ok_now += 1
            record_snmp_result(
                ip, enabled=1, proto=probe.get("proto"),
                community=probe.get("community"), v3_user=probe.get("v3_user"),
                diag=None, ok_now=True)
            if prior_enabled != 1:
                newly_fixed.append((ip, r["hostname"] or ip))
        else:
            diag = probe.get("error") or "snmp probe failed"
            record_snmp_result(
                ip, enabled=0, proto=None, community=None, v3_user=None,
                diag=diag, ok_now=False)
            still_broken.append((ip, r["hostname"] or ip, diag))

    log.info("--- SNMP Test Summary ---")
    log.info("Probed:        %d (%s)", total, scope)
    log.info("Working:       %d", ok_now)
    log.info("Newly fixed:   %d", len(newly_fixed))
    log.info("Still broken:  %d", len(still_broken))
    for ip, host in newly_fixed:
        log.info("  fixed:  %s (%s)", ip, host)
    for ip, host, diag in still_broken:
        log.info("  broken: %s (%s) — %s", ip, host, diag[:100])
    if still_broken:
        log.info("Run 'discover' for a full SSH-side SNMP config diff on the "
                 "broken devices.")


# ---------------------------------------------------------------------------
# Device backup helpers
# ---------------------------------------------------------------------------

def parse_devices(filepath):
    """Parse the device list file and return a list of IP addresses.

    Strips inline comments (e.g., '10.1.1.1  # hostname (mac)' -> '10.1.1.1').
    """
    ips = []
    with open(filepath, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            ip = line.split()[0]  # strip inline comments
            ips.append(ip)
    return ips


def parse_failed_file(filepath):
    """Parse failed_devices.txt, returning list of IPs (strips inline comments)."""
    ips = []
    with open(filepath, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            ip = line.split()[0]
            ips.append(ip)
    return ips






# Compiled regex patterns for secret sanitization.
# Each tuple is (pattern, replacement). Covers HP ProCurve/ArubaOS-Switch,
# Cisco IOS/IOS-XE, Palo Alto PAN-OS, and Juniper JunOS.
# Value matching uses (?:"[^"]*"|\S+) to handle both quoted and unquoted values.
_SANITIZE_PATTERNS = [
    # --- HP ProCurve / ArubaOS-Switch ---
    # password manager/operator [user-name "..."] plaintext/sha1/sha256 VALUE
    (re.compile(r'(password\s+(?:manager|operator)\s+(?:user-name\s+"[^"]*"\s+)?(?:plaintext|sha1|sha256)\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # encrypted-password manager/operator [user-name "..."] "VALUE"
    (re.compile(r'(encrypted-password\s+(?:manager|operator)\s+(?:user-name\s+"[^"]*"\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # radius-server [host IP] [encrypted-]key [TYPE] VALUE
    (re.compile(r'(radius-server\s+(?:host\s+\S+\s+)?(?:encrypted-)?key\s+)(?:\d\s+)?(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # tacacs-server [host IP] [encrypted-]key [plaintext|7] VALUE
    (re.compile(r'(tacacs-server\s+(?:host\s+\S+\s+)?(?:encrypted-)?key\s+(?:plaintext\s+|7\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # snmpv3 user NAME auth md5/sha "HASH" priv [aes/des] "HASH"
    (re.compile(r'(snmpv3\s+user\s+\S+\s+auth\s+(?:md5|sha)\s+)(?:"[^"]*"|\S+)(\s+priv\s+(?:(?:des|aes)\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***\2***'),
    # snmp-server community VALUE ...
    (re.compile(r'(snmp-server\s+community\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # snmp-server host IP COMMUNITY ...
    (re.compile(r'(snmp-server\s+host\s+\S+\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # sntp/ntp authentication key-id N key-value VALUE
    (re.compile(r'((?:sntp|ntp)\s+authentication\s+key-id\s+\d+\s+key-value\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # encrypt-credentials pre-shared-key plaintext/hex VALUE
    (re.compile(r'(encrypt-credentials\s+pre-shared-key\s+(?:plaintext|hex)\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # wpa-passphrase / wpa-preshared-key
    (re.compile(r'(wpa-(?:passphrase|preshared-key)\s+(?:(?:ascii|hex)\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),

    # --- ArubaOS-CX ---
    # user NAME [group GRP] password ciphertext/plaintext VALUE
    (re.compile(r'(password\s+(?:ciphertext|plaintext)\s+)\S+', re.IGNORECASE), r'\1***'),
    # radius-server/tacacs-server key ciphertext/plaintext VALUE
    (re.compile(r'((?:radius-server|tacacs-server)\s+.*?key\s+(?:ciphertext|plaintext)\s+)\S+', re.IGNORECASE), r'\1***'),

    # --- Cisco IOS / IOS-XE ---
    # enable secret/password [TYPE] VALUE
    (re.compile(r'(enable\s+(?:secret|password)\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # username NAME [privilege N] secret/password [TYPE] VALUE
    (re.compile(r'(username\s+\S+\s+(?:privilege\s+\d+\s+)?(?:secret|password)\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # crypto isakmp key [TYPE] VALUE
    (re.compile(r'(crypto\s+isakmp\s+key\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # pre-shared-key local/remote [TYPE] VALUE (Cisco IKEv2 keyring)
    (re.compile(r'(pre-shared-key\s+(?:local|remote)\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # pre-shared-key address IP key [TYPE] VALUE (Cisco IKEv1 keyring)
    (re.compile(r'(pre-shared-key\s+address\s+\S+\s+key\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # pre-shared-key ascii-text/hexadecimal VALUE (Juniper)
    (re.compile(r'(pre-shared-key\s+(?:ascii-text|hexadecimal)\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # neighbor IP password [TYPE] VALUE (BGP)
    (re.compile(r'(neighbor\s+\S+\s+password\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # server-private IP key [TYPE] VALUE (RADIUS/TACACS new-style)
    (re.compile(r'(server-private\s+\S+\s+.*?key\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # ntp authentication-key N md5 VALUE
    (re.compile(r'(ntp\s+authentication-key\s+\d+\s+md5\s+)\S+', re.IGNORECASE), r'\1***'),
    # standby N authentication [md5 key-string [TYPE]] VALUE
    (re.compile(r'(standby\s+\d+\s+authentication\s+(?:md5\s+key-string\s+(?:\d\s+)?)?)\S+', re.IGNORECASE), r'\1***'),

    # --- Cisco NX-OS (Nexus) ---
    # snmp-server user NAME ROLE auth md5|sha HASH priv [aes-128|aes-256|des] HASH localizedkey
    (re.compile(r'(snmp-server\s+user\s+\S+\s+\S+\s+auth\s+(?:md5|sha)\s+)\S+(\s+priv\s+(?:(?:aes-128|aes-256|des)\s+)?)\S+', re.IGNORECASE), r'\1***\2***'),
    # authNoPriv variant (auth hash only, no priv)
    (re.compile(r'(snmp-server\s+user\s+\S+\s+\S+\s+auth\s+(?:md5|sha)\s+)\S+', re.IGNORECASE), r'\1***'),

    # --- Shared: Cisco / HP / general ---
    # line password [TYPE] VALUE (skip "password manager/operator" already handled above)
    (re.compile(r'(^\s*password\s+)(?!manager\b|operator\b)(?:\d\s+)?\S+', re.IGNORECASE | re.MULTILINE), r'\1***'),
    # key-string [plaintext|ciphertext|TYPE] VALUE
    (re.compile(r'(key-string\s+(?:(?:plaintext|ciphertext|0|7)\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # key [TYPE] VALUE (inside radius/tacacs server block — Cisco new-style)
    (re.compile(r'(^\s*key\s+)(?:\d\s+)?\S+', re.IGNORECASE | re.MULTILINE), r'\1***'),
    # ip ospf authentication-key [TYPE] VALUE
    (re.compile(r'(ip\s+ospf\s+authentication-key\s+)(?:\d\s+)?(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # ip ospf message-digest-key N md5 [TYPE] VALUE
    (re.compile(r'(ip\s+ospf\s+message-digest-key\s+\d+\s+md5\s+)(?:\d\s+)?(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),

    # --- Palo Alto PAN-OS (XML format) ---
    (re.compile(r'(<(?:phash|password|secret|bind-password|pre-shared-key|private-key|key|auth-key|wmi-password|passphrase|snmp-community-string)>)[^<]+(</)', re.IGNORECASE), r'\1***\2'),
    # PAN-OS set format: keyword VALUE at end of line
    (re.compile(r'((?:phash|bind-password|wmi-password)\s+)\S+', re.IGNORECASE), r'\1***'),

    # --- Juniper JunOS ---
    # Quoted value followed by ## SECRET-DATA
    (re.compile(r'(\S+\s+)"[^"]*"(;\s*##\s*SECRET-DATA)', re.IGNORECASE), r'\1"***"\2'),
    # set format: encrypted-password / secret / auth keys "VALUE"
    (re.compile(r'((?:encrypted-password|secret|authentication-key|authentication-password|privacy-password|simple-password)\s+)"[^"]*"', re.IGNORECASE), r'\1"***"'),
]


# Config lines the DEVICE rewrites on its own (no operator involved).
# Stripped from every capture before storage/comparison so they can't
# churn the nightly backup archive or false-fire the intraday
# config-change alert. The canonical offender is Cisco IOS
# 'ntp clock-period', which IOS recalibrates continuously; IOS re-adds
# the line by itself if a stripped config is ever pasted back.
_VOLATILE_CONFIG_RE = re.compile(r"^ntp clock-period \d+[ \t]*\r?\n", re.M)


def _strip_volatile_lines(config_text):
    """Remove device-rewritten (volatile) lines from a captured config."""
    return _VOLATILE_CONFIG_RE.sub("", config_text)


def sanitize_config(config_text):
    """Remove passwords, keys, and secrets from device config text.

    Preserves usernames and config structure; replaces secret values with ***.
    """
    lines = config_text.splitlines(keepends=True)
    sanitized = []
    for line in lines:
        for pattern, replacement in _SANITIZE_PATTERNS:
            line = pattern.sub(replacement, line)
        sanitized.append(line)
    return "".join(sanitized)


def _detect_platform(child):
    """Detect device platform from prompt format.

    Returns 'fastiron' for Ruckus (SSH@hostname> / telnet@hostname>),
    'junos' for Juniper (user@hostname>), or 'procurve' for HP/default.
    """
    # 15s per attempt + one retry — was 5s with no retry, but under
    # credential-test-phase concurrency a slow device's prompt echo can
    # take >5s. The resulting TIMEOUT leaked out of test_one, leaving
    # platform=None / dev_id={} / is_switch=False and bucketing a real
    # switch as non-switch. Retry handles a single transient stall.
    for attempt in (1, 2):
        try:
            child.sendline("")
            child.expect(PROMPT_RE, timeout=15)
            break
        except pexpect.TIMEOUT:
            if attempt == 2:
                raise
            log.debug("_detect_platform: prompt timed out, retrying once")
    raw = (child.before or "") + (child.after or "")
    clean = _strip_ansi(raw)
    if re.search(r"(?:SSH|telnet)@[\w\-]+[#>]", clean):
        log.debug("  platform=fastiron  prompt=%r", clean.strip()[-60:])
        return "fastiron"
    if "@" in clean and re.search(r"\w+@[\w\-/\.]+[#>]", clean):
        log.debug("  platform=junos  prompt=%r", clean.strip()[-60:])
        return "junos"
    log.debug("  platform=procurve  prompt=%r", clean.strip()[-60:])
    return "procurve"


def _classify_platform(child):
    """Classify platform to the level monitoring needs:
    junos | aruba-cx | fastiron | procurve.

    Extends _detect_platform by running `show version` on non-junos / non-fastiron
    devices — ArubaOS-CX prints "ArubaOS-CX" in its version banner; ProCurve does
    not. Also catches Ruckus FastIron via banner content as a fallback when prompt
    detection missed (older firmwares with non-standard prompt formats).
    """
    base = _detect_platform(child)
    if base in ("junos", "fastiron"):
        return base
    try:
        out = send_command(child, "show version", timeout=10)
    except Exception as e:
        log.debug("  show version failed (%s) — defaulting to procurve", e)
        return "procurve"
    if "ArubaOS-CX" in out:
        return "aruba-cx"
    if "Ruckus" in out or "Brocade Communications" in out or re.search(r"\bICX\d", out):
        return "fastiron"
    if _is_cisco_banner(out):
        return "cisco-ios"
    return "procurve"


def backup_device(ip, password, usernames, sanitize=True, password_list=None,
                  known_username=None, known_password_hash=None,
                  user_passwords=None):
    """Detect, connect, and back up a single device.

    Returns (filename, config, info) where info is a dict with
    hostname, base_mac, serial, firmware, model, proto, username, password_hash.
    """
    child, proto, username, pw_hash, ssh_open, telnet_open = connect_device(
        ip, usernames, password, password_list=password_list,
        known_username=known_username, known_password_hash=known_password_hash,
        user_passwords=user_passwords, reason="backup")
    if child is None:
        raise ConnectionError(f"All connection attempts failed for {ip}")

    try:
        platform = _detect_platform(child)
        hostname = get_hostname(child)

        if platform == "junos":
            config = send_command(child, "show configuration | no-more", timeout=60)
            # Also get set-format config (can be pasted into a replacement device)
            config_set = send_command(child, "show configuration | display set | no-more", timeout=60)
            id_output = ""
            hardware_info = ""
            try:
                ver = send_command(child, "show version | no-more", timeout=15)
                hw = send_command(child, "show chassis hardware | no-more", timeout=15)
                mac_out = send_command(child, "show chassis mac-addresses | no-more", timeout=15)
                id_output = ver + "\n" + hw + "\n" + mac_out
                hardware_info = hw
            except Exception:
                pass
        elif platform == "fastiron":
            config = send_command(child, "show running-config", timeout=60)
            config_set = None
            id_output = ""
            hardware_info = ""
            try:
                ver_out = send_command(child, "show version", timeout=30)
                id_output = ver_out
                hardware_info = ver_out
            except Exception:
                pass
        else:
            config = send_command(child, "show running-config", timeout=60)
            config_set = None
            id_output = ""
            hardware_info = ""
            try:
                sys_out = send_command(child, "show system", timeout=10)
                ver_out = send_command(child, "show version", timeout=10)
                id_output = sys_out + "\n" + ver_out
                hardware_info = sys_out
                if _is_cisco_banner(ver_out):
                    platform = "cisco-ios"
                else:
                    # See test_one — show stacking + show modules cover
                    # both stack and standalone Aruba 2920/2930. Safe
                    # on non-applicable devices.
                    try:
                        stack_out = send_command(child, "show stacking",
                                                 timeout=10)
                        id_output = id_output + "\n" + stack_out
                    except Exception:
                        pass
                    try:
                        mod_out = send_command(child, "show modules",
                                               timeout=10)
                        id_output = id_output + "\n" + mod_out
                    except Exception:
                        pass
            except Exception:
                pass

        dev_id = get_device_id(id_output)
        if not looks_like_switch(platform, dev_id):
            # Belt-and-suspenders: run_backup already filters status="active"
            # so a non-switch row shouldn't reach here, but if the operator
            # manually invokes backup against a misclassified device (or an
            # iLO/BMC slipped in before reclassification), refuse cleanly
            # rather than store garbage.
            raise NotASwitchError(
                f"{ip}: target is not a recognized switch — refusing backup")
    finally:
        disconnect(child)

    # Use None (not "") so upsert_device preserves existing values if
    # identification came up empty for any field.
    base_mac = dev_id.get("base_mac") or None
    serial = dev_id.get("serial") or None
    firmware = dev_id.get("firmware") or None
    model = dev_id.get("model") or None

    # Refuse a capture taken over a PAGED session: the platform's
    # pager-disable command was rejected/unsupported (e.g. some FastIron
    # firmwares reject skip-page-display at the user prompt), so
    # send_command had to page through — and FastIron's line-redraw pager
    # merges lines and drops characters. The _netops_paged flag is the
    # root-cause signal (set the instant any pager prompt is consumed);
    # the string grep is a belt-and-suspenders check for residue that
    # survived scrubbing. Either way: discard, recorded as a per-device
    # failure, never archived or emailed as a config change.
    if getattr(child, "_netops_paged", False) or re.search(
            r"next page: Space, next line: |--More--|-- MORE --", config):
        raise RuntimeError(
            f"{ip}: capture taken over a paged session (pager not "
            f"disabled) — output unreliable, discarded")

    config = _strip_volatile_lines(config)
    if sanitize:
        config = sanitize_config(config)
        if config_set:
            config_set = sanitize_config(config_set)

    if hostname:
        filename = f"{hostname}_{ip}.cfg"
    else:
        filename = f"{ip}.cfg"

    info = {"hostname": hostname, "base_mac": base_mac, "serial": serial,
            "firmware": firmware, "model": model,
            "hardware_info": hardware_info, "proto": proto,
            "username": username, "password_hash": pw_hash,
            "ssh_open": ssh_open, "telnet_open": telnet_open,
            "config_set": config_set}
    # DEBUG-level so it doesn't collide with the \r progress bar during
    # multi-threaded backup runs. Still captured by --debug-file.
    log.debug("  detected: user=%s, proto=%s, hostname=%s, mac=%s",
              username, proto, hostname or "unknown", base_mac or "unknown")
    return filename, config, info


# ---------------------------------------------------------------------------
# Storage modes
# ---------------------------------------------------------------------------

def save_config_local(filename, config_text):
    """Save config using local-history mode (current/ + history/ when changed).

    Returns True if the config changed (or is new), False if identical.
    """
    current_dir = os.path.join(CONFIGS_DIR, "current")
    history_dir = os.path.join(CONFIGS_DIR, "history")
    os.makedirs(current_dir, exist_ok=True)

    current_path = os.path.join(current_dir, filename)
    stem = os.path.splitext(filename)[0]

    if os.path.exists(current_path):
        with open(current_path, encoding="utf-8") as f:
            old_config = f.read()

        if old_config == config_text:
            return False

        # Config changed — archive the old version
        device_history_dir = os.path.join(history_dir, stem)
        os.makedirs(device_history_dir, exist_ok=True)
        timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        archive_path = os.path.join(device_history_dir, f"{timestamp}.cfg")
        shutil.copy2(current_path, archive_path)
        log.info("  archived previous config to %s", archive_path)

    # Write new/updated config
    with open(current_path, "w", encoding="utf-8") as f:
        f.write(config_text)

    return True


def git_commit_changes(changed_labels, remote_url="", branch="main"):
    """Stage configs/ and commit if there are changes.

    When remote_url is set, also push to that remote on branch. Push
    failures are logged as warnings but don't raise — the local commit is
    authoritative and the next backup will retry the push.
    """
    # Setting storage_mode=git is an explicit opt-in to track configs/ —
    # if .gitignore happens to list it (auto-generated boilerplate, copied
    # template, etc.), 'git add' would refuse silently. Use -f so the
    # configured intent wins; warn if the .gitignore entry exists so the
    # operator knows what's happening.
    ignored = subprocess.run(
        ["git", "check-ignore", "-q", "configs/"],
        cwd=STATE_DIR, capture_output=True,
    )
    if ignored.returncode == 0:
        log.warning("configs/ is excluded by .gitignore but storage_mode=git "
                    "is set — using 'git add -f' to override. Remove configs/ "
                    "from .gitignore (or set storage_mode=local) to clean up.")
    subprocess.run(["git", "add", "-f", "configs/"], cwd=STATE_DIR, check=True)

    result = subprocess.run(
        ["git", "diff", "--cached", "--quiet"],
        cwd=STATE_DIR,
    )
    if result.returncode == 0:
        log.info("No config changes detected — skipping git commit.")
        return

    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    message = f"Config backup {timestamp}\n\nChanged devices:\n"
    for label in changed_labels:
        message += f"  - {label}\n"

    subprocess.run(["git", "commit", "-m", message], cwd=STATE_DIR, check=True)
    log.info("Git commit created.")

    if remote_url:
        _git_push_remote(remote_url, branch)


def _git_push_remote(remote_url, branch):
    """Ensure 'origin' points at remote_url and push the branch.

    Idempotent: adds origin if missing, updates it if the URL changed, and
    always uses -u to keep upstream tracking current. Non-fatal on failure.
    """
    try:
        existing = subprocess.run(
            ["git", "remote", "get-url", "origin"],
            cwd=STATE_DIR, capture_output=True, text=True)
        if existing.returncode != 0:
            # No 'origin' yet — add it.
            subprocess.run(["git", "remote", "add", "origin", remote_url],
                           cwd=STATE_DIR, check=True)
            log.info("Added git remote 'origin' -> %s", remote_url)
        elif existing.stdout.strip() != remote_url:
            subprocess.run(["git", "remote", "set-url", "origin", remote_url],
                           cwd=STATE_DIR, check=True)
            log.info("Updated git remote 'origin' -> %s", remote_url)
        push = subprocess.run(
            ["git", "push", "-u", "origin", branch],
            cwd=STATE_DIR, capture_output=True, text=True)
        if push.returncode == 0:
            log.info("Pushed backup commit to origin/%s", branch)
        else:
            log.warning(
                "Git push to origin/%s failed (exit %d): %s — local commit "
                "kept; next backup will retry.",
                branch, push.returncode,
                (push.stderr or push.stdout).strip())
    except Exception as e:
        log.warning("Git push skipped: %s", e)


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

# Shared help content used by both the direct-CLI path (on missing/invalid
# args, via _DeviceHelpParser below) and the console REPL (via '<cmd> ?' and
# 'help <cmd>'). Same text in both places.

COMMAND_SUMMARIES = [
    ("backup",          "Back up configs for all active devices"),
    ("discover",        "Scan subnets + test credentials"),
    ("scan",            "Port-scan only (no credential test)"),
    ("test",            "Test credentials (test ssh|snmp [retest])"),
    ("probe",           "TCP reachability check (no login)"),
    ("show",            "Show a category listing or device detail by IP"),
    ("add",             "Add device IP(s) to the DB"),
    ("remove",          "Remove device IP(s) from the DB"),
    ("mark",            "Tag a device (mark non-switch|unknown|switch <ip>)"),
    ("import",          "Import devices from a file (list or CSV)"),
    ("export",          "Export device lists from the DB to flat files"),
    ("monitor",         "Poll device state (monitor stp | flap | topology | config)"),
    ("digest",          "Email a digest (digest health | flap | stp | backup)"),
    ("clear",           "Clear stored counters (clear flap)"),
    ("silence",         "Mute alert emails for device(s), max 30 days "
                        "(silence <dev>|name <pat>|all <duration>)"),
    ("ignore",          "Permanent per-port exclusions "
                        "(ignore flap|flux <ip> <if>)"),
    ("reconcile",       "Reconcile duplicate switch IPs (reconcile duplicates)"),
    ("connect",         "Open a brokered SSH session to a switch via netopsd"),
    ("daemon",          "Run the netopsd broker daemon (systemd-managed)"),
    ("investigate",     "Gather context for an STP change (investigate stp <ip> <port>)"),
    ("refresh-dns",     "Reverse-DNS all devices in the DB"),
    ("test-email",      "Send a test email (test-email all | <address>)"),
    ("configure",       "Change settings (configure email add|remove|list)"),
    ("check-config",    "Show system limits and effective config"),
    ("manual",          "Print the full user manual to stdout (pipe to a pager)"),
]

# Commands that exist in the CLI but are hidden from the console.
# Rationale: the console runs over SSH with no scp/sftp/upload surface,
# so a subcommand whose only useful invocation requires placing a file
# on the server first (today: `import <path>`) is non-functional from
# the console — exposing it only creates oracle-shaped paths (rejected-
# line warnings echoing file content; pre-3.8.31). `daemon` is the
# systemd-managed broker process itself — never run by hand. CLI
# invocations (real shell) keep these subcommands.
# NOTE: `connect` is intentionally NOT hidden — it's the credential-
# isolated way to reach a switch, and it's gated by the [connect] ACL
# (group-based, SO_PEERCRED-verified), so console users who aren't in an
# authorized group are denied by the daemon even though they can see it.
# Filtered into NetopsShell._command_summaries.
CONSOLE_HIDDEN_COMMANDS = {"import", "daemon"}

MONITOR_TARGETS = [
    ("stp",  "Poll STP + port state; confirm STP changes; email alerts"),
    ("flap", "Poll port state only; count flaps; no STP analysis"),
    ("config", "Pull every config; archive + alert email on any change"),
]

SINGLE_FLAP_TARGETS = [("flap", "Port flap counters")]

HELP_DETAILS = {
    "backup": [
        "Usage: backup [-s SUBNET] [-u USER] [-p PW] [--no-sanitize]",
        "  Connects to every active device and saves running config.",
        "  -s SUBNET       override subnet(s) to scan (comma-separated)",
        "  -u USER         override username(s)",
        "  -p PW           override default password",
        "  --no-sanitize   save raw configs without redacting secrets",
    ],
    "discover": [
        "Usage: discover [-s SUBNET] [-u USER] [-p PW] [--dry-run]",
        "  Scan subnets for open 22/23 and test credentials.",
        "  --dry-run       show results without writing to DB",
    ],
    "scan": [
        "Usage: scan [-s SUBNET] [--dry-run]",
        "  Port-scan only. Writes responsive IPs + open-port flags to DB.",
    ],
    "test": [
        "Usage: test <ssh|snmp> [retest] [-u USER] [-p PW]",
        "  Re-check credentials against devices already in the DB. No backup,",
        "  no subnet scan.",
        "  test ssh           SSH login auth on every device (report only)",
        "  test ssh retest    only failed/inactive devices; promote passers",
        "  test snmp          SNMP community/v3 on every active device",
        "  test snmp retest   only the 'no-snmp' set (SNMP can't poll)",
        "  -u USER / -p PW    override login credentials (ssh only)",
    ],
    "probe": [
        "Usage: probe [--status {active,inactive,failed,all}] [--csv FILE]",
        "       [--timeout SEC] [--threads N]",
        "  TCP reachability check for devices already in the DB. No login",
        "  attempts — just checks if ports 22/23 respond. Updates ssh_open/",
        "  telnet_open columns; does NOT change status.",
        "  Default filter: all (except duplicates).",
    ],
    "connect": [
        "Usage: connect <IP_OR_HOSTNAME> [--check]",
        "  Open a password-free SSH session to a switch, brokered by netopsd.",
        "  The credential never leaves the daemon — you get the switch prompt,",
        "  not the password. Log out of the device normally to end the session.",
        "  An exact IP connects to THAT IP (reach a stack via a working",
        "  secondary IP when the primary SVI is down); a hostname -> primary.",
        "  --check   test authorization only (don't open a session)",
        "  Access is governed by the [connect] ACL in netops.conf (group-based).",
        "  'DENIED: not authorized' means your user/group has no matching rule;",
        "  ask an admin to add one. Requires netopsd to be running.",
    ],
    "show": [
        "Usage: show <TARGET> [--csv FILE]",
        "  show spanning-tree [blocked|disabled|mismatch|root]   (alias: stp)",
        "      bare 'show spanning-tree' prints all four states",
        "  show devices switch [active|all|failed|inactive]",
        "  show devices firewall                  list active firewalls",
        "  --- device-facet tree: <sel> = an IP, an exact hostname, or",
        "      ip|name|model|serial <pattern>; matches dedupe to physical",
        "      chassis (duplicate IPs resolve to their primary) ---",
        "  show devices <sel>                     summary of matches",
        "  show devices <sel> detail              full per-device detail",
        "  show devices <sel> config [set]        full latest saved config",
        "                                         (one device; set = Junos",
        "                                         set-format)",
        "  show devices <sel> config list         config change timeline,",
        "                                         Junos-rollback numbered",
        "                                         (0 = current)",
        "  show devices <sel> config diff [#|WHEN]  unified diff of one change",
        "  show devices <sel> config compare A B  cumulative diff between any",
        "                                         two versions (#, date, or",
        "                                         'current')",
        "  show devices <sel> lldp neighbors      LLDP links, both directions",
        "  show devices <sel> downstream          every switch whose STP path",
        "                                         to the root runs through",
        "                                         this one (one device)",
        "  show devices <sel> interface <port>    all recorded state for one",
        "                                         port: LLDP neighbor, STP,",
        "                                         flaps, errors, learned MACs",
        "  show backups | duplicates | telnet | stack-members",
        "  show port-flaps | mac-flux",
        "  show silences                          active alert-email mutes",
        "  show slots                             live SSH session slots from",
        "                                         the netopsd broker (cap,",
        "                                         active grants, waiters)",
        "  show flap-history [--ip IP] [--interface IF] [--since D] [--until D]",
        "                                         append-only per-flap timeline",
        "                                         (survives the weekly clear-flap)",
        "  show port-errors [--ip IP] [--interface IF] [--type KIND]",
        "                                         interface error/discard/CRC",
        "                                         history per port + error type",
        "  show mac <addr|prefix>                 locate a MAC in the harvested",
        "                                         FDB w/ per-port flap+error counts",
        "  show lldp <name|mac|ip|port-desc>      reverse LLDP search: which",
        "                                         switch/port a neighbor (AP,",
        "                                         phone, server) connects to",
        "  show no-snmp                           active devices SNMP can't",
        "                                         poll (fall back to SSH)",
        "  show lldp-coverage                     map coverage + why devices",
        "                                         are missing (alias: map-coverage)",
    ],
    "silence": [
        "Usage: silence <ip|hostname> <dur> [reason...] | silence name",
        "       <pattern> <dur> [reason...] | silence all <dur> [reason...]",
        "       | silence list | silence delete <id>|all",
        "  Mute alert EMAILS (STP changes, unreachable/recovery, MAC-flux,",
        "  config-change) for one device, a hostname pattern, or everything.",
        "  dur: Nd/Nh/Nm, HARD-CAPPED at 30d — a silence is never permanent.",
        "  Monitoring, backups, and digests are unaffected; every add/delete",
        "  is audit-logged. 'show silences' lists what's active.",
    ],
    "ignore": [
        "Usage: ignore flux <ip> <if> [reason...]",
        "       ignore flap <ip> <if> [until DATE|Nd] [reason...]",
        "       ignore list [flap|flux|all] | ignore delete flap|flux <ip> <if>",
        "  Permanent per-port exclusions: flux = never page the MAC-flux",
        "  security signal for an approved port (hub/AP); flap = keep a",
        "  known-noisy port (sleepy printer) out of digest flap. 'until'",
        "  (flap only) makes an entry time-bounded. For temporary,",
        "  device-wide alert muting use 'silence' instead.",
    ],
    "add": [
        "Usage: add <IP>",
        "  Add device IP(s) to the DB. Comma-separated for multiple.",
    ],
    "remove": [
        "Usage: remove <IP_OR_KEYWORD>",
        "  Remove device(s) from the DB.",
        "  Use 'failed' or 'duplicates' as keywords to bulk-remove by status.",
    ],
    "import": [
        "Usage: import [PATH] [--format {auto,list,csv}]",
        "  Import devices from a file. With no PATH, uses device_file from config.",
        "  CSV header columns (case-insensitive, only 'ip' required):",
        "    ip, hostname, dns_name, model, status",
        "  Existing devices: status/fail_reason preserved; other fields merged.",
    ],
    "export": [
        "Usage: export",
        "  Write active devices, failed devices, and telnet report to flat files.",
    ],
    "monitor": [
        "Usage: monitor <stp|flap|config> [--no-email]",
        "  stp   Poll STP + port state across all active devices,",
        "        confirm changes over two polls, email alerts, count",
        "        port flaps. Concurrency: [general] ssh_threads.",
        "  flap  Poll port state only; count flaps. Skips STP commands",
        "        and does not touch STP state or alerts.",
        "  config  Pull the running config from every active device,",
        "        archive any change (feeds 'show devices <dev> config",
        "        list'), and",
        "        email ONE batched alert with a per-device diff. An",
        "        unexpected midday change is a security signal. Runs",
        "        hourly via netops-monitor-config.timer.",
        "  --no-email  poll, record and log as usual but send nothing",
        "        (manual runs/testing). Applies to stp, flap and config.",
    ],
    "digest": [
        "Usage: digest <flap|stp|backup|health>",
        "  flap [--min-count N] [--reset]",
        "    Email a digest of port flap counters accumulated since the",
        "    last reset. By default, counters are preserved — manage",
        "    retention separately with 'clear flap'.",
        "    --min-count N   only report ports with >= N flaps (default 1)",
        "    --reset         also clear all counters after a successful send",
        "  stp [--detail]",
        "    Email a combined report of STP-disabled devices, STP-mode-",
        "    mismatched devices, and the current root-bridge view per",
        "    instance (flagged when switches disagree). Expected mode is",
        "    configurable via [monitor] stp_expected_mode (default: mstp).",
        "    No email when everything is clean; safe to run on a fixed schedule.",
        "  backup [--detail]",
        "    Weekly status report covering never-backed-up devices, stale",
        "    backups, and config changes in the past 7 days. Stale threshold",
        "    is [monitor] backup_stale_days (default: 7). Always emails when",
        "    there is at least one active device.",
        "    --detail        also list every device in a full inventory table",
        "  health",
        "    Weekly enterprise-health digest: fleet inventory, backup",
        "    success rate, STP/flap activity, reachability alerts fired,",
        "    monitor-stp tick health, SNMP/SSH latency stats by reason,",
        "    netops-box disk/log/db size, and operator action items.",
        "    Also prunes op_events rows older than 35 days.",
    ],
    "clear": [
        "Usage: clear flap [--min-count N] [--dry-run]",
        "  Clear port flap counters. Run on the cadence that matches",
        "  your reporting window (e.g. daily, weekly).",
        "  --min-count N   only clear rows with >= N flaps (default 0 = all)",
        "  --dry-run       show what would be cleared without deleting",
    ],
    "refresh-dns": [
        "Usage: refresh-dns",
        "  Reverse-DNS every device in the DB and update dns_name.",
    ],
    "test-email": [
        "Usage: test-email all | <address>",
        "  Send a canned test message to verify [email] delivery.",
        "  'all' sends to every configured recipient; or name one",
        "  configured address to test just that one.",
    ],
    "configure": [
        "Usage: configure email add <address>",
        "       configure email remove <address>",
        "       configure email list",
        "  Change settings in netops.conf. Currently the [email] 'to'",
        "  recipient list for alert and digest delivery; 'list' prints the",
        "  current recipients. Passwords are NOT configured here — edit",
        "  secrets.conf directly.",
    ],
    "check-config": [
        "Usage: check-config",
        "  Show system limits, thread caps, effective config values.",
    ],
}


def print_root_help():
    """Device-style top-level help — summary line per command."""
    print("Commands:")
    for c, summary in COMMAND_SUMMARIES:
        print(f"  {c:<14} {summary}")
    print("\nType '<command> ?' or '<command> --help' for details on a command.")


def print_command_help(command):
    """Device-style per-command help. Same output for CLI and REPL."""
    details = HELP_DETAILS.get(command)
    if not details:
        print(f"No help available for '{command}'")
        return
    for line in details:
        print(line)
    if command == "show":
        print("  Options:")
        print("    --csv FILE      write to CSV file instead of a table")
        print("  port-flaps filters:")
        print("    --ip IP         show only this device")
        print("    --interface IF  show only this interface (exact match)")
        print("    --min-count N   only rows with >= N flaps (default 1)")


# Subparser class that replaces argparse's usage dump with device-style help
# for commands whose failure mode is "missing required arg" or "bad choice".
# Triggered for the short list of commands below; other subparsers keep the
# default argparse behavior.
_DEVICE_HELP_COMMANDS = {"monitor", "clear", "digest", "show", "configure",
                         "test-email", "add", "remove"}


class _DeviceHelpParser(argparse.ArgumentParser):
    def error(self, message):
        cmd = self.prog.rsplit(" ", 1)[-1] if self.prog else ""
        if cmd in _DEVICE_HELP_COMMANDS:
            # Bad-choice errors get a short '% Invalid target' prefix so the
            # user knows *why* they're looking at help again.
            if "invalid choice" in message:
                match = re.search(r"invalid choice:\s*'([^']+)'", message)
                bad = match.group(1) if match else "?"
                print(f"% Invalid target '{bad}'.")
            print_command_help(cmd)
            self.exit(2)
        super().error(message)


def parse_args():
    # Shared options inherited by every subcommand so they can appear
    # either before or after the subcommand name on the command line.
    # Use argparse.SUPPRESS as default so the subparser doesn't overwrite the
    # parent parser's value when --db (or other shared flags) appear before
    # the subcommand name. This lets `netops --db X foo` and `netops foo --db X`
    # both work.
    _shared = argparse.ArgumentParser(add_help=False)
    _shared.add_argument(
        "--db", type=str, metavar="PATH", default=argparse.SUPPRESS,
        help="Use an alternate database file (created if missing). "
             f"Default: {DB_FILE}",
    )
    _shared.add_argument(
        "--debug", action="store_true", default=argparse.SUPPRESS,
        help="Enable verbose debug logging on console and in the log file",
    )
    _shared.add_argument(
        "--debug-file", nargs="?",
        const=os.path.join(LOG_DIR, "netops-debug.log"),
        default=argparse.SUPPRESS, metavar="PATH",
        help="Write DEBUG logging to a separate file (netops.log unchanged). "
             "With no PATH, defaults to netops-debug.log in the log dir "
             "(/var/log/netops on a packaged install).",
    )
    _shared.add_argument(
        "--log-mode", choices=["append", "truncate"], default=argparse.SUPPRESS,
        help="Log file open mode: 'append' (default) or 'truncate' (fresh each run).",
    )

    parser = argparse.ArgumentParser(
        prog="netops", parents=[_shared],
        description=f"netops v{__version__} — Network device backup, discovery, and monitoring.",
    )
    parser.add_argument(
        "--version", action="version",
        version=f"%(prog)s {__version__}",
    )

    sub = parser.add_subparsers(dest="command", metavar="COMMAND",
                                parser_class=_DeviceHelpParser)

    # --- Credential-consumer helpers (shared arg shape) ---
    def _add_cred_overrides(p):
        p.add_argument("-u", "--username", type=str,
                       help="Override username(s) (comma-separated)")
        p.add_argument("-p", "--password", type=str,
                       help="Override default password")

    def _add_subnet_override(p):
        p.add_argument("-s", "--subnet", type=str,
                       help="Override subnet(s) to scan (comma-separated)")

    # --- Connection / workflow commands ---
    p_backup = sub.add_parser("backup", parents=[_shared],
                              help="Back up configs for all active devices")
    _add_subnet_override(p_backup)
    _add_cred_overrides(p_backup)
    p_backup.add_argument("--no-sanitize", action="store_true",
                          help="Save raw configs without redacting secrets")

    p_discover = sub.add_parser("discover", parents=[_shared],
                                aliases=["discovery"],
                                help="Scan subnets + test credentials, store passing devices")
    _add_subnet_override(p_discover)
    _add_cred_overrides(p_discover)
    p_discover.add_argument("--dry-run", action="store_true",
                            help="Show what would be found/stored without writing")
    p_discover.add_argument("--email", action="store_true",
                            help="Email a diff (new/removed/changed devices) against "
                                 "the pre-run state. Intended for weekly cron; omit "
                                 "when running manually so you don't trigger alerts.")

    p_scan = sub.add_parser("scan", parents=[_shared],
                            help="Port-scan only (no credential test). Stores open-port IPs.")
    _add_subnet_override(p_scan)
    p_scan.add_argument("--dry-run", action="store_true",
                        help="Show results without writing to the DB")

    # `test` is a grouped verb: pick the credential type, optionally add
    # `retest` to re-check only the currently-broken devices.
    #   test ssh            SSH login auth on every device (report)
    #   test ssh retest     only failed/inactive; promote passers to active
    #   test snmp           SNMP community/v3 on every active device
    #   test snmp retest    only the 'no-snmp' set; no subnet scan
    p_test = sub.add_parser("test", parents=[_shared],
                            help="Test credentials (test ssh|snmp [retest])")
    p_test.add_argument("proto", choices=["ssh", "snmp"],
                        help="which credential type to test: ssh (login auth) "
                             "or snmp (community / v3 user)")
    p_test.add_argument("mode", nargs="?", choices=["retest"], default=None,
                        help="retest: re-check only the currently-broken "
                             "devices and promote fixes — SSH re-tests "
                             "failed/inactive, SNMP re-probes the 'no-snmp' "
                             "set (no subnet scan)")
    _add_cred_overrides(p_test)

    p_probe = sub.add_parser("probe", parents=[_shared],
                             help="TCP reachability check for devices already in the DB")
    p_probe.add_argument(
        "--status", choices=["active", "inactive", "failed", "duplicate", "all"],
        default="all",
        help="Filter which devices to probe (default: all except duplicates)",
    )
    p_probe.add_argument(
        "--csv", type=str, metavar="FILE",
        help="Write reachability results to a CSV file",
    )
    p_probe.add_argument(
        "--timeout", type=float, default=2.0,
        help="Per-port TCP connect timeout in seconds (default 2)",
    )
    p_probe.add_argument(
        "--threads", type=int, default=100,
        help="Max concurrent probes (default 100)",
    )

    p_reconcile = sub.add_parser("reconcile", parents=[_shared],
                                 help="Reconcile duplicate switch IPs (reconcile duplicates)")
    p_reconcile.add_argument("target", choices=["duplicates"])
    p_reconcile.add_argument(
        "--no-probe", action="store_true",
        help="Skip the TCP reachability probe; reclassify from stored state only",
    )

    # --- DB read-only queries ---
    # `show` is overloaded: a category name lists that category,
    # a bare IPv4 address shows full device detail.
    p_show = sub.add_parser("show", parents=[_shared],
                            help="Show a grouped listing, a category, or device detail by IP")
    p_show.add_argument("arg", nargs="+", metavar="TARGET",
                        help="e.g. 'spanning-tree blocked', 'devices switch active', "
                             "'devices name core', 'backups', "
                             "'devices <dev> config list', "
                             "'devices <dev> downstream'")
    p_show.add_argument(
        "--csv", type=str, metavar="FILE",
        help="Write the result to a CSV file instead of printing a table",
    )
    p_show.add_argument(
        "--ip", type=str, default=None,
        help="Filter port-flaps / flap-history to a specific device IP",
    )
    p_show.add_argument(
        "--interface", type=str, default=None,
        help="Filter port-flaps / flap-history to a specific interface (exact match)",
    )
    p_show.add_argument(
        "--min-count", type=int, default=1,
        help="port-flaps: rows with at least N flaps; port-errors: at least "
             "N total errors (default 1)",
    )
    p_show.add_argument(
        "--type", type=str, default=None, dest="error_type", metavar="KIND",
        help="port-errors: filter to one error_type (in_errors, out_errors, "
             "in_discards, out_discards, fcs_errors)",
    )
    p_show.add_argument(
        "--since", type=str, default=None, metavar="DATE",
        help="flap-history: only events at/after DATE (YYYY-MM-DD[ HH:MM:SS])",
    )
    p_show.add_argument(
        "--until", type=str, default=None, metavar="DATE",
        help="flap-history: only events at/before DATE (a bare date = end of that day)",
    )

    # --- DB mutations ---
    sub.add_parser("add", parents=[_shared],
                   help="Add device IP(s) to the DB").add_argument(
        "ip", metavar="IP",
        help="IP address (comma-separated for multiple)",
    )
    sub.add_parser("remove", parents=[_shared],
                   help="Remove device IP(s) from the DB").add_argument(
        "ip", metavar="IP_OR_KEYWORD",
        help="IP (comma-separated), or 'failed'/'duplicates' to bulk-remove by status",
    )
    p_import = sub.add_parser("import", parents=[_shared],
                              help="Import devices from a file into the DB")
    p_import.add_argument(
        "path", nargs="?", default=None,
        help="Path to import (plain list or CSV). "
             "Default: the configured device_file (devices.txt).",
    )
    p_import.add_argument(
        "--format", choices=["auto", "list", "csv"], default="auto",
        help="File format (default: auto — detect by extension/header)",
    )
    p_export = sub.add_parser("export", parents=[_shared],
        help="Export from the DB (export [devices] | export topology)")
    p_export.add_argument("target", nargs="?",
        choices=["devices", "topology"], default="devices",
        help="devices = flat device lists (default); "
             "topology = network map of the LLDP switch-to-switch backbone")
    p_export.add_argument("--format", dest="format",
        choices=["drawio"], default="drawio",
        help="(topology) output format — drawio = draw.io/mxGraph XML, "
             "imports into Lucidchart and diagrams.net (default: drawio)")
    p_export.add_argument("-o", "--output", default=None,
        help="(topology) output file path, or '-' for stdout "
             "(default: topology-<host>-<date>.drawio in the export dir)")
    p_export.add_argument("--max-age-hours", type=int, default=48,
        help="(topology) drop LLDP edges whose last_seen is older than N "
             "hours so stale cabling doesn't linger (default: 48)")
    p_export.add_argument("--include-unknown", action="store_true",
        help="(topology) add dashed ghost nodes for unmanaged LLDP "
             "infrastructure neighbors — APs, undiscovered switches, "
             "routers — typed from their LLDP capabilities (default: off)")
    p_export.add_argument("--include-endpoints", action="store_true",
        help="(topology) also include endpoint neighbors (phones, hosts); "
             "noisy on phone-heavy fleets, so off by default")
    p_export.add_argument("--expand-neighbors", action="store_true",
        help="(topology) draw each unmanaged AP/phone/host as its own node "
             "instead of collapsing them into a per-switch 'N APs' badge")
    sub.add_parser("wipe-db", parents=[_shared],
                   help="Delete the active DB (prompts for confirmation)")

    # --- Monitoring (device-style grouped verbs) ---
    p_monitor = sub.add_parser("monitor", parents=[_shared],
                               help="Poll device state (monitor stp | flap | topology | config)")
    p_monitor.add_argument("target", choices=["stp", "flap", "topology", "config"],
                           help="stp = STP + port state + flap counters; "
                                "flap = port state + flap counters only; "
                                "topology = fleet-wide LLDP scrape into topology_edges; "
                                "config = intraday config-change watch + alert email")
    p_monitor.add_argument("--detail", action="store_true",
                           help="(stp only) also log the majority/agreeing switches "
                                "in a root-bridge disagreement (default: suppressed)")
    p_monitor.add_argument("--no-email", action="store_true",
                           help="poll, record and log as usual but send no alert "
                                "email — for manual runs and testing")
    _add_cred_overrides(p_monitor)

    p_digest = sub.add_parser("digest", parents=[_shared],
                              help="Email a digest (digest flap | digest stp | digest backup)")
    p_digest.add_argument("target", choices=["flap", "stp", "backup", "health"])
    p_digest.add_argument("--min-count", type=int, default=1,
                          help="(flap only) only include ports with >= N flaps (default 1)")
    p_digest.add_argument("--reset", action="store_true",
                          help="(flap only) also clear counters after a successful send")
    p_digest.add_argument("--detail", action="store_true",
                          help="(stp/backup) also list agreeing/healthy devices in full "
                               "(default: collapsed)")

    p_clear = sub.add_parser("clear", parents=[_shared],
                             help="Clear stored counters (clear flap)")
    p_clear.add_argument("target", choices=["flap"])
    p_clear.add_argument("--min-count", type=int, default=0,
                         help="Only clear ports with at least N flaps (default 0 = all)")
    p_clear.add_argument("--dry-run", action="store_true",
                         help="Show what would be cleared without deleting")

    p_investigate = sub.add_parser("investigate", parents=[_shared],
                                   help="Gather context for an STP change (investigate stp <ip> <port>)")
    p_investigate.add_argument("target", choices=["stp"])
    p_investigate.add_argument("ip", help="Device IP address")
    p_investigate.add_argument("port", help="Interface name (e.g. ge-1/0/11.0, 1/1/24, lag4)")
    _add_cred_overrides(p_investigate)

    # netopsd broker. `daemon` is systemd-managed (CLI-only, hidden from the
    # console — it's the long-running process, never run by hand). `connect` is
    # the client and IS available in the console: it grants live switch access,
    # but that access is gated by the [connect] ACL (group-based, SO_PEERCRED-
    # verified), so an unauthorized console user is denied by the daemon.
    sub.add_parser("daemon", parents=[_shared],
                   help="Run the netopsd broker daemon (systemd-managed)")
    p_connect = sub.add_parser("connect", parents=[_shared],
                               help="Open a brokered SSH session to a switch via netopsd")
    p_connect.add_argument("device", help="Device IP or hostname")
    p_connect.add_argument("--check", action="store_true",
                           help="Only check authorization; do not open a session")

    p_ignore = sub.add_parser("ignore", parents=[_shared],
        help="Permanent per-port exclusions "
             "(ignore flap|flux <ip> <if> | ignore list | ignore delete ...)")
    p_ignore.add_argument("words", nargs="+", metavar="ARGS",
        help="flux <ip> <if> [reason...] — never page MAC-flux for this port; "
             "flap <ip> <if> [until DATE|Nd] [reason...] — keep a known-noisy "
             "port out of digest flap; "
             "list [flap|flux|all]; delete flap|flux <ip> <if>")

    p_silence = sub.add_parser("silence", parents=[_shared],
        help="Mute alert emails for device(s), max 30 days "
             "(silence <dev>|name <pat>|all <duration>)")
    p_silence.add_argument("words", nargs="+", metavar="ARGS",
        help="<ip|hostname> <Nd|Nh|Nm> [reason...]; name <pattern> <dur> "
             "[reason...]; all <dur> [reason...]; list; delete <id>|all. "
             "Hard 30-day cap — a silence is never permanent. Only alert "
             "EMAILS are muted; monitoring and digests are unaffected.")

    p_mark = sub.add_parser("mark", parents=[_shared],
        help="Manually tag a device "
             "(mark non-switch|unknown|switch|firewall <ip>)")
    p_mark.add_argument("target",
        choices=["non-switch", "unknown", "switch", "firewall"],
        help="non-switch = identified as not a switch (server BMC etc.) "
             "— exclude from backup/monitor/digest; "
             "unknown = port-open device whose credentials never worked "
             "(Linux box / SAN / VMware / unknown vendor); "
             "switch = set role=switch (and revert non-switch/unknown "
             "tag back to active); "
             "firewall = set role=firewall (reachability- and "
             "flap-monitored and backed up, but skipped by STP analysis).")
    p_mark.add_argument("ip", help="Device IP address")

    # --- Introspection / setup ---
    sub.add_parser("check-config", parents=[_shared],
                   help="Show system limits, config values, and effective settings")
    sub.add_parser("manual", parents=[_shared],
                   help="Print the bundled user manual to stdout "
                        "(e.g. 'netops manual | less'); works without man-db")
    p_configure = sub.add_parser("configure", parents=[_shared],
                                 help="Change settings (configure email add|remove|list)")
    p_configure.add_argument("tokens", nargs="+", metavar="...",
                             help="e.g. 'email add ops@example.com', "
                                  "'email remove old@example.com', 'email list'")
    p_testemail = sub.add_parser("test-email", parents=[_shared],
                                 help="Send a test email (test-email all | <address>)")
    p_testemail.add_argument("target", metavar="all|ADDRESS",
                             help="'all' for every configured recipient, or "
                                  "one configured email address")
    sub.add_parser("console", parents=[_shared],
                   help="Drop into an interactive netops shell")
    sub.add_parser("refresh-dns", parents=[_shared],
                   help="Reverse-DNS all devices in the DB and update dns_name")

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(0)

    args = parser.parse_args()
    if args.command is None:
        parser.print_help()
        sys.exit(0)
    return args


# ---------------------------------------------------------------------------
# CLI query and export handlers
# ---------------------------------------------------------------------------

_COLUMN_MAX_WIDTH = {"model": 20, "dns_name": 35}


def _print_table(headers, rows):
    """Print aligned columns from a list of header names and row dicts/tuples."""
    if not rows:
        return
    # Calculate column widths
    widths = [len(h) for h in headers]
    str_rows = []
    for row in rows:
        # Preserve 0 / False as their string form; only None/"" render blank.
        cells = [("" if row[h] is None else str(row[h])) for h in headers]
        for i, h in enumerate(headers):
            cap = _COLUMN_MAX_WIDTH.get(h)
            if cap and len(cells[i]) > cap:
                cells[i] = cells[i][:cap - 1] + "…"
        str_rows.append(cells)
        for i, cell in enumerate(cells):
            widths[i] = max(widths[i], len(cell))

    # Header
    header_line = "  ".join(h.upper().ljust(widths[i]) for i, h in enumerate(headers))
    print(header_line)
    print("  ".join("-" * w for w in widths))
    for cells in str_rows:
        print("  ".join(cells[i].ljust(widths[i]) for i in range(len(headers))))


def _write_csv(path, headers, rows):
    """Write rows to a CSV file. Uses full (untruncated) values."""
    import csv
    path = os.path.abspath(os.path.expanduser(path))
    with open(path, "w", encoding="utf-8", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(headers)
        for row in rows:
            writer.writerow([("" if row[h] is None else str(row[h])) for h in headers])
    print(f"Wrote {len(rows)} row(s) to {path}")


def _output(headers, rows, csv_path=None):
    """Dispatcher: write CSV if csv_path set, else print table."""
    if csv_path:
        _write_csv(csv_path, headers, rows)
    else:
        _print_table(headers, rows)


# --- `show` command grammar -------------------------------------------
# Junos-style hierarchical listing. Single source of truth for dispatch,
# help, and REPL tab-completion. The leaf values map to the internal
# category keys that handle_list() dispatches on.
_SHOW_STP_STATES = ("blocked", "disabled", "mismatch", "root")
_SHOW_DEVICE_STATUSES = ("active", "all", "failed", "inactive")
_SHOW_FLAT = ("backups", "duplicates", "telnet", "stack-members",
              "port-flaps", "flap-history", "port-errors", "mac-flux",
              "no-snmp", "silences", "slots")
_SHOW_COVERAGE = ("lldp-coverage", "map-coverage")
_SHOW_TOP = ("spanning-tree", "stp", "devices", "lldp",
             "mac") + _SHOW_FLAT + _SHOW_COVERAGE

_SHOW_STP_CATEGORY = {"blocked": "stp-blocked", "disabled": "stp-disabled",
                      "mismatch": "stp-mismatch", "root": "stp-root"}
_SHOW_DEVICE_CATEGORY = {"active": "devices", "all": "all",
                         "failed": "failed", "inactive": "inactive"}


def _resolve_show(words):
    """Resolve a `show` TARGET word-list to a (kind, value) dispatch action.

      ("category", <internal handle_list category>)
      ("stp-summary", None)                  -- bare `show spanning-tree`
      ("filter", (field, pattern, detail))   -- show devices ip|name <pat>
      ("error", <message to print>)
    """
    head, rest = words[0], words[1:]

    if head in ("spanning-tree", "stp"):
        if not rest:
            return ("stp-summary", None)
        if len(rest) == 1 and rest[0] in _SHOW_STP_CATEGORY:
            return ("category", _SHOW_STP_CATEGORY[rest[0]])
        return ("error", "show spanning-tree: expected one of "
                + ", ".join(_SHOW_STP_STATES))

    if head == "devices":
        if not rest:
            return ("category", "devices")
        if rest[0] in ("ip", "name", "model", "serial"):
            field = rest[0]
            if len(rest) < 2:
                return ("error", f"show devices {field}: give a pattern "
                                 f"(optionally followed by a facet — detail, "
                                 f"config [set|list|diff|compare], "
                                 f"lldp neighbors, downstream)")
            pat, tail = rest[1], rest[2:]
            if not tail:
                return ("filter", (field, pat, False))
            if tail == ["detail"]:
                return ("filter", (field, pat, True))
            facet = _parse_devices_facet(tail)
            if facet[0] == "error":
                return ("error", "show devices: " + facet[1])
            return ("devices-facet", (field, pat, facet))
        # 3.8.28: 'firewall' lists active devices with role='firewall'.
        if rest[0] == "firewall":
            if len(rest) > 1:
                return ("error", "show devices firewall: no further keywords")
            return ("category", "firewalls")
        if rest[0] == "switch":
            status = rest[1:]
            if not status:
                return ("category", "devices")
            if len(status) == 1 and status[0] in _SHOW_DEVICE_CATEGORY:
                return ("category", _SHOW_DEVICE_CATEGORY[status[0]])
            return ("error", "show devices switch: expected one of "
                    + ", ".join(_SHOW_DEVICE_STATUSES))
        # Bare selector: an IP or exact hostname, optionally with a facet
        # (3.19.0 device-facet tree — device-scoped facts live here).
        token, tail = rest[0], rest[1:]
        facet = _parse_devices_facet(tail)
        if facet[0] == "error":
            return ("error", "show devices: " + facet[1])
        return ("devices-facet", ("exact", token, facet))

    if head == "mac":
        if len(rest) != 1:
            return ("error", "show mac: expected one MAC or partial "
                             "(e.g. show mac aa:bb:cc:dd:ee:ff, or show mac aabbcc)")
        return ("mac", rest[0])

    if head == "lldp":
        # Reverse LLDP-neighbor search: locate a neighbor (AP, phone,
        # server, unmanaged switch) by name/chassis/IP/port-desc across
        # every switch's advertised neighbors — the "where does this AP
        # connect?" lookup. `lldp-coverage` (map completeness) is separate.
        if len(rest) != 1:
            return ("error", "show lldp: expected one search term — an AP/"
                             "neighbor name, chassis MAC, IP, or port-desc "
                             "fragment (e.g. show lldp AP-3F-12)")
        return ("lldp-search", rest[0])

    if head in _SHOW_COVERAGE:
        if rest:
            return ("error", f"show {head}: takes no further keywords")
        return ("coverage", None)

    if head in _SHOW_FLAT:
        if not rest:
            return ("category", head)
        if rest == ["all"] and head in ("mac-flux", "port-flaps"):
            return ("category-all", head)
        if head in ("mac-flux", "port-flaps"):
            return ("error", f"show {head}: expected nothing, or 'all'")
        return ("error", f"show {head}: takes no further keywords")

    # Convenience: a bare IP is device detail (`show 10.1.60.2` ==
    # `show devices 10.1.60.2 detail`).
    try:
        ipaddress.IPv4Address(head)
        facet = _parse_devices_facet(rest) if rest else ("detail",)
        if facet[0] != "error":
            return ("devices-facet", ("exact", head, facet))
    except ValueError:
        pass

    return ("error", f"show: unrecognized target '{' '.join(words)}' — "
                      "see 'show' in the help (or 'show ?' in the console).")


def _parse_devices_facet(tail):
    """Facet words after a `show devices` selector -> facet tuple.

    ('summary',) | ('detail',) | ('config',) | ('config-set',) |
    ('config-list',) | ('config-diff', WHEN) | ('config-compare', A, B) |
    ('lldp-neighbors',) | ('downstream',) | ('error', msg)
    """
    if not tail:
        return ("summary",)
    if tail == ["detail"]:
        return ("detail",)
    if tail[0] == "config":
        rest = tail[1:]
        if not rest:
            return ("config",)
        if rest == ["set"]:
            return ("config-set",)
        if rest == ["list"]:
            return ("config-list",)
        if rest[0] == "diff":
            return ("config-diff", " ".join(rest[1:]) or "latest")
        if rest[0] == "compare":
            if len(rest) != 3:
                return ("error", "config compare takes exactly two versions "
                                 "— each a rollback number, a date/timestamp "
                                 "(use _ instead of a space), or 'current'")
            return ("config-compare", rest[1], rest[2])
        return ("error", f"unknown config action {rest[0]!r} "
                          "(expected set, list, diff, or compare)")
    if tail in (["lldp", "neighbors"], ["neighbors"], ["connections"]):
        return ("lldp-neighbors",)
    if tail == ["downstream"]:
        return ("downstream",)
    if tail[0] in ("interface", "port"):
        if len(tail) != 2:
            return ("error", f"{tail[0]} takes one interface name "
                             "(e.g. interface ge-0/0/3)")
        return ("interface", tail[1])
    return ("error", f"unknown device facet '{' '.join(tail)}' — expected "
                      "detail, config [set|list|diff [#|WHEN]|compare A B], "
                      "lldp neighbors, downstream, or interface <port>")


def _complete_show(parts, text):
    """Keyword tab-completion candidates for `show` in the REPL.

    `parts` is the whitespace-split line before the cursor (including the
    leading 'show'); `text` is the fragment being completed. Returns the
    keyword candidates for the current cursor position — readline lists
    them on a double-Tab when more than one matches. Device-name
    completion for the first word is added by the caller.
    """
    if len(parts) <= 1:
        opts = _SHOW_TOP
    elif parts[1] in ("spanning-tree", "stp") and len(parts) == 2:
        opts = _SHOW_STP_STATES
    elif parts[1] == "devices" and len(parts) == 2:
        opts = ("switch", "firewall", "ip", "name", "model", "serial")
    elif parts[1] == "devices" and len(parts) == 3 and parts[2] == "switch":
        opts = _SHOW_DEVICE_STATUSES
    elif parts[1] == "devices" and len(parts) == 3 \
            and parts[2] not in ("ip", "name", "model", "serial"):
        # bare selector — facet keywords come next
        opts = ("detail", "config", "lldp", "neighbors", "downstream", "interface")
    elif (parts[1] == "devices" and len(parts) == 4
          and parts[2] in ("ip", "name", "model", "serial")):
        opts = ("detail", "config", "lldp", "neighbors", "downstream", "interface")
    elif (parts[1] == "devices" and parts[-1] != "config"
          and len(parts) >= 4 and parts[-1] == "lldp"):
        opts = ("neighbors",)
    elif (parts[1] == "devices" and len(parts) >= 4
          and parts[-1] == "config"):
        opts = ("set", "list", "diff", "compare")
    elif len(parts) == 2 and parts[1] in ("mac-flux", "port-flaps"):
        opts = ("all",)
    else:
        opts = ()
    return [o for o in opts if o.startswith(text)]


def handle_list(category, csv_path=None, ip=None, interface=None, min_count=1,
                show_all=False, since=None, until=None, etype=None):
    """Handle `show <category>` queries. If csv_path is set, write CSV instead of a table.

    ip/interface/min_count are only consulted for the 'port-flaps' category;
    ip/interface/since/until for 'flap-history'; all of those plus etype
    (error_type) for 'port-errors'.
    """
    if category == "devices":
        rows = get_devices(status="active")
        if not rows:
            print("No active devices in database.")
            return
        _output(["ip", "hostname", "dns_name", "model", "base_mac", "firmware", "proto", "username", "last_seen"], rows, csv_path)
        print(f"\n{len(rows)} active device(s)")

    elif category == "firewalls":
        rows = [r for r in get_devices(status="active")
                if (r["role"] if "role" in r.keys() else "switch") == "firewall"]
        if not rows:
            print("No active firewalls in database.")
            return
        _output(["ip", "hostname", "dns_name", "model", "firmware",
                 "proto", "username", "last_seen"], rows, csv_path)
        print(f"\n{len(rows)} active firewall(s)")

    elif category == "failed":
        rows = get_devices(status="failed")
        if not rows:
            print("No failed devices in database.")
            return
        _output(["ip", "dns_name", "fail_reason", "last_seen"], rows, csv_path)
        print(f"\n{len(rows)} failed device(s)")

    elif category == "inactive":
        rows = get_devices(status="inactive")
        if not rows:
            print("No inactive devices in database.")
            return
        _output(["ip", "hostname", "base_mac", "fail_reason", "last_seen"], rows, csv_path)
        print(f"\n{len(rows)} inactive device(s)")

    elif category == "backups":
        conn = _db()
        rows = conn.execute("""
            SELECT d.ip, d.hostname, d.base_mac,
                   b.backed_up_at, b.changed, b.config_hash
            FROM devices d
            LEFT JOIN backups b ON d.ip = b.ip
                AND b.id = (SELECT MAX(id) FROM backups WHERE ip = d.ip)
            WHERE d.status = 'active'
        """).fetchall()
        conn.close()
        rows = _sort_by_ip(rows)
        if not rows:
            print("No backup data in database.")
            return
        # Format for display
        display = []
        backed_up = 0
        total_versions = 0
        for r in rows:
            status = "never"
            if r["backed_up_at"]:
                backed_up += 1
                status = "changed" if r["changed"] else "unchanged"
            # Stored versions = the live current/ file plus every archived
            # copy in history/<stem>/ — i.e. what `show devices <dev>
            # config list` can enumerate (rollback 0..N).
            stem = (f"{r['hostname']}_{r['ip']}" if r["hostname"]
                    else r["ip"])
            versions = 0
            if os.path.isfile(os.path.join(CONFIGS_DIR, "current",
                                           stem + ".cfg")):
                versions = 1
            hist_dir = os.path.join(CONFIGS_DIR, "history", stem)
            if os.path.isdir(hist_dir):
                versions += sum(1 for n in os.listdir(hist_dir)
                                if n.endswith(".cfg"))
            total_versions += versions
            display.append({
                "ip": r["ip"],
                "hostname": r["hostname"] or "",
                "base_mac": r["base_mac"] or "",
                "last_backup": r["backed_up_at"] or "never",
                "versions": versions,
                "status": status,
            })
        _output(["ip", "hostname", "base_mac", "last_backup", "versions",
                 "status"], display, csv_path)
        print(f"\n{backed_up}/{len(rows)} device(s) backed up; "
              f"{total_versions} stored version(s) total — "
              f"'show devices <dev> config list' to browse one device's")

    elif category == "duplicates":
        rows = get_duplicates()
        if not rows:
            print("No duplicate IPs detected.")
            return
        # Group by MAC for display
        current_mac = None
        for r in rows:
            if r["base_mac"] != current_mac:
                current_mac = r["base_mac"]
                print(f"\n{r['base_mac']} ({r['hostname'] or 'unknown'}):")
            marker = " *" if r["status"] == "active" else "  "
            print(f"  {marker} {r['ip']}  ({r['status']})")
        dup_count = len(set(r["base_mac"] for r in rows))
        print(f"\n{dup_count} physical switch(es) with multiple IPs")

    elif category == "telnet":
        conn = _db()
        rows = conn.execute("""
            SELECT * FROM devices
            WHERE telnet_open = 1 AND status IN ('active', 'inactive', 'duplicate')
        """).fetchall()
        conn.close()
        rows = _sort_by_ip(rows)
        if not rows:
            print("No devices with telnet open in database.")
            return
        # Show proto and ssh_open so user can see SSH+telnet vs telnet-only
        display = []
        for r in rows:
            display.append({
                "ip": r["ip"],
                "hostname": r["hostname"] or "",
                "base_mac": r["base_mac"] or "",
                "proto": r["proto"] or "",
                "ssh": "yes" if r["ssh_open"] else "no",
                "telnet": "yes" if r["telnet_open"] else "no",
                "status": r["status"],
                "last_seen": r["last_seen"],
            })
        _output(["ip", "hostname", "base_mac", "proto", "ssh", "telnet", "status", "last_seen"], display, csv_path)
        ssh_and_telnet = sum(1 for r in rows if r["ssh_open"])
        telnet_only = len(rows) - ssh_and_telnet
        print(f"\n{len(rows)} device(s) with telnet open: {telnet_only} telnet-only, {ssh_and_telnet} SSH+telnet")

    elif category == "no-snmp":
        # Active devices SNMP can't poll — snmp_enabled is the tri-state set by
        # discover (NULL=never tested, 0=tested/failed, 1=works). These fall
        # back to the SSH collectors for LLDP/topology + ARP, so a gap here
        # explains missing map coverage.
        conn = _db()
        rows = conn.execute("""
            SELECT * FROM devices
            WHERE status = 'active' AND (snmp_enabled IS NULL OR snmp_enabled != 1)
        """).fetchall()
        conn.close()
        rows = _sort_by_ip(rows)
        if not rows:
            print("All active devices have working SNMP.")
            return
        display = []
        for r in rows:
            keys = r.keys()
            se = r["snmp_enabled"] if "snmp_enabled" in keys else None
            state = ("never tested" if se is None
                     else "works" if se == 1 else "failed/off")
            has_creds = bool((r["snmp_community"] if "snmp_community" in keys else None)
                             or (r["snmp_v3_user"] if "snmp_v3_user" in keys else None))
            display.append({
                "ip": r["ip"],
                "hostname": r["hostname"] or "",
                "platform": r["platform"] or "",
                "snmp": state,
                "snmp_creds": "yes" if has_creds else "no",
                "last_seen": r["last_seen"],
            })
        _output(["ip", "hostname", "platform", "snmp", "snmp_creds", "last_seen"],
                display, csv_path)
        never = sum(1 for d in display if d["snmp"] == "never tested")
        failed = sum(1 for d in display if d["snmp"] == "failed/off")
        print(f"\n{len(rows)} active device(s) without working SNMP "
              f"({never} never tested, {failed} failed/off) — LLDP/topology + "
              f"ARP fall back to SSH for these.")

    elif category == "stp-blocked":
        conn = _db()
        # Real STP blocks only: require role=Alternate (ArubaOS-CX and our
        # inferred ProCurve role) or Juniper's BLK+ALT combo. Ignores
        # role=Disabled ports that happen to be in a Blocking state in
        # some MST instance — those are link-down noise.
        rows = conn.execute("""
            SELECT s.ip, d.hostname, s.interface, s.role, s.state, s.last_seen
            FROM stp_state s
            LEFT JOIN devices d ON s.ip = d.ip
            WHERE
              (s.role = 'ALT' AND s.state = 'BLK')                  -- Juniper
              OR (s.role = 'Alternate' AND s.state = 'Blocking')    -- ArubaOS-CX / ProCurve
        """).fetchall()
        conn.close()
        # Sort by IP numerically, then interface
        rows = sorted(rows, key=lambda r: (_ip_sort_key(r["ip"]), r["interface"] or ""))
        if not rows:
            print("No ports currently in STP blocking state.")
            return
        display = [{
            "ip": r["ip"],
            "hostname": r["hostname"] or "",
            "interface": r["interface"],
            "role": r["role"],
            "state": r["state"],
            "last_seen": r["last_seen"],
        } for r in rows]
        _output(["ip", "hostname", "interface", "role", "state", "last_seen"], display, csv_path)
        devices = len({r["ip"] for r in rows})
        print(f"\n{len(rows)} blocked port(s) across {devices} device(s)")

    elif category == "port-flaps":
        conn = _db()
        conn.row_factory = sqlite3.Row
        sql = """
            SELECT ip, hostname, interface, flap_count, first_seen, last_seen
            FROM port_flaps
            WHERE flap_count >= ?
        """
        params = [min_count]
        if ip:
            sql += " AND ip = ?"
            params.append(ip)
        if interface:
            sql += " AND interface = ?"
            params.append(interface)
        sql += " ORDER BY flap_count DESC, hostname, interface"
        rows = conn.execute(sql, params).fetchall()
        # 3.8.10: annotate silenced rows (the user's call: keep them
        # visible so operators stay aware they're still flapping; just
        # not paging anyone). The digest is what filters them out.
        silenced = _get_flap_silenced_ports(conn)
        conn.close()
        if not rows:
            filt = []
            if ip: filt.append(f"ip={ip}")
            if interface: filt.append(f"interface={interface}")
            if min_count > 0: filt.append(f"flap_count>={min_count}")
            suffix = f" (filters: {', '.join(filt)})" if filt else ""
            print(f"No port flap rows in database{suffix}.")
            return
        display = [{
            "ip": r["ip"],
            "hostname": (r["hostname"] or "")
                + (" (silenced)" if (r["ip"], r["interface"]) in silenced
                   else ""),
            "interface": r["interface"],
            "flap_count": r["flap_count"],
            "first_seen": r["first_seen"],
            "last_seen": r["last_seen"],
        } for r in rows]
        _output(["ip", "hostname", "interface", "flap_count",
                 "first_seen", "last_seen"], display, csv_path)
        devices = len({r["ip"] for r in rows})
        total = sum(r["flap_count"] for r in rows)
        silenced_count = sum(1 for r in rows
                             if (r["ip"], r["interface"]) in silenced)
        sil_suffix = (f" ({silenced_count} silenced — filtered from digest)"
                      if silenced_count else "")
        print(f"\n{len(rows)} port(s) across {devices} device(s), "
              f"{total} total flap(s){sil_suffix}")

    elif category == "silences":
        # Active alert-email mutes — same rendering as `silence list`.
        return handle_silence(["list"])

    elif category == "slots":
        # Live SSH session slots from the netopsd broker.
        try:
            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            sock.settimeout(5)
            sock.connect(NETOPSD_SOCK)
            _send_frame(sock, {"op": "slots"})
            resp = _recv_frame(sock)
            sock.close()
        except Exception as e:
            print(f"slot broker unavailable ({e}) — netopsd not running? "
                  f"Jobs are using the legacy whole-job ssh lock.")
            return
        if not (isinstance(resp, dict) and resp.get("ok")):
            print(f"slot broker: "
                  f"{(resp or {}).get('reason', 'no response')}")
            return
        print(f"SSH session slots — cap {resp['cap']}, "
              f"active {resp['active']}, "
              f"bulk allowance {resp['bulk_allowance']}")
        grants = resp.get("grants") or []
        if grants:
            _output(["token", "ip", "cls", "job", "user", "at"],
                    grants, csv_path)
        else:
            print("(no active session slots)")
        waiting = resp.get("waiting") or {}
        if waiting:
            print("waiting: " + ", ".join(f"{k}={v}"
                                          for k, v in waiting.items()))

    elif category == "flap-history":
        # Append-only per-flap timeline (survives the weekly clear-flap). The
        # point-in-time forensics view: filter by --since/--until/--ip/--interface.
        conn = _db()
        conn.row_factory = sqlite3.Row
        sql = "SELECT flapped_at, ip, hostname, interface FROM flap_events WHERE 1=1"
        params = []
        if ip:
            sql += " AND ip = ?"; params.append(ip)
        if interface:
            sql += " AND interface = ?"; params.append(interface)
        if since:
            sql += " AND flapped_at >= ?"; params.append(since)
        if until:
            # A bare date as --until means "through the end of that day".
            params.append(until + " 23:59:59" if len(until) <= 10 else until)
            sql += " AND flapped_at <= ?"
        sql += " ORDER BY flapped_at DESC"
        rows = conn.execute(sql, params).fetchall()
        conn.close()
        if not rows:
            filt = []
            if ip: filt.append(f"ip={ip}")
            if interface: filt.append(f"interface={interface}")
            if since: filt.append(f"since={since}")
            if until: filt.append(f"until={until}")
            suffix = f" (filters: {', '.join(filt)})" if filt else ""
            print(f"No flap events in history{suffix}.")
            return
        display = [{
            "flapped_at": r["flapped_at"],
            "ip": r["ip"],
            "hostname": r["hostname"] or "",
            "interface": r["interface"],
        } for r in rows]
        _output(["flapped_at", "ip", "hostname", "interface"], display, csv_path)
        devices = len({r["ip"] for r in rows})
        print(f"\n{len(rows)} flap event(s) across {devices} device(s) "
              f"(newest first)")

    elif category == "port-errors":
        # Interface error/discard/CRC history, aggregated per (port, type) over
        # the window. The "is this port healthy?" forensic counterpart to
        # port-flaps: --ip/--interface to scope to a host, --type to one error
        # kind, --since/--until to a time range, --min-count for a floor.
        conn = _db()
        conn.row_factory = sqlite3.Row
        sql = ("SELECT pe.ip, COALESCE(d.hostname, pe.ip) AS hostname, "
               "       pe.interface, pe.error_type, "
               "       SUM(pe.delta) AS total, COUNT(*) AS events, "
               "       MIN(pe.at) AS first_seen, MAX(pe.at) AS last_seen "
               "FROM port_errors pe LEFT JOIN devices d ON d.ip = pe.ip "
               "WHERE 1=1")
        params = []
        if ip:
            sql += " AND pe.ip = ?"; params.append(ip)
        if interface:
            sql += " AND pe.interface = ?"; params.append(interface)
        if etype:
            sql += " AND pe.error_type = ?"; params.append(etype)
        if since:
            sql += " AND pe.at >= ?"; params.append(since)
        if until:
            params.append(until + " 23:59:59" if len(until) <= 10 else until)
            sql += " AND pe.at <= ?"
        sql += (" GROUP BY pe.ip, pe.interface, pe.error_type "
                " HAVING SUM(pe.delta) >= ? "
                " ORDER BY total DESC, hostname, pe.interface, pe.error_type")
        params.append(min_count)
        rows = conn.execute(sql, params).fetchall()
        conn.close()
        if not rows:
            filt = []
            if ip: filt.append(f"ip={ip}")
            if interface: filt.append(f"interface={interface}")
            if etype: filt.append(f"type={etype}")
            if since: filt.append(f"since={since}")
            if until: filt.append(f"until={until}")
            if min_count > 1: filt.append(f"min-count={min_count}")
            suffix = f" (filters: {', '.join(filt)})" if filt else ""
            print(f"No port errors in history{suffix}. (Error counters are "
                  "SNMP-harvested by 'monitor topology' — empty on no-SNMP "
                  "sites; or none in the retention window / range.)")
            return
        display = [{
            "ip": r["ip"],
            "hostname": r["hostname"] or "",
            "interface": r["interface"],
            "type": r["error_type"],
            "errors": r["total"],
            "events": r["events"],
            "first_seen": r["first_seen"],
            "last_seen": r["last_seen"],
        } for r in rows]
        _output(["ip", "hostname", "interface", "type", "errors", "events",
                 "first_seen", "last_seen"], display, csv_path)
        devices = len({r["ip"] for r in rows})
        ports = len({(r["ip"], r["interface"]) for r in rows})
        total = sum(r["total"] for r in rows)
        print(f"\n{len(rows)} port/type combo(s) on {ports} port(s) across "
              f"{devices} device(s), {total} total error(s) (worst first)")

    elif category == "stp-disabled":
        conn = _db()
        rows = conn.execute("""
            SELECT ip, hostname, dns_name, model, proto, username, stp_last_check AS last_checked
            FROM devices
            WHERE stp_enabled = 0 AND status IN ('active', 'inactive')
        """).fetchall()
        conn.close()
        rows = _sort_by_ip(rows)
        if not rows:
            print("No devices with STP disabled (run 'monitor stp' first).")
            return
        _output(["ip", "hostname", "dns_name", "model", "proto", "username", "last_checked"], rows, csv_path)
        print(f"\n{len(rows)} device(s) with STP disabled")

    elif category == "stp-mismatch":
        cfg = load_config()
        expected = (cfg.get("stp_expected_mode") or "mstp").lower()
        conn = _db()
        rows = conn.execute("""
            SELECT ip, hostname, dns_name, model, stp_mode, stp_last_check AS last_checked
            FROM devices
            WHERE stp_mode IS NOT NULL
              AND lower(stp_mode) != ?
              AND status IN ('active', 'inactive')
        """, (expected,)).fetchall()
        conn.close()
        rows = _sort_by_ip(rows)
        if not rows:
            print(f"No STP mode mismatches — all devices match '{expected}'.")
            return
        _output(["ip", "hostname", "dns_name", "model", "stp_mode", "last_checked"], rows, csv_path)
        print(f"\n{len(rows)} device(s) not running {expected}")

    elif category == "stp-root":
        conn = _db()
        rows = conn.execute("""
            SELECT s.ip, COALESCE(d.hostname, s.ip) AS hostname,
                   s.instance, s.root_priority, s.root_mac,
                   s.bridge_priority, s.bridge_mac, s.is_root,
                   s.tcn_count, s.last_tcn_seconds, s.updated_at
            FROM stp_root_state s
            LEFT JOIN devices d ON d.ip = s.ip
            ORDER BY s.instance, s.root_priority, s.root_mac, hostname
        """).fetchall()
        conn.close()
        if not rows:
            print("No root-bridge state recorded (run 'monitor stp' first).")
            return
        display = [{
            "ip": r["ip"],
            "hostname": r["hostname"],
            "instance": r["instance"],
            "root_priority": r["root_priority"],
            "root_mac": r["root_mac"],
            "bridge_priority": r["bridge_priority"],
            "bridge_mac": r["bridge_mac"],
            "is_root": "yes" if r["is_root"] else "",
            "tcn_count": r["tcn_count"],
            "last_tcn_s": r["last_tcn_seconds"],
            "updated_at": r["updated_at"],
        } for r in rows]
        _output(["ip", "hostname", "instance", "root_priority", "root_mac",
                 "bridge_priority", "bridge_mac", "is_root",
                 "tcn_count", "last_tcn_s", "updated_at"], display, csv_path)
        # Quick agreement summary so the operator knows at a glance whether
        # the network is converged.
        instances = {}
        for r in rows:
            instances.setdefault(r["instance"], set()).add(
                (r["root_priority"], r["root_mac"]))
        print()
        for inst, roots in sorted(instances.items()):
            if len(roots) == 1:
                prio, mac = next(iter(roots))
                print(f"{inst}: all {len(rows)} device(s) agree — root priority={prio} mac={mac}")
            else:
                print(f"{inst}: *** {len(roots)} distinct roots seen — see 'digest stp' or run 'monitor stp' for alert details ***")

    elif category == "stack-members":
        conn = _db()
        rows = conn.execute("""
            SELECT dm.ip, COALESCE(d.hostname, dm.ip) AS hostname,
                   d.chassis_type, dm.member_id, dm.role,
                   dm.serial, dm.mac, dm.model, dm.status, dm.updated_at
            FROM device_members dm
            LEFT JOIN devices d ON d.ip = dm.ip
            ORDER BY hostname, dm.member_id
        """).fetchall()
        conn.close()
        if not rows:
            print("No stack/chassis members recorded (run 'discover' or "
                  "'test ssh retest' to populate).")
            return
        display = [{
            "ip": r["ip"],
            "hostname": r["hostname"],
            "chassis_type": r["chassis_type"],
            "member_id": r["member_id"],
            "role": r["role"],
            "serial": r["serial"],
            "mac": r["mac"],
            "model": r["model"],
            "status": r["status"],
            "updated_at": r["updated_at"],
        } for r in rows]
        _output(["ip", "hostname", "chassis_type", "member_id", "role",
                 "serial", "mac", "model", "status", "updated_at"],
                display, csv_path)
        # Summary: count devices classified as stacks vs standalone-with-members.
        stacks = len({r["ip"] for r in rows if r["chassis_type"] == "stack"})
        standalones = len({r["ip"] for r in rows if r["chassis_type"] == "standalone"})
        print(f"\n{len(rows)} member(s) across {stacks} stack(s) + {standalones} "
              f"standalone device(s)")

    elif category == "mac-flux":
        # 3.8.9: surface mac_flux_alerts to operators directly. Default
        # shows ACTIVE alerts only ("what's firing right now?"); --all
        # includes cleared episodes for forensics. (3.8.10 fix: the
        # `--all` flag is now a proper argparse kwarg threaded through
        # handle_list — the 3.8.9 `bool(ip)` hack didn't work because
        # argparse rejected the bare positional with "unrecognized
        # arguments: all".)
        conn = _db(); conn.row_factory = sqlite3.Row
        try:
            sql = ("SELECT a.ip, COALESCE(d.hostname, a.ip) AS hostname, "
                   "       a.interface, a.distinct_macs, a.notified_at, "
                   "       COALESCE(a.cleared_at, '(active)') AS state "
                   "FROM mac_flux_alerts a "
                   "LEFT JOIN devices d ON d.ip = a.ip ")
            if not show_all:
                sql += "WHERE a.cleared_at IS NULL "
            sql += "ORDER BY a.cleared_at IS NULL DESC, a.notified_at DESC"
            rows = conn.execute(sql).fetchall()
        except sqlite3.OperationalError:
            print("(no mac_flux_alerts table — pre-3.8.0 DB?)")
            return
        if not rows:
            print("No active MAC-flux alerts.")
            if not show_all:
                print("(Use `show mac-flux all` to include cleared episodes.)")
            return
        _output(["ip", "hostname", "interface", "distinct_macs",
                 "notified_at", "state"], rows, csv_path)
        active = sum(1 for r in rows if r["state"] == "(active)")
        print(f"\n{len(rows)} alert(s) — {active} active, "
              f"{len(rows) - active} cleared")

    elif category == "all":
        rows = get_devices()
        if not rows:
            print("Database is empty.")
            return
        # list all sorts purely by IP (not status-grouped), for reports/CSV.
        rows = _sort_by_ip(rows)
        _output(["ip", "hostname", "dns_name", "base_mac", "firmware", "proto", "status", "fail_reason", "last_seen"], rows, csv_path)
        active = sum(1 for r in rows if r["status"] == "active")
        inactive = sum(1 for r in rows if r["status"] == "inactive")
        failed = sum(1 for r in rows if r["status"] == "failed")
        dupes = sum(1 for r in rows if r["status"] == "duplicate")
        print(f"\n{len(rows)} total: {active} active, {inactive} inactive, {failed} failed, {dupes} duplicate(s)")


_EXPORT_BANNER = (
    "# *** AUTO-GENERATED from database — do not edit manually ***\n"
    "# Changes here will be overwritten on next export.\n"
    "# Use the database (--list, -d, -r) to manage devices.\n"
    "#\n"
)


def handle_export(cfg):
    """Export device data from DB to flat files."""
    device_file = cfg["device_file"]
    base_dir = os.path.dirname(device_file)

    # Export active devices to devices.txt
    active = get_devices(status="active")
    if active:
        identity = {}
        for r in active:
            parts = []
            if r["hostname"]:
                parts.append(r["hostname"])
            if r["base_mac"]:
                parts.append(f"({r['base_mac']})")
            if parts:
                identity[r["ip"]] = " ".join(parts)
        ips = [r["ip"] for r in active]
        write_devices_file(device_file, ips, identity=identity)
        # Prepend export banner
        _prepend_export_banner(device_file)
        log.info("Exported %d active device(s) to %s", len(ips), device_file)
    else:
        log.info("No active devices to export.")

    # Export failed devices to failed_devices.txt
    failed = get_devices(status="failed")
    failed_file = os.path.join(base_dir, "failed_devices.txt")
    if failed:
        failures = [(r["ip"], r["fail_reason"] or "unknown") for r in failed]
        write_failed_file(failed_file, failures)
        _prepend_export_banner(failed_file)
        log.info("Exported %d failed device(s) to %s", len(failures), failed_file)

    # Export telnet report
    conn = _db()
    telnet_only = sorted(
        [r["ip"] for r in conn.execute(
            "SELECT ip FROM devices WHERE telnet_open=1 AND ssh_open=0 AND status='active'"
        ).fetchall()],
        key=_ip_sort_key,
    )
    telnet_and_ssh = sorted(
        [r["ip"] for r in conn.execute(
            "SELECT ip FROM devices WHERE telnet_open=1 AND ssh_open=1 AND status='active'"
        ).fetchall()],
        key=_ip_sort_key,
    )
    conn.close()
    telnet_report = os.path.join(base_dir, "telnet_devices.txt")
    write_telnet_report(telnet_report, telnet_only, telnet_and_ssh)
    if telnet_only or telnet_and_ssh:
        _prepend_export_banner(telnet_report)
        log.info("Exported telnet report to %s", telnet_report)


def _prepend_export_banner(filepath):
    """Prepend the auto-generated warning banner to an exported file."""
    with open(filepath, encoding="utf-8") as f:
        content = f.read()
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(_EXPORT_BANNER)
        f.write(content)


def _match_device_rows(field, pattern):
    """Device rows matching a selector pattern — substring semantics.

    'ip' matches the ip column; 'name' the hostname or resolved DNS name;
    'model' the model string; 'serial' the device serial OR any stacked-
    member serial (so a stack is found by a member's serial). Case-
    insensitive except 'ip'.
    """
    pat = pattern.lower()
    member_ips = set()
    if field == "serial":
        conn = _db()
        for row in conn.execute(
                "SELECT DISTINCT ip FROM device_members WHERE LOWER(serial) LIKE ?",
                (f"%{pat}%",)):
            member_ips.add(row[0])
        conn.close()
    matches = []
    for r in get_devices():
        keys = r.keys()
        if field == "ip":
            ok = pattern in (r["ip"] or "")
        elif field == "model":
            ok = pat in ((r["model"] if "model" in keys else "") or "").lower()
        elif field == "serial":
            ok = (pat in ((r["serial"] if "serial" in keys else "") or "").lower()
                  or r["ip"] in member_ips)
        else:  # name
            hn = (r["hostname"] or "").lower()
            dn = (r["dns_name"] if "dns_name" in keys else "") or ""
            ok = pat in hn or pat in dn.lower()
        if ok:
            matches.append(r)
    return matches


def _dedupe_physical(rows):
    """Collapse device rows to one per physical chassis.

    Group by hardware identity (base_mac, else serial, else the ip
    itself — the same identity `reconcile duplicates` uses); rows whose
    status is 'duplicate' resolve through duplicate_of to their primary.
    Within a group the active row wins, else the first seen. Order is
    preserved by first appearance.
    """
    out = []
    seen_keys = {}
    for r in rows:
        if (r["status"] or "") == "duplicate" and (r["duplicate_of"] or ""):
            primary = _lookup_device(r["duplicate_of"])
            if primary:
                r = primary
        keys = r.keys()
        key = ((r["base_mac"] if "base_mac" in keys else "") or
               (r["serial"] if "serial" in keys else "") or r["ip"])
        if key in seen_keys:
            prev_idx = seen_keys[key]
            if (out[prev_idx]["status"] != "active"
                    and (r["status"] or "") == "active"):
                out[prev_idx] = r
            continue
        seen_keys[key] = len(out)
        out.append(r)
    return out


def _select_devices(field, pattern):
    """Resolve a `show devices` selector to deduped physical device rows.

    field 'exact' = a bare token: IPv4 first, else exact hostname
    (case-insensitive; may legitimately hit several chassis that share a
    hostname). Other fields are the substring selectors of
    _match_device_rows.
    """
    if field == "exact":
        try:
            ipaddress.IPv4Address(pattern)
            row = _lookup_device(pattern)
            rows = [row] if row else []
        except ValueError:
            conn = _db()
            conn.row_factory = sqlite3.Row
            rows = conn.execute(
                "SELECT * FROM devices WHERE LOWER(hostname) = LOWER(?)",
                (pattern,)).fetchall()
            conn.close()
    else:
        rows = _match_device_rows(field, pattern)
    return _dedupe_physical(rows)


# Facets that print one artifact and therefore need the selector to
# resolve to exactly ONE physical device. Summarizing facets fan out
# over every match with a per-device section header instead.
_ARTIFACT_FACETS = {"config", "config-set", "config-diff", "config-compare",
                    "downstream", "interface"}


def handle_devices_facet(field, pattern, facet, cfg, csv_path=None,
                         since=None, until=None):
    """`show devices <selector> <facet>` — device-scoped facts.

    selector: a bare IP / exact hostname ('exact'), or the
    ip|name|model|serial substring patterns. Matches are deduped to
    physical chassis (base_mac/serial identity, duplicate_of followed).

    facets: summary (default listing), detail, config [set],
    config list, config diff [#|WHEN], config compare A B,
    lldp neighbors, downstream.
    """
    rows = _select_devices(field, pattern)
    if not rows:
        print(f"show devices: nothing matches "
              f"{field + ' ' if field != 'exact' else ''}'{pattern}'.")
        return
    kind = facet[0]

    if kind == "summary":
        _output(["ip", "hostname", "model", "serial", "base_mac", "status",
                 "last_seen"], rows, csv_path)
        print(f"\n{len(rows)} device(s)")
        return

    if kind in _ARTIFACT_FACETS and len(rows) > 1:
        names = ", ".join(f"{r['hostname'] or r['ip']} ({r['ip']})"
                          for r in rows)
        print(f"show devices: '{pattern}' matches {len(rows)} devices — "
              f"{names}. '{kind.replace('-', ' ')}' needs exactly one; "
              f"narrow the pattern.")
        return

    multi = len(rows) > 1
    for i, row in enumerate(rows):
        if multi:
            if i:
                print()
            print(f"===== {row['hostname'] or row['ip']} ({row['ip']}) =====")
        if kind == "detail":
            handle_show(row["ip"])
        elif kind in ("config", "config-set"):
            text = _read_current_config(row, set_format=(kind == "config-set"))
            if text is not None:
                sys.stdout.write(text)
                if text and not text.endswith("\n"):
                    sys.stdout.write("\n")
        elif kind == "config-list":
            handle_config_history(pattern, cfg, action=None, since=since,
                                  until=until,
                                  csv_path=csv_path if not multi else None,
                                  row=row)
        elif kind == "config-diff":
            handle_config_history(pattern, cfg, action=("diff", facet[1]),
                                  row=row)
        elif kind == "config-compare":
            handle_config_history(pattern, cfg,
                                  action=("compare", facet[1], facet[2]),
                                  row=row)
        elif kind == "lldp-neighbors":
            handle_neighbors(row["ip"],
                             csv_path=csv_path if not multi else None)
        elif kind == "downstream":
            handle_downstream(row, csv_path=csv_path)
        elif kind == "interface":
            handle_device_interface(row, facet[1], csv_path=csv_path)
    if multi and csv_path:
        print("(--csv ignored for a multi-device match — narrow the "
              "pattern to export one device)", file=sys.stderr)


def handle_device_interface(row, port, csv_path=None):
    """`show devices <sel> interface <port>` — everything netops has
    recorded about one switch port, consolidated: LLDP neighbor, STP
    role/state, flap count + last flap, error breakdown by type (with a
    recent daily rate), and learned-MAC count. The per-port counterpart
    to `show mac` — the "what's going on with THIS port?" view.

    Reads only harvested state (topology_edges / stp_state / port_flaps /
    flap_events / port_errors / port_macs). It doesn't SSH the switch for
    live counters — everything shown is as of the last relevant poll.
    """
    ip = row["ip"]
    host = row["hostname"] or ip
    conn = _db()
    conn.row_factory = sqlite3.Row

    edge = conn.execute(
        "SELECT neighbor_sysname, neighbor_chassis, neighbor_ip, "
        "       neighbor_port, neighbor_port_desc, neighbor_caps, last_seen "
        "FROM topology_edges WHERE src_ip=? AND src_port=?",
        (ip, port)).fetchone()
    stp = conn.execute(
        "SELECT role, state, last_seen FROM stp_state "
        "WHERE ip=? AND interface=?", (ip, port)).fetchone()
    flap = conn.execute(
        "SELECT flap_count FROM port_flaps WHERE ip=? AND interface=?",
        (ip, port)).fetchone()
    last_flap = conn.execute(
        "SELECT MAX(flapped_at) AS t FROM flap_events "
        "WHERE ip=? AND interface=?", (ip, port)).fetchone()
    errs = conn.execute(
        "SELECT error_type, SUM(delta) AS total, MAX(at) AS last_at, "
        "       SUM(CASE WHEN at > date('now','-7 days') THEN delta END) AS d7 "
        "FROM port_errors WHERE ip=? AND interface=? "
        "GROUP BY error_type ORDER BY total DESC", (ip, port)).fetchall()
    macs = conn.execute(
        "SELECT COUNT(*) AS n, COUNT(DISTINCT vlan) AS vlans, MAX(last_seen) AS t "
        "FROM port_macs WHERE ip=? AND interface=?", (ip, port)).fetchone()
    nb_port = (_lldp_resolve_port(conn, ip, edge["neighbor_ip"],
                                  edge["neighbor_port"], edge["neighbor_port_desc"])
               if edge else "")
    conn.close()

    if csv_path:
        display = [{"error_type": e["error_type"], "total": e["total"],
                    "per_day_7d": round((e["d7"] or 0) / 7.0, 1),
                    "last_at": e["last_at"]} for e in errs]
        _output(["error_type", "total", "per_day_7d", "last_at"],
                display, csv_path)
        return

    if not (edge or stp or flap or errs or (macs and macs["n"])):
        print(f"Interface {port} on {host} ({ip}): no recorded state. "
              f"(Unknown port name? netops uses the SNMP/LLDP interface "
              f"name — e.g. ge-0/0/3, 1/1/24, Trk4. Or the port has no "
              f"harvested flaps/errors/MACs/STP/LLDP yet.)")
        return

    print(f"\nInterface {port} on {host} ({ip})\n")
    if edge:
        nb = _clean_lldp_field(edge["neighbor_sysname"]) or \
             _clean_lldp_field(edge["neighbor_chassis"]) or "?"
        caps = _clean_lldp_field(edge["neighbor_caps"])
        nbip = edge["neighbor_ip"] or "unmanaged"
        print(f"  LLDP neighbor : {nb} ({nbip})"
              f"{'  [' + caps + ']' if caps else ''}"
              f"   port {nb_port}"
              f"   seen {edge['last_seen']}")
    else:
        print("  LLDP neighbor : (none recorded)")
    if stp:
        print(f"  STP           : {stp['role']}/{stp['state']}"
              f"   (as of {stp['last_seen']})")
    fc = flap["flap_count"] if flap else 0
    lf = last_flap["t"] if last_flap else None
    print(f"  Flaps         : {fc}"
          f"{'   last ' + lf if lf else ''}")
    if macs and macs["n"]:
        print(f"  MACs learned  : {macs['n']} across {macs['vlans']} vlan(s)"
              f"   (as of {macs['t']})")
    if errs:
        print("  Errors:")
        print(f"    {'type':<14} {'total':>10} {'~/day (7d)':>12}   last")
        for e in errs:
            rate = (e["d7"] or 0) / 7.0
            print(f"    {e['error_type']:<14} {e['total']:>10} "
                  f"{rate:>12.0f}   {e['last_at']}")
    else:
        print("  Errors        : none recorded")
    print()


def _stp_parent_map(conn):
    """child_ip -> (parent_ip, child_root_port, parent_port).

    Each device's parent is the LLDP neighbor on its STP Root port — the
    same directional signal _walk_topology_upstream follows one hop at a
    time. conn needs a sqlite3.Row row_factory.

    LAG root ports (ProCurve Trk2, ICX lag1, Junos ae0, ...): STP runs on
    the aggregate but LLDP edges are keyed by physical member port, so
    there's no direct edge to follow. Resolution: a LAG partner is any
    neighbor reached over >=2 parallel physical links; iterate, accepting
    a partner as the parent once it is the only candidate whose own
    parent chain doesn't lead back through this device (an access stack
    with one uplink LAG resolves immediately; a distribution switch
    resolves after its children are placed, which eliminates them as
    candidates). Devices that stay ambiguous just don't appear —
    downstream views and outage grouping degrade to less grouping, never
    to a wrong tree.
    """
    # Restrict to ACTIVE devices on both sides: stp_state and
    # topology_edges retain stale rows keyed by old duplicate SVI IPs
    # (pre-reconcile), which would otherwise flood the map with
    # zero-edge phantoms (observed on one site: 124 of 144 Root rows).
    root_rows = conn.execute(
        "SELECT s.ip, s.interface FROM stp_state s "
        "JOIN devices d ON d.ip = s.ip AND d.status = 'active' "
        "WHERE s.role IN ('Root', 'ROOT')").fetchall()
    edges = {}
    parallel = {}          # (src_ip, neighbor_ip) -> [physical ports]
    for e in conn.execute(
            "SELECT e.src_ip, e.src_port, e.neighbor_ip, e.neighbor_port "
            "FROM topology_edges e "
            "JOIN devices d ON d.ip = e.src_ip AND d.status = 'active' "
            "WHERE e.neighbor_ip IS NOT NULL").fetchall():
        edges[(e["src_ip"], e["src_port"])] = e
        parallel.setdefault((e["src_ip"], e["neighbor_ip"]),
                            []).append(e["src_port"])
    try:
        roots = {r["ip"] for r in conn.execute(
            "SELECT ip FROM stp_root_state WHERE is_root = 1").fetchall()}
    except sqlite3.OperationalError:
        roots = set()

    parent = {}
    lag_pending = []
    for r in root_rows:
        phys = r["interface"].split(".", 1)[0]
        e = edges.get((r["ip"], phys))
        if e:
            parent[r["ip"]] = (e["neighbor_ip"], phys, e["neighbor_port"])
        elif _LAG_IFACE_RE.search(phys):
            lag_pending.append((r["ip"], phys))

    def _chain_avoids(start, avoid):
        """True when start's current parent chain reaches the root bridge
        or dead-ends WITHOUT passing through `avoid`."""
        cur, hops = start, 0
        while hops < 16:
            if cur == avoid:
                return False
            if cur in roots:
                return True
            nxt = parent.get(cur)
            if nxt is None:
                return True
            cur = nxt[0]
            hops += 1
        return False

    for _ in range(6):
        progressed = False
        for ip, lag_port in lag_pending:
            if ip in parent:
                continue
            partners = [n for (s, n), ports in parallel.items()
                        if s == ip and len(ports) >= 2]
            if not partners:
                # Sparse LLDP harvest (e.g. one member link captured per
                # LAG on SSH-polled ICX): consider every device-neighbor;
                # the chain check below still eliminates descendants, so
                # ambiguity degrades to "unplaced", never a wrong parent.
                partners = [n for (s, n) in parallel if s == ip]
            viable = [p for p in partners if _chain_avoids(p, ip)]
            if len(viable) == 1:
                p = viable[0]
                member_ports = parallel[(ip, p)]
                pe = edges.get((ip, sorted(member_ports)[0]))
                parent[ip] = (p, lag_port,
                              pe["neighbor_port"] if pe else "(lag)")
                progressed = True
        if not progressed:
            break
    return parent


def handle_downstream(row, csv_path=None):
    """`show devices <dev> downstream` — every switch whose STP path to
    the root passes through this device, as an indented tree.

    Direction comes from the same signal the unreachable alert's
    upstream walk uses: each device's STP Root port edge in
    topology_edges points at its parent; inverting that parent map gives
    the downstream subtree. Devices with no Root-port/topology data
    can't be placed and simply don't appear.
    """
    conn = _db()
    conn.row_factory = sqlite3.Row
    parent = _stp_parent_map(conn)
    names = {r["ip"]: (r["hostname"] or "")
             for r in conn.execute(
                 "SELECT ip, hostname FROM devices").fetchall()}
    conn.close()

    children = {}
    placed = len(parent)
    for child, (parent_ip, child_port, parent_port) in parent.items():
        children.setdefault(parent_ip, []).append(
            (child, child_port, parent_port))

    start = row["ip"]
    label = row["hostname"] or start
    lines = []
    flat = []

    def walk(ip, depth, seen):
        kids = sorted(children.get(ip, []),
                      key=lambda c: (names.get(c[0]) or c[0]))
        for child, child_port, parent_port in kids:
            if child in seen:
                continue
            seen.add(child)
            child_label = names.get(child) or child
            lines.append(f"{'  ' * depth}└─ {child_label} ({child})  "
                         f"[{parent_port or '?'} ↔ {child_port}]")
            flat.append({"depth": depth + 1, "hostname": child_label,
                         "ip": child, "uplink_via": parent_port or "",
                         "root_port": child_port})
            walk(child, depth + 1, seen)

    walk(start, 0, {start})

    if csv_path:
        _output(["depth", "hostname", "ip", "uplink_via", "root_port"],
                flat, csv_path)
        return
    print(f"Downstream of {label} ({start}) — from STP root-port topology "
          f"[parent-port ↔ child-root-port]:")
    if not lines:
        print("  (none — no device's STP root path leads through this "
              "switch, or topology/STP data is missing; run "
              "'monitor topology' and check 'show lldp-coverage')")
        return
    for line in lines:
        print(f"  {line}")
    print(f"\n{len(flat)} downstream switch(es) "
          f"({placed} device(s) placeable in the topology overall)")


def handle_devices_filter(field, pattern, detail=False, csv_path=None):
    """`show devices ip|name|model|serial <pattern>` — substring filter.

    detail=True prints full per-device detail instead of a listing table.
    """
    matches = _match_device_rows(field, pattern)
    if not matches:
        print(f"No devices match {field} '{pattern}'.")
        return
    if detail:
        for i, r in enumerate(matches):
            if i:
                print()
            handle_show(r["ip"])
        return
    _output(["ip", "hostname", "model", "serial", "base_mac", "status",
             "last_seen"], matches, csv_path)
    print(f"\n{len(matches)} device(s) matching {field} '{pattern}'")


def _is_locally_administered(mac):
    """True if a MAC is locally administered (the 0x02 bit of the first
    octet) — i.e. software-assigned, not an IEEE-registered OUI. Modern
    phones use these as per-SSID randomized ('private') Wi-Fi addresses,
    so they have no vendor to look up."""
    hexm = re.sub(r"[^0-9a-fA-F]", "", mac or "")
    if len(hexm) < 2:
        return False
    try:
        return bool(int(hexm[:2], 16) & 0x02)
    except ValueError:
        return False


def _mac_vendor_display(mac, vendor):
    """Vendor label for a MAC: the OUI vendor if known; else
    '(randomized)' when the address is locally administered (a private
    Wi-Fi MAC has no real vendor); else blank (a genuinely unlisted OUI)."""
    if vendor:
        return vendor
    return "(randomized)" if _is_locally_administered(mac) else ""


def handle_mac(mac_query, csv_path=None, since=None, until=None):
    """Locate a MAC in the harvested FDB (port_macs) and report every switch /
    port it's on, annotated with that port's flap count + error count — i.e.
    "where is this MAC, and is its port having ANY issues?". Accepts a full MAC
    in any
    separator style, or a partial / OUI prefix (substring match). Narrow noisy
    results with --since/--until (filter on the per-location last_seen). FDB is
    SNMP-harvested by `monitor topology`, so it's thin on no-SNMP sites."""
    hexq = re.sub(r"[^0-9a-fA-F]", "", mac_query or "").lower()
    if not hexq:
        print("show mac: give a MAC or partial, e.g. "
              "'show mac aa:bb:cc:dd:ee:ff' or 'show mac aabbcc'")
        return
    if len(hexq) > 12:
        print(f"show mac: '{mac_query}' has too many hex digits for a MAC")
        return
    if len(hexq) == 12:
        canon = ":".join(hexq[i:i + 2] for i in range(0, 12, 2))
        where, params = "pm.mac = ?", [canon]
    else:
        # Partial / OUI prefix — compare against the separator-stripped MAC so
        # 'aabbcc' or 'aa:bb:cc' both match 'aa:bb:cc:dd:ee:ff'.
        where, params = "REPLACE(LOWER(pm.mac), ':', '') LIKE ?", ["%" + hexq + "%"]
    if since:
        where += " AND pm.last_seen >= ?"
        params.append(since)
    if until:
        # A bare date as --until means "through the end of that day".
        params.append(until + " 23:59:59" if len(until) <= 10 else until)
        where += " AND pm.last_seen <= ?"
    conn = _db()
    conn.row_factory = sqlite3.Row
    rows = conn.execute(f"""
        SELECT pm.mac, pm.ip, COALESCE(d.hostname, pm.ip) AS hostname,
               pm.interface, pm.vlan, pm.oui_vendor,
               pm.first_seen AS mac_first, pm.last_seen AS mac_last,
               COALESCE(pf.flap_count, 0) AS flaps,
               (SELECT COALESCE(SUM(pe.delta), 0) FROM port_errors pe
                  WHERE pe.ip = pm.ip AND pe.interface = pm.interface) AS errors
        FROM port_macs pm
        LEFT JOIN devices d ON d.ip = pm.ip
        LEFT JOIN port_flaps pf ON pf.ip = pm.ip AND pf.interface = pm.interface
        WHERE {where}
        ORDER BY pm.last_seen DESC
    """, params).fetchall()
    conn.close()
    if not rows:
        rng = []
        if since: rng.append(f"since={since}")
        if until: rng.append(f"until={until}")
        rsuffix = f" in range ({', '.join(rng)})" if rng else ""
        print(f"No FDB entry for MAC '{mac_query}'{rsuffix}. (FDB is "
              "SNMP-harvested by 'monitor topology' — empty on no-SNMP sites "
              "where SNMP is disabled; or the MAC isn't in the retention window / range.)")
        return
    display = [{
        "mac": r["mac"],
        "switch": r["hostname"],
        "port": r["interface"],
        "vlan": r["vlan"],
        "vendor": _mac_vendor_display(r["mac"], r["oui_vendor"])[:18],
        "first_seen": r["mac_first"],
        "last_seen": r["mac_last"],
        "flaps": r["flaps"],
        "errors": r["errors"],
    } for r in rows]
    # Ordered newest-last_seen first, so a MAC that moved A->B shows B then A
    # — its movement history across ports. flaps + errors are that port's
    # health signal: the "is this node's port having ANY issues?" answer.
    _output(["mac", "switch", "port", "vlan", "vendor", "first_seen",
             "last_seen", "flaps", "errors"], display, csv_path)
    flapping = [r for r in rows if r["flaps"] > 0]
    erroring = [r for r in rows if r["errors"] > 0]
    n_mac = len({r["mac"] for r in rows})
    issues = []
    if flapping:
        issues.append("%d on a FLAPPING port" % len(flapping))
    if erroring:
        issues.append("%d on a port with ERRORS" % len(erroring))
    if issues:
        suffix = ("; " + ", ".join(issues) + " — drill in with "
                  "`show flap-history` / `show port-errors --ip <ip> "
                  "--interface <port>`")
    else:
        suffix = "; no flaps or errors on any connected port"
    print(f"\n{len(rows)} location(s) for {n_mac} MAC(s){suffix}")


def handle_neighbors(ip, csv_path=None):
    """`show neighbors <ip>` (alias `connections`) — every LLDP link for one
    switch, both directions, straight from topology_edges (no map export).

    Forward: the neighbors this switch advertises. Reverse: switches that list
    this device as THEIR neighbor — useful when this switch can't be SSH-polled
    (e.g. an unsupported platform) but its peers still see it.
    """
    conn = _db()
    conn.row_factory = sqlite3.Row
    dev = conn.execute("SELECT hostname FROM devices WHERE ip = ?", (ip,)).fetchone()
    if not dev:
        conn.close()
        print(f"No device with IP {ip} in the database.")
        return
    host = dev["hostname"] or ip
    fwd = conn.execute(
        "SELECT src_port, neighbor_sysname, neighbor_ip, neighbor_port, "
        "       neighbor_port_desc, neighbor_chassis "
        "FROM topology_edges WHERE src_ip = ? ORDER BY src_port", (ip,)).fetchall()
    rev = conn.execute(
        "SELECT t.src_ip, d.hostname AS src_host, t.src_port, t.neighbor_port "
        "FROM topology_edges t LEFT JOIN devices d ON d.ip = t.src_ip "
        "WHERE t.neighbor_ip = ? ORDER BY t.src_ip", (ip,)).fetchall()
    fwd_ports = {r["src_port"]: _lldp_resolve_port(conn, ip, r["neighbor_ip"],
                 r["neighbor_port"], r["neighbor_port_desc"]) for r in fwd}
    # For the reverse view, 'to_neighbor_port' is OUR port index as the
    # other switch sees it — resolve it to our own interface name.
    rev_ports = {(r["src_ip"], r["src_port"]):
                 _lldp_resolve_port(conn, r["src_ip"], ip, r["neighbor_port"], None)
                 for r in rev}
    conn.close()

    print(f"\nConnections for {host} ({ip})\n")
    if fwd:
        disp = [{
            "local_port": r["src_port"] or "",
            "neighbor": (r["neighbor_sysname"] or r["neighbor_chassis"] or "?"),
            "neighbor_ip": r["neighbor_ip"] or "(unmanaged)",
            "neighbor_port": fwd_ports.get(r["src_port"], ""),
        } for r in fwd]
        managed = sum(1 for r in fwd if r["neighbor_ip"])
        print(f"  LLDP neighbors {host} reports ({len(fwd)}; {managed} managed):")
        _output(["local_port", "neighbor", "neighbor_ip", "neighbor_port"],
                disp, csv_path)
    else:
        print(f"  {host} reports no LLDP neighbors "
              "(LLDP off, unreachable, or an unsupported platform).")

    if rev:
        disp = [{
            "via_switch": (r["src_host"] or r["src_ip"]),
            "via_ip": r["src_ip"],
            "their_port": r["src_port"] or "",
            "to_neighbor_port": rev_ports.get((r["src_ip"], r["src_port"]),
                                              r["neighbor_port"] or ""),
        } for r in rev]
        print(f"\n  Switches that list {host} as a neighbor ({len(rev)}):")
        _output(["via_switch", "via_ip", "their_port", "to_neighbor_port"],
                disp, None)
    print()


def _clean_lldp_field(val):
    """LLDP TLVs occasionally carry non-printable bytes (a truncated
    port-ID subtype, a MAC rendered raw) that leak into a text field and
    corrupt table output. Strip control/non-ASCII bytes for display."""
    if val is None:
        return ""
    if isinstance(val, bytes):
        val = val.decode("ascii", "ignore")
    return re.sub(r"[^\x20-\x7e]", "", str(val)).strip()


def _lldp_port_display(port, port_desc):
    """Best human-readable port label for an LLDP neighbor. An AP often
    advertises its port-ID as a raw MAC (and older captures mangled that
    into U+FFFD garbage); the port DESCRIPTION ('eth0', 'Downlink to FL1')
    is both cleaner and more useful. Prefer port_desc when the port-ID is
    empty or non-text; otherwise show the (clean) port-ID."""
    p = port.decode("utf-8", "replace") if isinstance(port, bytes) else (port or "")
    desc = _clean_lldp_field(port_desc)
    garbled = bool(re.search(r"[^\x20-\x7e]", p))
    # A bare MAC port-ID (common on APs) is valid but unfriendly — the
    # description ('eth0') reads better. The MAC stays matchable by
    # `show lldp <mac>` regardless. Prefer the description over a MAC or
    # garbled/empty port-ID; keep a readable port name/number otherwise.
    is_mac = bool(re.fullmatch(r"(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}", p.strip()))
    if garbled or is_mac or not p.strip():
        return desc or _clean_lldp_field(port) or ""
    return p.strip()


def _lldp_resolve_port(conn, src_ip, neighbor_ip, port, port_desc):
    """Human port label for an LLDP neighbor, resolving a bare numeric
    port-ID to the neighbor's real interface name.

    Some switches advertise their LLDP port-ID as a numeric index (an
    ifIndex / dot1dBasePort) rather than a name — e.g. '606' instead of
    'ge-0/0/46'. When the neighbor is a managed device and there is a
    SINGLE reverse LLDP edge (the neighbor reporting THIS switch back),
    that reverse edge's local port IS the neighbor's real interface, so
    we substitute it. Ambiguous (parallel/LAG links -> multiple reverse
    edges) or unmanaged neighbors keep the raw value. MAC/garbled IDs
    still fall back to the port description via _lldp_port_display."""
    disp = _lldp_port_display(port, port_desc)
    if not neighbor_ip or not re.fullmatch(r"\d+", (disp or "").strip()):
        return disp
    revs = conn.execute(
        "SELECT DISTINCT src_port FROM topology_edges "
        "WHERE src_ip = ? AND neighbor_ip = ? AND src_port IS NOT NULL",
        (neighbor_ip, src_ip)).fetchall()
    if len(revs) == 1 and revs[0]["src_port"]:
        return revs[0]["src_port"]
    return disp


def handle_lldp_search(query, csv_path=None):
    """`show lldp <query>` — reverse LLDP-neighbor lookup: find every
    switch/port where a neighbor matching <query> is advertised.

    <query> is matched case-insensitively as a substring against the
    neighbor's system name, chassis ID, management IP, and port
    description. The 'where does this AP connect?' answer that
    `show devices <sw> lldp neighbors` (switch -> its neighbors) can't
    give you without knowing the switch first. Most useful for UNMANAGED
    neighbors — APs, phones, servers — that aren't netops devices, so
    the tunneled-wireless case resolves to the AP's access switchport.
    """
    q = (query or "").strip().lower()
    if not q:
        print("show lldp: give a neighbor name, chassis MAC, IP, or "
              "port-desc fragment (e.g. show lldp AP-3F-12)")
        return
    conn = _db()
    conn.row_factory = sqlite3.Row
    rows = conn.execute(
        "SELECT e.src_ip, COALESCE(d.hostname, e.src_ip) AS switch, "
        "       e.src_port, e.neighbor_sysname, e.neighbor_chassis, "
        "       e.neighbor_ip, e.neighbor_port, e.neighbor_port_desc, "
        "       e.neighbor_caps, e.last_seen "
        "FROM topology_edges e LEFT JOIN devices d ON d.ip = e.src_ip "
        "WHERE LOWER(COALESCE(e.neighbor_sysname,'')) LIKE ? "
        "   OR REPLACE(LOWER(COALESCE(e.neighbor_chassis,'')),':','') LIKE ? "
        "   OR LOWER(COALESCE(e.neighbor_ip,'')) LIKE ? "
        "   OR LOWER(COALESCE(e.neighbor_port_desc,'')) LIKE ? "
        "ORDER BY e.last_seen DESC",
        (f"%{q}%", f"%{q.replace(':','')}%", f"%{q}%", f"%{q}%")).fetchall()
    if not rows:
        conn.close()
        print(f"No LLDP neighbor matches '{query}'. (Topology is scraped by "
              "'monitor topology'; a neighbor only appears if it speaks LLDP "
              "and its switch is polled. Thin where LLDP is off or SNMP/SSH "
              "topology collection is unavailable.)")
        return
    display = [{
        "switch": r["switch"],
        "local_port": _clean_lldp_field(r["src_port"]),
        "neighbor": _clean_lldp_field(r["neighbor_sysname"])
                    or _clean_lldp_field(r["neighbor_chassis"]) or "?",
        "neighbor_ip": r["neighbor_ip"] or "(unmanaged)",
        "neighbor_port": _lldp_resolve_port(conn, r["src_ip"],
                                            r["neighbor_ip"],
                                            r["neighbor_port"],
                                            r["neighbor_port_desc"]),
        "caps": _clean_lldp_field(r["neighbor_caps"]),
        "last_seen": r["last_seen"],
    } for r in rows]
    conn.close()
    _output(["switch", "local_port", "neighbor", "neighbor_ip",
             "neighbor_port", "caps", "last_seen"], display, csv_path)
    switches = len({r["src_ip"] for r in rows})
    print(f"\n{len(rows)} LLDP edge(s) matching '{query}' across "
          f"{switches} switch(es) (newest first).")


def handle_show(ip):
    """Show full details for a single device including hardware inventory."""
    conn = _db()
    dev = conn.execute("SELECT * FROM devices WHERE ip = ?", (ip,)).fetchone()
    if not dev:
        print(f"Device {ip} not found in database.")
        conn.close()
        return

    # Basic info
    print(f"{'IP:':<18} {dev['ip']}")
    print(f"{'Hostname:':<18} {dev['hostname'] or '(unknown)'}")
    # dns_name column may be absent on very old DBs
    dns_name = dev["dns_name"] if "dns_name" in dev.keys() else None
    print(f"{'DNS name:':<18} {dns_name or '(unresolved)'}")
    print(f"{'Status:':<18} {dev['status']}")
    print(f"{'Model:':<18} {dev['model'] or '(unknown)'}")
    print(f"{'Serial:':<18} {dev['serial'] or '(unknown)'}")
    print(f"{'Firmware:':<18} {dev['firmware'] or '(unknown)'}")
    print(f"{'Base MAC:':<18} {dev['base_mac'] or '(unknown)'}")
    # chassis_type/member_count columns may be absent on very old DBs.
    chassis_type = dev["chassis_type"] if "chassis_type" in dev.keys() else None
    member_count = dev["member_count"] if "member_count" in dev.keys() else None
    if chassis_type or member_count:
        suffix = f" ({member_count} member{'s' if (member_count or 0) != 1 else ''})" if member_count else ""
        print(f"{'Chassis:':<18} {chassis_type or '(unknown)'}{suffix}")
    print(f"{'Protocol:':<18} {dev['proto'] or '(unknown)'}")
    print(f"{'Username:':<18} {dev['username'] or '(unknown)'}")
    print(f"{'First seen:':<18} {dev['first_seen']}")
    print(f"{'Last seen:':<18} {dev['last_seen']}")
    if dev["duplicate_of"]:
        print(f"{'Duplicate of:':<18} {dev['duplicate_of']}")
    if dev["fail_reason"]:
        print(f"{'Fail reason:':<18} {dev['fail_reason']}")

    # Backup history
    backups = conn.execute(
        "SELECT backed_up_at, changed, filename FROM backups WHERE ip = ? ORDER BY backed_up_at DESC LIMIT 5",
        (ip,)
    ).fetchall()
    if backups:
        print(f"\n--- Last {len(backups)} Backup(s) ---")
        for b in backups:
            status = "changed" if b["changed"] else "unchanged"
            print(f"  {b['backed_up_at']}  {status}  {b['filename']}")

    # Port flaps (physical up-from-down transitions since last 'clear flap')
    flap_rows = conn.execute("""
        SELECT interface, flap_count, first_seen, last_seen
        FROM port_flaps WHERE ip = ?
        ORDER BY flap_count DESC, interface
    """, (ip,)).fetchall()
    if flap_rows:
        total = sum(r["flap_count"] for r in flap_rows)
        print(f"\n--- Port Flaps ({len(flap_rows)} port(s), {total} flap(s)) ---")
        print(f"  {'FLAPS':>5}  {'INTERFACE':<16}  {'FIRST SEEN':<19}  {'LAST SEEN':<19}")
        for r in flap_rows:
            print(f"  {r['flap_count']:>5}  {r['interface']:<16}  "
                  f"{r['first_seen']:<19}  {r['last_seen']:<19}")

    # Stack / chassis members
    members = conn.execute("""
        SELECT member_id, role, serial, mac, model, status, updated_at
        FROM device_members WHERE ip = ? ORDER BY member_id
    """, (ip,)).fetchall()
    if members:
        print(f"\n--- Members ({len(members)}) ---")
        print(f"  {'#':>3}  {'ROLE':<12}  {'SERIAL':<15}  {'MAC':<18}  MODEL")
        for r in members:
            print(f"  {r['member_id']:>3}  "
                  f"{(r['role'] or '-'):<12}  "
                  f"{(r['serial'] or '-'):<15}  "
                  f"{(r['mac'] or '-'):<18}  "
                  f"{r['model'] or '-'}")

    # Hardware inventory
    hw = dev["hardware_info"]
    if hw:
        print(f"\n--- Hardware Inventory ---")
        print(hw)

    conn.close()


def handle_add(ip_arg):
    """Manually add device IP(s) to the database."""
    ips = [s.strip() for s in ip_arg.split(",") if s.strip()]
    added = 0
    for ip in ips:
        # Basic IP validation
        try:
            ipaddress.ip_address(ip)
        except ValueError:
            log.error("Invalid IP address: %s", ip)
            continue
        upsert_device(ip, status="active")
        log.info("Added %s to database", ip)
        added += 1
    if added:
        log.info("Added %d device(s) to database.", added)


def handle_mark(target, ip):
    """Manually tag a single device.

    target ∈ {"non-switch", "unknown", "switch", "firewall"}.

      non-switch / unknown   change the device's STATUS to that
                             string. Role is left alone.
      switch / firewall      set the device's ROLE accordingly. If
                             the device is currently in a not-active
                             status (non-switch, unknown, inactive,
                             failed), the status is also flipped to
                             'active' — the operator is declaring
                             that they know what this device is, so
                             treat it as live. An already-active row
                             keeps its active status.

    Provides operator-side override when the auto-classifier didn't
    catch a device (or got it wrong).
    """
    ip = (ip or "").strip()
    if not ip:
        log.error("mark: missing IP address")
        return
    conn = _db()
    row = conn.execute(
        "SELECT status, role, hostname FROM devices WHERE ip = ?",
        (ip,)).fetchone()
    if not row:
        log.error("mark: %s not found in database", ip)
        conn.close()
        return
    old_status = row["status"]
    old_role = row["role"]
    label = row["hostname"] or ip
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    if target in ("switch", "firewall"):
        new_role = target
        # Flip away from non-active states so monitor/digest paths
        # pick up the row; leave already-active rows alone.
        new_status = "active" if old_status != "active" else old_status
        reason = None
        conn.execute(
            "UPDATE devices SET status = ?, role = ?, fail_reason = ?, "
            "last_seen = COALESCE(last_seen, ?) WHERE ip = ?",
            (new_status, new_role, reason, now, ip))
        change = (f"{old_status}/{old_role} -> {new_status}/{new_role}")
    elif target == "non-switch":
        new_status = "non-switch"
        reason = "manually tagged as non-switch"
        conn.execute(
            "UPDATE devices SET status = ?, fail_reason = ?, "
            "last_seen = COALESCE(last_seen, ?) WHERE ip = ?",
            (new_status, reason, now, ip))
        change = f"{old_status} -> {new_status}"
    else:  # unknown
        new_status = "unknown"
        reason = "manually tagged as unknown"
        conn.execute(
            "UPDATE devices SET status = ?, fail_reason = ?, "
            "last_seen = COALESCE(last_seen, ?) WHERE ip = ?",
            (new_status, reason, now, ip))
        change = f"{old_status} -> {new_status}"
    conn.commit()
    conn.close()
    log.info("Marked %s (%s): %s", label, ip, change)


def handle_remove(ip_arg):
    """Remove device IP(s) from the database."""
    arg = ip_arg.strip().lower()
    # Special keywords: remove all by status bucket
    if arg in ("failed", "duplicates", "unknown", "non-switch"):
        status_filter = {"duplicates": "duplicate"}.get(arg, arg)
        count = remove_devices_by_status(status_filter)
        label = arg if arg != "duplicates" else "duplicate"
        if count:
            log.info("Removed %d %s device(s) from database.", count, label)
        else:
            log.info("No %s devices to remove.", label)
        return

    ips = [s.strip() for s in ip_arg.split(",") if s.strip()]
    removed = 0
    for ip in ips:
        if remove_device(ip):
            log.info("Removed %s from database", ip)
            removed += 1
        else:
            log.warning("%s not found in database", ip)
    if removed:
        log.info("Removed %d device(s) from database.", removed)


def _detect_import_format(path):
    """Return 'csv' or 'list' based on extension and first non-comment line."""
    if path.lower().endswith(".csv"):
        return "csv"
    try:
        with open(path, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#"):
                    continue
                # CSV header almost always contains 'ip' (case-insensitive)
                # AND at least one comma.
                if "," in line and "ip" in line.lower():
                    return "csv"
                return "list"
    except OSError:
        pass
    return "list"


def _parse_csv_devices(path):
    """Parse a CSV file. Returns list of dicts with keys: ip, hostname, dns_name,
    model, status (any may be missing). 'ip' is required.

    Recognizes columns case-insensitively: ip, hostname, dns_name/dns,
    model, status.
    """
    import csv
    rows = []
    with open(path, encoding="utf-8", newline="") as f:
        reader = csv.DictReader(f)
        # Normalize fieldnames (lowercase, strip whitespace)
        if reader.fieldnames:
            reader.fieldnames = [(n or "").strip().lower() for n in reader.fieldnames]
        if not reader.fieldnames or "ip" not in reader.fieldnames:
            # NOT including reader.fieldnames in the error — it would
            # echo back the parsed header row of the file, which leaks
            # content when import is pointed at a non-CSV file (e.g. a
            # config under /etc/). Counts/structure only.
            raise ValueError(
                f"CSV at {path} has no 'ip' column header.")
        for raw in reader:
            row = {k: (v or "").strip() for k, v in raw.items() if k}
            ip = row.get("ip", "").strip()
            if not ip:
                continue
            # Support dns or dns_name
            dns_name = row.get("dns_name") or row.get("dns") or ""
            rows.append({
                "ip": ip,
                "hostname": row.get("hostname") or "",
                "dns_name": dns_name,
                "model": row.get("model") or "",
                "status": row.get("status") or "active",
            })
    return rows


def handle_import(cfg, path=None, fmt="auto"):
    """Import devices from a file into the database.

    path: explicit file path (overrides the configured device_file).
    fmt:  "auto" | "list" | "csv".

    Security: any path resolving under /etc/ is refused, and rejected
    rows/lines are reported only as counts (never echoed back). This
    prevents the import surface from being turned into a credential
    oracle (an SSH-shell console user pointing import at secrets.conf
    would otherwise see each section header / key echoed in the
    "skipping invalid IP" warnings). See the trust-boundary memory.
    """
    device_file = path if path else cfg["device_file"]

    # Block any path under /etc/. The legitimate device file lives in
    # STATE_DIR (/var/lib/netops/devices.txt by default); there's no
    # workflow that imports from /etc/. realpath resolves symlinks so
    # a symlink elsewhere pointing at /etc/* is also blocked.
    try:
        resolved = os.path.realpath(device_file)
    except OSError:
        resolved = device_file
    if resolved == "/etc" or resolved.startswith("/etc/"):
        log.error("Refusing to import from %s: paths under /etc/ are "
                  "not permitted.", device_file)
        return

    if not os.path.exists(device_file):
        log.error("%s not found.", device_file)
        return

    if fmt == "auto":
        fmt = _detect_import_format(device_file)
    log.info("Importing from %s (format: %s)", device_file, fmt)

    if fmt == "csv":
        try:
            rows = _parse_csv_devices(device_file)
        except ValueError as e:
            log.error("%s", e)
            return
        if not rows:
            log.info("No devices found in %s.", device_file)
            return
        added = 0
        skipped = 0
        for row in rows:
            try:
                ipaddress.ip_address(row["ip"])
            except ValueError:
                # Deliberately NOT echoing the row content — a malformed
                # value could be a sensitive substring of an unintended
                # input file. Aggregate count only.
                skipped += 1
                continue
            upsert_device(
                row["ip"],
                hostname=row["hostname"] or None,
                dns_name=row["dns_name"] or None,
                model=row["model"] or None,
                status=row["status"] or "active",
                preserve_status=True,
            )
            added += 1
        log.info("Imported %d device(s) from %s%s.",
                 added, device_file,
                 f" ({skipped} row(s) without a valid IP, skipped)"
                 if skipped else "")
        return

    # --- plain list format ---
    ips = parse_devices(device_file)
    if not ips:
        log.info("No devices found in %s.", device_file)
        return
    added = 0
    skipped = 0
    for ip in ips:
        try:
            ipaddress.ip_address(ip)
        except ValueError:
            # See CSV-path comment above — never echo rejected content.
            skipped += 1
            continue
        upsert_device(ip, status="active", preserve_status=True)
        added += 1
    log.info("Imported %d device(s) from %s%s.",
             added, device_file,
             f" ({skipped} line(s) without a valid IP, skipped)"
             if skipped else "")

    # Also import failed_devices.txt if it exists (only when importing the
    # default device_file — not when an explicit path is given).
    if not path:
        failed_file = os.path.join(os.path.dirname(device_file), "failed_devices.txt")
        if os.path.exists(failed_file):
            failed_ips = parse_failed_file(failed_file)
            for fip in failed_ips:
                try:
                    ipaddress.ip_address(fip)
                except ValueError:
                    continue
                upsert_device(fip, status="failed",
                              fail_reason="imported from failed_devices.txt",
                              preserve_status=True)
            log.info("Imported %d failed device(s) from %s.",
                     len(failed_ips), failed_file)


# ---------------------------------------------------------------------------
# Subcommand handlers (pure functions taking cfg / args as needed)
# ---------------------------------------------------------------------------

def handle_wipe_db():
    """Delete the active DB file after confirmation."""
    if not _acquire_advisory_lock("wipe"):
        return
    if not os.path.exists(DB_FILE):
        log.info("No database to wipe at %s", DB_FILE)
        return
    try:
        conn = sqlite3.connect(DB_FILE, timeout=30)
        dev_count = conn.execute("SELECT COUNT(*) FROM devices").fetchone()[0]
        bk_count = conn.execute("SELECT COUNT(*) FROM backups").fetchone()[0]
        conn.close()
        print(f"About to delete {DB_FILE}")
        print(f"  devices:         {dev_count}")
        print(f"  backup records:  {bk_count}")
    except sqlite3.DatabaseError:
        print(f"About to delete {DB_FILE} (not a valid netops DB)")
    resp = input("Delete this database? [y/N]: ").strip().lower()
    if resp != "y":
        print("Aborted.")
        return
    os.remove(DB_FILE)
    log.info("Deleted %s", DB_FILE)


def _lookup_device(device):
    """The devices row for an IPv4 address or (case-insensitive) hostname.

    IPv4 is tried first so a numeric hostname can never shadow an IP.
    Returns the sqlite3.Row or None.
    """
    conn = _db()
    conn.row_factory = sqlite3.Row
    try:
        ipaddress.IPv4Address(device)
        row = conn.execute(
            "SELECT * FROM devices WHERE ip = ? LIMIT 1",
            (device,)).fetchone()
    except ValueError:
        row = conn.execute(
            "SELECT * FROM devices "
            "WHERE LOWER(hostname) = LOWER(?) LIMIT 1",
            (device,)).fetchone()
    conn.close()
    return row


def _read_current_config(row, set_format=False):
    """The most recent saved config text for a device row, or None.

    Tries hostname_ip.cfg then ip.cfg under configs/current/ (with the
    _set variants first when set_format). Errors go to stderr so stdout
    stays clean for 'ssh netops@host netops ... > local.cfg' usage.
    """
    ip = row["ip"]
    host = row["hostname"]
    candidates = []
    if host:
        if set_format:
            candidates.append(f"{host}_{ip}_set.cfg")
        candidates.append(f"{host}_{ip}.cfg")
    if set_format:
        candidates.append(f"{ip}_set.cfg")
    candidates.append(f"{ip}.cfg")
    for fname in candidates:
        path = os.path.join(CONFIGS_DIR, "current", fname)
        if os.path.isfile(path):
            with open(path, encoding="utf-8", errors="replace") as f:
                return f.read()
    print(f"no saved config found for {host or '?'} ({ip}) — "
          f"checked {', '.join(candidates)} in {CONFIGS_DIR}/current/",
          file=sys.stderr)
    return None


def _config_versions_local(stem):
    """Ordered oldest→newest config versions for one device in
    storage_mode=local, as [(became_at, label, path)].

    The nightly backup archives the outgoing config to
    history/<stem>/<timestamp>.cfg the moment a change is detected, so an
    archive's FILENAME timestamp is when its SUCCESSOR became current.
    became_at is therefore the previous entry's filename timestamp — None
    for the oldest version, whose start predates the archive trail.
    """
    entries = []
    hist_dir = os.path.join(CONFIGS_DIR, "history", stem)
    if os.path.isdir(hist_dir):
        for name in sorted(os.listdir(hist_dir)):
            if name.endswith(".cfg"):
                entries.append((name[:-4], os.path.join(hist_dir, name)))
    cur = os.path.join(CONFIGS_DIR, "current", stem + ".cfg")
    if os.path.isfile(cur):
        entries.append(("current", cur))

    versions = []
    prev_label = None
    for label, path in entries:
        became_at = None
        if prev_label:
            # archive filenames are YYYY-MM-DD_HH-MM-SS
            d, t = prev_label.split("_", 1)
            became_at = d + " " + t.replace("-", ":")
        versions.append((became_at, label, path))
        prev_label = label
    return versions


def _config_versions_git(fname):
    """Ordered oldest→newest config versions for one device in
    storage_mode=git, as [(became_at, label, commit_hash)].

    Each commit that touched configs/<fname> is one version; the commit
    time is when it became current. --follow survives hostname renames.
    """
    rel = os.path.join("configs", fname)
    out = subprocess.run(
        ["git", "-C", STATE_DIR, "log", "--follow", "--format=%H %ci",
         "--", rel],
        capture_output=True, text=True)
    if out.returncode != 0:
        return []
    versions = []
    for line in reversed(out.stdout.splitlines()):
        parts = line.split()
        if len(parts) < 3:
            continue
        commit, date, clock = parts[0], parts[1], parts[2]
        versions.append((date + " " + clock, commit[:10], commit))
    if versions:
        # The oldest commit is the initial capture, not a change.
        first = versions[0]
        versions[0] = (None, first[1], first[2])
    return versions


def _config_text(ref, storage_mode, fname):
    """Read one version's config text. ref is a path (local) or commit (git)."""
    if storage_mode == "local":
        with open(ref, encoding="utf-8", errors="replace") as f:
            return f.read()
    out = subprocess.run(
        ["git", "-C", STATE_DIR, "show",
         f"{ref}:{os.path.join('configs', fname)}"],
        capture_output=True, text=True)
    return out.stdout if out.returncode == 0 else ""


def handle_config_history(device, cfg, action=None, since=None, until=None,
                          csv_path=None, row=None):
    """`show devices <dev> config list|diff|compare` — when a device's config
    changed and what changed, from the nightly backup trail (history/
    archives or git log). Pass row= (an already-resolved primary device
    row from _select_devices) to skip the lookup; `device` is then only
    used in messages.

    Listing: one row per detected change, newest first, with +/- line
    counts against the previous version (--since/--until scope it like
    flap-history). Keyword actions, device-CLI style:
      action=("diff", WHEN)   unified diff of one change — WHEN='latest',
                              a rollback number (0 = the latest change,
                              N = the change that produced version N), or
                              a digit-prefix of a listed timestamp
                              ('2026-07-03' or a full timestamp both work)
      action=("compare", A, B) cumulative diff between two versions — each
                              side a rollback number (Junos mindset:
                              0 = current, higher = older), a
                              date/timestamp, or 'current'
    Version numbers follow the Junos rollback convention: the listing's
    '#' column is how many changes ago that version became current.
    """
    import difflib

    if row is None:
        row = _lookup_device(device)
        if not row:
            print(f"config-history: no device matches {device!r}")
            return
        if row["status"] == "duplicate" and (row["duplicate_of"] or ""):
            primary = _lookup_device(row["duplicate_of"])
            if primary:
                print(f"note: {row['ip']} is a duplicate IP of "
                      f"{primary['ip']} — showing the primary's history\n",
                      file=sys.stderr)
                row = primary
    ip, host = row["ip"], row["hostname"]
    stem = f"{host}_{ip}" if host else ip
    storage_mode = cfg["storage_mode"] if cfg else "local"

    if storage_mode == "local":
        versions = _config_versions_local(stem)
    else:
        versions = _config_versions_git(stem + ".cfg")
    if not versions:
        print(f"config-history: no saved config for {host or '?'} ({ip}) — "
              f"has 'backup' run against it yet? (see 'show backups')")
        return

    # Configs for the same box can exist under other stems — stale files
    # from demoted duplicate IPs, or a pre-rename hostname. Point at them
    # rather than silently showing a subset of the trail.
    siblings = set()
    for sub in ("current", "history"):
        d = os.path.join(CONFIGS_DIR, sub)
        if not os.path.isdir(d):
            continue
        for name in os.listdir(d):
            s = name[:-4] if name.endswith(".cfg") else name
            if s.endswith("_set"):
                continue
            if s != stem and (s.endswith(f"_{ip}")
                              or (host and s.startswith(f"{host}_"))):
                siblings.add(s)
    if siblings:
        print(f"note: older captures also exist under: "
              f"{', '.join(sorted(siblings))} (stale duplicate-IP or "
              f"renamed-host files)\n", file=sys.stderr)

    texts = [_config_text(ref, storage_mode, stem + ".cfg")
             for _, _, ref in versions]

    # One change event per version after the first: it replaced its
    # predecessor at became_at.
    events = []
    for i in range(1, len(versions)):
        became_at, label, _ = versions[i]
        old_lines = texts[i - 1].splitlines()
        new_lines = texts[i].splitlines()
        added = removed = 0
        for dline in difflib.unified_diff(old_lines, new_lines, lineterm=""):
            if dline.startswith("+") and not dline.startswith("+++"):
                added += 1
            elif dline.startswith("-") and not dline.startswith("---"):
                removed += 1
        events.append({
            "changed_at": became_at or "?",
            "added": added, "removed": removed,
            "lines": len(new_lines), "bytes": len(texts[i]),
            "old_idx": i - 1, "new_idx": i, "label": label,
        })

    if action is not None:
        digits = lambda s: re.sub(r"\D", "", s)

        def _version_index_at(spec):
            s = spec.strip().lower()
            if s in ("current", "now", "latest"):
                return len(versions) - 1
            # A short all-digit spec is a rollback number (Junos
            # mindset: 0 = current, higher = older); anything
            # date-shaped has 8+ digits.
            if s.isdigit() and len(s) < 8:
                n = int(s)
                if n >= len(versions):
                    print(f"config-history: rollback {n} doesn't exist "
                          f"— {len(versions)} version(s) on file "
                          f"(0 = current, {len(versions) - 1} = oldest).")
                    return None
                return len(versions) - 1 - n
            d = digits(s)
            if len(d) < 8:
                print(f"config-history: can't parse {spec!r} — give a "
                      f"rollback number, a date (YYYY-MM-DD; use _ "
                      f"instead of a space before the time), or "
                      f"'current'.")
                return None
            d = d.ljust(14, "0")
            as_of = (f"{d[0:4]}-{d[4:6]}-{d[6:8]} "
                     f"{d[8:10]}:{d[10:12]}:{d[12:14]}")
            idx = 0
            for i, (became_at, _label, _ref) in enumerate(versions):
                if became_at is not None and became_at <= as_of:
                    idx = i
            return idx

        if not events:
            print(f"config-history: no changes recorded for {host or '?'} "
                  f"({ip}) — nothing to diff.")
            return

        # `compare A B`: the config in effect at time A vs at time B —
        # cumulative across however many change events lie between.
        # 'current' means the live version, so
        # `compare 2026-06-01 current` reads naturally.
        if action[0] == "compare":
            a_spec, b_spec = action[1], action[2]
            ia, ib = _version_index_at(a_spec), _version_index_at(b_spec)
            if ia is None or ib is None:
                return
            la = (versions[ia][0] or f"first capture ({versions[ia][1]})") \
                + f" [rollback {len(versions) - 1 - ia}]"
            lb = (versions[ib][0] or f"first capture ({versions[ib][1]})") \
                + f" [rollback {len(versions) - 1 - ib}]"
            if ia == ib:
                print(f"config-history: {a_spec!r} and {b_spec!r} resolve "
                      f"to the same version ({la}) — nothing to compare.")
                return
            dlines = list(difflib.unified_diff(
                texts[ia].splitlines(), texts[ib].splitlines(),
                fromfile=f"{stem} as of {la}",
                tofile=f"{stem} as of {lb}", lineterm=""))
            if not dlines:
                print(f"config-history: no differences between {la} and "
                      f"{lb} (the config changed and changed back).")
                return
            for dline in dlines:
                print(dline)
            return

        # `diff [WHEN]`: one change event.
        when = action[1]
        if when == "latest":
            matches = [events[-1]]
        elif when.isdigit() and len(when) < 8:
            # Rollback number: the change that produced version N.
            n = int(when)
            if n >= len(events):
                print(f"config-history: no change numbered {n} — "
                      f"{len(events)} change(s) on file "
                      f"(0 = most recent, {len(events) - 1} = oldest).")
                return
            matches = [events[len(events) - 1 - n]]
        else:
            matches = [e for e in events
                       if digits(e["changed_at"]).startswith(digits(when))]
        if not matches:
            print(f"config-history: no change matches {when!r} — timestamps "
                  f"are: {', '.join(e['changed_at'] for e in events)}")
            return
        if len(matches) > 1:
            print(f"config-history: {when!r} matches {len(matches)} changes "
                  f"({', '.join(e['changed_at'] for e in matches)}) — "
                  f"give more of the timestamp.")
            return
        ev = matches[0]
        old_i, new_i = ev["old_idx"], ev["new_idx"]
        for dline in difflib.unified_diff(
                texts[old_i].splitlines(), texts[new_i].splitlines(),
                fromfile=f"{stem} before {ev['changed_at']}",
                tofile=f"{stem} after {ev['changed_at']}", lineterm=""):
            print(dline)
        return

    first_label = versions[0][1]
    if not events:
        print(f"No config changes recorded for {host or '?'} ({ip}) — one "
              f"capture on file ({first_label}: {len(texts[0].splitlines())} "
              f"lines, {len(texts[0])} bytes).")
        return

    # '#' = Junos-rollback-style version number: how many changes ago this
    # version became current (0 = the live config).
    n_latest = len(versions) - 1
    display = [{
        "#": n_latest - e["new_idx"],
        "changed_at": e["changed_at"],
        "+lines": f"+{e['added']}",
        "-lines": f"-{e['removed']}",
        "lines": e["lines"],
        "bytes": e["bytes"],
    } for e in reversed(events)]
    if since:
        display = [r for r in display if r["changed_at"] >= since]
    if until:
        u = until + " 23:59:59" if len(until) <= 10 else until
        display = [r for r in display if r["changed_at"] <= u]
    if not display:
        print(f"No config changes for {host or '?'} ({ip}) in that window "
              f"({len(events)} total on file).")
        return
    _output(["#", "changed_at", "+lines", "-lines", "lines", "bytes"],
            display, csv_path)
    newest = events[-1]
    print(f"\n{len(display)} config change(s) for {host or '?'} ({ip}), "
          f"newest first (earliest capture: {first_label}). Current config "
          f"= rollback 0, unchanged since {newest['changed_at']}. "
          f"'show devices {device} config diff <#|timestamp>' for one "
          f"change, '... config compare <A> <B>' between any two versions "
          f"(rollback numbers, dates, or 'current').")


def _email_recipients():
    """Current [email] 'to' recipients from netops.conf, as a list."""
    import configparser
    cp = configparser.ConfigParser(interpolation=None)
    cp.optionxform = str
    try:
        cp.read(CONFIG_FILE, encoding="utf-8")
    except (configparser.Error, OSError):
        return []
    raw = cp.get("email", "to", fallback="") if cp.has_section("email") else ""
    return [a.strip() for a in raw.split(",") if a.strip()]


def _set_email_recipients(recipients):
    """Rewrite the [email] 'to' line in netops.conf, preserving comments.

    netops.conf is 0660 root:netops so the netops group can write it.
    Returns True on success.
    """
    new_to = "to = " + ", ".join(recipients)
    try:
        with open(CONFIG_FILE, encoding="utf-8") as f:
            lines = f.read().split("\n")
    except OSError as e:
        print(f"configure: cannot read {CONFIG_FILE}: {e}")
        return False
    out = []
    in_email = False
    wrote = False
    for line in lines:
        s = line.strip()
        if s.startswith("[") and s.endswith("]"):
            if in_email and not wrote:        # leaving [email], no 'to' seen
                out.append(new_to)
                wrote = True
            in_email = (s == "[email]")
            out.append(line)
            continue
        if (in_email and not wrote and "=" in s
                and s.split("=", 1)[0].strip() == "to"):
            out.append(new_to)
            wrote = True
            continue
        out.append(line)
    if in_email and not wrote:                # [email] was the final section
        out.append(new_to)
        wrote = True
    if not wrote:                             # no [email] section at all
        if out and out[-1].strip():
            out.append("")
        out += ["[email]", new_to]
    try:
        with open(CONFIG_FILE, "w", encoding="utf-8") as f:
            f.write("\n".join(out))
    except OSError as e:
        print(f"configure: cannot write {CONFIG_FILE}: {e}")
        # Distinguish DAC from LSM (AppArmor) so the operator isn't sent
        # down the wrong path — the old "must be 0660 root:netops" message
        # is wrong when perms are correct and AppArmor enforce is blocking.
        try:
            mode = os.stat(CONFIG_FILE).st_mode & 0o777
        except OSError:
            mode = None
        if mode == 0o660:
            print("  file is 0660 (DAC OK) — likely AppArmor enforce. Check:")
            print("    sudo journalctl -k -n50 | grep apparmor.*netops")
        elif mode is not None:
            print(f"  file is 0{mode:o}; expected 0660 root:netops "
                  "(chown root:netops + chmod 0660).")
        else:
            print("  netops.conf must be group-writable (0660 root:netops).")
        return False
    return True


def handle_configure(tokens):
    """`configure` — change settings in netops.conf (currently email)."""
    if tokens[0] != "email":
        print(f"configure: unknown setting '{tokens[0]}' (expected 'email')")
        return
    rest = tokens[1:]
    if not rest:
        print("Usage: configure email {add <address>|remove <address>|list}")
        return
    action = rest[0]
    if action == "list":
        recips = _email_recipients()
        if recips:
            for r in recips:
                print(f"  {r}")
            print(f"\n{len(recips)} email recipient(s)")
        else:
            print("No email recipients configured ([email] 'to' is empty).")
        return
    if action not in ("add", "remove"):
        print(f"configure email: unknown action '{action}' (add|remove|list)")
        return
    if len(rest) != 2:
        print(f"Usage: configure email {action} <address>")
        return
    addr = rest[1]
    if "@" not in addr:
        print(f"configure email {action}: '{addr}' is not an email address")
        return
    recips = _email_recipients()
    if action == "add":
        if addr in recips:
            print(f"{addr} is already a recipient.")
            return
        recips.append(addr)
    else:
        if addr not in recips:
            print(f"{addr} is not a current recipient — "
                  "'configure email list' shows them.")
            return
        recips.remove(addr)
    if _set_email_recipients(recips):
        verb = "Added" if action == "add" else "Removed"
        print(f"{verb} {addr}.")
        print(f"Recipients now: {', '.join(recips) if recips else '(none)'}")


def handle_manual():
    """Print the bundled user manual to stdout — a man-db-independent way
    to read the docs (e.g. `netops manual | less`). Works on minimal/base
    installs where man-db isn't present. Finds user-manual.md under the
    packaged doc dir or next to the script (run-in-place), handling the
    gzip that dh_compress applies to packaged docs."""
    import gzip
    script_dir = os.path.dirname(os.path.abspath(__file__))
    search_dirs = ["/usr/share/doc/netops",
                   os.path.join(script_dir, "docs"),
                   script_dir]
    for d in search_dirs:
        for name in ("user-manual.md", "user-manual.md.gz"):
            path = os.path.join(d, name)
            if not os.path.isfile(path):
                continue
            try:
                if path.endswith(".gz"):
                    with gzip.open(path, "rt", encoding="utf-8",
                                   errors="replace") as f:
                        text = f.read()
                else:
                    with open(path, encoding="utf-8", errors="replace") as f:
                        text = f.read()
            except OSError as e:
                print(f"manual: cannot read {path}: {e}", file=sys.stderr)
                return
            sys.stdout.write(text)
            if not text.endswith("\n"):
                sys.stdout.write("\n")
            return
    print("manual: user-manual.md not found (looked in "
          + ", ".join(search_dirs) + "). On a packaged install it lives at "
          "/usr/share/doc/netops/; 'man netops' also covers the command "
          "reference where man-db is installed.", file=sys.stderr)
    sys.exit(1)


def handle_check_config(cfg):
    """Print effective config, system limits, and DB status."""
    caps = cfg["_thread_caps"]
    fd = caps["fd_limit"]
    mem = caps["total_mem_mb"]
    scan_cap = caps["scan_cap"]

    print(f"netops v{__version__}")
    print()
    print("--- System Limits ---")
    print(f"  OS:                  {sys.platform}")
    print(f"  ssh:                 {'found' if _HAS_SSH else 'NOT FOUND (required)'}")
    print(f"  telnet:              {'found' if _HAS_TELNET else 'not found (telnet fallback disabled)'}")
    print(f"  File descriptors:    {fd}  (raise with 'ulimit -n <value>')")
    print(f"  System memory:       {mem} MB  ({mem / 1024:.1f} GB)")
    print()
    print("--- Thread Limits ---")
    print(f"  scan_threads:")
    print(f"    configured:        {caps['raw_scan']}")
    print(f"    max (fd-based):    {scan_cap}  ({fd} fd - 100 reserved)")
    print(f"    effective:         {cfg['scan_threads']}")
    print(f"  ssh_threads:")
    fd_bk = max((fd - 100) // 3, 10)
    mem_bk = max(int(mem * _SSH_MEM_FRACTION / _SSH_MEM_MB), 10)
    print(f"    configured:        {caps['raw_ssh']}")
    print(f"    max (fd-based):    {fd_bk}  (({fd} fd - 100) / 3 fds per session)")
    print(f"    max (mem-based):   {mem_bk}  (25% of {mem} MB / {_SSH_MEM_MB} MB per session)")
    print(f"    effective:         {cfg['ssh_threads']}  (limited by {'fd' if fd_bk <= mem_bk else 'memory'})")
    print()
    print("--- Database ---")
    if os.path.exists(DB_FILE):
        conn = _db()
        dev_count = conn.execute("SELECT COUNT(*) FROM devices").fetchone()[0]
        bk_count = conn.execute("SELECT COUNT(*) FROM backups").fetchone()[0]
        conn.close()
        print(f"  path:                {DB_FILE}")
        print(f"  devices:             {dev_count}")
        print(f"  backup records:      {bk_count}")
    else:
        print(f"  path:                {DB_FILE} (not created yet)")
    print()
    print("--- Config File ---")
    print(f"  path:                {CONFIG_FILE}")
    print(f"  storage_mode:        {cfg['storage_mode']}")
    if cfg["storage_mode"] == "git":
        print(f"  git_remote_url:      {cfg.get('git_remote_url') or '(none — local only)'}")
        print(f"  git_branch:          {cfg.get('git_branch') or 'main'}")
    print(f"  log_format:          {cfg['log_format']}")
    print(f"  device_file:         {cfg['device_file']}")
    print(f"  subnets:             {', '.join(cfg['subnets']) or '(none)'}")
    print(f"  ports:               {', '.join(str(p) for p in cfg['ports'])}")
    print(f"  scan timeout:        {cfg['timeout']}s")
    print(f"  usernames:           {', '.join(cfg['usernames'])}")
    pw_src = cfg.get("_pw_source", "none")
    pw_count = len(cfg.get("password_list", []))
    if pw_src == "env":
        print(f"  passwords:           {pw_count} (from NETOPS_PASSWORDS env var)")
    elif pw_src == "secrets.conf":
        print(f"  passwords:           {pw_count} (from {SECRETS_FILE})")
    else:
        print(f"  passwords:           (not set — add them to {os.path.basename(SECRETS_FILE)})")
    overrides = [k for k in cfg["passwords"] if k != "default"]
    if overrides:
        print(f"  password overrides:  {', '.join(overrides)}")
    user_pw_map = cfg.get("user_passwords") or {}
    if user_pw_map:
        print(f"  user_passwords:      {len(user_pw_map)} user(s) mapped")
        for user, pws in user_pw_map.items():
            print(f"    {user}: {len(pws)} password(s) — bulk list skipped for this user")


def run_probe(cfg, status_filter="all", csv_path=None, timeout=2.0, threads=100):
    """TCP reachability check for IPs already in the DB.

    Updates ssh_open/telnet_open columns. Does NOT change 'status' — a
    previously failed device that's now reachable is still failed until
    re-tested. Writes nothing to the DB if there are no matching rows.
    """
    conn = _db()
    if status_filter == "all":
        rows = conn.execute(
            "SELECT ip, hostname, dns_name, status FROM devices "
            "WHERE status != 'duplicate'"
        ).fetchall()
    else:
        rows = conn.execute(
            "SELECT ip, hostname, dns_name, status FROM devices WHERE status = ?",
            (status_filter,),
        ).fetchall()
    conn.close()

    rows = _sort_by_ip(rows)
    if not rows:
        log.info("No devices match filter '%s'.", status_filter)
        return

    ports = cfg.get("ports") or [22, 23]
    total = len(rows)
    log.info("Probing %d device(s) on ports %s (timeout %.1fs)...",
             total, ",".join(str(p) for p in ports), timeout)

    # Worker: return (ip, list_of_open_ports)
    def _probe_one(ip):
        return (ip, check_ports(ip, ports, timeout))

    results_by_ip = {}
    done = 0
    with ThreadPoolExecutor(max_workers=threads) as pool:
        futures = {pool.submit(_probe_one, r["ip"]): r["ip"] for r in rows}
        for future in as_completed(futures):
            ip, open_ports = future.result()
            results_by_ip[ip] = open_ports
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rProbing... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    # Update DB + build summary
    conn = _db()
    reachable = 0
    unreachable = 0
    ssh_count = 0
    telnet_count = 0
    both_count = 0
    display_rows = []
    for r in rows:
        ip = r["ip"]
        open_ports = results_by_ip.get(ip, [])
        has_ssh = 22 in open_ports
        has_telnet = 23 in open_ports
        conn.execute(
            "UPDATE devices SET ssh_open = ?, telnet_open = ? WHERE ip = ?",
            (1 if has_ssh else 0, 1 if has_telnet else 0, ip),
        )
        if open_ports:
            reachable += 1
            if has_ssh:
                ssh_count += 1
            if has_telnet:
                telnet_count += 1
            if has_ssh and has_telnet:
                both_count += 1
            reach_flag = "yes"
        else:
            unreachable += 1
            reach_flag = "no"
        display_rows.append({
            "ip": ip,
            "hostname": r["hostname"] or "",
            "dns_name": r["dns_name"] or "" if "dns_name" in r.keys() else "",
            "status": r["status"],
            "reachable": reach_flag,
            "ssh": "yes" if has_ssh else "no",
            "telnet": "yes" if has_telnet else "no",
        })
    conn.commit()
    conn.close()

    log.info("--- Probe Summary ---")
    log.info("Reachable:    %d  (SSH: %d, telnet: %d, both: %d)",
             reachable, ssh_count, telnet_count, both_count)
    log.info("Unreachable:  %d", unreachable)
    log.info("Total:        %d", total)

    if csv_path:
        _write_csv(csv_path,
                   ["ip", "hostname", "dns_name", "status",
                    "reachable", "ssh", "telnet"],
                   display_rows)


def reconcile_duplicates(cfg, do_probe=True):
    """Reclassify the multiple IPs of one physical switch.

    Groups every device by hardware identity (base MAC, else serial). For each
    identity shared by more than one IP, keeps one primary (the active /
    best-status IP) and marks the rest status='duplicate' with duplicate_of set
    — overriding the failed/unknown/inactive states an L3 switch's secondary
    SVIs drift into when each is treated as a standalone device.

    When do_probe, TCP-probes each duplicate so a secondary IP that SHOULD be
    reachable on a healthy network but isn't gets flagged (fail_reason +
    surfaced in the weekly health digest). The weekly 'discover' refreshes each
    reachable device's serial before this pass, so regrouping on the fresh
    identity also re-verifies that a duplicate IP still belongs to the same
    switch — a re-IP'd SVI gets a new serial and falls out of the group.

    Returns a summary dict: {groups, marked, reachable, unreachable:[(ip,host,primary)]}.
    """
    conn = _db()
    rows = conn.execute(
        "SELECT * FROM devices WHERE status != 'non-switch'").fetchall()
    conn.close()

    groups = {}
    for r in rows:
        ident = _device_identity(r["base_mac"], r["serial"])
        if ident:
            groups.setdefault(ident, []).append(r)
    multi = {k: v for k, v in groups.items() if len(v) > 1}

    if not multi:
        log.info("Duplicate reconciliation: no shared-identity groups found.")
        return {"groups": 0, "marked": 0, "reachable": 0, "unreachable": []}

    ports = cfg.get("ports") or [22, 23]
    timeout = 2.0

    # Probe reachability of EVERY member IP up front (not just the duplicates)
    # so the primary choice can prefer a reachable IP: if the status-based
    # primary is down but a sibling SVI answers, promote the reachable one — so
    # netops (and `connect <hostname>`) targets a live IP instead of a dead
    # primary whose SVI was removed.
    reach = {}
    if do_probe:
        all_ips = sorted({m["ip"] for members in multi.values() for m in members})
        if all_ips:
            with ThreadPoolExecutor(max_workers=min(100, len(all_ips))) as pool:
                futs = {pool.submit(check_ports, ip, ports, timeout): ip
                        for ip in all_ips}
                for fut in as_completed(futs):
                    ip = futs[fut]
                    try:
                        reach[ip] = fut.result()
                    except Exception:
                        reach[ip] = []

    def _member_reachable(m):
        if do_probe:
            return bool(reach.get(m["ip"]))
        return bool(m["ssh_open"] or m["telnet_open"])

    # Decide primary + duplicates per group.
    plan = []         # (dup_row, primary_ip)
    fix_primary = []  # primary IPs whose own row needs un-flagging
    for members in multi.values():
        primary = _choose_primary(members)
        # Reachability-aware repromotion: if the sticky/status-based primary is
        # unreachable but a sibling IS reachable, promote the best reachable
        # sibling (the dead primary then gets demoted to a duplicate below).
        # Sticky otherwise — a reachable primary keeps winning, no churn.
        if not _member_reachable(primary):
            reachable_members = [m for m in members if _member_reachable(m)]
            if reachable_members:
                primary = _choose_primary(reachable_members)
        if primary["status"] == "duplicate" or (primary["duplicate_of"] or ""):
            # Chosen primary was itself flagged a dup (degenerate group, or a
            # just-repromoted reachable sibling): un-flag it so discover/test/
            # backup can re-promote it to active.
            fix_primary.append(primary["ip"])
        for m in members:
            if m["ip"] != primary["ip"]:
                plan.append((m, primary["ip"]))

    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    unreachable = []
    n_reachable = 0
    conn = _db()
    for ip in fix_primary:
        conn.execute(
            "UPDATE devices SET duplicate_of=NULL, "
            "status=CASE WHEN status='duplicate' THEN 'inactive' ELSE status END "
            "WHERE ip=?", (ip,))
    for m, primary_ip in plan:
        ip = m["ip"]
        if do_probe:
            open_ports = reach.get(ip, [])
        else:
            open_ports = [p for p, on in ((22, m["ssh_open"]),
                                          (23, m["telnet_open"])) if on]
        if open_ports:
            n_reachable += 1
            conn.execute(
                "UPDATE devices SET status='duplicate', duplicate_of=?, "
                "fail_reason=?, ssh_open=?, telnet_open=?, last_seen=? WHERE ip=?",
                (primary_ip, f"duplicate of {primary_ip}",
                 1 if 22 in open_ports else 0, 1 if 23 in open_ports else 0,
                 now, ip))
        else:
            unreachable.append((ip, m["hostname"], primary_ip))
            # last_seen intentionally NOT bumped — it should reflect the last
            # time the IP was actually reachable.
            conn.execute(
                "UPDATE devices SET status='duplicate', duplicate_of=?, "
                "fail_reason=?, ssh_open=0, telnet_open=0 WHERE ip=?",
                (primary_ip, f"duplicate of {primary_ip} — UNREACHABLE", ip))
    conn.commit()
    conn.close()

    marked = len(plan)
    log.info("--- Duplicate Reconciliation ---")
    log.info("Shared-identity switches: %d", len(multi))
    log.info("Duplicate IPs marked:     %d (%d reachable, %d unreachable)",
             marked, n_reachable, len(unreachable))
    for ip, host, primary_ip in unreachable:
        log.warning("  UNREACHABLE duplicate: %s (%s) — should be reachable; "
                    "duplicate of %s", ip, host or "?", primary_ip)
    return {"groups": len(multi), "marked": marked,
            "reachable": n_reachable, "unreachable": unreachable}


def run_test(cfg):
    """Test credentials against all devices (no backup)."""
    # SSH-heavy fleet sweep — serialize with the other SSH jobs (it
    # historically took no lock at all and could collide with monitor
    # ticks / backup).
    if not _ssh_job_gate(wait=90):
        print("test ssh: another SSH-heavy job holds the lock "
              "(waited 90s) — try again in a minute.")
        return
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        sys.exit(1)

    device_file = cfg["device_file"]
    db_devices = get_devices(status="active") + get_devices(status="inactive")
    if db_devices:
        ips = [r["ip"] for r in db_devices]
    elif os.path.exists(device_file):
        ips = parse_devices(device_file)
        if ips:
            log.info("Database empty — reading from %s", device_file)
    else:
        ips = []

    if not ips:
        log.info("No devices to test. Run 'discover' first or 'import'.")
        sys.exit(0)

    # Resolve DNS for any devices without a name yet (first-test or
    # post-import without a preceding discover/scan).
    ips_needing_dns = []
    if db_devices:
        for r in db_devices:
            existing_dns = r["dns_name"] if "dns_name" in r.keys() else None
            if not existing_dns:
                ips_needing_dns.append(r["ip"])
    else:
        # No DB rows yet — resolve all
        ips_needing_dns = list(ips)

    if ips_needing_dns:
        log.info("Resolving reverse DNS for %d device(s)...", len(ips_needing_dns))
        dns_map = resolve_dns_batch(ips_needing_dns)
        conn = _db()
        for ip, name in dns_map.items():
            if name:
                conn.execute("UPDATE devices SET dns_name = ? WHERE ip = ?",
                             (name, ip))
        conn.commit()
        conn.close()

    def test_one(ip):
        conn = _db()
        row = conn.execute(
            "SELECT username, password_hash FROM devices WHERE ip = ? AND status = 'active'",
            (ip,)).fetchone()
        conn.close()
        known_user = row["username"] if row else None
        known_pw_hash = row["password_hash"] if row else None

        child, proto, username, pw_hash, ssh_open, telnet_open = connect_device(
            ip, cfg["usernames"], passwords["default"],
            password_list=cfg.get("password_list"),
            known_username=known_user, known_password_hash=known_pw_hash,
            user_passwords=cfg.get("user_passwords"),
        )
        if child is not None:
            # Defensive: any pexpect/identification failure here (e.g. a
            # Ruckus FastIron `enable` reauth confusing the session into
            # User Name: / Password: state that doesn't match PROMPT_RE)
            # leaves a damaged session — _detect_platform raises TIMEOUT.
            # Catch it: credentials WORKED (connect_device returned), so
            # this is still a PASS, just with empty identification.
            hostname = ""
            base_mac = ""
            serial = ""
            firmware = ""
            model = ""
            id_output = ""
            platform = None
            dev_id = {}
            try:
                platform = _detect_platform(child)
                hostname = get_hostname(child)
                if platform == "junos":
                    ver = send_command(child, "show version | no-more", timeout=15)
                    hw = send_command(child, "show chassis hardware | no-more", timeout=15)
                    mac_out = send_command(child, "show chassis mac-addresses | no-more", timeout=15)
                    id_output = ver + "\n" + hw + "\n" + mac_out
                elif platform == "fastiron":
                    # Ruckus FastIron has no `show system`; `show version`
                    # alone contains HW (model), SW (firmware) and Serial #.
                    # 30s — multi-unit stacks paginate heavily.
                    id_output = send_command(child, "show version", timeout=30)
                else:
                    sys_out = send_command(child, "show system", timeout=10)
                    ver_out = send_command(child, "show version", timeout=10)
                    id_output = sys_out + "\n" + ver_out
                    # Cisco IOS lands in the procurve bucket by prompt
                    # alone (`hostname#`/`hostname>` is identical). Re-tag
                    # post-banner so downstream platform-keyed code (STP,
                    # FDB, future features) can dispatch correctly.
                    if _is_cisco_banner(ver_out):
                        platform = "cisco-ios"
                    else:
                        # Aruba 2920/2930F/2930M model isn't in `show
                        # system` or `show version`. Two probe commands
                        # cover both topologies:
                        #   show stacking — STACK model row:
                        #     "1  mac  Aruba JL322A 2930M-48G-PoE+
                        #      Switch  255 Commander"
                        #   show modules — STANDALONE chassis row:
                        #     "Chassis: 2930M-48G-PoE+  JL322A  Serial
                        #      Number: SG..."
                        # Either matches a regex in get_device_id; on
                        # non-applicable devices both return "Invalid
                        # input" or empty output and the regexes don't
                        # match — harmless.
                        try:
                            stack_out = send_command(child, "show stacking",
                                                     timeout=10)
                            id_output = id_output + "\n" + stack_out
                        except Exception:
                            pass
                        try:
                            mod_out = send_command(child, "show modules",
                                                   timeout=10)
                            id_output = id_output + "\n" + mod_out
                        except Exception:
                            pass
                dev_id = get_device_id(id_output)
                base_mac = dev_id.get("base_mac", "")
                serial = dev_id.get("serial", "")
                firmware = dev_id.get("firmware", "")
                model = dev_id.get("model", "")
                if not model and platform not in ("junos", "fastiron"):
                    for sys_line in id_output.splitlines():
                        sys_line = sys_line.strip()
                        lower = sys_line.lower()
                        # Skip header lines that name fields rather
                        # than values; show-modules expansion-card
                        # rows that would otherwise win the keyword
                        # race; and show-stacking member rows (we
                        # have a structured regex for those — if it
                        # missed, grabbing the raw row would store a
                        # truncated/columnated mess as the model).
                        if "system name" in lower or "module" in lower:
                            continue
                        if re.search(r"\s+\d{1,3}\s+"
                                     r"(?:Commander|Standby|Member)\s*$",
                                     sys_line):
                            continue
                        if any(kw in lower for kw in
                               ["procurve", "aruba", "cisco", "juniper",
                                "palo", "switch", "router", "firewall"]):
                            model = sys_line.strip("; ")
                            break
            except Exception as e:
                log.debug("test_one: %s — identification failed: %s", ip, e)
            finally:
                try:
                    disconnect(child)
                except Exception:
                    pass
            is_switch = looks_like_switch(platform, dev_id)
            label = hostname or ip
            if model:
                label = f"{label} ({model})"
            mac_info = f", mac={base_mac}" if base_mac else ""
            info = {"hostname": hostname, "base_mac": base_mac, "serial": serial,
                    "firmware": firmware, "model": model, "proto": proto,
                    "username": username, "password_hash": pw_hash,
                    "ssh_open": ssh_open, "telnet_open": telnet_open,
                    "is_switch": is_switch}
            if not is_switch:
                info["fail_reason"] = ("not a switch (no recognized vendor "
                                       "banner in show version)")
                return (ip, f"NON-SWITCH proto={proto}, user={username}, "
                            f"host={label} — excluded from backup",
                        True, info)
            return (ip, f"PASS proto={proto}, user={username}"
                        f"{mac_info}, host={label}", True, info)
        else:
            reason = classify_fail_reason(ssh_open, telnet_open)
            return (ip, f"FAIL — {reason}", False,
                    {"hostname": None, "base_mac": "", "serial": "",
                     "firmware": "", "model": "", "proto": None,
                     "username": None, "password_hash": None,
                     "ssh_open": ssh_open, "telnet_open": telnet_open,
                     "fail_reason": reason})

    results = []
    total = len(ips)
    done = 0
    with ThreadPoolExecutor(max_workers=cfg["ssh_threads"]) as pool:
        futures = {pool.submit(test_one, ip): ip for ip in ips}
        for future in as_completed(futures):
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rTesting credentials... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()
            ip_for_future = futures[future]
            try:
                result = future.result()
            except Exception as e:
                log.error("test_one(%s) raised unhandled %s: %s",
                          ip_for_future, type(e).__name__, e)
                result = (ip_for_future,
                          f"FAIL — internal error: {type(e).__name__}",
                          False,
                          {"hostname": None, "base_mac": "", "serial": "",
                           "firmware": "", "model": "", "proto": None,
                           "username": None, "password_hash": None,
                           "ssh_open": None, "telnet_open": None,
                           "fail_reason": f"internal error: {type(e).__name__}"})
            results.append(result)
            # Incremental upsert so a crash later in the run doesn't lose
            # progress. The final batch below re-upserts the same data
            # (idempotent).
            try:
                _ip, _msg, ok, info = result
                if ok:
                    _status = "active" if info.get("is_switch", True) else "non-switch"
                    upsert_device(_ip, base_mac=info.get("base_mac"),
                                  hostname=info.get("hostname"),
                                  model=info.get("model"),
                                  serial=info.get("serial"),
                                  firmware=info.get("firmware"),
                                  proto=info.get("proto"),
                                  username=info.get("username"),
                                  password_hash=info.get("password_hash"),
                                  ssh_open=info.get("ssh_open"),
                                  telnet_open=info.get("telnet_open"),
                                  status=_status,
                                  fail_reason=info.get("fail_reason"))
                else:
                    upsert_device(_ip, status="inactive",
                                  fail_reason=info.get("fail_reason")
                                              or "all connection attempts failed")
            except Exception as e:
                log.debug("test incremental upsert failed: %s", e)

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    results.sort(key=lambda r: ipaddress.ip_address(r[0]))
    passed_count = 0
    failed_count = 0
    mac_groups = {}
    for ip, result, ok, info in results:
        print(f"  {ip}: {result}")
        if ok:
            passed_count += 1
            base_mac = info["base_mac"]
            # Don't fold non-switch base_macs into the dedup map — a server
            # BMC's MAC has no relationship to a switch chassis MAC.
            if base_mac and info.get("is_switch", True):
                mac_groups.setdefault(base_mac, []).append(ip)
            _status = "active" if info.get("is_switch", True) else "non-switch"
            upsert_device(ip, base_mac=info["base_mac"],
                          hostname=info["hostname"], model=info.get("model"),
                          serial=info.get("serial"), firmware=info.get("firmware"),
                          hardware_info=info.get("hardware_info"),
                          proto=info["proto"], username=info["username"],
                          password_hash=info.get("password_hash"),
                          ssh_open=info.get("ssh_open"),
                          telnet_open=info.get("telnet_open"),
                          status=_status,
                          fail_reason=info.get("fail_reason"))
        else:
            failed_count += 1
            upsert_device(ip, status="inactive",
                          fail_reason=info.get("fail_reason")
                                      or "all connection attempts failed")

    dupes = {mac: ips_list for mac, ips_list in mac_groups.items() if len(ips_list) > 1}
    if dupes:
        log.info("--- Duplicate IPs (same physical switch) ---")
        for mac, ips_list in dupes.items():
            log.info("  %s: %s", mac, ", ".join(ips_list))

    log.info("Passed: %d  Failed: %d", passed_count, failed_count)


def run_backup(cfg, sanitize=True):
    """Back up configs for all devices in the active DB (+ inactive fallback)."""
    # Backup fires daily at 01:30 and collided at :30 with the per-minute
    # monitor-stp tick holding the 'ssh' lock. A non-blocking acquire made
    # the nightly backup silently skip whenever it lost that race. Wait up
    # to 90 s (one stp tick is worst-case ~80 s) rather than miss the run.
    # (Daemon mode: no whole-job lock — per-device slots govern.)
    if not _ssh_job_gate(wait=90):
        return
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        sys.exit(1)

    device_file = cfg["device_file"]
    storage_mode = cfg["storage_mode"]

    # Backup only targets devices that successfully authenticated at some point.
    # Devices in 'inactive' (failed auth) are skipped — use 'test ssh retest'
    # to promote them back to 'active' once the underlying issue is fixed.
    db_devices = get_devices(status="active")
    if db_devices:
        ips = [r["ip"] for r in db_devices]
    elif os.path.exists(device_file):
        ips = parse_devices(device_file)
        if ips:
            log.info("Database empty — reading from %s", device_file)
    else:
        ips = []

    if not ips:
        log.info("No devices to back up. Run 'discover' first or 'import'.")
        sys.exit(0)

    os.makedirs(CONFIGS_DIR, exist_ok=True)

    successes = []
    failures = []

    def backup_one(ip):
        conn = _db()
        row = conn.execute(
            "SELECT username, password_hash FROM devices WHERE ip = ? AND status = 'active'",
            (ip,)).fetchone()
        conn.close()
        known_user = row["username"] if row else None
        known_pw_hash = row["password_hash"] if row else None

        _t0 = time.monotonic()
        _started_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        try:
            filename, config, info = backup_device(
                ip, passwords["default"], cfg["usernames"],
                sanitize=sanitize,
                password_list=cfg.get("password_list"),
                known_username=known_user, known_password_hash=known_pw_hash,
                user_passwords=cfg.get("user_passwords"),
            )
            _record_op("backup_device",
                       started_at=_started_at,
                       duration_ms=int((time.monotonic() - _t0) * 1000),
                       success=True, ip=ip, reason="backup")
            return (ip, filename, config, None, info)
        except Exception as e:
            _record_op("backup_device",
                       started_at=_started_at,
                       duration_ms=int((time.monotonic() - _t0) * 1000),
                       success=False, ip=ip, reason="backup",
                       error=str(e)[:200])
            return (ip, None, None, e,
                    {"hostname": None, "base_mac": "", "serial": "",
                     "firmware": "", "model": "", "hardware_info": "",
                     "proto": None, "username": None, "password_hash": None})

    results = []
    total = len(ips)
    done = 0
    with ThreadPoolExecutor(max_workers=cfg["ssh_threads"]) as pool:
        futures = {pool.submit(backup_one, ip): ip for ip in ips}
        for future in as_completed(futures):
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rBacking up devices... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()
            results.append(future.result())

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    results.sort(key=lambda r: ipaddress.ip_address(r[0]))
    seen_macs = {}
    skipped = []
    for ip, filename, config, error, info in results:
        if error:
            # Non-switch detection during backup isn't a failure — it means
            # this device was misclassified as a switch. Reclassify and skip
            # without logging an error or adding to the failure list.
            if isinstance(error, NotASwitchError):
                log.info("  %s — not a switch, reclassifying as non-switch", ip)
                upsert_device(ip, status="non-switch",
                              fail_reason="not a switch (refused at backup)")
                continue
            log.error("  %s FAILED: %s", ip, error)
            failures.append(ip)
            upsert_device(ip, status="inactive", fail_reason=str(error))
            continue

        base_mac = info["base_mac"]

        if base_mac and base_mac in seen_macs:
            log.info("  %s SKIPPED (duplicate of %s, mac %s)",
                     ip, seen_macs[base_mac], base_mac)
            skipped.append(ip)
            upsert_device(ip, base_mac=base_mac, hostname=info["hostname"],
                          firmware=info.get("firmware"),
                          hardware_info=info.get("hardware_info"),
                          proto=info["proto"], username=info["username"],
                          password_hash=info.get("password_hash"),
                          ssh_open=info.get("ssh_open"),
                          telnet_open=info.get("telnet_open"),
                          status="duplicate", duplicate_of=seen_macs[base_mac])
            continue
        if base_mac:
            seen_macs[base_mac] = ip

        changed = False
        if storage_mode == "local":
            changed = save_config_local(filename, config)
            config_set = info.get("config_set")
            if config_set:
                set_filename = os.path.splitext(filename)[0] + "_set.cfg"
                save_config_local(set_filename, config_set)
            if changed:
                log.info("  %s OK (config updated)", ip)
            else:
                log.info("  %s OK (no changes)", ip)
        else:
            config_path = os.path.join(CONFIGS_DIR, filename)
            with open(config_path, "w", encoding="utf-8") as f:
                f.write(config)
            config_set = info.get("config_set")
            if config_set:
                set_path = os.path.splitext(config_path)[0] + "_set.cfg"
                with open(set_path, "w", encoding="utf-8") as f:
                    f.write(config_set)
            changed = True
            log.info("  %s OK", ip)

        successes.append(filename)

        config_hash = hashlib.sha256(config.encode()).hexdigest()
        record_backup(ip, base_mac=base_mac, hostname=info["hostname"],
                      filename=filename, config_hash=config_hash,
                      changed=changed)
        upsert_device(ip, base_mac=base_mac, hostname=info["hostname"],
                      serial=info.get("serial"), firmware=info.get("firmware"),
                      hardware_info=info.get("hardware_info"),
                      proto=info["proto"], username=info["username"],
                      password_hash=info.get("password_hash"),
                      ssh_open=info.get("ssh_open"),
                      telnet_open=info.get("telnet_open"),
                      status="active")

    if storage_mode == "git" and successes:
        try:
            git_commit_changes(
                successes,
                remote_url=cfg.get("git_remote_url", ""),
                branch=cfg.get("git_branch", "main"))
        except Exception as e:
            log.error("Git commit failed: %s", e)

    log.info("--- Summary ---")
    log.info("Success:    %d", len(successes))
    if skipped:
        log.info("Duplicates: %d (same switch, different IP)", len(skipped))
    log.info("Failed:     %d", len(failures))
    if failures:
        log.info("Failed devices: %s", ", ".join(failures))


# Cap the per-device diff quoted in a config-change alert. A rogue change
# is usually a handful of lines; a 2000-line diff means a wholesale
# replacement and the email only needs to say so, not carry all of it.
_CONFIG_ALERT_DIFF_LINES = 400


def run_monitor_config(cfg, email=True):
    """`monitor config` — intraday config-change watch.

    An unexpected running-config change in the middle of the day is a
    security signal (unauthorized access, out-of-process change), and the
    nightly backup would sit on it for up to 24h. This pulls the config
    from every active device through the same collection + sanitize path
    as 'backup', archives the outgoing version on change (the same trail
    'show config-history' reads), records the event in the backups table,
    and emails ONE batched [netops] alert carrying a unified diff per
    changed device.

    Deliberate differences from 'backup':
      - unchanged devices leave no backups-table row, so the nightly run
        stays the freshness signal for 'digest backup';
      - a first-ever capture is a baseline, not a change — no alert;
      - connect failures are logged and skipped; down-device alerting
        belongs to the unreachable path, not this one;
      - two-poll confirmation (the STP-alert philosophy applied to
        configs): a detected change is re-pulled before it is believed.
        ICX stacks have historically dropped whole config sections from
        `show running-config` under concurrent sessions — a transient
        capture glitch must not fire a security alert. Re-pull matches
        the old config => glitch, logged and ignored; matches the new =>
        confirmed; matches neither (operator mid-change) => the newest
        capture wins.
    """
    import difflib

    if not _ssh_job_gate(wait=90):
        return
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or "
                  "mappings in [user_passwords] (netops.conf or secrets.conf).")
        sys.exit(1)
    storage_mode = cfg["storage_mode"]
    db_devices = get_devices(status="active")
    if not db_devices:
        log.info("No active devices to check.")
        return
    os.makedirs(CONFIGS_DIR, exist_ok=True)

    def check_one(row):
        ip = row["ip"]
        _t0 = time.monotonic()
        _started_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        try:
            filename, config, info = backup_device(
                ip, passwords["default"], cfg["usernames"],
                sanitize=True,
                password_list=cfg.get("password_list"),
                known_username=row["username"],
                known_password_hash=row["password_hash"],
                user_passwords=cfg.get("user_passwords"),
            )
            _record_op("config_check",
                       started_at=_started_at,
                       duration_ms=int((time.monotonic() - _t0) * 1000),
                       success=True, ip=ip, reason="monitor_config")
            return (ip, filename, config, info, None)
        except Exception as e:
            _record_op("config_check",
                       started_at=_started_at,
                       duration_ms=int((time.monotonic() - _t0) * 1000),
                       success=False, ip=ip, reason="monitor_config",
                       error=str(e)[:200])
            return (ip, None, None, None, e)

    rows_by_ip = {r["ip"]: r for r in db_devices}
    results = []
    with ThreadPoolExecutor(max_workers=cfg["ssh_threads"]) as pool:
        futures = [pool.submit(check_one, r) for r in db_devices]
        for future in as_completed(futures):
            results.append(future.result())

    changes = []          # dicts: ip, hostname, added, removed, diff_lines
    baselines = []
    failed = []
    committed = []
    for ip, filename, config, info, error in sorted(
            results, key=lambda r: _ip_sort_key(r[0])):
        if error:
            failed.append(ip)
            log.debug("  %s config check failed: %s", ip, error)
            continue
        if storage_mode == "local":
            current_path = os.path.join(CONFIGS_DIR, "current", filename)
        else:
            current_path = os.path.join(CONFIGS_DIR, filename)
        old = None
        if os.path.isfile(current_path):
            with open(current_path, encoding="utf-8", errors="replace") as f:
                old = f.read()

        if old == config:
            log.debug("  %s unchanged", ip)
            continue

        if old is not None:
            row = rows_by_ip.get(ip)
            try:
                _fn2, config2, info2 = backup_device(
                    ip, passwords["default"], cfg["usernames"],
                    sanitize=True,
                    password_list=cfg.get("password_list"),
                    known_username=row["username"] if row else None,
                    known_password_hash=row["password_hash"] if row else None,
                    user_passwords=cfg.get("user_passwords"),
                )
            except Exception as e:
                log.warning("  %s: change seen but the confirm re-pull "
                            "failed (%s) — deferring to the next tick", ip, e)
                continue
            if config2 == old:
                log.warning("  %s: change NOT confirmed on re-pull — "
                            "transient capture glitch, ignored", ip)
                _record_op("config_check_glitch",
                           started_at=datetime.now().strftime(
                               "%Y-%m-%d %H:%M:%S"),
                           duration_ms=0, success=True, ip=ip,
                           reason="monitor_config",
                           error="unconfirmed change (capture glitch)")
                continue
            if config2 != config:
                # Re-pull differs from BOTH the stored config and the
                # first pull: an unstable capture (pager glitch, session
                # fallback, device mid-write). Alerting would email
                # garbage — skip, record, let the next tick decide.
                log.warning("  %s: capture UNSTABLE (re-pull differs from "
                            "stored AND first pull) — not archived, not "
                            "alerted; deferring to next tick", ip)
                _record_op("config_check_glitch",
                           started_at=datetime.now().strftime(
                               "%Y-%m-%d %H:%M:%S"),
                           duration_ms=0, success=True, ip=ip,
                           reason="monitor_config",
                           error="unstable capture (three-way mismatch)")
                continue

        if storage_mode == "local":
            save_config_local(filename, config)
            config_set = info.get("config_set")
            if config_set:
                save_config_local(
                    os.path.splitext(filename)[0] + "_set.cfg", config_set)
        else:
            with open(current_path, "w", encoding="utf-8") as f:
                f.write(config)
            config_set = info.get("config_set")
            if config_set:
                set_path = os.path.splitext(current_path)[0] + "_set.cfg"
                with open(set_path, "w", encoding="utf-8") as f:
                    f.write(config_set)
        committed.append(filename)
        record_backup(ip, base_mac=info.get("base_mac"),
                      hostname=info["hostname"], filename=filename,
                      config_hash=hashlib.sha256(config.encode()).hexdigest(),
                      changed=True)

        if old is None:
            baselines.append(ip)
            log.info("  %s baseline config captured", ip)
            continue

        # Alert only for a CONFIDENTLY-IDENTIFIED device. A row with no
        # base_mac AND no serial never produced a clean identifying
        # capture — which on a multi-IP L3 stack means it's an
        # un-deduped secondary SVI whose captures garble under concurrent
        # sessions to the same chassis (dropped chars, reordered
        # interfaces). Its "diff" is capture noise, not an operator
        # change, and it can't be a trustworthy security signal. Still
        # archive it (so a later clean capture identifies it and
        # reconcile dedupes it) — just don't email.
        row = rows_by_ip.get(ip)
        identified = bool((info.get("base_mac") or
                           (row["base_mac"] if row and "base_mac" in
                            row.keys() else None)) or
                          (info.get("serial") or
                           (row["serial"] if row and "serial" in
                            row.keys() else None)))
        if not identified:
            log.info("  %s config changed but device is UNIDENTIFIED "
                     "(no base_mac/serial) — archived, not alerted "
                     "(likely an un-deduped stack SVI)", ip)
            continue

        diff = list(difflib.unified_diff(
            old.splitlines(), config.splitlines(),
            fromfile=f"{filename} (before)", tofile=f"{filename} (after)",
            lineterm=""))
        added = sum(1 for l in diff
                    if l.startswith("+") and not l.startswith("+++"))
        removed = sum(1 for l in diff
                      if l.startswith("-") and not l.startswith("---"))
        if len(diff) > _CONFIG_ALERT_DIFF_LINES:
            omitted = len(diff) - _CONFIG_ALERT_DIFF_LINES
            diff = diff[:_CONFIG_ALERT_DIFF_LINES] + [
                f"... [{omitted} more diff line(s) omitted — full history: "
                f"netops show config-history {ip} --diff]"]
        changes.append({"ip": ip, "hostname": info["hostname"] or ip,
                        "added": added, "removed": removed, "diff": diff})
        log.warning("  %s CONFIG CHANGED (+%d/-%d lines)",
                    ip, added, removed)

    if storage_mode == "git" and committed:
        try:
            git_commit_changes(committed,
                               remote_url=cfg.get("git_remote_url", ""),
                               branch=cfg.get("git_branch", "main"))
        except Exception as e:
            log.error("Git commit failed: %s", e)

    log.info("monitor config: %d checked, %d changed, %d baseline, %d failed",
             len(results), len(changes), len(baselines), len(failed))
    if not changes:
        return

    for c in changes:
        _audit_log("config_change_detected", ip=c["ip"],
                   detail=f"+{c['added']}/-{c['removed']}")

    if not email:
        log.warning("monitor config: email suppressed (--no-email) — "
                    "%d change(s) NOT alerted", len(changes))
        return

    muted = _silenced_ip_set([c["ip"] for c in changes])
    if muted:
        for c in changes:
            if c["ip"] in muted:
                log.info("  %s config-change alert muted by silence "
                         "(change archived + recorded)", c["ip"])
        changes = [c for c in changes if c["ip"] not in muted]
        if not changes:
            return

    names = [c["hostname"] for c in changes]
    shown = ", ".join(names[:3]) + (", …" if len(names) > 3 else "")
    subject = (f"[netops] CONFIG CHANGED — {len(changes)} device(s): {shown}")
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    body_lines = [
        f"Running-config change(s) detected {now} by the intraday config",
        "watch (netops monitor config). If nobody was making changes,",
        "treat this as a possible security event.",
        "",
        "Verify: who-did-what audit trail on this server:",
        "  journalctl -t netops-audit --since today",
        "Full change history for a device:",
        "  netops show config-history <device> [--diff]",
        "",
    ]
    for c in changes:
        body_lines += [
            f"===== {c['hostname']} ({c['ip']}) — "
            f"+{c['added']}/-{c['removed']} line(s) =====",
        ]
        body_lines += c["diff"]
        body_lines.append("")
    ok, detail = _send_email(cfg, subject, "\n".join(body_lines))
    if ok:
        log.info("Config-change alert %s", detail)
    else:
        log.error("Config-change alert email failed: %s", detail)


def _format_outage_duration(alerted_at_str):
    """Render an alerted_at timestamp as '2h 35m' since now. '?' on parse failure."""
    try:
        t0 = datetime.strptime(alerted_at_str, "%Y-%m-%d %H:%M:%S")
        sec = max(0, int((datetime.now() - t0).total_seconds()))
        h, rem = divmod(sec, 3600)
        m, _ = divmod(rem, 60)
        return f"{h}h {m}m" if h else f"{m}m"
    except (TypeError, ValueError):
        return "?"


def _classify_reachability(ip, timeout=3.0):
    """Quick TCP probe to confirm/refute an SNMP-unreachable signal.

    Reuses check_ports (the same helper run_probe uses). Returns
    {'ssh': bool, 'telnet': bool}. Both False is the 'switch is dark'
    signal; either True means SNMP-specific issue (ACL, community,
    daemon) on an otherwise-reachable box.
    """
    open_ports = check_ports(ip, [22, 23], timeout)
    return {"ssh": 22 in open_ports, "telnet": 23 in open_ports}


def _send_unreachable_alert(cfg, downed):
    """Email a batched 'netops can't monitor' alert.

    Triggered after N consecutive failed polls. In auto poll mode this
    means BOTH the SNMP fast-path AND the SSH/CLI fallback failed to
    return STP data — not just a transient SNMP hiccup. The TCP probe
    (port 22 / 23 banner check) only confirms whether a daemon is
    listening; it does NOT mean SSH is usable for monitoring (creds,
    parser support, auth-rate limits can all still be broken).

    downed: list of dicts with keys ip, hostname, model, fails,
    last_ok, last_diag, probe (output of _classify_reachability).
    Partitions into 'no protocol responding' (TCP closed on both 22
    and 23) and 'daemon listens but unusable' (banner open, data
    collection still failing) so the operator can triage at a glance.
    """
    muted = _silenced_ip_set([d["ip"] for d in downed])
    if muted:
        for d in downed:
            if d["ip"] in muted:
                log.info("unreachable alert for %s muted by silence", d["ip"])
        downed = [d for d in downed if d["ip"] not in muted]
        if not downed:
            return

    # --- Outage correlation (3.19.0). When several devices are down at
    # once, group them by their highest DOWNED ancestor in the STP
    # root-port topology: a dead aggregation switch (or a powered-off
    # closet/building) shows as ONE outage with its casualty list, not a
    # pile of unrelated alerts. Power-loss scenario per the field report:
    # everything below the dead switch fails together.
    conn = _db()
    conn.row_factory = sqlite3.Row
    parent = _stp_parent_map(conn)
    conn.close()
    downed_ips = {d["ip"] for d in downed}
    by_ip = {d["ip"]: d for d in downed}

    def _outage_root(ip):
        cur, root, seen = ip, ip, {ip}
        while cur in parent:
            p = parent[cur][0]
            if p in seen:
                break
            if p in downed_ips:
                root = p
            seen.add(p)
            cur = p
        return root

    groups = {}
    for d in downed:
        groups.setdefault(_outage_root(d["ip"]), []).append(d["ip"])
    corr = {root: members for root, members in groups.items()
            if len(members) > 1}
    independent = [by_ip[m] for root, members in groups.items()
                   if len(members) == 1 for m in members]

    def _dev_lines(d, indent="  "):
        banner = []
        if d["probe"]["ssh"]:    banner.append("SSH/22")
        if d["probe"]["telnet"]: banner.append("Telnet/23")
        out = [f"{indent}{d['hostname'] or d['ip']} ({d['ip']})  "
               f"[{d['model'] or '?'}]  "
               f"TCP banner: {', '.join(banner) or 'none'}"]
        out.append(f"{indent}  consecutive-fails: {d['fails']}  "
                   f"last-ok: {d['last_ok'] or '(never)'}")
        if d["last_diag"]:
            out.append(f"{indent}  last-error: {d['last_diag']}")
        return out

    if corr:
        biggest_root = max(corr, key=lambda r: len(corr[r]))
        broot = by_ip[biggest_root]
        n_groups = len(corr)
        subject = (f"[netops] OUTAGE: {len(downed)} switch(es) down — "
                   f"likely root: {broot['hostname'] or broot['ip']} "
                   f"(+{len(corr[biggest_root]) - 1} downstream)")
        if n_groups > 1 or independent:
            subject += (f" [{n_groups} group(s)"
                        + (f" + {len(independent)} other" if independent
                           else "") + "]")
    elif len(downed) == 1:
        d = downed[0]
        subject = (f"[netops] switch UNREACHABLE: "
                   f"{d['hostname'] or d['ip']} ({d['ip']})")
    else:
        subject = f"[netops] {len(downed)} switch(es) unreachable"

    lines = []
    for root, members in sorted(corr.items(),
                                key=lambda kv: -len(kv[1])):
        rd = by_ip[root]
        casualties = [m for m in members if m != root]
        up = parent.get(root)
        up_note = ""
        if up:
            up_ip = up[0]
            up_state = "ALSO DOWN" if up_ip in downed_ips else "reachable"
            up_note = f"; its uplink neighbor {up_ip} is {up_state}"
        lines.append(f"=== Correlated outage — likely root: "
                     f"{rd['hostname'] or rd['ip']} ({rd['ip']}), "
                     f"{len(casualties)} downstream switch(es) also "
                     f"down{up_note} ===")
        lines.append("All of these sit below the root in the STP topology "
                     "and failed together — one failure domain (power, "
                     "uplink, or the root switch itself).")
        lines.extend(_dev_lines(rd))
        if casualties:
            lines.append(f"  downstream casualties ({len(casualties)}):")
            for m in sorted(casualties,
                            key=lambda m: (by_ip[m]["hostname"] or m)):
                md = by_ip[m]
                lines.append(f"    {md['hostname'] or m} ({m})  "
                             f"[{md['model'] or '?'}]")
        lines.append("")
        trace_lines = _format_topology_trace_lines(
            root, rd["hostname"], max_hops=4, activity_window_min=5)
        if trace_lines:
            lines.append(f"=== Upstream trace from the outage root ===")
            lines.extend(trace_lines)
            lines.append("")

    if corr and independent:
        lines.append(f"=== Not part of a correlated group "
                     f"({len(independent)}) ===")
        lines.append("")

    downed = independent if corr else downed
    no_banner = [d for d in downed
                 if not d["probe"]["ssh"] and not d["probe"]["telnet"]]
    banner_up = [d for d in downed
                 if d["probe"]["ssh"] or d["probe"]["telnet"]]

    if no_banner:
        lines.append(f"=== Unreachable — no protocol responding ({len(no_banner)}) ===")
        lines.append("SNMP poll failed AND TCP/22 + TCP/23 both closed. "
                     "Likely: switch off the network, mgmt VLAN/uplink down, or rebooting.")
        lines.append("")
        for d in no_banner:
            lines.append(f"  {d['hostname'] or d['ip']} ({d['ip']})  [{d['model'] or '?'}]")
            lines.append(f"    consecutive-fails: {d['fails']}  last-ok: {d['last_ok'] or '(never)'}")
            if d["last_diag"]:
                lines.append(f"    last-error: {d['last_diag']}")
        lines.append("")
    if banner_up:
        lines.append(f"=== Can't monitor — SNMP AND SSH/CLI both failed; "
                     f"TCP banner open ({len(banner_up)}) ===")
        lines.append("netops tried SNMP, then fell back to SSH/CLI; both failed. "
                     "TCP/22 (or /23) listening does NOT mean SSH is usable — "
                     "credentials may be wrong, the platform may be unsupported by "
                     "our parsers, or the device may be config-locked.")
        lines.append("Likely: missing/wrong creds, unsupported model, SNMP not "
                     "configured, or device-side config issue. Operator action "
                     "required — netops cannot see this switch's STP state.")
        lines.append("")
        for d in banner_up:
            banner = []
            if d["probe"]["ssh"]:    banner.append("SSH/22")
            if d["probe"]["telnet"]: banner.append("Telnet/23")
            lines.append(f"  {d['hostname'] or d['ip']} ({d['ip']})  [{d['model'] or '?'}]  "
                         f"TCP banner: {', '.join(banner)}")
            lines.append(f"    consecutive-fails: {d['fails']}  last-ok: {d['last_ok'] or '(never)'}")
            if d["last_diag"]:
                lines.append(f"    last-error: {d['last_diag']}")
        lines.append("")

    # Per-device upstream topology trace — Slice A. Walks the cached
    # topology_edges from each downed device toward the root bridge and
    # reports recent activity at each hop. Catches the common "the
    # uplink to this switch flapped" failure mode without the operator
    # needing to mentally walk the topology themselves.
    for d in downed:
        trace_lines = _format_topology_trace_lines(
            d["ip"], d["hostname"], max_hops=4, activity_window_min=5)
        if trace_lines:
            lines.append(f"=== Upstream trace from {d['hostname'] or d['ip']} ({d['ip']}) ===")
            lines.extend(trace_lines)
            lines.append("")

    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("Unreachable alert email %s (%d switch(es))", detail, len(downed))
    else:
        log.error("Unreachable alert email failed: %s", detail)


def _send_recovery_alert(cfg, recovered):
    """Email a batched 'switch recovered' alert.

    recovered: list of dicts with keys ip, hostname, model, alerted_at, downtime.
    """
    muted = _silenced_ip_set([d["ip"] for d in recovered])
    if muted:
        recovered = [d for d in recovered if d["ip"] not in muted]
        if not recovered:
            return
    subject = f"[netops] {len(recovered)} switch(es) recovered"
    lines = ["Previously-unreachable switch(es) are responding again.", ""]
    for d in recovered:
        lines.append(f"  {d['hostname'] or d['ip']} ({d['ip']})  [{d['model'] or '?'}]")
        lines.append(f"    down since: {d['alerted_at']}  downtime: {d['downtime']}")
    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("Recovery alert email %s (%d switch(es))", detail, len(recovered))
    else:
        log.error("Recovery alert email failed: %s", detail)


# ---------------------------------------------------------------------------
# Topology mapping — fleet-wide LLDP scrape, populates topology_edges so the
# alert path can walk one or more hops upstream from an affected device.
# ---------------------------------------------------------------------------

# CLI error/help/pager lines that must never be parsed as LLDP neighbors.
# A mis-detected platform (e.g. a FastIron ICX cached as 'junos') runs the
# wrong 'show lldp' command, and the device's '% Invalid input' / 'Type ...
# for a list of ...' / Junos 'warning:' reply would otherwise be turned into
# bogus edges (seen on one site: src_port = Invalid/Type/show/Error).
_LLDP_NOISE_RE = re.compile(
    r"invalid input|unknown command|ambiguous|incomplete|"
    r"%\s*error|^\s*error\b|type\s+.*\bfor a list|not needed by configuration|"
    r"--more--|^\s*\^\s*$|^\s*warning\b",
    re.IGNORECASE)


def _lldp_line_is_noise(line):
    """True for CLI error/help/pager lines that aren't LLDP neighbor rows."""
    return bool(_LLDP_NOISE_RE.search(line))


# LLDP-MIB lldpRemSysCapEnabled is a BITS field; bit 0 is the MSB of octet 1.
# Verified against live Aruba-CX gear: 0x30=bridge+wlanAP (an AP),
# 0x28=bridge+router (an L3 switch), 0x24=bridge+telephone (a phone).
_LLDP_CAP_BITS = (
    (0x80, "other"), (0x40, "repeater"), (0x20, "bridge"),
    (0x10, "wlan-access-point"), (0x08, "router"), (0x04, "telephone"),
    (0x02, "docsis"), (0x01, "station-only"),
)
# Words seen in CLI 'System/Enabled capabilities' lines (FastIron etc.),
# mapped to the same canonical capability names.
_LLDP_CAP_WORDS = {
    "bridge": "bridge", "wlan": "wlan-access-point", "wlan-access-point": "wlan-access-point",
    "access": "wlan-access-point", "wlanaccesspoint": "wlan-access-point",
    "router": "router", "telephone": "telephone", "phone": "telephone",
    "repeater": "repeater", "station": "station-only", "station-only": "station-only",
}


def _decode_lldp_caps(raw):
    """Decode an lldpRemSysCapEnabled SNMP value to a set of capability
    names. The value comes back as the raw bitmask octet — as an int, as
    bytes, or (the common case from our SNMP layer) a 1-char string whose
    ord() is the octet. Hex text ('30 00' / '0x30') is tolerated too."""
    if raw is None:
        return set()
    b = None
    if isinstance(raw, int):
        b = raw
    elif isinstance(raw, (bytes, bytearray)):
        b = raw[0] if raw else 0
    else:
        s = str(raw)
        if not s:
            return set()
        mh = re.match(r"^\s*(?:0x)?([0-9a-fA-F]{2})(?:[\s:]|$)", s)
        b = int(mh.group(1), 16) if (mh and len(s.strip()) >= 2) else ord(s[0])
    return {name for bit, name in _LLDP_CAP_BITS if b & bit}


def _caps_words_to_set(text):
    """Parse a CLI 'capabilities' line ('bridge, telephone') to cap names."""
    out = set()
    for tok in re.split(r"[,\s]+", (text or "").lower()):
        if tok in _LLDP_CAP_WORDS:
            out.add(_LLDP_CAP_WORDS[tok])
    return out


# Cisco IOS LLDP capability letter codes ('Enabled Capabilities: B,R') ->
# the same canonical names used elsewhere.
_CISCO_LLDP_CAP_LETTERS = {
    "B": "bridge", "R": "router", "T": "telephone",
    "W": "wlan-access-point", "C": "docsis", "S": "station-only",
    "P": "repeater", "O": "other",
}


def _cisco_caps_to_set(text):
    """Parse a Cisco 'Enabled Capabilities: B,R' line to canonical cap names."""
    out = set()
    for tok in re.split(r"[,\s]+", (text or "").strip()):
        name = _CISCO_LLDP_CAP_LETTERS.get(tok.upper())
        if name:
            out.add(name)
    return out


def _lldp_caps_to_kind(caps):
    """Classify an LLDP neighbor from its capability set. Precedence:
    AP > phone > switch > router > host > other — a phone advertises
    bridge+telephone (internal PC-port switch), so telephone wins over
    bridge; an AP advertises bridge+wlanAP, so wlanAP wins."""
    if not caps:
        return "unknown"
    if "wlan-access-point" in caps:
        return "ap"
    if "telephone" in caps:
        return "phone"
    if "bridge" in caps:
        return "switch"
    if "router" in caps:
        return "router"
    if "station-only" in caps:
        return "host"
    return "other"


def _fixed_width_spans(marker_line, drop_divider=True):
    """Column (start, end) char-spans inferred from a header or rule line.

    Each maximal run of non-space chars begins a column; the column runs to the
    next run's start (the final column to end-of-line, end=None). With
    drop_divider, a lone '+' run — the ProCurve LocalPort divider in the
    '--- + ---' rule line — is dropped so it doesn't become a column.
    """
    runs = [(mm.start(), mm.group()) for mm in re.finditer(r"\S+", marker_line)]
    spans = []
    for i, (start, txt) in enumerate(runs):
        end = runs[i + 1][0] if i + 1 < len(runs) else None
        if drop_divider and txt in ("+", "|"):
            continue
        spans.append((start, end))
    return spans


def _slice_columns(line, spans):
    """Slice a fixed-width row into stripped column values per _fixed_width_spans."""
    return [(line[s:e] if e is not None else line[s:]).strip() for (s, e) in spans]


def _parse_lldp_neighbors_bulk(text, platform):
    """Parse a fleet-style 'show lldp neighbors' output into a list of dicts.

    Each dict has: local_port, neighbor_chassis, neighbor_sysname,
    neighbor_port, neighbor_port_desc. Best-effort regex-based parsing —
    vendor table formats differ; we tolerate column-order changes by
    keying off the header row when we can detect one.

    Skips internal stack-member LLDP rows on Aruba-CX 6200/8325 stacks
    (interface format X/M/N where M is a member index — the stack
    members peer with each other internally and aren't useful for
    cross-switch topology).
    """
    rows = []
    if not text:
        return rows
    lines = text.splitlines()
    if platform == "junos":
        # Format:
        #   Local Interface    Parent Interface  Chassis Id          Port info        System Name
        #   ge-0/0/1           -                 ec:eb:b8:11:22:33   ge-1/0/1         AG-BDF-0
        # Junos returns logical iface names ('ge-0/0/1.0'); we canonicalize
        # to the physical name so the walk's stp_state lookup (which also
        # canonicalizes) finds the matching edge.
        for line in lines:
            line = line.rstrip()
            if not line or line.lstrip().startswith("Local Interface"):
                continue
            if _lldp_line_is_noise(line):
                continue
            m = re.match(
                r"^\s*((?:ge|xe|et|fe|me|ae|lo|ms)-\S+|\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s*(.*)$",
                line)
            if not m:
                continue
            local, parent, chassis, port, sysname = m.groups()
            # A real interface name always contains a digit — rejects CLI
            # noise tokens (warning:, Invalid, Type, show, Error, ...).
            if local in ("ge-X/Y/Z",) or not re.search(r"\d", local):
                continue
            local = local.split(".", 1)[0]
            rows.append({
                "local_port":         local,
                "neighbor_chassis":   chassis,
                "neighbor_sysname":   sysname.strip() or "",
                "neighbor_port":      port,
                "neighbor_port_desc": "",
            })
    elif platform == "aruba-cx":
        # AOS-CX 'show lldp neighbor-info' — a fixed-width table whose columns
        # come from the header:
        #   LOCAL-PORT  CHASSIS-ID         PORT-ID   PORT-DESC   TTL  SYS-NAME
        #   1/1/47      f8:60:f0:c3:1a:96  1/1/33    CORE     120  CORE-1
        # (Earlier code split on '|', a delimiter this firmware does not emit,
        # so it parsed nothing — every aruba-cx switch fell off the map unless
        # SNMP collected it.) PORT-DESC / SYS-NAME can contain spaces, so we
        # slice by column position rather than splitting on whitespace.
        spans = None
        for line in lines:
            line = line.rstrip()
            if _lldp_line_is_noise(line):
                continue
            if spans is None:
                up = line.upper()
                if "LOCAL-PORT" in up and "CHASSIS-ID" in up:
                    spans = _fixed_width_spans(line, drop_divider=False)
                continue
            if not line.strip() or re.match(r"^\s*-{3,}\s*$", line):
                continue
            cols = _slice_columns(line, spans)
            if len(cols) < 6:
                continue
            local, chassis, port_id, port_desc, _ttl, sysname = cols[:6]
            if not re.match(r"^\d+/\d+/\d+", local):
                continue
            rows.append({
                "local_port":         local,
                "neighbor_chassis":   chassis,
                "neighbor_sysname":   sysname,
                "neighbor_port":      port_id,
                "neighbor_port_desc": port_desc,
            })
    elif platform == "fastiron":
        # Ruckus/Brocade FastIron (ICX) 'show lldp neighbors detail'. Blocks:
        #   Local port: 1/2/1
        #     Neighbor: f860.f0c3.1a96, TTL 120 seconds
        #       + Chassis ID (...): <val>
        #       + Port ID (...): <val>
        #       + System name: <val>
        # A local port can list several Neighbor blocks (phones, APs, the
        # uplink switch). The 'Neighbor:' value is the chassis MAC (dotted),
        # which _normalize_mac canonicalizes for the devices.base_mac match.
        # The plain 'show lldp neighbors' table is too ragged to parse; the
        # detail form is used (see _collect_lldp_one).
        cur = None
        cur_local = None
        for line in lines:
            line = line.rstrip()
            if _lldp_line_is_noise(line):
                continue
            m = re.match(r"^\s*Local port:\s*(\S+)", line)
            if m:
                if cur and cur["neighbor_chassis"] and cur["local_port"]:
                    rows.append(cur)
                cur = None
                cur_local = m.group(1)
                continue
            m = re.match(r"^\s*Neighbor:\s*([0-9a-fA-F][0-9a-fA-F.:-]+)", line)
            if m:
                if cur and cur["neighbor_chassis"] and cur["local_port"]:
                    rows.append(cur)
                cur = {"local_port": cur_local, "neighbor_chassis": m.group(1),
                       "neighbor_sysname": "", "neighbor_port": "",
                       "neighbor_port_desc": "", "neighbor_caps": ""}
                continue
            if cur is None:
                continue
            m = re.match(r"^\s*\+\s*Port ID\b[^:]*:\s*(.+)$", line)
            if m and not cur["neighbor_port"]:
                cur["neighbor_port"] = m.group(1).strip().strip('"')
                continue
            m = re.match(r"^\s*\+\s*System name\s*:\s*(.+)$", line)
            if m:
                cur["neighbor_sysname"] = m.group(1).strip().strip('"')
                continue
            # '+ System capabilities : bridge, telephone' or
            # 'Enabled capabilities: bridge, wlan-access-point'
            m = re.match(r"^\s*(?:\+\s*)?(?:System|Enabled) capabilit(?:y|ies)\s*:"
                         r"\s*(.+)$", line)
            if m and not cur["neighbor_caps"]:
                caps = _caps_words_to_set(m.group(1))
                if caps:
                    cur["neighbor_caps"] = ",".join(sorted(caps))
                continue
        if cur and cur["neighbor_chassis"] and cur["local_port"]:
            rows.append(cur)
    elif platform == "cisco-ios":
        # Cisco IOS 'show lldp neighbors detail' — neighbor blocks split by a
        # dashed rule, each with 'Chassis id:', 'Port id:', 'System Name:',
        # 'Enabled Capabilities:'. Older Catalysts (3560/3750) omit the
        # 'Local Intf:' line, so when it's absent we synthesize a stable local
        # port from the neighbor chassis to keep each neighbor a distinct edge.
        cur = None
        for line in lines:
            if re.match(r"^\s*-{4,}\s*$", line):
                if cur and (cur["neighbor_chassis"] or cur["neighbor_sysname"]):
                    rows.append(cur)
                cur = {"local_port": "", "neighbor_chassis": "", "neighbor_sysname": "",
                       "neighbor_port": "", "neighbor_port_desc": "", "neighbor_caps": ""}
                continue
            if cur is None:
                continue
            m = re.match(r"^\s*Local Intf:\s*(.+)$", line)
            if m:
                cur["local_port"] = m.group(1).strip(); continue
            m = re.match(r"^\s*Chassis id:\s*(.+)$", line)
            if m:
                cur["neighbor_chassis"] = m.group(1).strip(); continue
            m = re.match(r"^\s*Port id:\s*(.+)$", line)
            if m:
                cur["neighbor_port"] = m.group(1).strip(); continue
            m = re.match(r"^\s*Port Description:\s*(.+)$", line)
            if m and "not advertised" not in m.group(1):
                cur["neighbor_port_desc"] = m.group(1).strip(); continue
            m = re.match(r"^\s*System Name:\s*(.+)$", line)
            if m:
                cur["neighbor_sysname"] = m.group(1).strip(); continue
            m = re.match(r"^\s*Enabled Capabilities:\s*(.+)$", line)
            if m and not cur["neighbor_caps"]:
                caps = _cisco_caps_to_set(m.group(1))
                if caps:
                    cur["neighbor_caps"] = ",".join(sorted(caps))
                continue
        if cur and (cur["neighbor_chassis"] or cur["neighbor_sysname"]):
            rows.append(cur)
        for r in rows:
            if not r.get("local_port"):
                r["local_port"] = "lldp:" + (_normalize_mac(r["neighbor_chassis"])
                                             or r["neighbor_sysname"] or "?")
    else:  # procurve — ArubaOS-Switch / ProCurve 'show lldp info remote-device'
        # Fixed-width table; columns come from the '--- + ---' rule line:
        #   LocalPort | ChassisId          PortId             PortDescr SysName
        #   1/A1      | f860f0-c31a96      1/1/1              SW-ADM-1  CORE-1
        # ChassisId/PortId render per LLDP subtype — the chassis may be a MAC
        # ('xxxxxx-xxxxxx'), an IP, or a hostname, and the peer MAC often lands
        # in PortId — so slice by column rather than assuming a leading hex MAC.
        # Downstream resolution keys on the chassis MAC or the SysName, so the
        # managed-switch uplinks resolve even when phones/APs in the same table
        # carry an IP or hostname chassis. (The old leading-hex regex matched
        # none of this firmware's rows, dropping every SSH-collected ProCurve.)
        spans = None
        for line in lines:
            line = line.rstrip()
            if _lldp_line_is_noise(line):
                continue
            if spans is None:
                # Spans come from the header ('LocalPort | ChassisId ...') or
                # the '--- + ---' rule line — both yield identical columns; the
                # header is the firmware-independent anchor (no '+' required).
                if ("LocalPort" in line and "ChassisId" in line) or \
                   ("+" in line and re.search(r"-{3,}\s+\+\s+-{3,}", line)):
                    spans = _fixed_width_spans(line, drop_divider=True)
                continue
            if "|" not in line or not line.strip():
                continue
            cols = _slice_columns(line, spans)
            if len(cols) < 5:
                continue
            local, chassis, port_id, port_desc, sysname = cols[:5]
            if not re.match(r"^[0-9A-Za-z]+(?:/[0-9A-Za-z]+)*$", local):
                continue
            rows.append({
                "local_port":         local,
                "neighbor_chassis":   chassis,
                "neighbor_sysname":   sysname,
                "neighbor_port":      port_id,
                "neighbor_port_desc": port_desc,
            })
    return rows


def _resolve_edge_neighbor_ip(neighbor_chassis, neighbor_sysname, conn):
    """Return the IP of a topology neighbor if it's in our devices table.

    Match priority: chassis MAC -> base_mac, then sysname -> hostname.
    Same approach as Phase A's _resolve_neighbor_in_db but takes an
    already-open connection.
    """
    norm = _normalize_mac(neighbor_chassis or "")
    if norm:
        for r in conn.execute(
            "SELECT ip, base_mac FROM devices "
            "WHERE base_mac IS NOT NULL AND status IN ('active','inactive')"
        ).fetchall():
            if _normalize_mac(r["base_mac"]) == norm:
                return r["ip"]
    if neighbor_sysname:
        r = conn.execute(
            "SELECT ip FROM devices WHERE LOWER(hostname) = LOWER(?) "
            "AND status IN ('active','inactive') LIMIT 1",
            (neighbor_sysname.strip(),)).fetchone()
        if r:
            return r["ip"]
    return None


def _collect_lldp_snmp(ip, cred, snmp_cfg):
    """Walk LLDP-MIB::lldpRemTable via SNMP and return the same row dicts
    as _parse_lldp_neighbors_bulk. Works regardless of platform / login
    shell — only needs a working SNMP cred. Empty list on any failure.

    lldpRemTable is indexed by (lldpRemTimeMark, lldpRemLocalPortNum,
    lldpRemIndex). We map lldpRemLocalPortNum → human port name via
    ifName (preferred) or ifDescr (fallback) from IF-MIB. Most modern
    gear uses ifIndex as the local-port-num so this lookup just works.
    """
    timeout = int(snmp_cfg.get("timeout") or 5)
    retries = int(snmp_cfg.get("retries") or 1)
    try:
        # 1) Walk port-num → name (try ifName first, fall back to ifDescr)
        try:
            port_names = _snmp_walk_dict(cred, ip, "1.3.6.1.2.1.31.1.1.1.1",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)
        except SnmpCollectError:
            port_names = _snmp_walk_dict(cred, ip, "1.3.6.1.2.1.2.2.1.2",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)

        # 2) Walk the LLDP remote table — one walk per column we need
        chassis_ids   = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.5",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20, hex_octets=True)
        chassis_subt  = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.4",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)
        port_ids      = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.7",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20, hex_octets=True)
        port_id_subt  = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.6",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)
        try:
            port_descs = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.8",
                                          timeout=timeout, retries=retries,
                                          op_timeout=20)
        except SnmpCollectError:
            port_descs = {}
        try:
            sys_names  = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.9",
                                          timeout=timeout, retries=retries,
                                          op_timeout=20)
        except SnmpCollectError:
            sys_names = {}
        try:
            # lldpRemSysCapEnabled — types the neighbor (AP/phone/switch/...).
            sys_caps  = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.12",
                                          timeout=timeout, retries=retries,
                                          op_timeout=20)
        except SnmpCollectError:
            sys_caps = {}
    except SnmpCollectError as e:
        log.debug("lldp snmp walk failed on %s: %s", ip, e)
        return []
    if not chassis_ids and not port_ids:
        return []

    # 3) Build rows. Index keys look like '0.514.1' — split to extract the
    # local port number (the middle component).
    rows = []
    for key in chassis_ids.keys():
        parts = str(key).split(".")
        if len(parts) != 3:
            continue
        try:
            local_port_num = int(parts[1])
        except ValueError:
            continue
        local_port = port_names.get(local_port_num) or f"port{local_port_num}"
        # Junos returns logical names ('ge-0/0/1.0'); canonicalize.
        local_port = local_port.split(".", 1)[0] if "-" in local_port else local_port
        chassis_raw = chassis_ids.get(key)
        chassis_str = _format_lldp_id(chassis_raw, chassis_subt.get(key),
                                      kind="chassis")
        port_raw = port_ids.get(key)
        port_str = _format_lldp_id(port_raw, port_id_subt.get(key),
                                   kind="port")
        rows.append({
            "local_port":         local_port,
            "neighbor_chassis":   chassis_str,
            "neighbor_sysname":   (sys_names.get(key) or "").strip()
                                    if isinstance(sys_names.get(key), str) else "",
            "neighbor_port":      port_str,
            "neighbor_port_desc": (port_descs.get(key) or "").strip()
                                    if isinstance(port_descs.get(key), str) else "",
            "neighbor_caps":      ",".join(sorted(
                                    _decode_lldp_caps(sys_caps.get(key)))),
        })
    return rows


def _format_lldp_id(raw, subtype, kind="chassis"):
    """Best-effort conversion of an LLDP chassis-id / port-id to a string.

    The macAddress subtype value DIFFERS between the two ID types:
    chassis-id macAddress = 4, but port-id macAddress = 3 (and
    networkAddress is chassis 5 / port 4). `kind` selects the right
    enumeration. A macAddress ID becomes 'aa:bb:cc:dd:ee:ff'; a bare
    6-byte binary blob is treated as a MAC even if the subtype is
    mislabelled; other binary is hex-encoded; text is used as-is. IDs are
    walked with -Ox (hex output), so a text port-ID like an interface
    name arrives as bytes and decodes cleanly back to text here."""
    mac_subtype = 4 if kind == "chassis" else 3
    if raw is None:
        return ""
    if isinstance(raw, bytes):
        if (subtype == mac_subtype or subtype is None) and len(raw) == 6 \
                and any(b > 0x7e or b < 0x20 for b in raw):
            return ":".join(f"{b:02x}" for b in raw)
        # Printable bytes → text (an interface-name / local port-ID).
        if all(0x20 <= b <= 0x7e for b in raw):
            return raw.decode("ascii").rstrip("\x00").strip()
        # Non-printable but not a 6-byte MAC → hex.
        if len(raw) == 6:
            return ":".join(f"{b:02x}" for b in raw)
        return raw.hex()
    # Already a str (e.g. a value that came back without -Ox). If it still
    # holds the U+FFFD wreckage of a UTF-8-mangled MAC, we cannot recover
    # the bytes — leave it for the display layer to fall back to port_desc.
    return str(raw).strip()


# Cap distinct MACs per port before treating it as a trunk/uplink and
# dropping it from port_macs. Access/edge ports (the only ones the flux
# signal and last-known-MAC fallback care about) sit far below this; a
# port above it is structurally a switch-link already covered by the LLDP
# topology graph. Keeps port_macs small on a 365-device fleet.
_FDB_PORT_MAC_CAP = 64


def _collect_fdb_snmp(ip, cred, snmp_cfg):
    """Walk the bridge forwarding DB via SNMP -> [(ifname, mac, vlan), ...]
    for access/edge ports only.

    Q-BRIDGE dot1qTpFdbTable first (VLAN-aware; index <vlan>.<6 MAC
    octets>), else classic BRIDGE-MIB dot1dTpFdbTable (index = 6 MAC
    octets, vlan=0). Table value is a dot1dBasePort, mapped
    base->ifIndex->ifName with the same chain the SNMP STP collector
    uses. Ports with > _FDB_PORT_MAC_CAP MACs are dropped as
    trunks/uplinks. Empty list on any failure — best-effort."""
    timeout = int(snmp_cfg.get("timeout") or 5)
    retries = int(snmp_cfg.get("retries") or 1)
    try:
        base_to_ifidx = _snmp_walk_dict(cred, ip, _OID_DOT1D_BASE_PORT_IFIDX,
                                        timeout=timeout, retries=retries,
                                        op_timeout=20)
        ifidx_to_name = _snmp_walk_dict(cred, ip, _OID_IF_NAME,
                                        timeout=timeout, retries=retries,
                                        op_timeout=20)
        vlan_aware = True
        try:
            fdb = _snmp_walk_dict(cred, ip, _OID_DOT1Q_TPFDB_PORT,
                                  timeout=timeout, retries=retries,
                                  op_timeout=30)
        except SnmpCollectError:
            fdb = {}
        if not fdb:
            vlan_aware = False
            fdb = _snmp_walk_dict(cred, ip, _OID_DOT1D_TPFDB_PORT,
                                  timeout=timeout, retries=retries,
                                  op_timeout=30)
    except SnmpCollectError as e:
        log.debug("fdb snmp walk failed on %s: %s", ip, e)
        return []
    if not fdb:
        return []

    by_port = {}   # ifname -> list[(mac, vlan)]
    for idx, base_port in fdb.items():
        octets = str(idx).split(".")
        if vlan_aware:
            if len(octets) < 7:
                continue
            try:
                vlan = int(octets[0])
            except ValueError:
                continue
            # Junos EX (EX4100 and newer) report dot1qFdbId as the VLAN
            # SHIFTED LEFT 16 bits — e.g. index component 458752 (0x70000
            # = 7<<16) for VLAN 7, 131072 (2<<16) for VLAN 2 — rather than
            # the bare VLAN older EX models return. Undo the shift when the
            # value is out of the 1-4094 range with its low 16 bits clear;
            # anything still unrecognizable becomes 0 (unknown) so a bogus
            # >4094 VLAN never reaches the FDB (it would desync show mac and
            # inflate the per-port VLAN count the MAC-flux signal uses).
            if vlan > 4094:
                if (vlan & 0xFFFF) == 0 and 1 <= (vlan >> 16) <= 4094:
                    vlan = vlan >> 16
                else:
                    vlan = 0
        else:
            if len(octets) < 6:
                continue
            vlan = 0
        try:
            ovals = [int(o) for o in octets[-6:]]
        except ValueError:
            continue
        if any(o < 0 or o > 255 for o in ovals):
            continue
        if all(o == 0 for o in ovals):
            continue
        if ovals[0] & 1:               # group bit set -> multicast/broadcast
            continue
        try:
            bp = int(base_port)
        except (TypeError, ValueError):
            continue
        if bp <= 0:
            continue
        # _snmp_walk_dict stores all-digit trailing indexes as int keys
        # (base port, ifIndex), so look up with the int first; tolerate a
        # str key too in case that ever changes.
        ifidx = base_to_ifidx.get(bp, base_to_ifidx.get(str(bp)))
        if ifidx is None:
            continue
        name = ifidx_to_name.get(ifidx, ifidx_to_name.get(str(ifidx)))
        if not name:
            continue
        if "-" in name and "." in name:   # Junos 'ge-0/0/1.0' -> 'ge-0/0/1'
            name = name.split(".", 1)[0]
        mac = ":".join("%02x" % o for o in ovals)
        by_port.setdefault(name, []).append((mac, vlan))

    out = []
    for name, entries in by_port.items():
        if len(entries) > _FDB_PORT_MAC_CAP:
            continue                   # trunk/uplink — LLDP graph covers it
        for mac, vlan in entries:
            out.append((name, mac, vlan))
    return out


def _decode_arp_mac(mac_val):
    """Convert an SNMP ARP-table value into 'aa:bb:cc:dd:ee:ff' or None
    when malformed/multicast/broadcast/all-zero. Tolerates both bytes
    (the usual Hex-STRING parse) and space-separated hex strings."""
    if isinstance(mac_val, bytes):
        mb = list(mac_val)
    elif isinstance(mac_val, str):
        try:
            mb = [int(b, 16) for b in mac_val.split()]
        except ValueError:
            return None
    else:
        return None
    if len(mb) != 6 or any(b < 0 or b > 255 for b in mb):
        return None
    if all(b == 0 for b in mb):
        return None
    if mb[0] & 1:                      # group bit -> multicast/broadcast
        return None
    return ":".join("%02x" % b for b in mb)


def _collect_arp_snmp(ip, cred, snmp_cfg):
    """Walk ARP/neighbor cache -> [(ifindex, ip_str, mac_str), ...].

    Tries ipNetToPhysicalTable first (modern dual-stack — Aruba CX,
    recent Cisco/Juniper); falls back to ipNetToMediaTable for older
    devices (Juniper EX, ProCurve) that still ship the legacy IPv4-only
    table. IPv4-only here regardless of table — the launch plan defers
    IPv6 enrichment. Empty list on any failure or on pure L2 access
    switches that expose neither — best-effort, like _collect_fdb_snmp.

    Closes the identification chain: a MAC harvested into port_macs is
    joined here to find its IP (the device's ARP cache learned the
    IP<->MAC pair when the host first ARP'd its gateway), which then
    feeds reverse-DNS for a hostname in alert email enrichment.
    """
    timeout = int(snmp_cfg.get("timeout") or 5)
    retries = int(snmp_cfg.get("retries") or 1)
    out = []
    # ipNetToPhysicalTable: index = '<ifindex>.<addrType>.<addrLen>.<addr>'.
    # IPv4 rows have addrType=1, addrLen=4, so the dot-string has exactly
    # 7 components: '<ifindex>.1.4.<a>.<b>.<c>.<d>'. IPv6 rows (addrType=2,
    # addrLen=16) have many more components and are skipped here.
    try:
        modern = _snmp_walk_dict(cred, ip, _OID_IP_NET_TO_PHYSICAL_PHYS,
                                  timeout=timeout, retries=retries,
                                  op_timeout=30)
    except SnmpCollectError as e:
        log.debug("ipNetToPhysical walk failed on %s: %s", ip, e)
        modern = {}
    for idx, mac_val in modern.items():
        parts = str(idx).split(".")
        if len(parts) != 7 or parts[1] != "1" or parts[2] != "4":
            continue
        try:
            ifindex = int(parts[0])
            octets  = [int(o) for o in parts[3:7]]
        except ValueError:
            continue
        if ifindex <= 0 or any(o < 0 or o > 255 for o in octets):
            continue
        mac_str = _decode_arp_mac(mac_val)
        if not mac_str:
            continue
        out.append((ifindex, ".".join(str(o) for o in octets), mac_str))
    if out:
        return out
    # Legacy fallback: ipNetToMediaTable index = '<ifindex>.<a.b.c.d>'.
    try:
        legacy = _snmp_walk_dict(cred, ip, _OID_IP_NET_TO_MEDIA_PHYS,
                                  timeout=timeout, retries=retries,
                                  op_timeout=30)
    except SnmpCollectError as e:
        log.debug("ipNetToMedia walk failed on %s: %s", ip, e)
        return []
    for idx, mac_val in legacy.items():
        parts = str(idx).split(".")
        if len(parts) < 5:
            continue
        try:
            ifindex = int(parts[0])
            octets  = [int(o) for o in parts[-4:]]
        except ValueError:
            continue
        if ifindex <= 0 or any(o < 0 or o > 255 for o in octets):
            continue
        mac_str = _decode_arp_mac(mac_val)
        if not mac_str:
            continue
        out.append((ifindex, ".".join(str(o) for o in octets), mac_str))
    return out


# topology_edges is upsert-only; edges older than this are pruned each
# topology run so moved/dark cables and one-off bad parses don't linger.
_TOPOLOGY_EDGE_TTL_DAYS = 30


def _effective_platform(platform, model):
    """Correct an obviously mis-cached platform from the model string.

    Ruckus/Brocade ICX gear is sometimes cached as 'junos' (mis-detected
    before FastIron support, or a prompt quirk) — seen on one site, where
    ICX7450/ICX7750 rows carried platform='junos' and ran the Junos LLDP
    command, producing only CLI-error garbage. The model is authoritative.
    """
    if model and re.match(r"\s*ICX", model, re.IGNORECASE):
        return "fastiron"
    # Cisco Catalyst/Nexus (e.g. WS-C3560G, WS-C3750X, C9300, Nexus N9K) that
    # discovery left as procurve or unclassified — the model is authoritative.
    if model and re.match(r"\s*(WS-C|C9\d|Catalyst|Nexus|N[3579]K)", model,
                          re.IGNORECASE):
        return "cisco-ios"
    return platform


def _collect_iface_errors_snmp(ip, cred, snmp_cfg):
    """Walk the IF-MIB error/discard counters + EtherLike FCS (CRC) counter
    -> {ifname: {error_type: counter_value}}.

    Values are the cumulative counters as the device reports them; the caller
    (_record_port_errors) diffs them against the stored snapshot to derive the
    per-interval delta. A device that doesn't expose the EtherLike-MIB simply
    contributes no fcs_errors entries (that OID's walk is skipped). Empty dict
    on any failure — best-effort, like _collect_fdb_snmp."""
    timeout = int(snmp_cfg.get("timeout") or 5)
    retries = int(snmp_cfg.get("retries") or 1)
    try:
        ifidx_to_name = _snmp_walk_dict(cred, ip, _OID_IF_NAME,
                                        timeout=timeout, retries=retries,
                                        op_timeout=20)
    except SnmpCollectError as e:
        log.debug("iface-error ifname walk failed on %s: %s", ip, e)
        return {}
    if not ifidx_to_name:
        return {}
    out = {}
    for etype, oid in _IFACE_ERROR_OIDS:
        try:
            walk = _snmp_walk_dict(cred, ip, oid, timeout=timeout,
                                   retries=retries, op_timeout=20)
        except SnmpCollectError:
            continue                      # OID/table unsupported on this device
        for ifidx, val in walk.items():
            name = ifidx_to_name.get(ifidx, ifidx_to_name.get(str(ifidx)))
            if not name:
                continue
            if "-" in name and "." in name:   # Junos ge-0/0/1.0 -> ge-0/0/1
                name = name.split(".", 1)[0]
            try:
                cval = int(val)
            except (TypeError, ValueError):
                continue
            if cval < 0:
                continue
            # Stripping the Junos logical-unit suffix can collide the physical
            # port (real error count) with its .0 unit (~0). Keep the larger —
            # cumulative counters only grow, so max() is the physical's true
            # total.
            slot = out.setdefault(name, {})
            if cval > slot.get(etype, -1):
                slot[etype] = cval
    return out


def _collect_lldp_one(ip, hostname, platform, cfg, password_cache):
    """Return {"lldp": [...], "fdb": [...], "arp": [...], "errors": {...}} for
    one device.

    LLDP tries SNMP first (works on more devices, no SSH session) then
    SSH fallback. FDB (port<->MAC) and ARP (IP<->MAC) are SNMP-only and
    harvested opportunistically on the same cached SNMP cred — SSH-only
    devices contribute neither. ARP is empty on pure L2 access switches
    (which is the common case) — we walk every device because flagging
    "L3 devices" out of band would miss freshly-promoted SVIs. Empty
    lists on failure — best-effort, like run_backup."""
    snmp_cfg = cfg.get("snmp") or {}
    result = {"lldp": [], "fdb": [], "arp": [], "errors": {}}
    conn = _db()
    conn.row_factory = sqlite3.Row
    row = conn.execute(
        "SELECT username, password_hash, snmp_enabled, snmp_proto, "
        "       snmp_community, snmp_v3_user, model "
        "FROM devices WHERE ip = ?", (ip,)
    ).fetchone()
    conn.close()
    # Correct mis-cached ICX-as-junos rows before choosing the SSH command.
    platform = _effective_platform(platform, row["model"] if row else None)

    # SNMP fast path — LLDP + FDB + ARP share the one cached SNMP cred.
    if row and row["snmp_enabled"] == 1:
        try:
            cred = _resolve_snmp_cred(row, snmp_cfg)
            result["lldp"] = _collect_lldp_snmp(ip, cred, snmp_cfg) or []
            try:
                result["fdb"] = _collect_fdb_snmp(ip, cred, snmp_cfg) or []
            except SnmpCollectError as e:
                log.debug("fdb snmp on %s: %s", ip, e)
            try:
                result["arp"] = _collect_arp_snmp(ip, cred, snmp_cfg) or []
            except SnmpCollectError as e:
                log.debug("arp snmp on %s: %s", ip, e)
            try:
                result["errors"] = _collect_iface_errors_snmp(
                    ip, cred, snmp_cfg) or {}
            except SnmpCollectError as e:
                log.debug("iface-error snmp on %s: %s", ip, e)
            if result["lldp"]:
                return result
        except SnmpCollectError as e:
            log.debug("lldp snmp on %s: %s — falling back to SSH", ip, e)

    # SSH fallback — LLDP only (FDB keeps whatever SNMP produced, if any).
    if platform not in ("junos", "aruba-cx", "procurve", "fastiron", "cisco-ios"):
        return result
    if not row or not row["username"] or not row["password_hash"]:
        return result
    usernames = cfg.get("usernames") or []
    password_list = cfg.get("password_list") or []
    user_passwords = cfg.get("user_passwords") or {}
    child, proto, user, _, ssh_open, telnet_open = connect_device(
        ip, usernames, None, timeout=15,
        password_list=password_list, user_passwords=user_passwords,
        known_username=row["username"], known_password_hash=row["password_hash"],
        reason="monitor_topology_fallback")
    if child is None:
        return result
    try:
        if platform == "junos":
            cmd = "show lldp neighbors | no-more"
        elif platform == "aruba-cx":
            cmd = "show lldp neighbor-info"
        elif platform in ("fastiron", "cisco-ios"):
            cmd = "show lldp neighbors detail"
        else:  # procurve
            cmd = "show lldp info remote-device"
        out = _strip_cli_noise(send_command(child, cmd, timeout=30))
    except Exception as e:
        log.debug("lldp ssh collect failed for %s: %s", ip, e)
        return result
    finally:
        disconnect(child)
    result["lldp"] = _parse_lldp_neighbors_bulk(out, platform)
    return result


def _record_port_errors(conn, ip, hostname, iface_errors, now):
    """Diff this poll's interface error counters against the stored snapshot;
    append a port_errors row for every counter that INCREASED and refresh
    port_error_state. Returns the number of error rows written.

    First sighting of a (port, error_type) seeds the baseline WITHOUT logging
    — we can't tell how old the device's cumulative count is, so logging the
    whole thing as one delta would be a lie. A drop (counter < stored) is a
    device reboot / 32-bit wrap: rebaseline silently rather than log a bogus
    negative. Only genuine inter-poll increases become events."""
    written = 0
    for iface, counters in iface_errors.items():
        for etype, cval in counters.items():
            prev = conn.execute(
                "SELECT counter_value FROM port_error_state "
                "WHERE ip = ? AND interface = ? AND error_type = ?",
                (ip, iface, etype)).fetchone()
            if prev is not None:
                delta = cval - prev["counter_value"]
                if delta > 0:
                    conn.execute(
                        "INSERT INTO port_errors (ip, hostname, interface, "
                        "error_type, delta, counter_value, at) "
                        "VALUES (?, ?, ?, ?, ?, ?, ?)",
                        (ip, hostname, iface, etype, delta, cval, now))
                    written += 1
            conn.execute(
                "INSERT INTO port_error_state (ip, interface, error_type, "
                "counter_value, updated_at) VALUES (?, ?, ?, ?, ?) "
                "ON CONFLICT(ip, interface, error_type) DO UPDATE SET "
                "counter_value = excluded.counter_value, "
                "updated_at = excluded.updated_at",
                (ip, iface, etype, cval, now))
    return written


def run_topology(cfg):
    """Fleet-wide LLDP scrape — populates topology_edges.

    Per-device path: SNMP first (works on any device with snmp_enabled=1,
    no SSH session required), SSH fallback for devices without SNMP or
    whose SNMP LLDP table is empty. Best-effort — a single device's
    failure doesn't abort the run.
    """
    # Topology fires every 30 min and historically COLLIDED at :00/:30
    # with the per-minute monitor-stp tick that also holds the 'ssh' lock.
    # On larger fleets stp's tick is 30-42 s so the non-blocking acquire
    # silently lost the race the majority of the time (a 55-75 % miss
    # rate across sites over a representative 12-hour window).
    # Wait up to 90 s for the lock (much shorter than a 30-min cycle) and
    # record a skip op_event if the wait expires so the miss isn't silent.
    _topo_started = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    if not _ssh_job_gate(wait=90):
        _record_op("monitor_topology_tick",
                   started_at=_topo_started,
                   duration_ms=0, success=False,
                   reason="ssh-lock-held",
                   extra="skipped=lock_held wait_s=90")
        return
    _topo_t0 = time.monotonic()
    _snmp_timing_start()
    db_devices = get_devices(status="active")
    # SNMP works on any device that's been probed (regardless of platform);
    # SSH only on the four platforms we have CLI parsers for. Including
    # snmp_enabled=1 devices broadens our coverage past the SSH-only set.
    targets = [r for r in db_devices
               if (_effective_platform(r["platform"], r["model"])
                   in ("junos", "aruba-cx", "procurve", "fastiron", "cisco-ios"))
               or r["snmp_enabled"] == 1]
    if not targets:
        log.info("No supported devices to scrape LLDP from.")
        return

    threads = min(cfg.get("ssh_threads", 50), max(1, len(targets)))
    log.info("Collecting LLDP from %d device(s) using %d threads...",
             len(targets), threads)

    results_by_ip = {}
    done = 0
    total = len(targets)
    with ThreadPoolExecutor(max_workers=threads) as pool:
        futures = {
            pool.submit(_collect_lldp_one, r["ip"], r["hostname"],
                        r["platform"], cfg, None): r
            for r in targets
        }
        for fut in as_completed(futures):
            r = futures[fut]
            try:
                rows = fut.result()
            except Exception as e:
                log.debug("lldp collect exception on %s: %s", r["ip"], e)
                rows = {"lldp": [], "fdb": [], "arp": [], "errors": {}}
            results_by_ip[r["ip"]] = rows
            done += 1
            sys.stdout.write(f"\rLLDP scrape... ({done}/{total})  ")
            sys.stdout.flush()
    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    conn.row_factory = sqlite3.Row
    edges_written = 0
    macs_written = 0
    arps_written = 0
    errors_written = 0
    devices_with_arp = 0
    devices_with_neighbors = 0
    host_by_ip = {r["ip"]: r["hostname"] for r in targets}
    for src_ip, res in results_by_ip.items():
        lldp_rows = (res or {}).get("lldp") or []
        fdb_rows = (res or {}).get("fdb") or []
        arp_rows = (res or {}).get("arp") or []
        err_map = (res or {}).get("errors") or {}
        if lldp_rows:
            devices_with_neighbors += 1
            for row in lldp_rows:
                neigh_ip = _resolve_edge_neighbor_ip(
                    row["neighbor_chassis"], row["neighbor_sysname"], conn)
                conn.execute("""
                    INSERT INTO topology_edges
                        (src_ip, src_port, neighbor_chassis, neighbor_sysname,
                         neighbor_port, neighbor_port_desc, neighbor_ip,
                         neighbor_caps, last_seen)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(src_ip, src_port) DO UPDATE SET
                        neighbor_chassis = excluded.neighbor_chassis,
                        neighbor_sysname = excluded.neighbor_sysname,
                        neighbor_port = excluded.neighbor_port,
                        neighbor_port_desc = excluded.neighbor_port_desc,
                        neighbor_ip = excluded.neighbor_ip,
                        neighbor_caps = excluded.neighbor_caps,
                        last_seen = excluded.last_seen
                """, (src_ip, row["local_port"],
                      row["neighbor_chassis"] or None,
                      row["neighbor_sysname"] or None,
                      row["neighbor_port"] or None,
                      row["neighbor_port_desc"] or None,
                      neigh_ip, (row.get("neighbor_caps") or None), now))
                edges_written += 1
        # FDB -> port_macs history (last-known MAC fallback + flux signal).
        for ifname, mac, vlan in fdb_rows:
            conn.execute("""
                INSERT INTO port_macs
                    (ip, interface, mac, vlan, oui_vendor,
                     first_seen, last_seen, times_seen)
                VALUES (?, ?, ?, ?, ?, ?, ?, 1)
                ON CONFLICT(ip, interface, mac, vlan) DO UPDATE SET
                    last_seen = excluded.last_seen,
                    times_seen = port_macs.times_seen + 1,
                    oui_vendor = excluded.oui_vendor
            """, (src_ip, ifname, mac, vlan, _oui_vendor_hint(mac), now, now))
            macs_written += 1
        # ARP -> ip_arp (3.8.31 ID-loop). Access switches contribute empty
        # walks (fast); L3 devices supply the IPs that reverse-DNS turns
        # into hostnames in alert email enrichment.
        if arp_rows:
            devices_with_arp += 1
        for ifindex, learned_ip, mac in arp_rows:
            conn.execute("""
                INSERT INTO ip_arp
                    (device_ip, ifindex, ip, mac, first_seen, last_seen)
                VALUES (?, ?, ?, ?, ?, ?)
                ON CONFLICT(device_ip, ifindex, ip, mac) DO UPDATE SET
                    last_seen = excluded.last_seen
            """, (src_ip, ifindex, learned_ip, mac, now, now))
            arps_written += 1
        # Interface error/discard/CRC counters -> port_errors timeline. Delta
        # vs the stored snapshot; a row per counter that rose this interval.
        if err_map:
            errors_written += _record_port_errors(
                conn, src_ip, host_by_ip.get(src_ip), err_map, now)
    conn.commit()
    # Prune edges not seen in _TOPOLOGY_EDGE_TTL_DAYS. topology_edges is
    # upsert-only, so a moved/dark cable (or a one-off bad parse from a
    # mis-detected platform) would otherwise linger forever. The export
    # already filters by freshness; this bounds table growth and clears
    # truly stale rows from the graph.
    try:
        pruned = conn.execute(
            "DELETE FROM topology_edges WHERE last_seen < "
            "datetime('now','localtime',?)",
            (f"-{_TOPOLOGY_EDGE_TTL_DAYS} days",)).rowcount
        if pruned:
            conn.commit()
            log.debug("Pruned %d stale topology edge(s) (> %dd old).",
                      pruned, _TOPOLOGY_EDGE_TTL_DAYS)
    except Exception as e:
        log.debug("topology edge prune skipped: %s", e)
    # Edge-triggered non-trunk MAC-flux security notice. Evaluated here so
    # detection latency is <= one 30-min FDB harvest. Returns only the
    # ports that just crossed (suppressed until the episode clears); email
    # is sent after the DB handle is closed so SMTP doesn't hold it.
    try:
        new_flux = _evaluate_mac_flux_alerts(conn, cfg, now)
    except Exception as e:
        log.debug("mac-flux evaluation skipped: %s", e)
        new_flux = []
    conn.close()
    if new_flux:
        _send_flux_alert_email(cfg, new_flux)
    log.info("LLDP scrape complete: %d edge(s), %d port-MAC(s), %d ARP(s), "
             "%d port-error event(s) recorded across %d device(s) (%d w/ ARP)",
             edges_written, macs_written, arps_written, errors_written,
             devices_with_neighbors, devices_with_arp)

    # Tick-level op_event with SNMP timing rollup
    _snmp_stats = _snmp_timing_stop()
    _topo_extra = {
        "edges_written": edges_written,
        "macs_written": macs_written,
        "arps_written": arps_written,
        "errors_written": errors_written,
        "devices_with_neighbors": devices_with_neighbors,
        "devices_with_arp": devices_with_arp,
        "targets": len(targets),
    }
    if _snmp_stats:
        _topo_extra.update(_snmp_stats)
    _record_op("monitor_topology_tick",
               started_at=_topo_started,
               duration_ms=int((time.monotonic() - _topo_t0) * 1000),
               success=True,
               extra=_topo_extra)


def _walk_topology_upstream(start_ip, max_hops=4):
    """Walk topology_edges upstream from start_ip, following each device's
    current STP root port. Returns (hops, stop_reason) where hops is a
    list of dicts (one per *successfully walked* upstream device) and
    stop_reason is a string describing why we stopped.

    Each hop dict: {ip, hostname, model, via_port (on previous hop's side),
    neighbor_port (this hop's side), edge_last_seen, is_root}.

    "Upstream" = follow the port currently in STP Root role.
    """
    hops = []
    visited = {start_ip}
    cur_ip = start_ip
    stop_reason = None
    conn = _db()
    conn.row_factory = sqlite3.Row
    try:
        for _ in range(max_hops):
            # Find current device's root port (CIST instance preferred
            # if multiple). stp_state stores role per port; look for exact
            # 'Root' (Junos) or 'ROOT' (Aruba-CX / ProCurve).
            root_port_row = conn.execute(
                "SELECT interface FROM stp_state "
                "WHERE ip = ? AND role IN ('Root', 'ROOT') "
                "ORDER BY interface LIMIT 1", (cur_ip,)).fetchone()
            if root_port_row is None:
                is_root_row = conn.execute(
                    "SELECT is_root FROM stp_root_state "
                    "WHERE ip = ? AND instance = 'CIST'", (cur_ip,)).fetchone()
                stop_reason = (
                    f"{cur_ip} appears to be the root bridge (no Root-port)"
                    if is_root_row and is_root_row["is_root"]
                    else f"no STP Root-port recorded for {cur_ip}"
                )
                break
            via_port = root_port_row["interface"]
            # Junos STP reports logical iface ('ge-0/0/8.0'); topology_edges
            # is keyed on the physical name ('ge-0/0/8').
            via_port_phys = via_port.split(".", 1)[0]
            edge = conn.execute(
                "SELECT neighbor_ip, neighbor_sysname, neighbor_port, "
                "       neighbor_chassis, last_seen "
                "FROM topology_edges WHERE src_ip = ? AND src_port = ?",
                (cur_ip, via_port_phys)).fetchone()
            if edge is None:
                stop_reason = (f"no topology edge cached for {cur_ip} "
                               f"{via_port_phys} — run 'monitor topology'")
                break
            if edge["neighbor_ip"] is None:
                stop_reason = (
                    f"{cur_ip} {via_port_phys}'s LLDP neighbor "
                    f"(chassis={edge['neighbor_chassis']!r}, "
                    f"sysname={edge['neighbor_sysname']!r}) is not in "
                    f"netops devices — endpoint or undiscovered switch")
                break
            if edge["neighbor_ip"] in visited:
                stop_reason = (f"cycle detected — neighbor {edge['neighbor_ip']} "
                               f"already on the path")
                break
            neigh = conn.execute(
                "SELECT hostname, model FROM devices WHERE ip = ?",
                (edge["neighbor_ip"],)).fetchone()
            hops.append({
                "ip": edge["neighbor_ip"],
                "hostname": neigh["hostname"] if neigh else None,
                "model":    neigh["model"]    if neigh else None,
                "via_port": via_port_phys,
                "neighbor_port": edge["neighbor_port"],
                "edge_last_seen": edge["last_seen"],
                "is_root": False,
            })
            # Did we just land on the root bridge? If so, mark and stop.
            is_root_row = conn.execute(
                "SELECT is_root FROM stp_root_state "
                "WHERE ip = ? AND instance = 'CIST'",
                (edge["neighbor_ip"],)).fetchone()
            if is_root_row and is_root_row["is_root"]:
                hops[-1]["is_root"] = True
                stop_reason = "reached root bridge"
                break
            visited.add(edge["neighbor_ip"])
            cur_ip = edge["neighbor_ip"]
        else:
            stop_reason = f"reached max_hops={max_hops}"
    finally:
        conn.close()
    return hops, stop_reason


def _format_topology_trace_lines(start_ip, start_hostname, max_hops=4,
                                  activity_window_min=5):
    """Render the multi-hop upstream trace block for an alert email.

    For each hop, attach a one-line summary of recent (in the last N
    minutes) port flaps + STP changes + root-bridge changes on that
    device. Operator can see which hop had nearby activity at the time
    of the alert — that's usually the proximate cause.

    Returns an empty list if there's no useful trace (e.g. no
    topology_edges data populated yet).
    """
    hops, stop_reason = _walk_topology_upstream(start_ip, max_hops=max_hops)
    lines = [f"--- Upstream topology trace (toward root, up to {max_hops} hops) ---"]
    lines.append(f"  hop 0  this device  {start_hostname or '?'} ({start_ip})")
    conn = _db()
    conn.row_factory = sqlite3.Row
    try:
        for i, h in enumerate(hops, start=1):
            label = h["hostname"] or "?"
            via = h["via_port"] or "?"
            n_port = h["neighbor_port"] or "?"
            head = (f"  hop {i}  via {via} ↔ {n_port}  "
                    f"{label} ({h['ip']})")
            if h.get("model"):
                head += f"  [{h['model']}]"
            if h.get("is_root"):
                head += "  ★ ROOT BRIDGE"
            lines.append(head)
            # Recent activity on THIS hop's device in the activity window.
            window_clause = f"-{int(activity_window_min)} minutes"
            flaps = conn.execute("""
                SELECT interface, flap_count, last_seen FROM port_flaps
                WHERE ip = ? AND last_seen >= datetime('now','localtime',?)
                ORDER BY last_seen DESC LIMIT 5
            """, (h["ip"], window_clause)).fetchall()
            stp_chg = conn.execute("""
                SELECT interface, old_role, old_state, new_role, new_state, changed_at
                FROM stp_changes
                WHERE ip = ? AND changed_at >= datetime('now','localtime',?)
                ORDER BY changed_at DESC LIMIT 5
            """, (h["ip"], window_clause)).fetchall()
            root_chg = conn.execute("""
                SELECT instance, changed_at FROM stp_root_changes
                WHERE ip = ? AND changed_at >= datetime('now','localtime',?)
                ORDER BY changed_at DESC LIMIT 3
            """, (h["ip"], window_clause)).fetchall()
            if not flaps and not stp_chg and not root_chg:
                lines.append(f"         no port-flap / STP / root-change activity in last {activity_window_min}m")
            for r in flaps:
                lines.append(f"         port_flap  {r['interface']:14s} "
                             f"count={r['flap_count']:>4d} last={r['last_seen']}")
            for r in stp_chg:
                lines.append(f"         stp_change {r['interface']:14s} "
                             f"{r['old_role']}/{r['old_state']} -> "
                             f"{r['new_role']}/{r['new_state']} @ {r['changed_at']}")
            for r in root_chg:
                lines.append(f"         root_change inst={r['instance']} @ {r['changed_at']}")
    finally:
        conn.close()
    if stop_reason:
        lines.append(f"  trace stopped: {stop_reason}")
    return lines


# ---------------------------------------------------------------------------
# Network map export — render the LLDP-discovered switch-to-switch backbone
# from topology_edges to an open charting format. draw.io (mxGraph) XML
# imports directly into Lucidchart (File > Import) and diagrams.net. This is
# the first consumer of topology_edges that produces a whole-fleet artifact
# rather than the per-device upstream walk used by the alert paths.
# ---------------------------------------------------------------------------

def _build_topology_graph(conn, max_age_hours=48, include_unknown=False,
                          include_endpoints=False, expand_neighbors=False):
    """Build the LLDP topology graph from topology_edges.

    Returns (nodes, edges):
      nodes: list of {id, ip, hostname, model, platform, is_root, kind,
                      ghost} — ghost=False for our own devices; ghost=True
             for unmanaged LLDP neighbors, with kind typed from the
             advertised capabilities (ap / phone / switch / router / host /
             unknown). A ghost may also be a per-switch summary badge
             ({badge: True, count: N}) standing in for N leaf neighbors.
      edges: list of {a_id, a_port, b_id, b_port, last_seen, unknown}

    Resolved edges (neighbor matched to one of our devices) always form the
    managed backbone; the two directional rows of a physical link collapse
    into one undirected edge carrying both port names. Edges older than
    max_age_hours are dropped (topology_edges is upsert-only). Unmanaged
    neighbors become typed ghost nodes: include_unknown adds infrastructure
    (AP / switch / router / unknown) so APs and undiscovered switches show;
    include_endpoints additionally adds phones/hosts (off by default —
    otherwise a phone-heavy access fleet floods the map).

    Leaf neighbors (AP / phone / host) are collapsed per switch into one
    count badge per kind ("12 APs") so a switch with dozens of APs is one
    badge, not dozens of nodes. expand_neighbors=True draws each one
    individually instead. Undiscovered switches/routers are always drawn
    individually — they're real topology and can multi-home.
    """
    conn.row_factory = sqlite3.Row
    cutoff = f"-{int(max_age_hours)} hours"
    has_caps = any(r[1] == "neighbor_caps" for r in
                   conn.execute("PRAGMA table_info(topology_edges)"))

    # Device metadata + root flags, loaded once.
    dev = {}
    for r in conn.execute(
        "SELECT ip, hostname, model, platform FROM devices "
        "WHERE status IN ('active','inactive')"):
        dev[r["ip"]] = {"hostname": r["hostname"], "model": r["model"],
                        "platform": r["platform"]}
    root_ips = {r["ip"] for r in conn.execute(
        "SELECT ip FROM stp_root_state WHERE instance='CIST' AND is_root=1")}

    caps_col = "neighbor_caps" if has_caps else "'' AS neighbor_caps"
    edge_rows = conn.execute(
        f"SELECT src_ip, src_port, neighbor_ip, neighbor_chassis, "
        f"       neighbor_sysname, neighbor_port, {caps_col}, last_seen "
        f"FROM topology_edges "
        f"WHERE last_seen >= datetime('now','localtime', ?)", (cutoff,)
    ).fetchall()

    nodes = {}          # id -> node dict
    edge_map = {}       # canonical key -> edge dict
    badges = {}         # (switch_ip, kind) -> set of neighbor keys (leaf rollup)

    def _ensure_switch(ip):
        if ip in nodes:
            return
        d = dev.get(ip, {})
        nodes[ip] = {
            "id": ip, "ip": ip,
            "hostname": d.get("hostname"),
            "model": d.get("model"),
            "platform": d.get("platform"),
            "is_root": ip in root_ips,
            "kind": "switch", "ghost": False,
        }

    for row in edge_rows:
        a_ip = row["src_ip"]
        a_port = (row["src_port"] or "").split(".", 1)[0]
        if row["neighbor_ip"]:
            b_ip = row["neighbor_ip"]
            b_port = (row["neighbor_port"] or "").split(".", 1)[0]
            # Canonical undirected key: both directional rows of one
            # physical link map to the same (endpoint, endpoint) pair when
            # LLDP advertises ifName as the port-id (the common case the
            # FDB/STP collectors already rely on). Asymmetric port naming
            # at worst over-draws a second line — never mislabels.
            ends = sorted([(a_ip, a_port), (b_ip, b_port)])
            key = (ends[0], ends[1])
            existing = edge_map.get(key)
            if existing:
                if row["last_seen"] > existing["last_seen"]:
                    existing["last_seen"] = row["last_seen"]
                continue
            _ensure_switch(a_ip)
            _ensure_switch(b_ip)
            edge_map[key] = {
                "a_id": ends[0][0], "a_port": ends[0][1],
                "b_id": ends[1][0], "b_port": ends[1][1],
                "last_seen": row["last_seen"], "unknown": False,
            }
        elif include_unknown or include_endpoints:
            # Unmanaged neighbor — type it from LLDP capabilities and apply
            # the infra/endpoint filter before drawing.
            caps = {c for c in (row["neighbor_caps"] or "").split(",") if c}
            kind = _lldp_caps_to_kind(caps)
            is_endpoint = kind in ("phone", "host")
            if is_endpoint and not include_endpoints:
                continue
            if (not is_endpoint) and not include_unknown:
                continue
            # Key on chassis MAC, else sysname, so several uplinks to the
            # same undiscovered device collapse to one node.
            gkey = _normalize_mac(row["neighbor_chassis"] or "") or \
                   (row["neighbor_sysname"] or "").strip().lower()
            if not gkey:
                continue
            # Leaf neighbors (AP/phone/host) roll up into a per-switch count
            # badge unless the operator asked to expand them.
            if kind in ("ap", "phone", "host") and not expand_neighbors:
                _ensure_switch(a_ip)
                badges.setdefault((a_ip, kind), set()).add(gkey)
                continue
            gid = "ghost:" + gkey
            if gid not in nodes:
                nodes[gid] = {
                    "id": gid, "ip": None,
                    "hostname": (row["neighbor_sysname"] or "").strip() or None,
                    "model": None, "platform": None,
                    "is_root": False, "kind": kind, "ghost": True,
                    "chassis": row["neighbor_chassis"],
                }
            _ensure_switch(a_ip)
            b_port = (row["neighbor_port"] or "").split(".", 1)[0]
            key = ((a_ip, a_port), (gid, b_port))
            if key in edge_map:
                continue
            edge_map[key] = {
                "a_id": a_ip, "a_port": a_port,
                "b_id": gid, "b_port": b_port,
                "last_seen": row["last_seen"], "unknown": True,
            }

    # Bundle parallel links between the same node pair into one logical
    # edge. Multiple physical links between two switches are a LAG (or
    # redundant uplinks); drawing them as one line — labelled with the
    # member count and ports — uses the LAG reality to cut clutter and the
    # crossings that N stacked parallel lines would add.
    bundles = {}
    for e in edge_map.values():
        pair = frozenset((e["a_id"], e["b_id"]))
        b = bundles.get(pair)
        if b is None:
            b = {"a_id": e["a_id"], "b_id": e["b_id"], "members": [],
                 "last_seen": e["last_seen"], "unknown": e["unknown"]}
            bundles[pair] = b
        b["members"].append((e["a_port"], e["b_port"]))
        if e["last_seen"] > b["last_seen"]:
            b["last_seen"] = e["last_seen"]
    edge_list = sorted(bundles.values(), key=lambda b: (b["a_id"], b["b_id"]))

    # Per-switch leaf badges: one summary node per (switch, kind) standing
    # in for its rolled-up APs/phones/hosts, with a plain edge to the switch.
    _plural = {"ap": "APs", "phone": "phones", "host": "hosts"}
    badge_edges = []
    for (sw, kind), keys in sorted(badges.items()):
        n = len(keys)
        bid = f"badge:{sw}:{kind}"
        nodes[bid] = {
            "id": bid, "ip": None,
            "hostname": f"{n} {_plural.get(kind, kind)}",
            "model": None, "platform": None, "is_root": False,
            "kind": kind, "ghost": True, "badge": True, "count": n,
        }
        badge_edges.append({"a_id": sw, "b_id": bid, "members": [],
                            "unknown": True, "badge": True})
    edge_list.extend(badge_edges)

    node_list = sorted(nodes.values(),
                       key=lambda n: (n["ghost"],
                                      (n["hostname"] or n["id"]).lower()))
    return node_list, edge_list


def _topology_layout(nodes, edges, node_w=180, node_h=60, x_gap=70, y_gap=130):
    """Assign (x, y) per node id with a layered, crossing-reduced layout.

    BFS depth from the root bridge gives the rows (root on top, then
    distribution, access, ...). Within each row the order is then refined
    by the median heuristic over a few down/up sweeps: each node is pulled
    toward the median position of its neighbours in the adjacent row, so
    children cluster under their parents and the parent->child links stop
    crossing — important when there are several distribution switches whose
    subtrees would otherwise interleave. Returns id -> (x, y).
    """
    from collections import deque
    adj = {}
    for n in nodes:
        adj.setdefault(n["id"], set())
    for e in edges:
        adj.setdefault(e["a_id"], set()).add(e["b_id"])
        adj.setdefault(e["b_id"], set()).add(e["a_id"])

    roots = [n["id"] for n in nodes if n.get("is_root")]
    if not roots and nodes:
        # No STP root known — anchor on the highest-degree node.
        roots = [max(nodes, key=lambda n: len(adj.get(n["id"], ())))["id"]]

    depth = {}
    dq = deque()
    for r in roots:
        depth[r] = 0
        dq.append(r)
    while dq:
        cur = dq.popleft()
        for nb in sorted(adj.get(cur, ())):
            if nb not in depth:
                depth[nb] = depth[cur] + 1
                dq.append(nb)
    max_depth = max(depth.values()) if depth else 0
    for n in nodes:                       # disconnected -> trailing row
        depth.setdefault(n["id"], max_depth + 1)

    # Rows, initialised in a stable (alphabetical) order.
    rows = {}
    for n in nodes:
        rows.setdefault(depth[n["id"]], []).append(n["id"])
    for d in rows:
        rows[d].sort(key=lambda i: i.lower())
    depths = sorted(rows)

    def _reorder(target_d, ref_d):
        """Reorder rows[target_d] by the median index of each node's
        neighbours in rows[ref_d]. Nodes with no neighbour in the reference
        row fall back to their current index, so they stay put (stable)."""
        ref_pos = {nid: i for i, nid in enumerate(rows[ref_d])}
        keyed = []
        for i, nid in enumerate(rows[target_d]):
            idxs = sorted(ref_pos[nb] for nb in adj.get(nid, ()) if nb in ref_pos)
            if idxs:
                k = len(idxs)
                med = (idxs[k // 2] if k % 2
                       else (idxs[k // 2 - 1] + idxs[k // 2]) / 2.0)
            else:
                med = i
            keyed.append((med, i, nid))
        rows[target_d] = [nid for _, _, nid in
                          sorted(keyed, key=lambda t: (t[0], t[1]))]

    for _ in range(4):
        for k in range(1, len(depths)):            # down sweep
            _reorder(depths[k], depths[k - 1])
        for k in range(len(depths) - 2, -1, -1):   # up sweep
            _reorder(depths[k], depths[k + 1])

    widest = max((len(v) for v in rows.values()), default=1)
    pos = {}
    col_step = node_w + x_gap
    row_step = node_h + y_gap
    for d in depths:
        ids = rows[d]
        offset = (widest - len(ids)) * col_step / 2.0
        for col, nid in enumerate(ids):
            pos[nid] = (int(offset + col * col_step), int(d * row_step))
    return pos


def _xesc(s):
    """XML-escape text for use inside an attribute value.

    Strips characters that are illegal in XML 1.0 first — some LLDP
    port-/chassis-id values arrive as raw binary octets (e.g. a MAC-subtype
    port id), and embedding their control bytes would produce a .drawio
    that Lucidchart/diagrams.net reject as 'not well-formed'."""
    s = "" if s is None else str(s)
    s = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "", s)
    return (s.replace("&", "&amp;").replace("<", "&lt;")
             .replace(">", "&gt;").replace('"', "&quot;"))


_DRAWIO_STYLE_SWITCH = ("rounded=1;whiteSpace=wrap;html=1;"
                        "fillColor=#dae8fc;strokeColor=#6c8ebf;")
_DRAWIO_STYLE_ROOT   = ("rounded=1;whiteSpace=wrap;html=1;fontStyle=1;"
                        "fillColor=#ffe6cc;strokeColor=#d79b00;")
_DRAWIO_STYLE_GHOST  = ("rounded=1;whiteSpace=wrap;html=1;dashed=1;"
                        "fillColor=#f5f5f5;strokeColor=#999999;fontColor=#666666;")
_DRAWIO_STYLE_EDGE   = "endArrow=none;html=1;rounded=0;fontSize=9;fontColor=#666666;"
_DRAWIO_STYLE_GEDGE  = ("endArrow=none;html=1;rounded=0;dashed=1;"
                        "strokeColor=#999999;fontSize=9;fontColor=#999999;")
_DRAWIO_STYLE_TITLE  = "text;html=1;align=left;verticalAlign=top;fontSize=12;fontStyle=1;"
# Typed (dashed) styles for unmanaged LLDP neighbors, keyed by kind.
_DRAWIO_GHOST_STYLES = {
    "ap":     "rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#d5e8d4;strokeColor=#82b366;",
    "switch": "rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#dae8fc;strokeColor=#6c8ebf;",
    "router": "rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#ffe6cc;strokeColor=#d79b00;",
    "phone":  "rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#f5f5f5;strokeColor=#b3b3b3;fontColor=#666666;",
    "host":   "rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#fafafa;strokeColor=#cccccc;fontColor=#888888;",
}
_DRAWIO_KIND_TAG = {"ap": "AP", "switch": "switch?", "router": "router",
                    "phone": "phone", "host": "host", "other": "?",
                    "unknown": "?"}


def _render_topology_drawio(nodes, edges, title, node_w=180, node_h=60):
    """Render the topology graph to draw.io (mxGraph) XML.

    Imports directly into Lucidchart (File > Import) and diagrams.net.
    Managed switches are blue rounded boxes, the STP root bridge is gold
    and bold. Unmanaged (ghost, dashed) neighbors are typed from their LLDP
    capabilities: APs green, undiscovered switches blue, routers orange,
    phones/hosts grey, each tagged [AP]/[switch?]/[phone]/... Multi-line
    labels use <br> (escaped) with html=1 so they render on import.
    """
    pos = _topology_layout(nodes, edges, node_w=node_w, node_h=node_h)
    cell_id = {}                    # node id -> mxCell id
    out = ['<?xml version="1.0" encoding="UTF-8"?>',
           '<mxfile host="netops" type="device">',
           f'  <diagram id="topology" name="{_xesc(title)}">',
           '    <mxGraphModel dx="800" dy="600" grid="1" gridSize="10" '
           'guides="1" tooltips="1" connect="1" arrows="1" fold="1" '
           'page="1" pageScale="1" pageWidth="1600" pageHeight="1200" '
           'math="0" shadow="0">',
           '      <root>',
           '        <mxCell id="0" />',
           '        <mxCell id="1" parent="0" />',
           f'        <mxCell id="title" value="{_xesc(title)}" '
           f'style="{_DRAWIO_STYLE_TITLE}" vertex="1" parent="1">',
           '          <mxGeometry x="0" y="-60" width="900" height="40" as="geometry" />',
           '        </mxCell>']

    for i, n in enumerate(nodes):
        cid = f"v{i}"
        cell_id[n["id"]] = cid
        if n.get("ghost"):
            kind = n.get("kind") or "unknown"
            style = _DRAWIO_GHOST_STYLES.get(kind, _DRAWIO_STYLE_GHOST)
            if n.get("badge"):
                # Per-switch rollup, e.g. "12 APs" — bold, no [tag] prefix.
                style = style + "fontStyle=1;"
                parts = [n.get("hostname") or "?"]
            else:
                tag = _DRAWIO_KIND_TAG.get(kind, "?")
                parts = [f"[{tag}] {n.get('hostname') or '(undiscovered)'}"]
                if n.get("chassis"):
                    parts.append(n["chassis"])
        else:
            style = _DRAWIO_STYLE_ROOT if n.get("is_root") else _DRAWIO_STYLE_SWITCH
            parts = [n.get("hostname") or "(unnamed)"]
            if n.get("ip"):
                parts.append(n["ip"])
            if n.get("model"):
                parts.append(n["model"])
            if n.get("is_root"):
                parts.append("★ root bridge")
        value = _xesc("<br>".join(p for p in parts if p))
        x, y = pos.get(n["id"], (0, 0))
        out.append(f'        <mxCell id="{cid}" value="{value}" '
                   f'style="{style}" vertex="1" parent="1">')
        out.append(f'          <mxGeometry x="{x}" y="{y}" '
                   f'width="{node_w}" height="{node_h}" as="geometry" />')
        out.append('        </mxCell>')

    for i, e in enumerate(edges):
        src = cell_id.get(e["a_id"])
        tgt = cell_id.get(e["b_id"])
        if not src or not tgt:
            continue
        if e.get("badge"):
            # Switch -> rollup badge: plain unlabelled dashed line.
            label, style = "", _DRAWIO_STYLE_GEDGE
            out.append(f'        <mxCell id="e{i}" value="" '
                       f'style="{style}" edge="1" parent="1" '
                       f'source="{src}" target="{tgt}">')
            out.append('          <mxGeometry relative="1" as="geometry" />')
            out.append('        </mxCell>')
            continue
        members = e.get("members") or [(e.get("a_port"), e.get("b_port"))]
        if len(members) == 1:
            a_p, b_p = members[0]
            label = _xesc(f"{a_p or '?'} <-> {b_p or '?'}")
            style = _DRAWIO_STYLE_GEDGE if e.get("unknown") else _DRAWIO_STYLE_EDGE
        else:
            # LAG / redundant bundle — one thick line, member ports listed.
            a_ports = ",".join(sorted({mp[0] for mp in members if mp[0]}))
            label = _xesc(f"LAG x{len(members)}: {a_ports}")
            base = _DRAWIO_STYLE_GEDGE if e.get("unknown") else _DRAWIO_STYLE_EDGE
            style = base + "strokeWidth=3;"
        out.append(f'        <mxCell id="e{i}" value="{label}" '
                   f'style="{style}" edge="1" parent="1" '
                   f'source="{src}" target="{tgt}">')
        out.append('          <mxGeometry relative="1" as="geometry" />')
        out.append('        </mxCell>')

    out += ['      </root>', '    </mxGraphModel>', '  </diagram>', '</mxfile>']
    return "\n".join(out) + "\n"


def handle_export_topology(cfg, fmt="drawio", output=None,
                           max_age_hours=48, include_unknown=False,
                           include_endpoints=False, expand_neighbors=False):
    """`export topology` — render the LLDP topology to a chart file."""
    conn = _db()
    try:
        nodes, edges = _build_topology_graph(
            conn, max_age_hours=max_age_hours, include_unknown=include_unknown,
            include_endpoints=include_endpoints,
            expand_neighbors=expand_neighbors)
    finally:
        conn.close()

    if not nodes:
        log.warning("No topology edges within the last %dh — run "
                    "'monitor topology' first (or raise --max-age-hours).",
                    max_age_hours)
        return

    switch_n = sum(1 for n in nodes if not n.get("ghost"))
    ghost_nodes = [n for n in nodes if n.get("ghost")]
    # A badge node stands in for count leaf neighbors; individual ghosts = 1.
    unmanaged_n = sum(n.get("count", 1) for n in ghost_nodes)
    ap_n = sum(n.get("count", 1) for n in ghost_nodes if n["kind"] == "ap")
    host = socket.gethostname().split(".")[0] or "netops"
    stamp = datetime.now().strftime("%Y-%m-%d %H:%M")
    if unmanaged_n:
        ghost_suffix = f", {unmanaged_n} unmanaged" + (f" incl {ap_n} AP" if ap_n else "")
    else:
        ghost_suffix = ""
    title = (f"netops topology — {host} — {stamp} — "
             f"{switch_n} switches, {len(edges)} links{ghost_suffix}")

    if fmt == "drawio":
        content = _render_topology_drawio(nodes, edges, title)
        ext = "drawio"
    else:
        raise ValueError(f"unsupported topology export format: {fmt!r}")

    if output == "-":
        sys.stdout.write(content)
        log.info("Wrote topology (%d switches, %d links%s) to stdout.",
                 switch_n, len(edges), ghost_suffix)
        return

    if output:
        path = output
    else:
        base_dir = os.path.dirname(cfg["device_file"])
        date = datetime.now().strftime("%Y%m%d")
        path = os.path.join(base_dir, f"topology-{host}-{date}.{ext}")

    with open(path, "w", encoding="utf-8") as f:
        f.write(content)
    log.info("Exported topology map: %d switches, %d links%s -> %s",
             switch_n, len(edges), ghost_suffix, path)
    log.info("Import into Lucidchart via File > Import, or open at "
             "https://app.diagrams.net/.")


# ---------------------------------------------------------------------------
# Neighbor-discovery compliance — the map is only as complete as LLDP (or
# CDP) enablement across the fleet. These helpers scan each device's saved
# config so the coverage report can say *why* a switch is missing from the
# graph, and what to turn on to fix it. Diagnostic aid, not ground truth:
# defaults aren't in configs and "enabled in config" != "currently
# advertising" — the authoritative signal is whether edges actually appear.
# ---------------------------------------------------------------------------

def _latest_config_text(ip, hostname):
    """Return the most recent saved hierarchical config for a device, or
    None if there's no saved config. Mirrors cat-config's file resolution
    (uses the plain .cfg, not the Junos _set.cfg)."""
    candidates = []
    if hostname:
        candidates.append(f"{hostname}_{ip}.cfg")
    candidates.append(f"{ip}.cfg")
    for fname in candidates:
        path = os.path.join(CONFIGS_DIR, "current", fname)
        if os.path.isfile(path):
            try:
                with open(path, encoding="utf-8", errors="replace") as f:
                    return f.read()
            except OSError:
                return None
    return None


def _config_neighbor_proto_status(platform, text):
    """Inspect a saved config for neighbor-discovery protocol state.

    Returns {"lldp": s, "cdp": s} with s in 'enabled' | 'disabled' |
    'unknown'. Defaults differ by platform:
      - Junos: LLDP off until a 'protocols lldp' stanza configures it; no
        CDP support at all (cdp -> 'unknown').
      - Aruba-CX: LLDP on by default — 'no lldp' (global) disables it; no
        meaningful CDP (cdp -> 'unknown').
      - ProCurve: LLDP and CDP both on by default — look for the explicit
        'no lldp run' / 'no cdp run' negation.
      - FastIron (Ruckus/Brocade ICX): LLDP on by default — 'no lldp run'
        disables it (this is why ICX produce neighbors with no explicit
        config). CDP not meaningfully used (cdp -> 'unknown').
    """
    res = {"lldp": "unknown", "cdp": "unknown"}
    if not text:
        return res
    low = text.lower()
    if platform == "junos":
        configured = (re.search(r"(?m)^\s*lldp\s*\{", text) is not None
                      or "set protocols lldp" in low
                      or re.search(r"protocols\s*\{[^}]*\blldp\b", low,
                                   re.DOTALL) is not None)
        disabled = ("set protocols lldp disable" in low
                    or re.search(r"\blldp\s*\{\s*disable\b", low) is not None)
        res["lldp"] = "enabled" if (configured and not disabled) else "disabled"
        res["cdp"] = "unknown"          # Junos has no CDP
    elif platform == "aruba-cx":
        res["lldp"] = ("disabled" if re.search(r"(?im)^\s*no lldp\s*$", text)
                       else "enabled")
        res["cdp"] = "unknown"
    elif platform == "procurve":
        res["lldp"] = ("disabled" if re.search(r"(?im)^\s*no lldp run\b", text)
                       else "enabled")
        res["cdp"] = ("disabled" if re.search(r"(?im)^\s*no cdp run\b", text)
                      else "enabled")
    elif platform == "fastiron":
        res["lldp"] = ("disabled" if re.search(r"(?im)^\s*no lldp run\b", text)
                       else "enabled")
        res["cdp"] = "unknown"
    elif platform == "cisco-ios":
        # Cisco IOS: LLDP is OFF by default — 'lldp run' (global) enables it;
        # 'no lldp run' is the explicit disable. CDP is ON by default.
        res["lldp"] = ("enabled" if re.search(r"(?im)^\s*lldp run\b", text)
                       else "disabled")
        res["cdp"] = ("disabled" if re.search(r"(?im)^\s*no cdp run\b", text)
                      else "enabled")
    return res


def _lldp_coverage_report(conn, max_age_hours=48):
    """Classify every active device that is NOT on the topology map.

    A device is "on the map" if it is an endpoint of a *resolved* edge
    (the src or the neighbor of a topology_edges row whose neighbor_ip is
    set, newer than max_age_hours). Being merely a src of edges that all
    point at unresolved/unmanaged neighbors does NOT count — that switch
    isn't linked to the managed backbone, and counting it overstated
    coverage (e.g. a switch seeing only ESXi/phone neighbors). For each
    active switch that isn't on the map, scan its saved config to bucket why:
      lldp_off            — LLDP disabled/unconfigured: enable it
      cdp_only            — CDP enabled but LLDP off: enable LLDP
      lldp_on_no_neighbor — LLDP on but no resolved edge: leaf, or down link
      no_config           — no saved config to scan: back it up first

    Firewalls (role='firewall') are reported separately, not counted in the
    headline switch total — they're L3 edges and LLDP on them is optional
    (and must stay off external/DMZ interfaces).

    Returns {total, on_map, buckets:{name:[(ip,hostname),...]}, firewalls}.
    """
    conn.row_factory = sqlite3.Row
    cutoff = f"-{int(max_age_hours)} hours"
    on_map = set()
    for r in conn.execute(
        "SELECT DISTINCT src_ip AS ip FROM topology_edges "
        "WHERE neighbor_ip IS NOT NULL "
        "AND last_seen >= datetime('now','localtime',?)", (cutoff,)):
        on_map.add(r["ip"])
    for r in conn.execute(
        "SELECT DISTINCT neighbor_ip AS ip FROM topology_edges "
        "WHERE neighbor_ip IS NOT NULL "
        "AND last_seen >= datetime('now','localtime',?)", (cutoff,)):
        on_map.add(r["ip"])

    active = conn.execute(
        "SELECT ip, hostname, platform, role FROM devices "
        "WHERE status='active' ORDER BY hostname, ip").fetchall()
    buckets = {"lldp_off": [], "cdp_only": [],
               "lldp_on_no_neighbor": [], "lldp_unknown": [], "no_config": []}
    firewalls = []
    switch_ips = set()
    for d in active:
        if (d["role"] if "role" in d.keys() else "switch") == "firewall":
            if d["ip"] not in on_map:
                firewalls.append((d["ip"], d["hostname"]))
            continue
        switch_ips.add(d["ip"])
        if d["ip"] in on_map:
            continue
        text = _latest_config_text(d["ip"], d["hostname"])
        if text is None:
            buckets["no_config"].append((d["ip"], d["hostname"]))
            continue
        st = _config_neighbor_proto_status(d["platform"], text)
        if st["lldp"] == "enabled":
            buckets["lldp_on_no_neighbor"].append((d["ip"], d["hostname"]))
        elif st["lldp"] == "disabled" and st["cdp"] == "enabled":
            buckets["cdp_only"].append((d["ip"], d["hostname"]))
        elif st["lldp"] == "disabled":
            buckets["lldp_off"].append((d["ip"], d["hostname"]))
        else:  # unknown — don't claim it's off (e.g. platform we can't scan)
            buckets["lldp_unknown"].append((d["ip"], d["hostname"]))
    return {"total": len(switch_ips),
            "on_map": len(on_map & switch_ips),
            "buckets": buckets,
            "firewalls": firewalls}


_COVERAGE_BUCKET_LABELS = [
    ("lldp_off",            "LLDP off in config — enable LLDP"),
    ("cdp_only",            "speaks CDP, not LLDP — enable LLDP"),
    ("no_config",           "no saved config to scan — back it up first"),
    ("lldp_unknown",        "LLDP state unclear from saved config — verify on device"),
    ("lldp_on_no_neighbor", "LLDP on but no neighbor — leaf/edge or down link"),
]


def _format_coverage_lines(report, indent="  ", per_bucket=10):
    """Render an _lldp_coverage_report dict to a list of text lines shared
    by `digest health` and `show lldp-coverage`."""
    lines = []
    total, on_map = report["total"], report["on_map"]
    missing = total - on_map
    pct = (on_map * 100.0 / total) if total else 100.0
    lines.append(f"{indent}map coverage: {on_map}/{total} active devices "
                 f"on the LLDP graph ({pct:.0f}%); {missing} missing")
    for key, label in _COVERAGE_BUCKET_LABELS:
        rows = report["buckets"].get(key) or []
        if not rows:
            continue
        lines.append(f"{indent}  {len(rows)}  {label}:")
        for ip, host in rows[:per_bucket]:
            lines.append(f"{indent}    - {host or '?'} ({ip})")
        if len(rows) > per_bucket:
            lines.append(f"{indent}    ... and {len(rows) - per_bucket} more")
    fw = report.get("firewalls") or []
    if fw:
        lines.append(f"{indent}  {len(fw)}  firewall(s) not on the map — LLDP "
                     f"optional; if enabled, TRUSTED/internal interfaces only, "
                     f"never external/DMZ:")
        for ip, host in fw[:per_bucket]:
            lines.append(f"{indent}    - {host or '?'} ({ip})")
        if len(fw) > per_bucket:
            lines.append(f"{indent}    ... and {len(fw) - per_bucket} more")
    return lines


def handle_lldp_coverage(max_age_hours=48):
    """`show lldp-coverage` — on-demand neighbor-discovery compliance."""
    conn = _db()
    try:
        report = _lldp_coverage_report(conn, max_age_hours=max_age_hours)
    finally:
        conn.close()
    print("=== LLDP / topology map coverage ===")
    for line in _format_coverage_lines(report, indent="", per_bucket=1000):
        print(line)
    if report["total"] == report["on_map"]:
        print("every active device is on the map 🎉")


def run_monitor(cfg, mode, detail=False, email=True):
    """Poll device state across active devices.

    mode="stp":  poll STP + interface link state, confirm STP changes across
                 two polls, email alerts, and count port flaps.
    mode="flap": poll interface link state only, count port flaps. No STP
                 commands are sent; existing STP state is left untouched.

    The per-device platform (junos|aruba-cx|procurve) is cached on the
    `devices` row; a cache miss triggers a `show version` classification
    that is then stored for subsequent polls.

    email=False (--no-email) polls, records and logs exactly as usual but
    sends nothing — and leaves unreachable_alerted_at unstamped, so a manual
    run can't swallow an outage alert the next scheduled tick still owes.
    """
    if mode not in ("stp", "flap"):
        raise ValueError(f"run_monitor: invalid mode {mode!r}")
    # Legacy mode keeps the deliberate NON-blocking skip (missing one
    # minute is cheap); daemon mode needs no whole-job lock at all.
    if not (_slot_daemon_available() or _acquire_advisory_lock("ssh")):
        return
    label = "STP" if mode == "stp" else "port"
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        sys.exit(1)

    # Per-tick instrumentation — accumulates SNMP query timing across all
    # worker threads, rolled up to one op_events row at tick end.
    _tick_t0 = time.monotonic()
    _tick_started_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    _snmp_timing_start()

    db_devices = get_devices(status="active")
    if not db_devices:
        log.info("No active devices. Run 'discover' first.")
        sys.exit(0)

    # Firewalls (role='firewall') stay in the stp tick for REACHABILITY and
    # port-flap tracking; only the STP command/analysis phase is skipped —
    # STP on a firewall's handful of switching ports isn't worth digest
    # noise, and a pure-L3 box just answers 'Invalid input'. (3.8.28
    # excluded firewalls from this tick entirely, expecting 'monitor flap'
    # to cover them — but no flap timer ships, so firewalls silently fell
    # out of ALL scheduled monitoring: their unreachable state could
    # neither fire nor clear, and the device whose death takes a whole
    # site with it was the one device nothing watched.) A firewall with no
    # supported cached platform is still excluded: probing-to-classify an
    # unknown OS would soft-fail every tick and march the device into a
    # false unreachable alert. To fully STP-monitor a firewall, set its
    # role to 'switch'.
    firewall_ips = set()
    if mode == "stp":
        supported_platforms = ("junos", "aruba-cx", "procurve", "cisco-ios")
        kept = []
        skipped_firewalls = []
        for r in db_devices:
            if (r["role"] if "role" in r.keys() else "switch") != "firewall":
                kept.append(r)
                continue
            platform = r["platform"] if "platform" in r.keys() else None
            if platform in supported_platforms:
                firewall_ips.add(r["ip"])
                kept.append(r)
            else:
                skipped_firewalls.append(r["hostname"] or r["ip"])
        db_devices = kept
        if skipped_firewalls:
            log.debug("monitor stp: skipping %d firewall(s) with no "
                      "supported platform: %s",
                      len(skipped_firewalls), ", ".join(skipped_firewalls))

    ips = [r["ip"] for r in db_devices]
    hostnames = {r["ip"]: r["hostname"] or r["ip"] for r in db_devices}
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    # Pre-resolve passwords for known-good connections.
    # Monitoring should ONLY use stored credentials — never sweep all combos,
    # because Juniper (and others) impose progressive login delays after
    # failed attempts, causing timeouts that cascade.
    all_pws = list(cfg.get("password_list") or [])
    for u, pws in (cfg.get("user_passwords") or {}).items():
        all_pws.extend(pws)
    pw_by_hash = {hashlib.sha256(pw.encode()).hexdigest(): pw for pw in all_pws}

    snmp_cfg = cfg.get("snmp") or {}
    poll_mode_cfg = cfg.get("stp_poll_mode", "auto")  # auto|snmp|ssh

    def poll_one(ip):
        """Poll a single device. Transport is chosen per device:
          - SNMP when cfg.stp_mode allows it and the device has snmp_enabled=1
            (cached cred from discover). Single-roundtrip per OID column,
            no RADIUS auth, sub-second per device.
          - SSH otherwise — legacy path. Same return shape.

        Returns (ip, ports, ifaces, stp_mode, root_entries, last_flapped) where:
          ports        = list of STP port dicts (empty in flap mode or when STP disabled)
          ifaces       = {interface: link_up_bool}
          stp_mode     = lowercased mode string ('mstp'/'rstp'/'stp'); None when
                         not determinable from this transport (existing DB value
                         is preserved by the caller).
          root_entries = list of root-state dicts (one per STP instance; currently
                         just CIST). Empty in flap mode or when STP is disabled.
          ports is None when the poll failed (credentials/connection/parse).
        """
        conn = _db()
        row = conn.execute(
            "SELECT username, password_hash, platform, snmp_enabled, snmp_proto,"
            " snmp_community, snmp_v3_user, iface_terse "
            "FROM devices WHERE ip = ? AND status = 'active'",
            (ip,)).fetchone()
        conn.close()
        known_user = row["username"] if row else None
        known_pw_hash = row["password_hash"] if row else None
        cached_platform = row["platform"] if row and "platform" in row.keys() else None

        # ----- SNMP fast path -----
        # Per-device decision: SNMP when allowed, the device is probed-good,
        # and we have a cached platform (snmp_collect_stp needs platform up
        # front; we can't run 'show version' without an SSH session).
        snmp_enabled = (row["snmp_enabled"] if row and "snmp_enabled" in row.keys()
                        else None)
        snmp_allowed = poll_mode_cfg in ("auto", "snmp") and snmp_enabled == 1
        if snmp_allowed and cached_platform in ("junos", "aruba-cx", "procurve"):
            try:
                cred = _resolve_snmp_cred(row, snmp_cfg)
                ports, ifaces, stp_mode_v, root_entries, last_flapped = \
                    snmp_collect_stp(ip, cred, cached_platform)
                log.debug(
                    "monitor-%s: %s — snmp poll: %d STP port(s), %d iface(s), "
                    "mode=%s, root_instances=%d, %d flap timestamps",
                    mode, ip, len(ports), len(ifaces), stp_mode_v,
                    len(root_entries),
                    sum(1 for v in last_flapped.values() if v is not None))
                return (ip, ports, ifaces, stp_mode_v, root_entries, last_flapped)
            except SnmpCollectError as e:
                # Hard failure: stp_mode='snmp' treats this as failure for the
                # poll (no SSH fallback). 'auto' falls through to SSH below so
                # transient SNMP issues don't drop the device from monitoring.
                log.debug("monitor-%s: %s — snmp failed: %s", mode, ip, e)
                if poll_mode_cfg == "snmp":
                    return (ip, None, {}, None, [], {})
                # else fall through to SSH

        if poll_mode_cfg == "snmp":
            # snmp-only mode but device isn't snmp-capable yet (or platform
            # missing). Skip rather than reverting to SSH — the operator chose
            # this mode explicitly.
            log.debug("monitor-%s: %s — snmp-only mode but device not "
                      "snmp-ready (snmp_enabled=%s platform=%s); skipping",
                      mode, ip, snmp_enabled, cached_platform)
            return (ip, None, {}, None, [], {})

        if not known_user or not known_pw_hash:
            log.debug("monitor-%s: %s — no stored credentials, skipping", mode, ip)
            return (ip, None, {}, None, [], {})

        known_pw = pw_by_hash.get(known_pw_hash)
        if not known_pw:
            log.debug("monitor-%s: %s — stored password hash not in current config, skipping", mode, ip)
            return (ip, None, {}, None, [], {})

        # ssh_connect handles modern-first negotiation and falls back to
        # legacy algos only when the device's negotiation error advertises
        # them. Algorithm negotiation failures don't count as failed
        # logins on Juniper (the auth phase hasn't started), so the smart
        # retry loop is safe even on Junos.
        #
        # Soft-failure retry: when the connect SUCCEEDS but a command
        # fails or times out, re-run the whole poll once on a fresh
        # session before counting a failure. A slow-but-healthy device
        # (large virtual chassis, RADIUS auth) can overrun a command
        # timeout on a busy tick; without the retry a few such ticks
        # march a reachable device across the unreachable threshold and
        # email a false outage. A genuinely down device fails at
        # connect and still fails fast — no retry.
        for attempt in (1, 2):
            _ssh_t0 = time.monotonic()
            _ssh_started = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            child = ssh_connect(ip, known_user, known_pw, timeout=30)
            _record_op("ssh_connect",
                       started_at=_ssh_started,
                       duration_ms=int((time.monotonic() - _ssh_t0) * 1000),
                       success=(child is not None),
                       ip=ip, reason="monitor_stp_fallback")
            if child is None:
                log.debug("monitor-%s: %s — SSH failed as %s", mode, ip, known_user)
                return (ip, None, {}, None, [], {})

            ports = []
            ifaces = {}
            stp_mode = None
            root_entries = []
            try:
                platform = cached_platform
                if platform not in ("junos", "aruba-cx", "procurve", "cisco-ios"):
                    platform = _classify_platform(child)
                    c = _db()
                    c.execute("UPDATE devices SET platform = ? WHERE ip = ?", (platform, ip))
                    c.commit()
                    c.close()
                    log.debug("monitor-%s: %s — classified platform=%s", mode, ip, platform)

                # Firewalls skip the STP command phase — reachability +
                # flap tracking only (see the firewall_ips partition above).
                if mode == "stp" and ip not in firewall_ips:
                    if platform == "junos":
                        # 60s: a large virtual chassis emits ~17KB here and
                        # intermittently needs well over 15s (RADIUS auth +
                        # member relay). Generous caps only affect
                        # connected-but-slow devices — a down device fails
                        # at connect, never at the command.
                        out = send_command(child, "show spanning-tree interface | no-more", timeout=60)
                        ports = parse_stp_junos(out)
                        # 'show spanning-tree bridge' gives us mode + CIST root-bridge
                        # info in one shot.
                        bridge_out = send_command(child, "show spanning-tree bridge | no-more", timeout=30)
                        stp_mode = parse_stp_mode_junos(bridge_out)
                        root_entries = parse_stp_root_junos(bridge_out)
                    else:
                        out = send_command(child, "show spanning-tree", timeout=15)
                        if "Spanning-tree is disabled" in out:
                            log.debug("monitor-%s: %s — STP disabled", mode, ip)
                            disconnect(child)
                            return (ip, [], {}, None, [], {})
                        if platform == "aruba-cx":
                            ports = parse_stp_aruba_cx(out)
                            stp_mode = parse_stp_mode_aruba_cx(out)
                            root_entries = parse_stp_root_aruba_cx(out)
                        elif platform == "cisco-ios":
                            ports = parse_stp_cisco(out)
                            stp_mode = parse_stp_mode_cisco(out)
                            # One root_entry per VLAN (PVST) or MST instance (MSTP);
                            # mode-agnostic so it survives a future MSTP migration.
                            root_entries = parse_stp_root_cisco(out)
                        else:
                            ports = parse_stp_procurve(out)
                            stp_mode = parse_stp_mode_procurve(out)
                            root_entries = parse_stp_root_procurve(out)

                # Pull link state AND per-port last-flapped in one command where
                # vendor supports it. Junos filtered form keeps output compact
                # (2 lines per port); ArubaOS-CX's full 'show interface' is
                # larger (~500KB for a VSF stack) but parseable. ProCurve lacks
                # a native last-flapped field so last_flapped stays empty.
                last_flapped = {}
                if platform == "junos":
                    use_terse = (attempt > 1 or bool(
                        row["iface_terse"] if "iface_terse" in row.keys()
                        else 0))
                    if use_terse:
                        iface_out = send_command(
                            child, "show interfaces terse | no-more",
                            timeout=30)
                        ifaces = parse_ifaces_junos_terse(iface_out)
                    else:
                        try:
                            iface_out = send_command(
                                child,
                                'show interfaces | match "^Physical|^  Last flapped" | no-more',
                                timeout=60)
                        except pexpect.TIMEOUT:
                            # A large virtual chassis relays per-interface
                            # stats across members and can be UNABLE to
                            # finish this listing (observed: a 4-member
                            # stack aborts the relay at ~61s and returns
                            # 388B of truncated output — no timeout fixes
                            # that). Flag the device so future polls go
                            # straight to terse, then let the retry loop
                            # finish this poll on a fresh session.
                            c = _db()
                            c.execute(
                                "UPDATE devices SET iface_terse = 1 "
                                "WHERE ip = ?", (ip,))
                            c.commit()
                            c.close()
                            log.debug(
                                "monitor-%s: %s — full interface listing "
                                "timed out; device flagged for 'terse'",
                                mode, ip)
                            raise
                        ifaces = parse_ifaces_junos(iface_out)
                        last_flapped = parse_junos_iface_flaps(iface_out)
                elif platform == "aruba-cx":
                    iface_out = send_command(child, "show interface", timeout=40)
                    ifaces = parse_ifaces_aruba_cx(iface_out)
                    last_flapped = parse_aruba_cx_iface_flaps(iface_out)
                elif platform == "cisco-ios":
                    iface_out = send_command(child, "show interfaces status", timeout=20)
                    ifaces = parse_ifaces_cisco(iface_out)
                    # Cisco has no clean per-port last-flapped here — Signal-1 only.
                else:
                    iface_out = send_command(child, "show interfaces brief", timeout=20)
                    ifaces = parse_ifaces_procurve(iface_out)
            except Exception as e:
                disconnect(child)
                if attempt == 1:
                    log.debug("monitor-%s: %s — soft poll failure (%s); "
                              "retrying once on a fresh session", mode, ip, e)
                    continue
                log.debug("monitor-%s: %s — poll failed after retry: %s",
                          mode, ip, e)
                return (ip, None, {}, None, [], {})
            disconnect(child)
            log.debug("monitor-%s: %s — %d STP port(s), %d interface(s) polled, "
                      "mode=%s, root_instances=%d, %d flap timestamps",
                      mode, ip, len(ports), len(ifaces), stp_mode,
                      len(root_entries), sum(1 for v in last_flapped.values() if v is not None))
            return (ip, ports, ifaces, stp_mode, root_entries, last_flapped)

    # --- Poll all devices ---
    results = []
    total = len(ips)
    done = 0
    threads = cfg["ssh_threads"]
    log.info("Using %d concurrent threads for %s polling", threads, label)
    with ThreadPoolExecutor(max_workers=threads) as pool:
        futures = {pool.submit(poll_one, ip): ip for ip in ips}
        for future in as_completed(futures):
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rPolling {label} state... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()
            results.append(future.result())

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    # --- Compare against baseline and store ---
    expected_mode = (cfg.get("stp_expected_mode") or "mstp").lower()
    conn = _db()
    changes = []              # confirmed STP changes (emit alerts for these)
    new_pending = 0           # counter: new STP changes buffered this poll
    dropped_pending = 0       # counter: pending changes that reverted (transient)
    flap_suppressed = 0       # counter: confirmed STP changes dropped as flap artifacts
    classifier_suppressed = 0 # counter: confirmed STP changes tagged as snmp_transient
    suspected_fp = []         # changes the classifier tagged as suspected false-positive
    root_changes = []         # confirmed root-bridge changes (per device/instance)
    new_root_pending = 0      # counter: new root changes buffered this poll
    dropped_root_pending = 0  # counter: root-change pending that reverted
    flap_events = 0           # counter: observed up-from-down transitions
    polled = 0
    failed = 0
    stp_disabled = 0
    stp_mismatched = 0
    # Reachability tracking: collect devices that cross the unreachable
    # threshold this tick (first email) and devices that recover (clearing
    # email). Both are batched into single emails at end of monitor so a
    # site-wide outage doesn't spam N separate messages.
    unreachable_threshold = int(cfg.get("snmp_unreachable_polls", 3))
    pending_stamped_this_tick = []   # ips newly stamped pending this poll
    newly_recovered = []     # [{ip, hostname, model, alerted_at, downtime}]

    def _on_poll_success(ip_):
        """Reset fail counter; if device was previously alerted as down,
        clear the alert flag and queue a recovery email."""
        r = conn.execute(
            "SELECT hostname, model, snmp_consecutive_fails, "
            "unreachable_alerted_at FROM devices WHERE ip = ?", (ip_,)
        ).fetchone()
        if not r:
            return
        was_alerted = r["unreachable_alerted_at"]
        if (r["snmp_consecutive_fails"] or 0) > 0 or was_alerted:
            # Also clears a pending-but-never-alerted outage stamp: a
            # transient that recovered inside the coalescing hold produces
            # neither an unreachable email nor a recovery email.
            conn.execute(
                "UPDATE devices SET snmp_consecutive_fails = 0, "
                "unreachable_alerted_at = NULL, "
                "unreachable_pending_since = NULL WHERE ip = ?", (ip_,))
        if was_alerted:
            newly_recovered.append({
                "ip": ip_,
                "hostname": r["hostname"],
                "model": r["model"],
                "alerted_at": was_alerted,
                "downtime": _format_outage_duration(was_alerted),
            })

    def _on_poll_fail(ip_):
        """Increment fail counter; if it just crossed the threshold and we
        haven't already alerted for this outage, queue the device for the
        unreachable email (TCP probe runs after the loop)."""
        r = conn.execute(
            "SELECT hostname, model, snmp_consecutive_fails, "
            "unreachable_alerted_at, unreachable_pending_since, "
            "snmp_last_ok, snmp_diag "
            "FROM devices WHERE ip = ?", (ip_,)
        ).fetchone()
        if not r:
            return
        new_count = (r["snmp_consecutive_fails"] or 0) + 1
        conn.execute(
            "UPDATE devices SET snmp_consecutive_fails = ? WHERE ip = ?",
            (new_count, ip_))
        if new_count >= unreachable_threshold and not r["unreachable_alerted_at"]:
            # Coalescing (3.19.0): don't queue the email yet — stamp the
            # device pending. The batch releases once the outage stops
            # GROWING (no new pending device for one poll) or after
            # unreachable_coalesce_min, so one power event = one
            # topology-grouped email instead of a trickle.
            keys = r.keys()
            already_pending = (r["unreachable_pending_since"]
                               if "unreachable_pending_since" in keys
                               else None)
            if not already_pending:
                conn.execute(
                    "UPDATE devices SET unreachable_pending_since = ? "
                    "WHERE ip = ?", (now, ip_))
                pending_stamped_this_tick.append(ip_)

    for ip, ports, ifaces, stp_mode, root_entries, last_flapped in results:
        if ports is None:
            failed += 1
            _on_poll_fail(ip)
            # Don't touch stp_enabled — keep last known value
            continue
        _on_poll_success(ip)
        hostname = hostnames.get(ip, ip)
        if mode == "stp" and ip not in firewall_ips:
            # Empty ports alone does NOT mean STP is off: some platforms report
            # the mode and per-instance root without a per-port table (e.g.
            # ArubaOS-Switch RPVST, whose per-port states need per-VLAN queries
            # we don't run). Only treat the device as STP-disabled when there's
            # no STP signal at all — no ports, no mode, and no root entries.
            if not ports and not stp_mode and not root_entries:
                stp_disabled += 1
                # stp_disabled_since: only set if not already set. Preserves
                # the original transition timestamp across successive
                # consecutive-disabled polls so the digest can require a
                # sustained-disabled duration before listing (avoids
                # false-positives from transient empty-SNMP-table polls).
                conn.execute(
                    "UPDATE devices SET stp_enabled = 0, stp_mode = NULL, "
                    "stp_last_check = ?, "
                    "stp_disabled_since = COALESCE(stp_disabled_since, ?) "
                    "WHERE ip = ?",
                    (now, now, ip),
                )
                # An STP-disabled device has no meaningful root info either;
                # clear any stale entries so 'show spanning-tree root' isn't misleading.
                conn.execute("DELETE FROM stp_root_state WHERE ip = ?", (ip,))
                continue
            polled += 1
            # --- Persist root-bridge state (one row per STP instance) ---
            # root_priority/root_mac are two-phase confirmed (same shape as
            # stp_pending_changes): a new reading goes into stp_root_pending and
            # only overwrites stp_root_state after the NEXT poll still sees it.
            # bridge_*, tcn_count, last_tcn_seconds, updated_at update in place
            # every poll (they're not alert drivers and stale values would lie
            # to the 'show spanning-tree root' query and to the cross-switch consensus).
            for re_ in root_entries:
                inst = re_["instance"]
                new_t = (re_.get("root_priority"), re_.get("root_mac"))
                bridge_prio = re_.get("bridge_priority")
                bridge_mac = re_.get("bridge_mac")
                tcn = re_.get("tcn_count")
                last_tcn = re_.get("last_tcn_seconds")

                prev_row = conn.execute(
                    "SELECT root_priority, root_mac FROM stp_root_state "
                    "WHERE ip = ? AND instance = ?", (ip, inst)).fetchone()
                prev_t = (prev_row["root_priority"], prev_row["root_mac"]) if prev_row else None

                pend_row = conn.execute(
                    "SELECT old_priority, old_mac, new_priority, new_mac "
                    "FROM stp_root_pending WHERE ip = ? AND instance = ?",
                    (ip, inst)).fetchone()

                # Decide what root_priority/root_mac to store in stp_root_state
                # this poll. Only "confirmed" values are kept there; the cross-
                # switch consensus query reads from stp_root_state, so it too
                # sees only confirmed roots.
                confirmed_t = prev_t
                if pend_row:
                    pend_old = (pend_row["old_priority"], pend_row["old_mac"])
                    pend_new = (pend_row["new_priority"], pend_row["new_mac"])
                    if new_t == pend_new:
                        # Confirmed — alert and move state forward
                        root_changes.append({
                            "ip": ip, "hostname": hostname, "instance": inst,
                            "old_priority": pend_old[0], "old_mac": pend_old[1],
                            "new_priority": new_t[0], "new_mac": new_t[1],
                        })
                        conn.execute(
                            "DELETE FROM stp_root_pending WHERE ip = ? AND instance = ?",
                            (ip, inst))
                        confirmed_t = new_t
                    elif new_t == pend_old:
                        # Reverted — silent, drop pending
                        conn.execute(
                            "DELETE FROM stp_root_pending WHERE ip = ? AND instance = ?",
                            (ip, inst))
                        dropped_root_pending += 1
                        confirmed_t = pend_old
                    else:
                        # Rebased — keep original old, update new to latest reading
                        conn.execute(
                            "UPDATE stp_root_pending SET new_priority = ?, new_mac = ? "
                            "WHERE ip = ? AND instance = ?",
                            (new_t[0], new_t[1], ip, inst))
                        confirmed_t = pend_old
                else:
                    if prev_t is None:
                        # First sighting of this (ip, instance) — record as
                        # baseline without alerting.
                        confirmed_t = new_t
                    elif new_t != prev_t:
                        # New change — enter pending for next-poll confirmation.
                        conn.execute("""
                            INSERT INTO stp_root_pending
                                (ip, instance, old_priority, old_mac,
                                 new_priority, new_mac, first_seen)
                            VALUES (?, ?, ?, ?, ?, ?, ?)
                        """, (ip, inst, prev_t[0], prev_t[1],
                              new_t[0], new_t[1], now))
                        new_root_pending += 1
                        confirmed_t = prev_t

                conf_prio, conf_mac = confirmed_t if confirmed_t else (None, None)
                is_root = 1 if (conf_mac and bridge_mac and conf_mac == bridge_mac) else 0
                conn.execute("""
                    INSERT INTO stp_root_state
                        (ip, instance, root_priority, root_mac,
                         bridge_priority, bridge_mac, is_root,
                         tcn_count, last_tcn_seconds, updated_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(ip, instance) DO UPDATE SET
                        root_priority = excluded.root_priority,
                        root_mac = excluded.root_mac,
                        bridge_priority = excluded.bridge_priority,
                        bridge_mac = excluded.bridge_mac,
                        is_root = excluded.is_root,
                        tcn_count = excluded.tcn_count,
                        last_tcn_seconds = excluded.last_tcn_seconds,
                        updated_at = excluded.updated_at
                """, (ip, inst, conf_prio, conf_mac,
                      bridge_prio, bridge_mac, is_root,
                      tcn, last_tcn, now))
            # Only overwrite stp_mode when we successfully parsed a value; a
            # failed parse leaves the previous value intact (conservative, so
            # a transient output glitch doesn't flip a device in/out of the
            # mismatch list on a single poll).
            up_count = sum(1 for v in ifaces.values() if v)
            if stp_mode is not None:
                # Clear stp_disabled_since on the 0→1 transition (or first
                # poll). Idempotent: NULL stays NULL on enabled→enabled.
                conn.execute(
                    "UPDATE devices SET stp_enabled = 1, stp_mode = ?, "
                    "stp_last_check = ?, link_up_count = ?, "
                    "stp_disabled_since = NULL WHERE ip = ?",
                    (stp_mode, now, up_count, ip),
                )
                if stp_mode != expected_mode:
                    stp_mismatched += 1
            else:
                conn.execute(
                    "UPDATE devices SET stp_enabled = 1, stp_last_check = ?, "
                    "link_up_count = ?, stp_disabled_since = NULL "
                    "WHERE ip = ?",
                    (now, up_count, ip),
                )
        else:
            polled += 1

        # --- Port flap tracking (physical link-up/down) ---
        prev_port_rows = conn.execute(
            "SELECT interface, link_up, last_seen FROM port_state WHERE ip = ?", (ip,)
        ).fetchall()
        prev_link = {r["interface"]: bool(r["link_up"]) for r in prev_port_rows}
        # Per-iface previous poll timestamp; used to compute the gap between
        # polls so we can compare it against the vendor's 'last flapped X ago'
        # value and detect sub-poll flaps our up-from-down observation misses.
        prev_last_seen = {r["interface"]: r["last_seen"] for r in prev_port_rows}
        # Set of interfaces that completed a flap cycle during THIS poll —
        # used below to suppress STP state-change alerts whose transition is
        # a byproduct of the flap rather than a real topology event.
        flapped_this_poll = set()
        now_dt = datetime.strptime(now, "%Y-%m-%d %H:%M:%S")
        for iface, link_up in ifaces.items():
            was_up = prev_link.get(iface)
            observed_flap = (was_up is False and link_up)
            # Inferred flap: vendor says the port's last flap was AFTER our
            # previous poll on this iface. Means the port flapped during the
            # gap — we missed it because the up-from-down transition didn't
            # straddle two polls. Only applies to iface that already had a
            # prev_last_seen (i.e., we polled it before).
            inferred_flap = False
            flap_sec = last_flapped.get(iface)
            if (flap_sec is not None and flap_sec > 0
                    and not observed_flap
                    and iface in prev_last_seen):
                try:
                    prev_dt = datetime.strptime(prev_last_seen[iface],
                                                "%Y-%m-%d %H:%M:%S")
                    gap_sec = int((now_dt - prev_dt).total_seconds())
                    if 0 < flap_sec < gap_sec:
                        inferred_flap = True
                except (TypeError, ValueError):
                    pass
            if observed_flap or inferred_flap:
                # one completed flap cycle
                conn.execute("""
                    INSERT INTO port_flaps (ip, hostname, interface, flap_count, first_seen, last_seen)
                    VALUES (?, ?, ?, 1, ?, ?)
                    ON CONFLICT(ip, interface) DO UPDATE SET
                        hostname = excluded.hostname,
                        flap_count = flap_count + 1,
                        last_seen = excluded.last_seen
                """, (ip, hostname, iface, now, now))
                # Append-only timeline (survives clear-flap; `show flap-history`).
                conn.execute(
                    "INSERT INTO flap_events (ip, hostname, interface, flapped_at) "
                    "VALUES (?, ?, ?, ?)", (ip, hostname, iface, now))
                flap_events += 1
                flapped_this_poll.add(iface)
            conn.execute("""
                INSERT INTO port_state (ip, interface, link_up, last_seen)
                VALUES (?, ?, ?, ?)
                ON CONFLICT(ip, interface) DO UPDATE SET
                    link_up = excluded.link_up,
                    last_seen = excluded.last_seen
            """, (ip, iface, 1 if link_up else 0, now))

        # STP analysis only runs in stp mode; flap mode and firewalls
        # (reachability + flap tracking only) skip it entirely.
        if mode != "stp" or ip in firewall_ips:
            continue

        # --- STP change detection with 1-poll confirmation ---
        prev_rows = conn.execute(
            "SELECT interface, role, state, edge_expiry FROM stp_state WHERE ip = ?",
            (ip,)
        ).fetchall()
        prev = {r["interface"]: (r["role"], r["state"]) for r in prev_rows}

        pending_rows = conn.execute("""
            SELECT interface, old_role, old_state, new_role, new_state, first_seen,
                   edge_expiry_at_first_seen, tcn_at_first_seen
            FROM stp_pending_changes WHERE ip = ?
        """, (ip,)).fetchall()
        pending = {r["interface"]: (
            (r["old_role"], r["old_state"]),
            (r["new_role"], r["new_state"]),
            r["first_seen"],
            r["edge_expiry_at_first_seen"],
            r["tcn_at_first_seen"],
        ) for r in pending_rows}

        # --- Phantom-evidence inputs (Junos SNMP path only) ---
        # Per-port edge_expiry from this poll's SNMP walk; bridge-wide
        # CIST TC count from root_entries. Both are None for non-Junos
        # devices and for any platform/path where the SNMP walk didn't
        # populate them — the downstream comparison treats None as
        # 'unknown' and skips logging rather than mis-classifying.
        current_expiry = {p["interface"]: p.get("edge_expiry") for p in ports}
        current_tcn = None
        for re_ in root_entries:
            if re_["instance"] == "CIST" and re_.get("tcn_count") is not None:
                current_tcn = re_["tcn_count"]
                break

        # Snapshot flap last-seen times for this device so the confirmation
        # branch can check whether a recorded flap falls inside the pending
        # window. Recovery alerts ('BLK -> FWD' after the link came back up)
        # don't coincide with an up-from-down transition in the confirmation
        # poll itself, so in-poll checks alone miss them.
        flap_last_seen = {r["interface"]: r["last_seen"] for r in conn.execute(
            "SELECT interface, last_seen FROM port_flaps WHERE ip = ?", (ip,)
        ).fetchall()}

        current = {p["interface"]: (p["role"], p["state"]) for p in ports}
        GONE = ("GONE", "GONE")

        # Union of all interfaces we've ever seen for this device that are
        # still in play (current poll, last confirmed state, or pending).
        all_ifaces = set(current) | set(prev) | set(pending)

        for iface in all_ifaces:
            new_t = current.get(iface, GONE)
            if iface in pending:
                (old_t, pend_new_t, pend_first_seen,
                 pend_edge_expiry, pend_tcn) = pending[iface]
                if new_t == pend_new_t:
                    # Confirmed — same change persisted to this poll. Before
                    # alerting, check whether this transition is a link-flap
                    # artifact and should be suppressed. Three signals:
                    #   1. Port is currently down.
                    #   2. Port completed an up-from-down cycle in this poll.
                    #   3. port_flaps.last_seen >= pending.first_seen — a flap
                    #      was observed at or after we first noticed the STP
                    #      change. Catches the recovery side ('BLK -> FWD'
                    #      after the link came back up a poll ago), which
                    #      signals 1 and 2 miss by the time the port is
                    #      stable-up again.
                    # STP reports Junos logical names ('ge-1/0/0.0') but our
                    # ifaces/flap tracking keys off the physical port name
                    # ('ge-1/0/0'), so strip the '.N' subunit before lookup.
                    iface_phys = iface.split(".", 1)[0]
                    link_down = ifaces.get(iface_phys) is False
                    in_poll_flap = iface_phys in flapped_this_poll
                    last_flap = flap_last_seen.get(iface_phys)
                    in_window_flap = (last_flap is not None and pend_first_seen
                                      and last_flap >= pend_first_seen)
                    if link_down or in_poll_flap or in_window_flap:
                        conn.execute("DELETE FROM stp_pending_changes WHERE ip = ? AND interface = ?",
                                     (ip, iface))
                        flap_suppressed += 1
                        # Leave stp_state at old_t so a later recovery to old_t
                        # is silent and a later true transition still compares
                        # against the last stable reference.
                        continue
                    # Phantom-evidence log: when a state change confirms,
                    # compare the per-port state-machine activity counter
                    # (edge_expiry) and bridge-wide TC count between
                    # pending-creation and now. delta=0 on both means the
                    # state machine never ran and no TC propagated, yet
                    # the MIB reported a state transition — direct
                    # evidence of a Junos MIB-cache vs state-machine race.
                    cur_expiry = current_expiry.get(iface)
                    if (pend_edge_expiry is not None
                            and cur_expiry is not None):
                        edge_delta = cur_expiry - pend_edge_expiry
                    else:
                        edge_delta = None
                    if pend_tcn is not None and current_tcn is not None:
                        tcn_delta = current_tcn - pend_tcn
                    else:
                        tcn_delta = None
                    log.info(
                        "phantom-evidence: %s (%s) %s  %s/%s -> %s/%s  "
                        "edge_delta=%s tcn_delta=%s",
                        ip, hostname, iface,
                        old_t[0], old_t[1], new_t[0], new_t[1],
                        edge_delta if edge_delta is not None else "?",
                        tcn_delta if tcn_delta is not None else "?",
                    )
                    changes.append({
                        "ip": ip, "hostname": hostname, "interface": iface,
                        "old_role": old_t[0], "old_state": old_t[1],
                        "new_role": new_t[0], "new_state": new_t[1],
                    })
                    conn.execute("DELETE FROM stp_pending_changes WHERE ip = ? AND interface = ?",
                                 (ip, iface))
                    if new_t == GONE:
                        conn.execute("DELETE FROM stp_state WHERE ip = ? AND interface = ?",
                                     (ip, iface))
                    else:
                        conn.execute("""
                            INSERT INTO stp_state (ip, interface, role, state, last_seen, edge_expiry)
                            VALUES (?, ?, ?, ?, ?, ?)
                            ON CONFLICT(ip, interface) DO UPDATE SET
                                role = excluded.role, state = excluded.state,
                                last_seen = excluded.last_seen,
                                edge_expiry = excluded.edge_expiry
                        """, (ip, iface, new_t[0], new_t[1], now,
                              current_expiry.get(iface)))
                elif new_t == old_t:
                    # Reverted — transient, silently drop
                    conn.execute("DELETE FROM stp_pending_changes WHERE ip = ? AND interface = ?",
                                 (ip, iface))
                    dropped_pending += 1
                else:
                    # State changed to yet another value — re-check the rebased
                    # (old_t, new_t) pair. If it's no longer alertable (e.g. the
                    # original pending caught a transient step that now rolled into
                    # a link-down), drop the pending just like a revert; otherwise
                    # rebase new_t and keep old_t so a later revert still detects.
                    if should_alert_stp_change(old_t, new_t):
                        conn.execute("""
                            UPDATE stp_pending_changes
                            SET new_role = ?, new_state = ?
                            WHERE ip = ? AND interface = ?
                        """, (new_t[0], new_t[1], ip, iface))
                    else:
                        conn.execute("DELETE FROM stp_pending_changes WHERE ip = ? AND interface = ?",
                                     (ip, iface))
                        dropped_pending += 1
            else:
                old_t = prev.get(iface)
                if old_t is None:
                    # First time we've seen this port — just record it, no alert.
                    # Skip transient states (LRN/LIS) so we don't latch a mid-
                    # convergence snapshot as the reference point.
                    if new_t != GONE and not _is_stp_transient_state(*new_t):
                        conn.execute("""
                            INSERT INTO stp_state (ip, interface, role, state, last_seen, edge_expiry)
                            VALUES (?, ?, ?, ?, ?, ?)
                        """, (ip, iface, new_t[0], new_t[1], now,
                              current_expiry.get(iface)))
                elif new_t != old_t:
                    if should_alert_stp_change(old_t, new_t):
                        conn.execute("""
                            INSERT INTO stp_pending_changes
                                (ip, hostname, interface, old_role, old_state,
                                 new_role, new_state, first_seen,
                                 edge_expiry_at_first_seen, tcn_at_first_seen)
                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                        """, (ip, hostname, iface,
                              old_t[0], old_t[1], new_t[0], new_t[1], now,
                              current_expiry.get(iface), current_tcn))
                        new_pending += 1
                    # else: filtered (link-flap artifact) — no pending, no alert,
                    # and stp_state stays at old_t so recovery is also silent.

        # Refresh edge_expiry for every port we polled this round, not just
        # the ones whose state changed. The phantom-evidence delta needs a
        # *fresh* prior value (the one captured into stp_pending_changes
        # at first-seen time gets compared to *this* poll's reading); if
        # we only wrote edge_expiry on state changes, stable ports would
        # carry NULL forever and the delta would be uncomputable. Cheap:
        # a single UPDATE per port, indexed lookup, no I/O on the hot path.
        for iface, expiry in current_expiry.items():
            if expiry is not None:
                conn.execute(
                    "UPDATE stp_state SET edge_expiry = ? "
                    "WHERE ip = ? AND interface = ?",
                    (expiry, ip, iface),
                )

    # Record confirmed changes
    for ch in changes:
        conn.execute("""
            INSERT INTO stp_changes (ip, hostname, interface, old_role, old_state,
                                     new_role, new_state, changed_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (ch["ip"], ch["hostname"], ch["interface"],
              ch["old_role"], ch["old_state"],
              ch["new_role"], ch["new_state"], now))

    for ch in root_changes:
        conn.execute("""
            INSERT INTO stp_root_changes
                (ip, hostname, instance, old_priority, old_mac,
                 new_priority, new_mac, changed_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (ch["ip"], ch["hostname"], ch["instance"],
              ch["old_priority"], ch["old_mac"],
              ch["new_priority"], ch["new_mac"], now))

    conn.commit()
    conn.close()

    # --- Investigation + post-investigation flap filter ---
    # Runs before the summary report so the logged counts reflect the
    # final, post-filter numbers. Investigation happens once per affected
    # switch (one SSH session per IP, multiple ports per session).
    investigations = {}
    if mode == "stp" and changes:
        by_switch = {}
        for ch in changes:
            by_switch.setdefault(ch["ip"], []).append(ch["interface"])
        for inv_ip, inv_ports in by_switch.items():
            for port, info in investigate_stp_ports(inv_ip, inv_ports, cfg).items():
                investigations[(inv_ip, port)] = info

        # If the vendor's own 'last flapped' field says the port flapped
        # within FLAP_SUPPRESS_SEC, the STP transition is a flap artifact.
        # Authoritative source of truth — catches sub-poll flaps our
        # port_flaps table can't see.
        FLAP_SUPPRESS_SEC = 300
        kept = []
        for ch in changes:
            info = investigations.get((ch["ip"], ch["interface"]))
            flap_sec = None
            if info and info.get("ok"):
                out = info.get("interface_stats", "") or ""
                flap_sec = (parse_last_flapped_junos(out)
                            if "Last flapped" in out
                            else parse_last_flapped_aruba_cx(out))
            if flap_sec is not None and flap_sec < FLAP_SUPPRESS_SEC:
                log.info("flap-suppressed (investigation): %s (%s) %s — "
                         "last flapped %ds ago",
                         ch["ip"], ch["hostname"], ch["interface"], flap_sec)
                flap_suppressed += 1
                continue
            kept.append(ch)
        changes = kept

        # --- Classifier-based suspected-false-positive partition ---
        # The heuristic classifier (see _classify_stp_event) inspects the
        # investigation output and tags each surviving change. The
        # 'snmp_transient' label is the strongest false-positive signal we
        # have — it fires only when the port has *no* port_flaps row, *no*
        # BPDU input, link is up, and no recent topology change. On Junos
        # this is the documented MIB-cache-vs-state-machine race for
        # edge-boundary ports. We *partition* rather than drop so the
        # operator still sees the events; the alert email renders them in
        # a separate "Suspected false positives" section and subject line
        # reflects only the actionable count.
        suspected_fp = []
        kept = []
        for ch in changes:
            info = investigations.get((ch["ip"], ch["interface"]))
            label = LIKELY_UNCLASSIFIED
            if info and info.get("ok") and info.get("parsed"):
                pf, tcn_s = _stp_event_context(ch["ip"], ch["interface"])
                label, _conf, _ev = _classify_stp_event(
                    info["parsed"], pf, tcn_s,
                    old_role=ch.get("old_role"), new_role=ch.get("new_role"))
            ch["_classifier_label"] = label
            if label == LIKELY_SNMP_TRANSIENT:
                suspected_fp.append(ch)
            else:
                kept.append(ch)
        changes = kept
        classifier_suppressed = len(suspected_fp)

    # --- Report ---
    if mode == "stp":
        conn = _db()
        blocked_count = conn.execute("""
            SELECT COUNT(*) FROM stp_state
            WHERE (role = 'ALT' AND state = 'BLK')
               OR (role = 'Alternate' AND state = 'Blocking')
        """).fetchone()[0]
        root_mismatches, not_in_service, unassigned = _detect_root_mismatches(
            conn, now, cfg.get("stp_domains") or {})
        # Maps each disagreeing pair to its first_seen when the breakdown is
        # unchanged since the last tick, or None when it's new/changed and so
        # worth logging in full.
        root_disagree_seen = _sync_root_disagree_state(conn, root_mismatches, now)
        conn.close()

        log.info("--- STP Monitor Summary ---")
        log.info("Polled:       %d device(s)", polled)
        log.info("STP disabled: %d device(s)", stp_disabled)
        log.info("Mismatched:   %d device(s) not running %s", stp_mismatched, expected_mode)
        log.info("Failed:       %d device(s)", failed)
        log.info("Confirmed:    %d STP change(s)", len(changes))
        log.info("Pending:      %d new change(s) awaiting next poll", new_pending)
        log.info("Transient:    %d pending change(s) dropped (reverted)", dropped_pending)
        log.info("Flap-suppress: %d confirmed change(s) dropped as flap artifacts", flap_suppressed)
        log.info("Classifier-fp: %d confirmed change(s) tagged as snmp_transient (suspected false positive)",
                 classifier_suppressed)
        log.info("Flaps:        %d port flap(s) recorded", flap_events)
        log.info("Blocked:      %d port(s) currently in blocking (run 'show spanning-tree blocked' for details)",
                 blocked_count)
        log.info("Root changes: %d confirmed, %d pending, %d reverted",
                 len(root_changes), new_root_pending, dropped_root_pending)
        log.info("Root disagree: %d (domain, instance) pair(s) with root-bridge disagreement",
                 len(root_mismatches))
        log.info("Not in service: %d device(s) excluded from consensus (no data-plane links)",
                 not_in_service)
        if unassigned:
            log.info("Unassigned: %d device(s) — IPs not matched by any [stp_domains] prefix",
                     unassigned)

        if changes or suspected_fp:
            # Sort by hostname; stable sort preserves per-device interface order
            # from the switch's own output (natural numbering like ge-0/0/2
            # before ge-0/0/10, which a string sort would flip).
            changes.sort(key=lambda c: c["hostname"].lower())
            suspected_fp.sort(key=lambda c: c["hostname"].lower())
            if changes:
                log.info("--- Confirmed STP Changes ---")
                for ch in changes:
                    blocking = " *** BLOCKING ***" if is_stp_blocking(ch) else ""
                    log.info("  %s (%s) %s: %s/%s -> %s/%s%s",
                             ch["ip"], ch["hostname"], ch["interface"],
                             ch["old_role"], ch["old_state"],
                             ch["new_role"], ch["new_state"],
                             blocking)
            if suspected_fp:
                log.info("--- Suspected False Positives (snmp_transient) ---")
                for ch in suspected_fp:
                    log.info("  %s (%s) %s: %s/%s -> %s/%s",
                             ch["ip"], ch["hostname"], ch["interface"],
                             ch["old_role"], ch["old_state"],
                             ch["new_role"], ch["new_state"])
            # Email only when there's at least one actionable change. All-FP
            # ticks stay in the log + the weekly digest summary.
            if changes and email:
                _send_stp_alert(cfg, changes, investigations,
                                suspected_fp=suspected_fp)

        if root_changes:
            root_changes.sort(key=lambda c: (c["instance"], c["hostname"].lower()))
            log.info("--- Confirmed STP Root-Bridge Changes ---")
            for ch in root_changes:
                log.info("  %s (%s) %s: %s/%s -> %s/%s",
                         ch["ip"], ch["hostname"], ch["instance"],
                         ch["old_priority"], ch["old_mac"],
                         ch["new_priority"], ch["new_mac"])
            if email:
                _send_root_change_alert(cfg, root_changes)

        if root_mismatches:
            # Log only — the weekly 'digest stp' email is the alert channel for
            # root disagreement (cadence matches STP-disabled / mode-mismatch
            # config issues, which also surface only through the digest).
            log.info("--- STP Root-Bridge Disagreement (see 'digest stp' for alert) ---")
            for mm in root_mismatches:
                # Disagreement is a standing condition, not an event: the same
                # breakdown would otherwise be re-logged every minute for as
                # long as it lasts (the dominant source of log growth on a
                # large fleet). Print it in full when it's new or has changed,
                # and collapse it to one line while it holds steady. --detail
                # always expands, so an operator can still see it on demand.
                first_seen = root_disagree_seen.get((mm["domain"], mm["instance"]))
                if first_seen and not detail:
                    log.info("  domain=%s instance=%s — %d distinct roots, "
                             "unchanged since %s (pass --detail to list)",
                             mm["domain"], mm["instance"], len(mm["groups"]),
                             first_seen)
                    continue
                log.info("  domain=%s instance=%s — %d distinct roots:",
                         mm["domain"], mm["instance"], len(mm["groups"]))
                # Groups are ordered smallest-first; the LAST group is the
                # majority/agreeing one. Collapse its device list to a count
                # unless --detail was passed.
                majority_idx = len(mm["groups"]) - 1
                for i, grp in enumerate(mm["groups"]):
                    is_majority = (i == majority_idx)
                    suffix = " agree (pass --detail to list)" if is_majority and not detail else ""
                    log.info("    priority=%s mac=%s — %d switch(es)%s",
                             grp["root_priority"], grp["root_mac"],
                             len(grp["devices"]), suffix)
                    if is_majority and not detail:
                        continue
                    for ip, hostname in grp["devices"]:
                        log.info("      %s (%s)", hostname, ip)
    else:
        log.info("--- Flap Monitor Summary ---")
        log.info("Polled:       %d device(s)", polled)
        log.info("Failed:       %d device(s)", failed)
        log.info("Flaps:        %d port flap(s) recorded", flap_events)

    # --- Reachability alerts ---
    # Coalesced release: pending devices (crossed the threshold, not yet
    # alerted) are emailed once the outage stops growing — no NEW pending
    # device this poll — or once the oldest pending stamp is
    # unreachable_coalesce_min old. One power event = one grouped email.
    newly_unreachable = []
    conn_p = _db()
    conn_p.row_factory = sqlite3.Row
    pending = conn_p.execute(
        "SELECT ip, hostname, model, "
        "       snmp_consecutive_fails AS fails, "
        "       snmp_last_ok AS last_ok, snmp_diag AS last_diag, "
        "       unreachable_pending_since "
        "FROM devices WHERE unreachable_pending_since IS NOT NULL "
        "AND unreachable_alerted_at IS NULL").fetchall()
    conn_p.close()
    if pending:
        coalesce_min = int(cfg.get("unreachable_coalesce_min", 3))
        oldest = min(p["unreachable_pending_since"] for p in pending)
        try:
            age_min = ((datetime.now()
                        - datetime.strptime(oldest, "%Y-%m-%d %H:%M:%S"))
                       .total_seconds() / 60.0)
        except ValueError:
            age_min = coalesce_min
        if not pending_stamped_this_tick or age_min >= coalesce_min:
            newly_unreachable = [dict(p) for p in pending]
        else:
            log.info("unreachable coalescing: holding %d pending device(s) "
                     "— outage still growing this poll", len(pending))

    # Per-device TCP probe runs only for devices in a released batch, so
    # an idle minute does zero extra network I/O. Recovery emails fire on
    # the first successful poll after an alert.
    if newly_unreachable or newly_recovered:
        if newly_unreachable:
            log.info("--- Unreachable: %d device(s) crossed threshold ---",
                     len(newly_unreachable))
            probe_threads = min(20, max(1, len(newly_unreachable)))
            with ThreadPoolExecutor(max_workers=probe_threads) as pool:
                futures = {pool.submit(_classify_reachability, d["ip"]): d
                           for d in newly_unreachable}
                for fut in as_completed(futures):
                    futures[fut]["probe"] = fut.result()
            if email:
                # Mark alerted_at first so a same-tick email failure can't
                # cause re-alerting next minute (the operator sees one log
                # line per failed-email attempt; the alarm itself fires once
                # per outage). Under --no-email the stamp is skipped along
                # with the send, leaving the outage owed to the next tick.
                now_ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                conn2 = _db()
                for d in newly_unreachable:
                    conn2.execute(
                        "UPDATE devices SET unreachable_alerted_at = ?, "
                        "unreachable_pending_since = NULL WHERE ip = ?",
                        (now_ts, d["ip"]))
                conn2.commit()
                conn2.close()
                _send_unreachable_alert(cfg, newly_unreachable)
            else:
                log.warning("monitor %s: email suppressed (--no-email) — "
                            "%d unreachable device(s) left unalerted",
                            mode, len(newly_unreachable))
        if newly_recovered:
            log.info("--- Recovered: %d device(s) ---", len(newly_recovered))
            if email:
                _send_recovery_alert(cfg, newly_recovered)
            else:
                log.warning("monitor %s: email suppressed (--no-email) — "
                            "recovery notice for %d device(s) dropped (the "
                            "poll already cleared their outage state)",
                            mode, len(newly_recovered))

    # --- Tick instrumentation — write one op_event row for this tick ---
    _tick_duration_ms = int((time.monotonic() - _tick_t0) * 1000)
    _snmp_stats = _snmp_timing_stop()
    _tick_extra = {
        "polled": polled,
        "failed": failed,
        "flap_events": flap_events,
        "confirmed": len(changes),
    }
    if mode == "stp":
        _tick_extra["stp_disabled"] = stp_disabled
    if _snmp_stats:
        _tick_extra.update(_snmp_stats)
    _record_op(f"monitor_{mode}_tick",
               started_at=_tick_started_at,
               duration_ms=_tick_duration_ms,
               success=True,
               extra=_tick_extra)


_EMAIL_LOGO_CACHE = ".email-logo.png"
_EMAIL_LOGO_BOX = (240, 60)   # max (w, h) px for the header logo


def _prepare_email_logo(cfg):
    """Normalize the configured [email] logo (ANY format/size) into a small,
    email-safe PNG cached in the state dir, and return its path — or None when
    no usable logo. So the operator just drops any image and netops resizes +
    transcodes it; we convert ONCE and reuse the cache until the source file
    changes (mtime check). Needs Pillow (python3-pil); without it, a source
    that's already an embeddable raster (png/jpg/gif) is attached as-is,
    otherwise None. Best-effort — never raises into the send path."""
    src = (cfg.get("email_logo") or "").strip()
    if not src or not os.path.isfile(src):
        return None
    cache = os.path.join(STATE_DIR, _EMAIL_LOGO_CACHE)
    try:
        if (os.path.isfile(cache)
                and os.path.getmtime(cache) >= os.path.getmtime(src)):
            return cache                      # already normalized + current
    except OSError:
        pass
    try:
        from PIL import Image
        with Image.open(src) as im:
            im = im.convert("RGBA")           # flatten odd modes; keep alpha
            im.thumbnail(_EMAIL_LOGO_BOX)     # fit within the box, keep aspect
            im.save(cache, "PNG", optimize=True)
        return cache
    except ImportError:
        log.debug("Pillow (python3-pil) not installed — email logo used as-is")
    except Exception as e:
        log.debug("email logo normalize failed (%s) — using source as-is", e)
    # Fallback: source is already an embeddable raster -> attach unmodified.
    if os.path.splitext(src)[1].lower() in (".png", ".jpg", ".jpeg", ".gif"):
        return src
    return None


def _send_email(cfg, subject, body, html_body=None):
    """Send an email via the [email] config. Returns (ok, error_message).

    When html_body is given, the message is multipart/alternative: the plain
    `body` stays as the fallback (shown by clients that strip HTML, and it's
    the exhaustive version) and html_body is the rich rendering. Plain-text-only
    callers (every alert path) are unaffected — they omit html_body."""
    import smtplib
    from email.mime.text import MIMEText
    from email.mime.multipart import MIMEMultipart

    smtp_server = cfg.get("smtp_server")
    if not smtp_server:
        return False, "no [email] section configured (smtp_server missing)"

    smtp_port = int(cfg.get("smtp_port", 587))
    from_addr = cfg.get("email_from", "netops@localhost")
    to_addrs = [a.strip() for a in cfg.get("email_to", "").split(",") if a.strip()]
    if not to_addrs:
        return False, "no recipients configured in [email] to="

    if html_body:
        alt = MIMEMultipart("alternative")
        alt.attach(MIMEText(body, "plain"))      # fallback first (lower priority)
        alt.attach(MIMEText(html_body, "html"))  # preferred last
        # Optional client logo: embed as a CID part (NOT a base64 data-URI —
        # Gmail/Outlook strip those) so the HTML's <img src="cid:netops-logo">
        # resolves. multipart/related wraps the alternative + the image. Only
        # when a logo is configured AND the HTML actually references it (the
        # renderer omits the <img> when no logo). Best-effort.
        logo_path = _prepare_email_logo(cfg)   # normalized PNG (cached) or None
        if logo_path and "cid:netops-logo" in html_body:
            try:
                from email.mime.image import MIMEImage
                ext = os.path.splitext(logo_path)[1].lower().lstrip(".")
                subtype = {"jpg": "jpeg", "jpeg": "jpeg", "gif": "gif"}.get(ext, "png")
                with open(logo_path, "rb") as lf:
                    img = MIMEImage(lf.read(), _subtype=subtype)
                img.add_header("Content-ID", "<netops-logo>")
                img.add_header("Content-Disposition", "inline",
                               filename=os.path.basename(logo_path))
                msg = MIMEMultipart("related")
                msg.attach(alt)
                msg.attach(img)
            except Exception as e:
                log.debug("digest logo embed failed (%s) — sending without it", e)
                msg = alt
        else:
            msg = alt
    else:
        msg = MIMEText(body)
    msg["Subject"] = subject
    msg["From"] = from_addr
    msg["To"] = ", ".join(to_addrs)

    try:
        smtp_user = cfg.get("smtp_username")
        smtp_pass = cfg.get("smtp_password")
        with smtplib.SMTP(smtp_server, smtp_port, timeout=30) as server:
            server.ehlo()
            if smtp_port != 25:
                server.starttls()
            if smtp_user and smtp_pass:
                server.login(smtp_user, smtp_pass)
            server.sendmail(from_addr, to_addrs, msg.as_string())
        return True, f"sent to {', '.join(to_addrs)}"
    except Exception as e:
        return False, f"SMTP error: {e}"


_JUNOS_PROMPT_RE = re.compile(r"^\w+@\S*[>#]?$")
_JUNOS_MODE_RE = re.compile(r"^\{(master|backup)(?::\d+)?\}$")
# Junos displays a line-wrapped command as 'user@host> ...<rest-of-command>'.
# Not substantive output — matches only when preceded by a prompt.
_JUNOS_ECHO_CONT_RE = re.compile(r"^\w+@\S+>\s*\.\.\.")


def _strip_cli_noise(output):
    """Strip leading command-echo noise and trailing prompt/mode-tag lines
    left behind by Junos's multi-line command display. Conservative — only
    matches prompt-shaped lines, never substantive output.
    """
    if not output:
        return output
    lines = output.splitlines()
    while lines:
        s = lines[0].strip()
        if (not s
                or _JUNOS_PROMPT_RE.match(s)
                or _JUNOS_ECHO_CONT_RE.match(s)
                or s.startswith(("show ", "set ", "request ", "restart "))):
            lines.pop(0)
            continue
        break
    while lines:
        s = lines[-1].strip()
        if (not s
                or _JUNOS_PROMPT_RE.match(s)
                or _JUNOS_MODE_RE.match(s)):
            lines.pop()
            continue
        break
    return "\n".join(lines)


def investigate_stp_ports(ip, ports, cfg):
    """Re-connect to a switch and gather context for one or more alerted
    STP ports in a single SSH session. Returns {port: info_dict} where
    info_dict has keys: ok, error, stp_detail, lldp_neighbor, interface_stats.

    Output is captured raw (not parsed) — the reader is a human looking at
    an email or terminal; structured extraction would add work for no value.
    """
    results = {p: {"ok": False, "error": None,
                   "stp_detail": "", "lldp_neighbor": "", "interface_stats": "",
                   "stp_stats": "", "mac_table": "", "parsed": {}}
               for p in ports}

    conn = _db()
    row = conn.execute(
        "SELECT hostname, username, password_hash, platform FROM devices WHERE ip = ?",
        (ip,)).fetchone()
    conn.close()
    if not row:
        err = f"device {ip} not in DB"
        for p in ports:
            results[p]["error"] = err
        return results

    platform = row["platform"]
    if platform not in ("junos", "aruba-cx", "procurve"):
        err = f"unsupported platform: {platform!r}"
        for p in ports:
            results[p]["error"] = err
        return results

    usernames = cfg.get("usernames") or []
    password_list = cfg.get("password_list") or []
    user_passwords = cfg.get("user_passwords") or {}
    known_user = row["username"]
    known_pw_hash = row["password_hash"]

    child, proto, user, _, ssh_open, telnet_open = connect_device(
        ip, usernames, None, timeout=15,
        password_list=password_list, user_passwords=user_passwords,
        known_username=known_user, known_password_hash=known_pw_hash,
        reason="investigate_stp")
    if child is None:
        err = f"SSH connect failed (ssh_open={ssh_open}, telnet_open={telnet_open})"
        for p in ports:
            results[p]["error"] = err
        return results

    try:
        for port in ports:
            # Junos STP output reports logical port names ('ge-1/0/11.0'); the
            # operational CLI commands take the physical form ('ge-1/0/11').
            port_cli = port.split(".")[0] if platform == "junos" else port
            stp_stats_cmd = None
            if platform == "junos":
                stp_cmd = f"show spanning-tree interface {port_cli} detail | no-more"
                # Junos's per-interface form is already verbose (it shows
                # Chassis-ID, Port-ID, System name when the neighbor sends
                # it). The 'detail' keyword isn't valid on this command —
                # parser falls back to chassis-MAC matching for endpoints
                # that don't advertise System name.
                lldp_cmd = f"show lldp neighbors interface {port_cli} | no-more"
                # Plain 'show interfaces' (not 'extensive') — has physical link,
                # last flapped, carrier transitions, and error counters without
                # the per-queue/CoS noise that makes 'extensive' ~90 lines.
                iface_cmd = f"show interfaces {port_cli} | no-more"
                # Junos doesn't include BPDU counters in 'spanning-tree
                # interface detail' — the classifier needs them, so pull
                # them with a separate one-line statistics command.
                stp_stats_cmd = f"show spanning-tree statistics interface {port_cli} | no-more"
                mac_cmd = f"show ethernet-switching table interface {port_cli} | no-more"
            elif platform == "procurve":
                stp_cmd = f"show spanning-tree {port_cli} detail"
                lldp_cmd = f"show lldp info remote-device {port_cli}"
                iface_cmd = f"show interfaces {port_cli}"
                mac_cmd = f"show mac-address {port_cli}"
            else:  # aruba-cx
                stp_cmd = f"show spanning-tree interface {port_cli}"
                lldp_cmd = f"show lldp neighbor-info {port_cli}"
                iface_cmd = f"show interface {port_cli}"
                mac_cmd = f"show mac-address-table interface {port_cli}"

            info = results[port]
            try:
                info["stp_detail"] = _strip_cli_noise(send_command(child, stp_cmd, timeout=20))
            except Exception as e:
                info["stp_detail"] = f"(error: {e})"
            try:
                info["lldp_neighbor"] = _strip_cli_noise(send_command(child, lldp_cmd, timeout=20))
            except Exception as e:
                info["lldp_neighbor"] = f"(error: {e})"
            try:
                info["interface_stats"] = _strip_cli_noise(send_command(child, iface_cmd, timeout=25))
            except Exception as e:
                info["interface_stats"] = f"(error: {e})"
            if stp_stats_cmd:
                try:
                    info["stp_stats"] = _strip_cli_noise(send_command(child, stp_stats_cmd, timeout=15))
                except Exception as e:
                    info["stp_stats"] = f"(error: {e})"
            try:
                info["mac_table"] = _strip_cli_noise(send_command(child, mac_cmd, timeout=20))
            except Exception as e:
                info["mac_table"] = f"(error: {e})"

            # Parse the raw CLI for the heuristic classifier. Parser
            # failure is non-fatal — caller still gets the raw text.
            try:
                if platform == "junos":
                    info["parsed"] = _parse_junos_investigation(
                        info["stp_detail"], info["interface_stats"], info["stp_stats"])
                elif platform == "aruba-cx":
                    info["parsed"] = _parse_arubacx_investigation(
                        info["stp_detail"], info["interface_stats"])
                else:  # procurve
                    info["parsed"] = _parse_procurve_investigation(
                        info["stp_detail"], info["interface_stats"])
            except Exception as e:
                info["parsed"] = {}
                log.debug("classifier parse failed for %s %s: %s", ip, port, e)

            info["ok"] = True
    finally:
        disconnect(child)
    return results


def _format_root_bridge_lines(ip):
    """Read this switch's current view of the root bridge from stp_root_state
    (populated by the same monitor-stp poll). Returns a list of formatted
    lines for inclusion in the investigation block. Empty list if we have
    no root data for this device (e.g. STP disabled or first poll)."""
    conn = _db()
    rows = conn.execute("""
        SELECT instance, root_priority, root_mac,
               bridge_priority, bridge_mac, is_root,
               tcn_count, last_tcn_seconds
        FROM stp_root_state WHERE ip = ? ORDER BY instance
    """, (ip,)).fetchall()
    conn.close()
    if not rows:
        return []
    out = []
    for r in rows:
        rp = r["root_priority"]; rm = r["root_mac"]
        bp = r["bridge_priority"]; bm = r["bridge_mac"]
        is_root = "yes" if r["is_root"] else "no"
        line = (f"{r['instance']}: root priority={rp} mac={rm}  "
                f"this bridge priority={bp} mac={bm}  is_root={is_root}")
        if r["tcn_count"] is not None:
            tcn_bit = f"tcn_count={r['tcn_count']}"
            if r["last_tcn_seconds"] is not None:
                tcn_bit += f", last tcn {r['last_tcn_seconds']}s ago"
            line += f"  ({tcn_bit})"
        out.append(line)
    return out


# ============================================================
# STP-event classifier — heuristic, runs at investigate time
# ============================================================
# Operates on data we already collect (port_flaps, stp_root_state) plus
# CLI fields parsed from the live investigate_stp_ports SSH session.
# Output is one extra block at the top of the investigation: likely cause
# + suggested action + the evidence that fired the rule.

# Stable identifiers; copy lives in _STP_CLASSIFIER_COPY for easy edit.
LIKELY_PHYSICAL_INSTABILITY = "physical_link_instability"
LIKELY_ONE_WAY_LINK         = "one_way_link"
LIKELY_SNMP_TRANSIENT       = "snmp_transient_or_collector"
LIKELY_TC_FLUSH             = "topology_change_flush"
LIKELY_REAL_STP_EVENT       = "real_stp_topology_change"
LIKELY_UNCLASSIFIED         = "unclassified"

_STP_CLASSIFIER_COPY = {
    LIKELY_PHYSICAL_INSTABILITY: (
        "Physical link instability",
        "Check the cable, SFP/optic, and port hardware on this interface.",
    ),
    LIKELY_ONE_WAY_LINK: (
        "Suspected one-way link",
        "TX is active but RX is silent. Inspect the fiber strand pair, "
        "swap optics on either end, and verify the device on the other side.",
    ),
    LIKELY_SNMP_TRANSIENT: (
        "Likely SNMP/MIB transient — no real STP event",
        "No physical flap, no BPDU input, port has been up for an extended "
        "period. Operator action probably not required; flag for tool review "
        "if the pattern persists.",
    ),
    LIKELY_TC_FLUSH: (
        "Topology-change flush",
        "A TCN propagated through this bridge recently. The port itself is "
        "stable; brief learning state is expected during MAC-table refresh.",
    ),
    LIKELY_REAL_STP_EVENT: (
        "Genuine STP topology change",
        "Investigate which device became the new designated bridge upstream "
        "and why.",
    ),
    LIKELY_UNCLASSIFIED: (
        "Unable to classify automatically",
        "Manual review recommended; the available signals don't fit a known "
        "pattern.",
    ),
}

# Compact tags for the weekly digest's per-port line; the full label is too
# long for a fixed-width inline display. Keep widths consistent — we render
# them in a left-padded column.
_STP_CLASSIFIER_SHORT_TAG = {
    LIKELY_PHYSICAL_INSTABILITY: "physical",
    LIKELY_ONE_WAY_LINK:         "one_way",
    LIKELY_SNMP_TRANSIENT:       "snmp_transient",
    LIKELY_TC_FLUSH:             "tc_flush",
    LIKELY_REAL_STP_EVENT:       "real_event",
    LIKELY_UNCLASSIFIED:         "unknown",
}

# Tunables. Module-level so they're easy to find and adjust.
_HIGH_FLAP_THRESHOLD   = 5
_TC_FLUSH_WINDOW_S     = 300        # 5 minutes
_HIGH_EDGE_EXPIRY_RATE = 100        # absolute count, used as a noise floor


def _parse_junos_investigation(stp_detail, interface_stats, stp_stats):
    """Junos: extract BPDU/edge/link signals from the three CLI dumps.
    Missing fields → None; callers must tolerate."""
    p = {}
    m = re.search(r"Edge delay while expiry count\s*:\s*(\d+)", stp_detail or "")
    p["edge_delay_expiry"] = int(m.group(1)) if m else None
    m = re.search(r"Rcvd info while expiry count\s*:\s*(\d+)", stp_detail or "")
    p["rcvd_info_expiry"] = int(m.group(1)) if m else None
    m = re.search(r"Boundary port\s*:\s*(\w+)", stp_detail or "")
    p["boundary_port"] = (m.group(1).lower() == "yes") if m else None
    m = re.search(r"Link type\s*:\s*(\S+)", stp_detail or "")
    p["link_type"] = m.group(1) if m else None
    p["is_edge"] = bool(p["link_type"] and "EDGE" in p["link_type"].upper())

    m = re.search(r"Physical link is\s+(Up|Down)", interface_stats or "")
    p["link_up"] = (m.group(1) == "Up") if m else None
    m = re.search(r"Last flapped\s*:\s*.+?\(([^)]+) ago\)", interface_stats or "")
    p["last_flapped_text"] = m.group(1) if m else None
    m = re.search(r"Input rate\s*:\s*(\d+)\s*bps", interface_stats or "")
    p["input_rate_bps"] = int(m.group(1)) if m else None
    m = re.search(r"Output rate\s*:\s*(\d+)\s*bps", interface_stats or "")
    p["output_rate_bps"] = int(m.group(1)) if m else None

    # show spanning-tree statistics interface — single data row after header.
    m = re.search(r"^\s*\S+\s+(\d+)\s+(\d+)", stp_stats or "", re.M)
    if m:
        p["bpdus_sent"]     = int(m.group(1))
        p["bpdus_received"] = int(m.group(2))
    else:
        p["bpdus_sent"]     = None
        p["bpdus_received"] = None
    return p


def _parse_arubacx_investigation(stp_detail, interface_stats):
    """Aruba-CX: 'show spanning-tree interface' already exposes BPDU
    counters, so no separate stats command is needed."""
    p = {}
    m = re.search(r"BPDU Tx Count\s*:\s*(\d+)", stp_detail or "")
    p["bpdus_sent"] = int(m.group(1)) if m else None
    m = re.search(r"BPDU Rx Count\s*:\s*(\d+)", stp_detail or "")
    p["bpdus_received"] = int(m.group(1)) if m else None
    m = re.search(r"Admin Edge Port\s*:\s*(\S+)", stp_detail or "")
    aep = m.group(1).lower() if m else None
    p["is_edge"] = (aep in ("admin-edge", "auto-edge")) if aep is not None else None
    m = re.search(r"Link Type\s*:\s*(\S+)", stp_detail or "")
    p["link_type"] = m.group(1) if m else None
    # AOS-CX doesn't expose the equivalent edge-delay timer counter.
    p["edge_delay_expiry"] = None
    p["boundary_port"]     = None

    # 'Interface 1/1/1 is up' / 'is down'
    m = re.search(r"Interface\s+\S+\s+is\s+(up|down)", interface_stats or "", re.I)
    p["link_up"] = (m.group(1).lower() == "up") if m else None
    # 'Link state: up for X' / 'Link state: down for X'
    m = re.search(r"Link state\s*:\s*(?:up|down)\s+for\s+(.+?)(?:\s+\(|\n|$)",
                  interface_stats or "", re.I)
    p["last_flapped_text"] = m.group(1).strip() if m else None
    # Three numeric columns (RX, TX, Total) on the Mbits/sec row.
    m = re.search(r"Mbits / sec\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)",
                  interface_stats or "")
    if m:
        p["input_rate_bps"]  = int(float(m.group(1)) * 1_000_000)
        p["output_rate_bps"] = int(float(m.group(2)) * 1_000_000)
    else:
        p["input_rate_bps"]  = None
        p["output_rate_bps"] = None
    return p


def _parse_procurve_investigation(stp_detail, interface_stats):
    """ProCurve: BPDU counts are in 'show spanning-tree <port> detail'
    (six-column MST/CFG/TCN Tx/Rx table). Sums MST_Tx+CFG_Tx for sent and
    MST_Rx+CFG_Rx for received — TCN is excluded since it counts notifications,
    not regular BPDU traffic, and would inflate the figure for ports that
    don't normally carry STP traffic.
    """
    p = {}
    m = re.search(
        r"-{5,}\s+-{5,}\s+-{5,}\s+-{5,}\s+-{5,}\s+-{5,}\s+"
        r"(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)",
        stp_detail or ""
    )
    if m:
        mst_tx, mst_rx, cfg_tx, cfg_rx, _tcn_tx, _tcn_rx = (int(x) for x in m.groups())
        p["bpdus_sent"]     = mst_tx + cfg_tx
        p["bpdus_received"] = mst_rx + cfg_rx
    else:
        p["bpdus_sent"]     = None
        p["bpdus_received"] = None

    m = re.search(r"OperEdgePort\s*:\s*(\w+)", stp_detail or "")
    p["is_edge"] = (m.group(1).lower() == "yes") if m else None
    m = re.search(r"MST Region Boundary\s*:\s*(\w+)", stp_detail or "")
    p["boundary_port"] = (m.group(1).lower() == "yes") if m else None
    p["link_type"]         = None
    p["edge_delay_expiry"] = None  # not exposed in ProCurve CLI

    m = re.search(r"Link Status\s*:\s*(Up|Down)", interface_stats or "", re.I)
    p["link_up"] = (m.group(1).lower() == "up") if m else None
    p["last_flapped_text"] = None  # not in 'show interfaces' output

    # 'Total Rx(Kbps) : N    Total Tx(Kbps) : N' → bps
    m = re.search(r"Total Rx\(Kbps\)\s*:\s*(\d+)", interface_stats or "")
    p["input_rate_bps"] = int(m.group(1)) * 1000 if m else None
    m = re.search(r"Total Tx\(Kbps\)\s*:\s*(\d+)", interface_stats or "")
    p["output_rate_bps"] = int(m.group(1)) * 1000 if m else None
    return p


def _stp_event_context(ip, interface):
    """Pull port_flaps row + most-recent TCN at this bridge from the DB.
    last_tcn_seconds = MIN across instances (smallest seconds-ago = most
    recent topology change)."""
    conn = _db()
    pf = conn.execute(
        "SELECT * FROM port_flaps WHERE ip = ? AND interface = ?",
        (ip, interface)
    ).fetchone()
    tcn_row = conn.execute(
        "SELECT MIN(last_tcn_seconds) AS tcn_s FROM stp_root_state "
        "WHERE ip = ? AND last_tcn_seconds IS NOT NULL",
        (ip,)
    ).fetchone()
    conn.close()
    return pf, (tcn_row["tcn_s"] if tcn_row and tcn_row["tcn_s"] is not None else None)


def _classify_stp_event(parsed, port_flap_row, last_tcn_seconds,
                        old_role=None, new_role=None):
    """Pure-function classifier. Returns (label, confidence, evidence_lines).
    Confidence ('high'|'medium'|'low') is rendered for emphasis only — it
    never gates control flow.

    old_role/new_role: when both are provided and differ, the snmp_transient
    rule is gated off — a role change (e.g. DESG -> BACKUP) is a strong
    signal of a real STP topology event regardless of BPDU history. The
    'snmp_transient' rule is *only* meant to catch state-only flickers
    where the JUNIPER MIB cache reports a stale state for a port whose
    actual role and link are unchanged.
    """
    ev = []
    flap_count   = port_flap_row["flap_count"] if port_flap_row else 0
    last_flap_at = port_flap_row["last_seen"]  if port_flap_row else None
    bpdus_rx     = parsed.get("bpdus_received")
    role_changed = (old_role is not None and new_role is not None
                    and old_role != new_role)

    # 1) Real STP topology change via role flip — DESG <-> BACKUP, DESG
    # <-> ALT, etc. The role machine actually moved, regardless of BPDU
    # history. Highest priority because a role flip is unambiguous: the
    # other rules (one_way, physical, snmp_transient) are heuristics that
    # can fire spuriously on a port that just got promoted/demoted (no
    # BPDUs received on a brand-new DESG, edge_delay_expiry accumulating
    # over weeks, etc.). When the device tells us the role flipped, we
    # trust it.
    if role_changed:
        ev.append(f"role changed: {old_role} -> {new_role} "
                  "(real STP role-machine event)")
        if (bpdus_rx or 0) > 0:
            ev.append(f"BPDUs received={bpdus_rx} (active state machine)")
        return (LIKELY_REAL_STP_EVENT, "medium", ev)

    # 2) Physical link instability — observed flapping at the port.
    # Outranks the one_way rule because flap_count is observed history
    # (real-world transitions recorded by the cron). An actively-flapping
    # port can momentarily satisfy one_way's conditions (link 'up' between
    # flaps, BPDUs not received during a down moment), and we don't want
    # to misdiagnose a flapping cable as a one-way link.
    if flap_count >= _HIGH_FLAP_THRESHOLD:
        ev.append(f"port_flaps.flap_count={flap_count} "
                  f"(≥{_HIGH_FLAP_THRESHOLD} threshold)")
        if last_flap_at:
            ev.append(f"last flap @ {last_flap_at}")
        return (LIKELY_PHYSICAL_INSTABILITY, "high", ev)

    # 3) One-way link — link up, TX flowing, no RX, no BPDUs in. Gated on
    # *evidence of recent state-machine flux* so a one-shot input_rate_bps
    # snapshot of 0 bps on a steady port doesn't trip the rule. The gate
    # is: flap_count > 0 OR edge_delay_expiry above noise floor. Either
    # one means the port has been actively transitioning recently — both
    # are zero on a port that's just sitting there with no real traffic.
    edge_expiry = parsed.get("edge_delay_expiry") or 0
    state_churn = (flap_count > 0) or (edge_expiry > _HIGH_EDGE_EXPIRY_RATE)
    if (parsed.get("link_up") is True
            and (parsed.get("output_rate_bps") or 0) > 0
            and parsed.get("input_rate_bps") == 0
            and bpdus_rx == 0
            and state_churn):
        ev.append(f"link up; output={parsed['output_rate_bps']} bps; input=0 bps")
        ev.append("BPDUs received=0")
        if flap_count > 0:
            ev.append(f"port_flaps.flap_count={flap_count}")
        if edge_expiry > _HIGH_EDGE_EXPIRY_RATE:
            ev.append(f"edge_delay_expiry={edge_expiry} (high)")
        return (LIKELY_ONE_WAY_LINK, "high", ev)

    # 4) TC flush — recent topology change drove a brief learning state.
    if last_tcn_seconds is not None and last_tcn_seconds < _TC_FLUSH_WINDOW_S:
        ev.append(f"TC at this bridge {last_tcn_seconds}s ago "
                  f"(< {_TC_FLUSH_WINDOW_S}s window)")
        return (LIKELY_TC_FLUSH, "medium", ev)

    # 5) SNMP/MIB phantom — no flap row, no BPDU input, port stable, no TC.
    # Role-changed events are caught by rule 3 above and never land here.
    if (port_flap_row is None
            and bpdus_rx == 0
            and parsed.get("link_up") is True
            and (last_tcn_seconds is None
                 or last_tcn_seconds >= _TC_FLUSH_WINDOW_S)):
        ev.append("no port_flaps row (no observed link transitions)")
        ev.append("BPDUs received=0")
        if parsed.get("last_flapped_text"):
            ev.append(f"port up for {parsed['last_flapped_text']}")
        if parsed.get("is_edge") or parsed.get("boundary_port"):
            ev.append("edge/boundary port")
        return (LIKELY_SNMP_TRANSIENT, "medium", ev)

    # 6) Active state machine — BPDUs flowing without a role change.
    # Lower confidence than rule 3 because there's no specific event
    # signal beyond "BPDUs are arriving" — operator should still look but
    # the diagnosis is more "investigate" than "definite topology shift".
    if (bpdus_rx or 0) > 0 and parsed.get("link_up"):
        ev.append(f"BPDUs received={bpdus_rx} (active state machine)")
        return (LIKELY_REAL_STP_EVENT, "low", ev)

    return (LIKELY_UNCLASSIFIED, "low", ev)


def _format_classification_block(label, confidence, evidence_lines):
    """Render the classifier output as a fixed-width block matching the
    other investigation sections (--- Title --- header style)."""
    title, action = _STP_CLASSIFIER_COPY[label]
    out = [f"--- Likely cause ({confidence} confidence): {title} ---"]
    if evidence_lines:
        for line in evidence_lines:
            out.append(f"  • {line}")
    else:
        out.append("  (no signals matched any rule)")
    out.append(f"Suggested action: {action}")
    return out


# Vendor OUI hints for the per-port MAC table snapshot. Not exhaustive —
# covers the common gear an on-call admin needs to recognize at a glance
# (phones / APs / network kit / endpoint OSes). For an unmapped OUI we
# print "(unknown vendor)" rather than failing; the operator can still
# look it up manually in the IEEE OUI registry.
_OUI_VENDOR_HINTS = {
    # Cisco (selection — Cisco owns hundreds of OUIs)
    "0000": "Xerox/older", "0001": "Cisco?", "00000c": "Cisco",
    "001011": "Cisco", "00142a": "Cisco", "001b67": "Cisco",
    "001c0e": "Cisco", "001e4a": "Cisco", "001e7a": "Cisco",
    "0024c4": "Cisco", "0026cb": "Cisco", "00405d": "Cisco",
    "446d57": "Cisco", "503de5": "Cisco", "70b3d5": "Cisco",
    # Juniper
    "001a82": "Juniper", "002083": "Juniper", "002586": "Juniper",
    "0026ce": "Juniper", "5c5e89": "Juniper", "78fe3d": "Juniper",
    "9051b6": "Juniper", "a8d0e5": "Juniper",
    # HPE/Aruba Networking (acquired Aruba 2015; OUIs span both eras)
    "001635": "HP",        "001871": "HP",        "001e0b": "HP",
    "0024a8": "HP",        "002655": "HP",        "00306e": "HP",
    "00184e": "HP",        "00246c": "Aruba/HPE", "002a4b": "HPE Networking",
    "6cf37f": "HPE Networking", "84d47e": "HPE Networking",
    "94f128": "HPE Networking", "c0917e": "HPE Networking",
    "ecebb8": "HPE Networking",
    # Aruba IAP / APs (older OUI block before merger)
    "001a1e": "Aruba",     "0024fd": "Aruba",     "ac1610": "Aruba AP",
    # IP phones — Polycom, Mitel, Yealink
    "0004f2": "Polycom", "64167f": "Polycom",
    "00085d": "Mitel",   "001565": "Mitel/Yealink",  "805ec0": "Yealink",
    # Apple
    "000393": "Apple", "000a27": "Apple", "001ec2": "Apple",
    "002608": "Apple", "002500": "Apple", "0026bb": "Apple",
    "0c74c2": "Apple", "14c213": "Apple", "78fd94": "Apple",
    # Microsoft (Hyper-V virtual NICs, Surface, etc.)
    "00125a": "Microsoft", "00155d": "Microsoft (Hyper-V)",
    "001dd8": "Microsoft", "0050f2": "Microsoft", "7c1e52": "Microsoft",
    # VMware
    "000569": "VMware", "000c29": "VMware",
    "001456": "VMware", "005056": "VMware",
    # Intel (NICs)
    "0002b3": "Intel", "001b21": "Intel", "001e64": "Intel",
    "0019d1": "Intel", "1c697a": "Intel", "8c1645": "Intel",
    # Dell
    "001422": "Dell", "00188b": "Dell", "001ec9": "Dell", "002219": "Dell",
    "78ac44": "Dell", "b083fe": "Dell", "b4e9b0": "Dell", "f48e38": "Dell",
    # Lenovo
    "001cb3": "Lenovo", "0021cc": "Lenovo", "002435": "Lenovo",
    "447766": "Lenovo", "705a0f": "Lenovo",
    # Printers
    "001ba9": "Brother", "30055c": "Brother",
    "0007e9": "Canon",   "001e8f": "Canon",
    "00194b": "Lexmark", "002564": "Lexmark",
    # Ubiquiti
    "002722": "Ubiquiti", "00154d": "Ubiquiti",
    "78a351": "Ubiquiti", "fc92bc": "Ubiquiti", "78cb6e": "Ubiquiti",
    # Misc
    "00114c": "Aerohive",
    "0090fb": "Portwell",
    "001a4b": "HP/Compaq",
}


def _normalize_mac(mac):
    """Return a lowercase hex-only MAC (no separators), or '' if not a MAC."""
    if not mac:
        return ""
    s = re.sub(r"[^0-9a-fA-F]", "", mac).lower()
    return s if len(s) == 12 else ""


# Full IEEE OUI assignments — loaded lazily from the first available source
# (Debian ieee-data is the freshest; the .deb-bundled CSV is the floor so
# this always works even without the package; Wireshark's manuf as a third
# alternative). Parsed once per process and cached. Empty dict means we
# couldn't find any source.
_OUI_DB = None  # tuple (db_36, db_28, db_24): {hex_prefix_lower: vendor}


def _load_oui_db():
    """Return cached (db_36, db_28, db_24) — IEEE OUI assignments by tier.

    IEEE publishes three OUI registries (MA-S 36-bit, MA-M 28-bit, MA-L
    24-bit). Smaller-block vendors and many recent allocations live in
    MA-M/MA-S, so a 24-bit-only lookup misses them. We bundle all three
    (always fresh at release time) and consult them longest-prefix-first
    in _oui_vendor_hint.

    Priority: bundled (always fresh) > apt-installed ieee-data (24-bit
    only, may be old) > Wireshark manuf (24-bit only). The bundled CSVs
    are the floor — netops works correctly even without any other OUI
    package installed. Cached after first parse; {} dicts on any failure.
    """
    global _OUI_DB
    if _OUI_DB is not None:
        return _OUI_DB
    db_36, db_28, db_24 = {}, {}, {}

    def _load_csv(path, dst, hexlen):
        try:
            import csv
            with open(path, encoding="utf-8", errors="replace") as f:
                r = csv.reader(f)
                next(r, None)  # header
                for row in r:
                    if len(row) >= 3 and len(row[1]) == hexlen:
                        dst[row[1].lower()] = row[2].strip()
            return True
        except Exception as e:
            log.debug("OUI CSV load %s: %s", path, e)
            return False

    # Bundled first — always fresh at .deb release time, covers all three tiers.
    bundled_ok = False
    for path, dst, hexlen in (("/usr/share/netops/oui36.csv", db_36, 9),
                              ("/usr/share/netops/mam.csv",   db_28, 7),
                              ("/usr/share/netops/oui.csv",   db_24, 6)):
        if os.path.exists(path) and _load_csv(path, dst, hexlen):
            bundled_ok = True

    # If bundled missing (dev checkout, future-stripped package, etc.) fall
    # back to system-installed 24-bit lists. These are 24-bit-only — no
    # MA-M/MA-S coverage in this path. Better than nothing.
    if not bundled_ok:
        hex_line = re.compile(
            r'^([0-9A-Fa-f]{2})[-:]([0-9A-Fa-f]{2})[-:]([0-9A-Fa-f]{2})'
            r'\s+(?:\(hex\)\s+)?([^#\n]+?)\s*(?:#.*)?$')
        for path in ("/usr/share/ieee-data/oui.txt",
                     "/usr/share/wireshark/manuf"):
            if not os.path.exists(path):
                continue
            try:
                with open(path, encoding="utf-8", errors="replace") as f:
                    for line in f:
                        m = hex_line.match(line)
                        if m:
                            db_24[(m.group(1) + m.group(2)
                                   + m.group(3)).lower()] = m.group(4).strip()
                if db_24:
                    break
            except Exception as e:
                log.debug("OUI fallback %s: %s", path, e)

    _OUI_DB = (db_36, db_28, db_24)
    return _OUI_DB


def _oui_vendor_hint(mac):
    """Look up the operator-facing vendor name for a MAC. The curated
    _OUI_VENDOR_HINTS dict wins first (its hand-picked short names are
    nicer in alerts than the verbose IEEE org names), then we fall back to
    the full IEEE OUI assignments DB by longest-prefix match: MA-S
    (36-bit, 9 hex) -> MA-M (28-bit, 7 hex) -> MA-L (24-bit, 6 hex).
    Also tries 4-hex against the curated dict for legacy short-OUI
    entries we registered.

    Returns None when no vendor can be found. Callers that write the
    result into port_macs.oui_vendor get a NULL — that way a future
    registry update reaches the cached row via the live-lookup fallback
    instead of being shadowed by a stale literal '(unknown vendor)' that
    has to be backfilled out by hand."""
    norm = _normalize_mac(mac)
    if not norm:
        return None
    for plen in (6, 4):
        v = _OUI_VENDOR_HINTS.get(norm[:plen])
        if v:
            return v
    db_36, db_28, db_24 = _load_oui_db()
    v = (db_36.get(norm[:9]) if len(norm) >= 9 else None) \
        or (db_28.get(norm[:7]) if len(norm) >= 7 else None) \
        or db_24.get(norm[:6])
    return v  # may be None — display callers add the "(unknown vendor)" label


# Match the three common MAC string formats: aa:bb:cc:dd:ee:ff,
# aabb-ccdd-eeff (HP/ProCurve), aabb.ccdd.eeff (Cisco dot).
_MAC_RE = re.compile(
    r"(?:[0-9a-fA-F]{2}[:-]){5}[0-9a-fA-F]{2}"
    r"|[0-9a-fA-F]{6}-[0-9a-fA-F]{6}"
    r"|[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}"
)


def _format_mac_table_section(mac_table_text, limit=10):
    """Extract MACs from CLI output and render an operator-friendly list.

    Strips obvious self-MAC noise (e.g. the switch's own base MAC) is left
    to the caller — vendor CLIs don't typically include the switch MAC in
    a per-port table query, so we don't try to guess.

    limit caps the number of MACs rendered (trunk ports can have hundreds).
    """
    if not mac_table_text:
        return ["(no MAC table output)"]
    macs = []
    seen = set()
    for raw in _MAC_RE.findall(mac_table_text):
        norm = _normalize_mac(raw)
        if not norm or norm in seen:
            continue
        seen.add(norm)
        macs.append(raw)
    if not macs:
        return ["(no MACs learned on this port)"]
    lines = []
    for i, mac in enumerate(macs[:limit]):
        vendor = _oui_vendor_hint(mac) or "(unknown vendor)"
        lines.append(f"  {mac}  [{vendor}]")
    if len(macs) > limit:
        lines.append(f"  ... and {len(macs) - limit} more MAC(s)")
    return lines


# 3.8.31 reverse-DNS cache TTLs. Hits live longer; misses retry sooner so
# a transient DNS failure doesn't suppress hostnames for a full day.
_DNS_CACHE_HIT_TTL_SEC  = 86400        # 24h
_DNS_CACHE_MISS_TTL_SEC = 3600         # 1h


def _cached_reverse_dns(ip, conn=None):
    """Reverse-resolve `ip` via dns_cache (sqlite); falls back to a live
    lookup with a 2s timeout when the cached entry is missing or expired.
    Returns the hostname or '' (NXDOMAIN/timeout). conn may be a caller-
    supplied sqlite3.Connection to avoid open/close churn when this is
    invoked many times during a single email render."""
    owns_conn = False
    if conn is None:
        try:
            conn = _db()
            conn.row_factory = sqlite3.Row
            owns_conn = True
        except Exception:
            return reverse_dns(ip)     # DB unavailable -> live lookup, no cache
    now_dt = datetime.now()
    try:
        row = conn.execute(
            "SELECT hostname, looked_up_at FROM dns_cache WHERE ip = ?", (ip,)
        ).fetchone()
    except sqlite3.OperationalError:
        if owns_conn:
            conn.close()
        return reverse_dns(ip)         # pre-3.8.31 DB without dns_cache
    if row is not None:
        try:
            seen = datetime.strptime(row["looked_up_at"], "%Y-%m-%d %H:%M:%S")
            age = (now_dt - seen).total_seconds()
        except (ValueError, TypeError):
            age = _DNS_CACHE_HIT_TTL_SEC + 1   # force refresh
        ttl = _DNS_CACHE_HIT_TTL_SEC if row["hostname"] else _DNS_CACHE_MISS_TTL_SEC
        if age < ttl:
            if owns_conn:
                conn.close()
            return row["hostname"] or ""
    hostname = reverse_dns(ip)
    now_str = now_dt.strftime("%Y-%m-%d %H:%M:%S")
    try:
        conn.execute(
            "INSERT INTO dns_cache (ip, hostname, looked_up_at) "
            "VALUES (?, ?, ?) "
            "ON CONFLICT(ip) DO UPDATE SET "
            "  hostname = excluded.hostname, "
            "  looked_up_at = excluded.looked_up_at",
            (ip, hostname or None, now_str))
        conn.commit()
    except sqlite3.OperationalError:
        pass
    if owns_conn:
        conn.close()
    return hostname


def _arp_lookup_for_mac(mac, conn=None):
    """Return [(ip, hostname), ...] for a MAC from ip_arp, newest first
    and deduped on IP. Hostnames via _cached_reverse_dns; '' when no PTR.
    Returns [] when the MAC has never been ARP'd (or pre-3.8.31 DB).

    Caller may pass an open sqlite3.Connection to amortize DB churn when
    enriching many MACs in one email render."""
    owns_conn = False
    if conn is None:
        try:
            conn = _db()
            conn.row_factory = sqlite3.Row
            owns_conn = True
        except Exception:
            return []
    try:
        rows = conn.execute(
            "SELECT ip, MAX(last_seen) AS last_seen FROM ip_arp "
            "WHERE mac = ? GROUP BY ip ORDER BY last_seen DESC",
            (mac,)
        ).fetchall()
    except sqlite3.OperationalError:
        if owns_conn:
            conn.close()
        return []
    out = []
    for r in rows:
        host = _cached_reverse_dns(r["ip"], conn=conn)
        out.append((r["ip"], host))
    if owns_conn:
        conn.close()
    return out


def _format_arp_id_suffix(mac, conn=None, max_ips=2):
    """Render the ' -> ip [host]' tail appended to a port_macs history
    line. Returns '' when the MAC has no known IP. Shows the newest IP
    by default; '(+N more)' when the MAC carries multiple IPs (rare —
    secondary IPs, VRRP, dual-stack).

    Format: ' -> 10.1.10.42 desktop-jdoe.corp.local' or
            ' -> 10.1.10.42 desktop-jdoe.corp.local (+1 more IP)'."""
    hits = _arp_lookup_for_mac(mac, conn=conn)
    if not hits:
        return ""
    ip, host = hits[0]
    tail = f" -> {ip}"
    if host:
        tail += f" {host}"
    extra = len(hits) - 1
    if extra > 0:
        s = "" if extra == 1 else "s"
        tail += f" (+{extra} more IP{s})"
    return tail


def _format_port_macs_history(ip, interface, limit=10):
    """Last-known MAC(s) on a port from the port_macs FDB history, newest
    first, with OUI vendor + age + (3.8.31) IP/hostname from ip_arp when
    known. The STP-alert / flap-digest fallback for when the live FDB is
    empty because the port is down/flapping — the exact case where the
    operator most needs to know what *was* there.

    Returns [] on miss (no history, or pre-3.8.0 DB without the table)."""
    try:
        conn = _db()
        conn.row_factory = sqlite3.Row
        rows = conn.execute(
            "SELECT mac, vlan, oui_vendor, last_seen "
            "FROM port_macs WHERE ip = ? AND interface = ? "
            "ORDER BY last_seen DESC LIMIT ?",
            (ip, interface, limit)
        ).fetchall()
    except Exception:
        return []
    if not rows:
        conn.close()
        return []
    now = datetime.now()
    out = []
    for r in rows:
        try:
            seen = datetime.strptime(r["last_seen"], "%Y-%m-%d %H:%M:%S")
            secs = max(0, int((now - seen).total_seconds()))
            if secs < 3600:
                age = f"{secs // 60}m ago"
            elif secs < 86400:
                age = f"{secs // 3600}h ago"
            else:
                age = f"{secs // 86400}d ago"
        except (ValueError, TypeError):
            age = "age unknown"
        vendor = r["oui_vendor"] or _oui_vendor_hint(r["mac"]) or "(unknown vendor)"
        vlan = f", vlan {r['vlan']}" if r["vlan"] else ""
        id_tail = _format_arp_id_suffix(r["mac"], conn=conn)
        out.append(f"{r['mac']}  [{vendor}]  (last seen {age}{vlan}){id_tail}")
    conn.close()
    return out


def _port_macs_digest_hint(ip, interface):
    """One-line last-known-MAC hint for the flap digest (newest entry +
    'and N more'), or '' when there's no port_macs history. Lets an admin
    identify the flapping device without logging into the switch."""
    hist = _format_port_macs_history(ip, interface, limit=4)
    if not hist:
        return ""
    extra = f"   (+{len(hist) - 1} more recent MAC(s))" if len(hist) > 1 else ""
    return f"         last-known MAC: {hist[0]}{extra}"


# LAG / trunk-named interfaces across our vendors: ProCurve Trk4,
# Junos ae0, Aruba-CX lag1, Cisco-ish Po1 / Port-Channel1.
_LAG_IFACE_RE = re.compile(
    r'(?:trk|ae|lag|po|port-?channel)\s*\d+$', re.IGNORECASE)


def _compute_mac_flux(conn, threshold, ceiling, max_vlans,
                       max_times_seen=0, max_span_sec=0):
    """Access/edge ports whose ROTATING-MAC count over the trailing 24h is
    in [threshold, ceiling]. Hybrid non-trunk classifier — a port is
    excluded when ANY of:
      * the interface is LAG/trunk-named (Trk/ae/lag/po/port-channel),
      * it has ANY LLDP neighbor (present in topology_edges at all — a
        switch/AP link, not a bare access host),
      * its distinct-MAC count exceeds the uplink ceiling (structurally an
        aggregation/uplink, not a single rogue device),
      * its MACs span more than max_vlans VLANs — a true access port is
        one data VLAN (+ maybe a voice VLAN); a port carrying many VLANs
        is a trunk or a virtualization host (e.g. a VMware vSwitch uplink,
        which legitimately shows many VM MACs and would otherwise false-
        alarm on every vMotion). VLAN-unaware (dot1d, vlan=0) ports count
        as a single VLAN and are unaffected.

    Rotation filter (3.8.7+) — only MACs that look like brief passers-
    through count toward the threshold; stable residents do not. A MAC
    qualifies as "rotating" when BOTH of these hold:
      * its `times_seen` (= number of 30-min FDB harvests it appeared in)
        is <= max_times_seen, AND
      * its (last_seen - first_seen) span is < max_span_sec.
    Either parameter set to 0 (or non-positive) disables that part of
    the filter and falls back to the 3.8.6 "count all 24h distincts"
    semantics — preserved so operators with a legacy config still see
    the original behavior until they opt in. The classifier name
    matches its semantic intent: a port where the MAC neighborhood is
    ROTATING (rogue switch with rotating clients, hot-desk wall jack)
    rather than where many devices are STABLY present (conference-room
    AV stack, multi-device desk).

    Returns [{ip, hostname, interface, distinct, vlans}], busiest first.
    threshold <= 0 disables. conn needs a sqlite3.Row row_factory."""
    if threshold <= 0:
        return []
    since = (datetime.now() - timedelta(hours=24)).strftime("%Y-%m-%d %H:%M:%S")
    where = ["last_seen >= ?"]
    params = [since]
    if max_times_seen and max_times_seen > 0:
        where.append("times_seen <= ?")
        params.append(max_times_seen)
    if max_span_sec and max_span_sec > 0:
        # span in seconds — julianday() returns days, *86400 -> seconds.
        where.append(
            "(julianday(last_seen) - julianday(first_seen)) * 86400 < ?")
        params.append(max_span_sec)
    params.append(threshold)
    try:
        rows = conn.execute(f"""
            SELECT ip, interface,
                   COUNT(DISTINCT mac)  AS distinct_macs,
                   COUNT(DISTINCT vlan) AS vlan_count
            FROM port_macs
            WHERE {' AND '.join(where)}
            GROUP BY ip, interface
            HAVING distinct_macs >= ?
        """, params).fetchall()
    except sqlite3.OperationalError:
        return []                       # pre-3.8.0 DB without port_macs
    if not rows:
        return []
    # ANY LLDP neighbor => switch/AP link, not a bare access host.
    linked = {(e["src_ip"], e["src_port"]) for e in conn.execute(
        "SELECT src_ip, src_port FROM topology_edges").fetchall()}
    out = []
    for r in rows:
        if ceiling and ceiling > 0 and r["distinct_macs"] > ceiling:
            continue                    # structurally an uplink/aggregation
        if max_vlans and max_vlans > 0 and r["vlan_count"] > max_vlans:
            continue                    # multi-VLAN trunk / hypervisor uplink
        if (r["ip"], r["interface"]) in linked:
            continue                    # has an LLDP neighbor (switch/AP)
        if _LAG_IFACE_RE.match((r["interface"] or "").strip()):
            continue                    # LAG / trunk-named interface
        hn = conn.execute("SELECT hostname FROM devices WHERE ip = ?",
                           (r["ip"],)).fetchone()
        out.append({
            "ip": r["ip"],
            "hostname": (hn["hostname"] if hn and hn["hostname"] else r["ip"]),
            "interface": r["interface"],
            "distinct": r["distinct_macs"],
            "vlans": r["vlan_count"],
        })
    out.sort(key=lambda d: (-d["distinct"], d["hostname"], d["interface"]))
    return out


_SILENCE_MAX_MINUTES = 30 * 24 * 60      # 30 days — silences are never forever


def _parse_silence_duration(token):
    """'7d' / '12h' / '90m' -> minutes, or None if unparseable."""
    m = re.fullmatch(r"(\d+)([dhm])", (token or "").lower())
    if not m:
        return None
    n = int(m.group(1))
    if n <= 0:
        return None
    return n * {"d": 1440, "h": 60, "m": 1}[m.group(2)]


def _gc_expired_silences(conn):
    """Drop expired alert_silences rows. Called opportunistically from the
    silence verb — the enforcement helper filters by expires_at anyway, so
    this is bookkeeping, not correctness."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    try:
        cur = conn.execute(
            "DELETE FROM alert_silences WHERE expires_at <= ?", (now,))
        conn.commit()
        if cur.rowcount:
            log.info("Garbage-collected %d expired alert silence(s).",
                     cur.rowcount)
    except sqlite3.OperationalError:
        pass


def _silenced_ip_set(ips):
    """Subset of `ips` whose alert emails are currently muted.

    Matches active alert_silences rows: kind 'all' mutes everything;
    'ip' matches the exact IP; 'name' is a case-insensitive substring of
    the device hostname (same semantics as `show devices name <pat>`).
    Data collection is never affected — callers only use this to filter
    what gets EMAILED.
    """
    if not ips:
        return set()
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    conn.row_factory = sqlite3.Row
    try:
        sils = conn.execute(
            "SELECT kind, pattern FROM alert_silences WHERE expires_at > ?",
            (now,)).fetchall()
    except sqlite3.OperationalError:
        conn.close()
        return set()
    if not sils:
        conn.close()
        return set()
    if any(s["kind"] == "all" for s in sils):
        conn.close()
        return set(ips)
    hostnames = {r["ip"]: (r["hostname"] or "")
                 for r in conn.execute(
                     "SELECT ip, hostname FROM devices").fetchall()}
    conn.close()
    muted = set()
    for ip in ips:
        hn = hostnames.get(ip, "").lower()
        for s in sils:
            if s["kind"] == "ip" and s["pattern"] == ip:
                muted.add(ip)
                break
            if s["kind"] == "name" and s["pattern"].lower() in hn:
                muted.add(ip)
                break
    return muted


def _any_all_silence():
    """True when an active 'all' silence exists (gates fleet-level alerts
    like a root-bridge change that aren't attributable to one device)."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    try:
        row = conn.execute(
            "SELECT 1 FROM alert_silences WHERE kind = 'all' "
            "AND expires_at > ? LIMIT 1", (now,)).fetchone()
    except sqlite3.OperationalError:
        row = None
    conn.close()
    return row is not None


def handle_silence(words):
    """`silence` — mute alert emails for device(s), up to 30 days.

      silence <ip|hostname> <duration> [reason ...]
      silence name <pattern> <duration> [reason ...]
      silence all <duration> [reason ...]
      silence list
      silence delete <id>|all

    duration: Nd / Nh / Nm (e.g. 7d, 12h, 90m), hard-capped at 30 days —
    a silence is never permanent. Monitoring, backups, and the weekly
    digest are unaffected; only alert EMAILS are muted (STP changes,
    unreachable/recovery, MAC-flux, config-change). Every add/delete is
    audit-logged.
    """
    conn = _db()
    conn.row_factory = sqlite3.Row
    _gc_expired_silences(conn)
    now_dt = datetime.now()
    now = now_dt.strftime("%Y-%m-%d %H:%M:%S")

    if words[0] == "list":
        rows = conn.execute(
            "SELECT * FROM alert_silences WHERE expires_at > ? "
            "ORDER BY expires_at", (now,)).fetchall()
        conn.close()
        if not rows:
            print("No active silences.")
            return
        display = []
        for r in rows:
            exp = datetime.strptime(r["expires_at"], "%Y-%m-%d %H:%M:%S")
            mins = int((exp - now_dt).total_seconds() // 60)
            left = (f"{mins // 1440}d{(mins % 1440) // 60}h" if mins >= 1440
                    else f"{mins // 60}h{mins % 60}m" if mins >= 60
                    else f"{mins}m")
            scope = {"ip": "device", "name": "name~", "all": "ALL"}[r["kind"]]
            display.append({
                "id": r["id"], "scope": scope, "pattern": r["pattern"],
                "expires_at": r["expires_at"], "left": left,
                "by": r["created_by"] or "", "reason": r["reason"] or "",
            })
        _output(["id", "scope", "pattern", "expires_at", "left", "by",
                 "reason"], display, None)
        print(f"\n{len(rows)} active silence(s). "
              f"'silence delete <id>' to lift one early.")
        return

    if words[0] == "delete":
        if len(words) != 2:
            print("silence delete: give a silence id (from 'silence list') "
                  "or 'all'")
            conn.close()
            return
        if words[1] == "all":
            cur = conn.execute("DELETE FROM alert_silences")
        elif words[1].isdigit():
            cur = conn.execute("DELETE FROM alert_silences WHERE id = ?",
                               (int(words[1]),))
        else:
            print(f"silence delete: {words[1]!r} is not an id or 'all'")
            conn.close()
            return
        conn.commit()
        conn.close()
        if cur.rowcount:
            print(f"deleted {cur.rowcount} silence(s) — alerts resume "
                  f"immediately")
            _audit_log("silence_delete", detail=words[1])
        else:
            print("no matching silence")
        return

    # --- add forms ---
    if words[0] == "all":
        if len(words) < 2:
            print("silence all: give a duration (Nd/Nh/Nm, max 30d)")
            conn.close()
            return
        kind, pattern, label = "all", "*", "ALL devices"
        dur_tok, reason_words = words[1], words[2:]
    elif words[0] == "name":
        if len(words) < 3:
            print("silence name: give a pattern and a duration "
                  "(e.g. silence name SW 7d)")
            conn.close()
            return
        kind, pattern = "name", words[1]
        label = f"devices matching name '{pattern}'"
        dur_tok, reason_words = words[2], words[3:]
    else:
        if len(words) < 2:
            print("silence: give a duration (Nd/Nh/Nm, max 30d) — "
                  "e.g. silence SW1 7d")
            conn.close()
            return
        row = _lookup_device(words[0])
        if not row:
            print(f"silence: no device matches {words[0]!r}")
            conn.close()
            return
        if row["status"] == "duplicate" and (row["duplicate_of"] or ""):
            primary = _lookup_device(row["duplicate_of"])
            if primary:
                row = primary
        kind, pattern = "ip", row["ip"]
        label = f"{row['hostname'] or row['ip']} ({row['ip']})"
        dur_tok, reason_words = words[1], words[2:]

    minutes = _parse_silence_duration(dur_tok)
    if minutes is None:
        print(f"silence: can't parse duration {dur_tok!r} — use Nd/Nh/Nm "
              f"(e.g. 7d, 12h, 90m)")
        conn.close()
        return
    if minutes > _SILENCE_MAX_MINUTES:
        print(f"silence: {dur_tok} exceeds the 30-day cap — silences are "
              f"never forever. Use 30d or less (re-silence later if needed).")
        conn.close()
        return

    reason = " ".join(reason_words) or None
    created_by = (os.environ.get("SUDO_USER")
                  or os.environ.get("USER") or "unknown")
    expires_at = ((now_dt + timedelta(minutes=minutes))
                  .strftime("%Y-%m-%d %H:%M:%S"))
    conn.execute(
        "INSERT INTO alert_silences "
        "(kind, pattern, reason, created_at, created_by, expires_at) "
        "VALUES (?, ?, ?, ?, ?, ?)",
        (kind, pattern, reason, now, created_by, expires_at))
    conn.commit()
    conn.close()
    msg = f"alert emails for {label} silenced until {expires_at}"
    if reason:
        msg += f" — {reason}"
    print(msg + f"  [{created_by}]")
    _audit_log("silence_add", detail=f"kind={kind} pattern={pattern} "
                                     f"until={expires_at}")


def _get_flap_silenced_ports(conn):
    """Currently-active port-flap silences as a set of (ip, interface)
    tuples. Excludes expired entries (`expires_at IS NOT NULL AND
    expires_at < now`). Callers that filter the digest also want
    expired rows garbage-collected periodically — see
    _gc_expired_flap_silences()."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    try:
        rows = conn.execute(
            "SELECT ip, interface FROM port_flap_whitelist "
            "WHERE expires_at IS NULL OR expires_at > ?",
            (now,)).fetchall()
    except sqlite3.OperationalError:
        return set()                    # pre-3.8.10 DB without the table
    return {(r["ip"], r["interface"]) for r in rows}


def _gc_expired_flap_silences(conn):
    """Delete port_flap_whitelist rows whose expires_at has passed.
    Called from digest_flap so the table stays bounded without a
    dedicated cleanup timer. Safe to call when the table doesn't
    exist (pre-3.8.10 DB) — swallows the OperationalError."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    try:
        cur = conn.execute(
            "DELETE FROM port_flap_whitelist "
            "WHERE expires_at IS NOT NULL AND expires_at < ?",
            (now,))
        conn.commit()
        if cur.rowcount:
            log.info("Garbage-collected %d expired flap silence(s).",
                     cur.rowcount)
    except sqlite3.OperationalError:
        pass


def _get_excluded_ports(conn, cfg):
    """Union of the cfg-managed and DB-managed flux-whitelist sets.

    Returns a set of (ip, interface) tuples. The cfg key
    [monitor] mac_flux_excluded_ports (3.8.7) is the version-controlled
    surface; the mac_flux_whitelist DB table (3.8.8) is the operator-
    managed surface edited via `netops ignore flux`. Either
    or both may be populated; their union is the exclusion list."""
    excluded = set(cfg.get("mac_flux_excluded_ports", set()))
    try:
        for r in conn.execute(
                "SELECT ip, interface FROM mac_flux_whitelist").fetchall():
            excluded.add((r["ip"], r["interface"]))
    except sqlite3.OperationalError:
        pass                       # pre-3.8.8 DB without the table
    return excluded


def handle_ignore(words, cfg=None):
    """`ignore` — permanent per-port exclusions (device-CLI keyword form).

      ignore flux <ip> <interface> [reason ...]
          approved port that must never page the MAC-flux security signal
      ignore flap <ip> <interface> [until DATE|Nd] [reason ...]
          known-noisy port kept out of `digest flap` (sleepy printers);
          `until` makes it time-bounded — accepts YYYY-MM-DD[_HH:MM:SS]
          or a duration (Nd/Nh/Nm)
      ignore list [flap|flux|all]        (default: all)
      ignore delete flap|flux <ip> <interface>

    `ignore` is deliberately allowed to be permanent — that's what
    separates it from `silence`, which mutes ALERT EMAILS device-wide and
    is hard-capped at 30 days.
    """
    w = list(words)
    ns = argparse.Namespace(ip=None, interface=None, reason=None,
                            flap=False, flux=False, all=False, until=None)
    if w[0] == "list":
        ns.action = "list"
        cat = w[1] if len(w) > 1 else "all"
        if cat not in ("flap", "flux", "all"):
            print("ignore list: expected flap, flux, or all")
            return
        setattr(ns, cat, True)
    elif w[0] == "delete":
        if len(w) != 4 or w[1] not in ("flap", "flux"):
            print("ignore delete: usage — ignore delete flap|flux "
                  "<ip> <interface>")
            return
        ns.action = "remove"
        setattr(ns, w[1], True)
        ns.ip, ns.interface = w[2], w[3]
    elif w[0] in ("flap", "flux"):
        if len(w) < 3:
            print(f"ignore {w[0]}: usage — ignore {w[0]} <ip> <interface> "
                  f"{'[until DATE|Nd] ' if w[0] == 'flap' else ''}[reason ...]")
            return
        ns.action = "add"
        setattr(ns, w[0], True)
        ns.ip, ns.interface = w[1], w[2]
        rest = w[3:]
        if rest and rest[0] == "until":
            if len(rest) < 2:
                print("ignore: 'until' needs a date or duration value")
                return
            mins = _parse_silence_duration(rest[1])
            if mins is not None:
                ns.until = ((datetime.now() + timedelta(minutes=mins))
                            .strftime("%Y-%m-%d %H:%M:%S"))
            else:
                ns.until = rest[1].replace("_", " ")
            rest = rest[2:]
        if rest:
            ns.reason = " ".join(rest)
    else:
        print("ignore: expected flap, flux, list, or delete "
              "(e.g. ignore flap 10.1.60.2 1/1/24 sleepy printer)")
        return
    if cfg:
        ns._cfg_excluded = set(cfg.get("mac_flux_excluded_ports", set()))
    if ns.action in ("add", "remove"):
        _audit_log(f"ignore_{ns.action}",
                   detail=f"{'flap' if ns.flap else 'flux'} "
                          f"{ns.ip} {ns.interface}")
    return _ignore_impl(ns)


def _ignore_impl(args):
    """Table operations behind `ignore` (mac_flux_whitelist +
    port_flap_whitelist). Takes a namespace: action, ip, interface,
    reason, flap/flux/all, until."""
    action = args.action
    # Category selector — backward-compat: bare CLI = flux.
    want_flap = bool(getattr(args, "flap", False))
    want_flux = bool(getattr(args, "flux", False)) or \
                (not want_flap and not getattr(args, "all", False))
    want_all  = bool(getattr(args, "all", False))
    until     = getattr(args, "until", None)
    if until and not want_flap:
        print("ignore: 'until' only applies to flap entries; "
              "mac-flux ignores are permanent. Ignoring it.")
        until = None

    # ------------------------------------------------------------------
    if action == "list":
        conn = _db(); conn.row_factory = sqlite3.Row
        printed_any = False
        if want_flux or want_all:
            try:
                rows = conn.execute(
                    "SELECT ip, interface, reason, added_at, added_by "
                    "FROM mac_flux_whitelist ORDER BY ip, interface"
                ).fetchall()
                print("=== MAC-flux ignores (mac_flux_whitelist, permanent) ===")
                if rows:
                    _print_table(
                        ["ip", "interface", "added_at", "added_by", "reason"],
                        [{"ip": r["ip"], "interface": r["interface"],
                          "added_at": r["added_at"],
                          "added_by": r["added_by"] or "",
                          "reason": r["reason"] or ""} for r in rows])
                    printed_any = True
                else:
                    print("(empty)")
                cfg_excl = sorted(getattr(args, "_cfg_excluded", set()))
                if cfg_excl:
                    print("\nAlso excluded via [monitor] mac_flux_excluded_ports:")
                    for ip, iface in cfg_excl:
                        print(f"  {ip}  {iface}")
            except sqlite3.OperationalError:
                print("(no mac_flux_whitelist table — initialize DB first)")
        if want_flap or want_all:
            if want_all:
                print()
            try:
                rows = conn.execute(
                    "SELECT ip, interface, reason, added_at, added_by, expires_at "
                    "FROM port_flap_whitelist ORDER BY ip, interface"
                ).fetchall()
                print("=== Port-flap ignores (port_flap_whitelist) ===")
                if rows:
                    _print_table(
                        ["ip", "interface", "added_at", "added_by",
                         "expires_at", "reason"],
                        [{"ip": r["ip"], "interface": r["interface"],
                          "added_at": r["added_at"],
                          "added_by": r["added_by"] or "",
                          "expires_at": r["expires_at"] or "(permanent)",
                          "reason": r["reason"] or ""} for r in rows])
                    printed_any = True
                else:
                    print("(empty)")
            except sqlite3.OperationalError:
                print("(no port_flap_whitelist table — initialize DB first)")
        return

    # ------------------------------------------------------------------
    # add / remove: require ip + interface
    if not args.ip or not args.interface:
        print(f"ignore {action}: ip and interface are required")
        sys.exit(2)
    ip, iface = args.ip.strip(), args.interface.strip()
    added_by = (os.environ.get("SUDO_USER")
                or os.environ.get("USER") or "unknown")
    added_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    # Normalize --until: accept YYYY-MM-DD or YYYY-MM-DD HH:MM:SS.
    expires_at = None
    if until:
        u = until.strip()
        if len(u) == 10:                # bare date -> end of day
            u = f"{u} 23:59:59"
        try:
            datetime.strptime(u, "%Y-%m-%d %H:%M:%S")
            expires_at = u
        except ValueError:
            print(f"ignore: until {until!r} must be YYYY-MM-DD, "
                  f"'YYYY-MM-DD HH:MM:SS', or a duration like 30d")
            sys.exit(2)

    table  = "port_flap_whitelist" if want_flap else "mac_flux_whitelist"
    label  = "flap ignore"           if want_flap else "MAC-flux ignore"
    alerts = "(no live-alert clearing for flap silences)" if want_flap \
             else None
    conn = _db()

    if action == "add":
        if want_flap:
            conn.execute(
                "INSERT OR REPLACE INTO port_flap_whitelist "
                "(ip, interface, reason, added_at, added_by, expires_at) "
                "VALUES (?, ?, ?, ?, ?, ?)",
                (ip, iface, args.reason, added_at, added_by, expires_at))
        else:
            conn.execute(
                "INSERT OR REPLACE INTO mac_flux_whitelist "
                "(ip, interface, reason, added_at, added_by) "
                "VALUES (?, ?, ?, ?, ?)",
                (ip, iface, args.reason, added_at, added_by))
            # If the port currently has an active alert, mark it cleared so
            # operator intent is reflected immediately rather than waiting
            # up to 30 min for the next topology tick to reconcile.
            conn.execute(
                "UPDATE mac_flux_alerts SET cleared_at = ? "
                "WHERE ip = ? AND interface = ? AND cleared_at IS NULL",
                (added_at, ip, iface))
        conn.commit()
        msg = f"added {ip} {iface} to {label}"
        if args.reason:
            msg += f" — {args.reason}"
        if expires_at:
            msg += f" (expires {expires_at})"
        msg += f"  [{added_by}]"
        print(msg)
        return

    if action == "remove":
        cur = conn.execute(
            f"DELETE FROM {table} WHERE ip = ? AND interface = ?",
            (ip, iface))
        conn.commit()
        if cur.rowcount:
            print(f"removed {ip} {iface} from {label}")
        else:
            cfg_hint = ("if it's listed in [monitor] mac_flux_excluded_ports "
                        "in netops.conf, edit that file instead") \
                       if not want_flap else \
                       "no entry to remove"
            print(f"(no entry for {ip} {iface} in {table}; {cfg_hint})")
        return


def _evaluate_mac_flux_alerts(conn, cfg, now):
    """Reconcile current non-trunk MAC-flux against the mac_flux_alerts
    dedup table. Edge-triggered: returns the ports that JUST crossed the
    threshold this episode (so the caller emails once), suppresses while
    an episode is ongoing, and marks cleared_at when a port drops back
    under threshold so a later re-crossing re-alerts. Mirrors the
    stp_pending / *_alerted_at preservation pattern. conn needs a
    sqlite3.Row row_factory."""
    threshold = cfg.get("mac_flux_threshold_24h", 5)
    flux = _compute_mac_flux(conn, threshold,
                               cfg.get("mac_flux_uplink_ceiling", 24),
                               cfg.get("mac_flux_max_vlans", 2),
                               cfg.get("mac_flux_rotation_max_times_seen", 3),
                               cfg.get("mac_flux_rotation_max_span_sec", 7200))
    excluded = _get_excluded_ports(conn, cfg)
    if excluded:
        flux = [p for p in flux if (p["ip"], p["interface"]) not in excluded]
    try:
        existing = {(r["ip"], r["interface"]): r for r in conn.execute(
            "SELECT ip, interface, cleared_at FROM mac_flux_alerts"
        ).fetchall()}
    except sqlite3.OperationalError:
        return []                       # pre-3.8.0 DB without the table
    cur_keys = set()
    new_alerts = []
    for c in flux:
        key = (c["ip"], c["interface"])
        cur_keys.add(key)
        ex = existing.get(key)
        if ex is None or ex["cleared_at"] is not None:
            conn.execute(
                "INSERT OR REPLACE INTO mac_flux_alerts "
                "(ip, interface, distinct_macs, notified_at, cleared_at) "
                "VALUES (?, ?, ?, ?, NULL)",
                (c["ip"], c["interface"], c["distinct"], now))
            new_alerts.append(c)
        else:
            conn.execute(
                "UPDATE mac_flux_alerts SET distinct_macs = ? "
                "WHERE ip = ? AND interface = ?",
                (c["distinct"], c["ip"], c["interface"]))
    for key, ex in existing.items():
        if key not in cur_keys and ex["cleared_at"] is None:
            conn.execute(
                "UPDATE mac_flux_alerts SET cleared_at = ? "
                "WHERE ip = ? AND interface = ?", (now, key[0], key[1]))
    conn.commit()
    return new_alerts


def _send_flux_alert_email(cfg, new_alerts):
    """One-shot security notice for non-trunk ports that just crossed the
    24h MAC-flux threshold. Best-effort; logs the outcome."""
    muted = _silenced_ip_set([c["ip"] for c in new_alerts])
    if muted:
        for c in new_alerts:
            if c["ip"] in muted:
                log.info("MAC-flux alert for %s %s muted by silence",
                         c["ip"], c["interface"])
        new_alerts = [c for c in new_alerts if c["ip"] not in muted]
        if not new_alerts:
            return
    thr = cfg.get("mac_flux_threshold_24h", 5)
    subject = (f"[netops] SECURITY: MAC flux on {len(new_alerts)} "
               "non-trunk port(s)")
    lines = [
        f"{len(new_alerts)} non-trunk port(s) crossed the MAC-flux "
        f"threshold (>= {thr} distinct MACs in the trailing 24h).",
        "Possible unauthorized mini-switch/hub, an AP, MAC spoofing, or "
        "rapid device swapping. One notice per episode — no re-page until "
        "the port settles and the flux recurs.",
        "",
    ]
    for c in new_alerts:
        lines.append(f"=== {c['hostname']} ({c['ip']}) {c['interface']} "
                     f"— {c['distinct']} distinct MAC(s)/24h"
                     f", {c.get('vlans', 1)} vlan(s) ===")
        for h in _format_port_macs_history(c["ip"], c["interface"], limit=12):
            lines.append(f"  {h}")
        lines.append("")
    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("MAC-flux security email %s (%d port(s))",
                 detail, len(new_alerts))
    else:
        log.error("MAC-flux security email failed: %s", detail)


def _lookup_port_description(ip, hostname, interface, platform):
    """Find the operator's description/name for this port from the most
    recent saved config. Returns '' on miss (no backup, missing port, or
    no description configured). Best-effort — fast and silent on failure.
    """
    candidates = []
    if hostname:
        candidates.append(f"{hostname}_{ip}.cfg")
    candidates.append(f"{ip}.cfg")
    config_path = None
    for fname in candidates:
        path = os.path.join(CONFIGS_DIR, "current", fname)
        if os.path.isfile(path):
            config_path = path
            break
    if not config_path:
        return ""
    iface_phys = interface.split(".", 1)[0]
    try:
        # For Junos prefer the _set.cfg sibling (line-per-statement, simple
        # regex). For hierarchical Junos / Aruba-CX / ProCurve we walk the
        # interface block.
        if platform == "junos":
            stem = os.path.splitext(os.path.basename(config_path))[0]
            set_path = os.path.join(os.path.dirname(config_path),
                                    f"{stem}_set.cfg")
            if os.path.isfile(set_path):
                with open(set_path, encoding="utf-8", errors="replace") as f:
                    for line in f:
                        m = re.match(
                            rf"^set interfaces {re.escape(iface_phys)} "
                            rf'description "?([^"\n]+?)"?\s*$', line)
                        if m:
                            return m.group(1).strip()
                return ""
            # Hierarchical fallback
            with open(config_path, encoding="utf-8", errors="replace") as f:
                text = f.read()
            m = re.search(
                rf"\b{re.escape(iface_phys)}\b\s*\{{[^}}]*?"
                rf'description\s+"?([^";\n]+?)"?\s*;', text, re.DOTALL)
            return m.group(1).strip() if m else ""
        elif platform == "aruba-cx":
            with open(config_path, encoding="utf-8", errors="replace") as f:
                lines = f.readlines()
            in_iface = False
            for line in lines:
                stripped = line.strip()
                if stripped.startswith("interface "):
                    in_iface = (stripped == f"interface {iface_phys}")
                    continue
                if in_iface and stripped.startswith("description "):
                    return stripped[len("description "):].strip().strip('"')
                if in_iface and stripped and not stripped.startswith(" ") \
                        and not line.startswith(" ") and not line.startswith("\t"):
                    # New top-level stanza — exit interface block
                    if not stripped.startswith("!"):
                        in_iface = False
            return ""
        else:  # procurve
            with open(config_path, encoding="utf-8", errors="replace") as f:
                lines = f.readlines()
            in_iface = False
            for line in lines:
                stripped = line.strip()
                if stripped.startswith("interface "):
                    rest = stripped[len("interface "):].strip()
                    in_iface = (rest == iface_phys)
                    continue
                if in_iface and stripped.startswith("name "):
                    return stripped[len("name "):].strip().strip('"')
                if in_iface and stripped == "exit":
                    in_iface = False
            return ""
    except OSError:
        return ""


def _parse_lldp_neighbor(lldp_text, platform):
    """Best-effort extraction of LLDP-neighbor fields from per-port detail.

    Returns dict with keys (any can be empty string):
      system_name, system_description, port_id, port_description,
      chassis_id, chassis_type, capabilities.

    Vendor output formats differ; we use the same loose key-value scan for
    all three and let missing keys come back empty. A missing system_name
    is normal for endpoint devices that don't run LLDP-MED with the host
    name TLV — the DB cross-reference falls back to chassis MAC in that
    case.
    """
    out = {"system_name": "", "system_description": "", "port_id": "",
           "port_description": "", "chassis_id": "", "chassis_type": "",
           "capabilities": ""}
    if not lldp_text:
        return out
    # Junos / Aruba-CX / ProCurve all use a flavor of "Key  : Value".
    # ProCurve uses single-space separation in some lines; we match both
    # ": " and just ":" with surrounding whitespace.
    # Each pattern allows the separator between word parts to be space,
    # hyphen, underscore, or nothing (covers "System Name", "SysName",
    # "Chassis-ID", "ChassisId", etc.). Case-insensitive via re.IGNORECASE
    # at match time.
    kv_patterns = {
        "system_name":        r"(?:System[-_ ]?Name|SysName|Chassis[-_ ]?Name)\s*:?\s*(\S.*?)\s*$",
        "system_description": r"(?:System[-_ ]?Descr(?:iption)?|Chassis[-_ ]?Description|SystemDescr)\s*:?\s*(\S.*?)\s*$",
        "port_id":            r"^\s*Port[-_ ]?Id\s*:?\s*(\S.*?)\s*$",
        "port_description":   r"^\s*Port[-_ ]?Descr(?:iption|iption)?\s*:?\s*(\S.*?)\s*$",
        "chassis_id":         r"^\s*Chassis[-_ ]?Id\s*:?\s*(\S.*?)\s*$",
        "chassis_type":       r"^\s*Chassis[-_ ]?Type\s*:?\s*(\S.*?)\s*$",
    }
    cap_re = re.compile(r"(?:System\s*)?[Cc]apabilities", re.MULTILINE)
    for line in lldp_text.splitlines():
        for field, pat in kv_patterns.items():
            if out[field]:
                continue
            m = re.search(pat, line, re.IGNORECASE)
            if m:
                out[field] = m.group(1).strip()
                break
    # ProCurve writes ChassisId with internal spaces: "ec eb b8 11 22 33".
    # Collapse whitespace in chassis_id so downstream MAC matching works.
    if out["chassis_id"]:
        out["chassis_id"] = re.sub(r"\s+", "", out["chassis_id"]).strip()
    # System capabilities arrive as a multi-line block on Junos/Aruba-CX;
    # we just record whether 'Bridge' appears, which is the bit we use to
    # tell "endpoint" from "another switch" when the DB lookup misses.
    if cap_re.search(lldp_text):
        out["capabilities"] = ("bridge" if re.search(
            r"[Bb]ridge", lldp_text) else "non-bridge")
    return out


def _resolve_neighbor_in_db(parsed):
    """Return the matched devices row (or None) for an LLDP neighbor.

    Match priority:
      1) chassis MAC (12 hex chars) → devices.base_mac
      2) system_name → devices.hostname (case-insensitive)
    A miss usually means the neighbor is an endpoint (workstation / phone /
    AP) not tracked by netops, or a switch we haven't discovered yet.
    """
    chassis_norm = _normalize_mac(parsed.get("chassis_id"))
    sys_name = (parsed.get("system_name") or "").strip()
    if not chassis_norm and not sys_name:
        return None
    conn = _db()
    conn.row_factory = sqlite3.Row
    row = None
    if chassis_norm:
        # base_mac is stored lowercase, hyphenated as "aaaa-aaaa-aaaa"
        # (ProCurve style) or plain hex. Normalize both sides.
        norm_target = chassis_norm
        for r in conn.execute(
            "SELECT * FROM devices WHERE base_mac IS NOT NULL "
            "AND status IN ('active','inactive')"
        ).fetchall():
            stored = _normalize_mac(r["base_mac"])
            if stored and stored == norm_target:
                row = r
                break
    if row is None and sys_name:
        row = conn.execute(
            "SELECT * FROM devices "
            "WHERE LOWER(hostname) = LOWER(?) "
            "AND status IN ('active','inactive') LIMIT 1",
            (sys_name,)).fetchone()
    conn.close()
    return row


def _format_upstream_context_lines(ch, info):
    """Build the 'Upstream context (one hop)' lines for the email.

    Phase A: one-hop trace. Parse the LLDP neighbor on the alerted port,
    cross-reference against our devices DB, and report:
      - what's on the other side (endpoint vs tracked switch),
      - if a tracked switch, that switch's recent STP / root-change activity
        in the last 60 minutes.

    Empty list if there's no LLDP output at all (the caller already shows
    '(no neighbor detected on this port)' in the raw section).
    """
    raw = (info.get("lldp_neighbor") or "").strip()
    if not raw or raw.startswith("(error:"):
        return []
    conn = _db()
    conn.row_factory = sqlite3.Row
    plat_row = conn.execute(
        "SELECT platform FROM devices WHERE ip = ?", (ch["ip"],)
    ).fetchone()
    conn.close()
    platform = plat_row["platform"] if plat_row and plat_row["platform"] else ""
    parsed = _parse_lldp_neighbor(raw, platform)
    lines = ["--- Upstream context (one hop via LLDP) ---"]
    if not parsed["chassis_id"] and not parsed["system_name"]:
        lines.append("  (LLDP output didn't include a chassis-id or system-name "
                     "we could parse — see raw section below)")
        return lines

    # Identifier line — what we extracted from the neighbor's LLDP TLVs.
    ident_bits = []
    if parsed["system_name"]:
        ident_bits.append(f"system={parsed['system_name']}")
    if parsed["chassis_id"]:
        # Render as the same string format we usually display MACs in if
        # it parses as a MAC; otherwise show as-is (some endpoints set
        # chassis to a serial number).
        mac_norm = _normalize_mac(parsed["chassis_id"])
        if mac_norm:
            mac_disp = ":".join(mac_norm[i:i+2] for i in range(0, 12, 2))
            vendor = _oui_vendor_hint(mac_norm)
            ident_bits.append(f"chassis={mac_disp} [{vendor}]")
        else:
            ident_bits.append(f"chassis={parsed['chassis_id']}")
    if parsed["port_id"]:
        ident_bits.append(f"neighbor-port={parsed['port_id']}")
    lines.append("  " + " · ".join(ident_bits))
    if parsed["port_description"]:
        lines.append(f"  neighbor-port-desc: {parsed['port_description']}")

    matched = _resolve_neighbor_in_db(parsed)
    if matched is None:
        # No DB match — almost always an endpoint (printer / phone / AP /
        # workstation). The capabilities TLV usually tells us, but the
        # MAC's OUI vendor hint is the most useful one-line summary.
        cap = parsed.get("capabilities", "")
        if cap == "bridge":
            lines.append("  not tracked in netops devices — capabilities include "
                         "Bridge, so this is likely a switch we haven't discovered yet")
        else:
            lines.append("  not tracked in netops devices — likely an endpoint "
                         "(workstation / phone / AP / printer)")
        return lines

    # Matched a known device — report its current STP-related activity.
    lines.append(f"  matched netops device: {matched['hostname'] or '?'} "
                 f"({matched['ip']})  platform={matched['platform'] or '?'}  "
                 f"model={matched['model'] or '?'}")

    # Recent neighbor STP changes (last 60 min) — limited to its side of the link
    # if we can identify it, otherwise the full set.
    conn = _db()
    rows = conn.execute("""
        SELECT interface, old_role, old_state, new_role, new_state, changed_at
        FROM stp_changes
        WHERE ip = ?
          AND changed_at >= datetime('now', 'localtime', '-60 minutes')
        ORDER BY changed_at DESC
        LIMIT 8
    """, (matched["ip"],)).fetchall()
    if rows:
        lines.append(f"  STP changes on this neighbor in last 60 min ({len(rows)}):")
        for r in rows:
            lines.append(f"    {r['changed_at']}  {r['interface']:14s} "
                         f"{r['old_role']}/{r['old_state']} -> "
                         f"{r['new_role']}/{r['new_state']}")
    else:
        lines.append("  STP changes on this neighbor in last 60 min: none")
    root_rows = conn.execute("""
        SELECT instance, old_priority, old_mac, new_priority, new_mac, changed_at
        FROM stp_root_changes
        WHERE ip = ?
          AND changed_at >= datetime('now', 'localtime', '-60 minutes')
        ORDER BY changed_at DESC
        LIMIT 4
    """, (matched["ip"],)).fetchall()
    if root_rows:
        lines.append(f"  Root-bridge changes on this neighbor in last 60 min ({len(root_rows)}):")
        for r in root_rows:
            lines.append(f"    {r['changed_at']}  inst={r['instance']}  "
                         f"prio {r['old_priority']}->{r['new_priority']}  "
                         f"mac {r['old_mac']}->{r['new_mac']}")
    else:
        lines.append("  Root-bridge changes on this neighbor in last 60 min: none")
    conn.close()
    return lines


def _format_investigation_block(ch, info):
    """Format one investigation block for the alert email / stdout.

    Ordering is top-down — big picture first: root bridge -> port STP
    detail -> LLDP neighbor -> physical interface. The reader gets the
    'where is root?' context before drilling into port-level detail.
    """
    lines = []
    header = f"=== {ch['hostname']} ({ch['ip']}) {ch['interface']} — investigation ==="
    lines.append("")
    lines.append(header)
    if not info.get("ok"):
        lines.append(f"  (investigation failed: {info.get('error') or 'unknown error'})")
        return lines

    # Port description from the most recent saved config — surfaces the
    # operator's own label ("Room 204 phone", "AP-Lobby-East") right under
    # the header so the reader knows what's on this port before reading
    # any other section. Best-effort lookup; empty on miss.
    conn = _db()
    plat_row = conn.execute(
        "SELECT platform FROM devices WHERE ip = ?", (ch["ip"],)
    ).fetchone()
    conn.close()
    if plat_row and plat_row["platform"]:
        desc = _lookup_port_description(
            ch["ip"], ch["hostname"], ch["interface"], plat_row["platform"])
        if desc:
            lines.append(f"  port description (from saved config): {desc}")

    # Heuristic classifier verdict — operator-friendly summary at the top,
    # before the raw CLI dumps. Falls back gracefully when 'parsed' is
    # missing or empty (e.g. older code paths that don't populate it).
    if info.get("parsed"):
        pf, tcn_s = _stp_event_context(ch["ip"], ch["interface"])
        label, conf, evidence = _classify_stp_event(
            info["parsed"], pf, tcn_s,
            old_role=ch.get("old_role"), new_role=ch.get("new_role"))
        lines.extend(_format_classification_block(label, conf, evidence))
        lines.append("")

    # One-hop upstream context — parse LLDP, cross-reference the neighbor
    # against our devices table, and surface that neighbor's recent STP /
    # root-change activity. For endpoints (workstation/phone/AP/printer)
    # we just note that nothing else needs investigating one hop out.
    upstream_lines = _format_upstream_context_lines(ch, info)
    if upstream_lines:
        lines.extend(upstream_lines)
        lines.append("")

    root_lines = _format_root_bridge_lines(ch["ip"])
    if root_lines:
        lines.append("--- Root bridge (this switch's view) ---")
        for rl in root_lines:
            lines.append(f"  {rl}")
        lines.append("")
    lines.append("--- STP interface detail ---")
    lines.append((info["stp_detail"] or "").strip() or "(no output)")
    lines.append("")
    lines.append("--- LLDP neighbor ---")
    lldp = (info["lldp_neighbor"] or "").strip()
    lines.append(lldp or "(no neighbor detected on this port)")
    lines.append("")
    lines.append("--- MAC table (this port, with OUI vendor hints) ---")
    mac_lines = _format_mac_table_section(info.get("mac_table") or "")
    for ml in mac_lines:
        lines.append(ml)
    # Live FDB is empty exactly when the port is down/flapping — fall back
    # to the port_macs history so the operator still sees what was there.
    if mac_lines in (["(no MACs learned on this port)"],
                     ["(no MAC table output)"]):
        hist = _format_port_macs_history(ch["ip"], ch["interface"])
        if hist:
            lines.append("  last-known on this port (FDB history):")
            for hl in hist:
                lines.append(f"    {hl}")
    lines.append("")
    lines.append("--- Interface status ---")
    lines.append((info["interface_stats"] or "").strip() or "(no output)")
    return lines


def _send_stp_alert(cfg, changes, investigations=None, suspected_fp=None):
    """Send email alert for STP state changes if [email] is configured.

    investigations: optional {(ip, interface): info_dict} produced by
    investigate_stp_ports; when provided, the email body appends a
    per-port investigation block below the summary.

    suspected_fp: optional list of changes the classifier tagged as
    snmp_transient — rendered in a separate "Suspected false positives"
    section so the operator can verify the classification without acting
    on them. Subject line counts only `changes`.
    """
    if not cfg.get("smtp_server"):
        log.debug("No [email] config — skipping email alert")
        return

    suspected_fp = suspected_fp or []
    muted = _silenced_ip_set([c["ip"] for c in changes]
                             + [c["ip"] for c in suspected_fp])
    if muted:
        for c in changes:
            if c["ip"] in muted:
                log.info("STP-change alert for %s %s muted by silence",
                         c["ip"], c["interface"])
        changes = [c for c in changes if c["ip"] not in muted]
        suspected_fp = [c for c in suspected_fp if c["ip"] not in muted]
        if not changes and not suspected_fp:
            return
    subject = f"[netops] STP changes confirmed — {len(changes)} port(s)"
    if suspected_fp:
        subject += f" (+ {len(suspected_fp)} suspected false-positive)"
    body_lines = [f"STP state changes (confirmed across two consecutive polls) at "
                  f"{datetime.now():%Y-%m-%d %H:%M}:\n"]
    for ch in changes:
        blocking = " *** BLOCKING ***" if is_stp_blocking(ch) else ""
        body_lines.append(
            f"  {ch['hostname']} ({ch['ip']}) {ch['interface']}: "
            f"{ch['old_role']}/{ch['old_state']} -> {ch['new_role']}/{ch['new_state']}{blocking}"
        )

    if investigations:
        for ch in changes:
            info = investigations.get((ch["ip"], ch["interface"]))
            if info is not None:
                body_lines.extend(_format_investigation_block(ch, info))

    if suspected_fp:
        body_lines.append("")
        body_lines.append(
            f"=== Suspected false positives ({len(suspected_fp)}) ===")
        body_lines.append(
            "These changes confirmed across two polls but the classifier "
            "flagged them as snmp_transient (no port_flaps row, no BPDU "
            "input, link stable, no recent TC). Probably MIB-cache vs "
            "state-machine race on Junos edge/boundary ports. Verify the "
            "investigation block agrees before acting.")
        for ch in suspected_fp:
            body_lines.append(
                f"  {ch['hostname']} ({ch['ip']}) {ch['interface']}: "
                f"{ch['old_role']}/{ch['old_state']} -> "
                f"{ch['new_role']}/{ch['new_state']}"
            )
        if investigations:
            for ch in suspected_fp:
                info = investigations.get((ch["ip"], ch["interface"]))
                if info is not None:
                    body_lines.extend(_format_investigation_block(ch, info))

    ok, detail = _send_email(cfg, subject, "\n".join(body_lines))
    if ok:
        log.info("Email alert %s", detail)
    else:
        log.error("Email alert failed: %s", detail)


def _send_root_change_alert(cfg, changes):
    """Email alert when a switch's view of the STP root bridge changes
    (confirmed across two consecutive polls)."""
    if not cfg.get("smtp_server"):
        log.debug("No [email] config — skipping root-change email")
        return
    # A root-bridge change is fleet-level signal, not one device's — only
    # an 'all'-scope silence (or every reporting device being silenced)
    # mutes it.
    muted = _silenced_ip_set([c["ip"] for c in changes])
    if _any_all_silence() or (muted and all(c["ip"] in muted
                                            for c in changes)):
        log.info("root-change alert muted by silence")
        return
    subject = (f"[netops] STP root-bridge change confirmed — "
               f"{len(changes)} device(s)")
    lines = [f"STP root-bridge changes (confirmed across two consecutive polls) at "
             f"{datetime.now():%Y-%m-%d %H:%M}:\n"]
    for ch in changes:
        lines.append(
            f"  {ch['hostname']} ({ch['ip']}) {ch['instance']}: "
            f"{ch['old_priority']}/{ch['old_mac']} -> "
            f"{ch['new_priority']}/{ch['new_mac']}"
        )
    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("Root-change email %s", detail)
    else:
        log.error("Root-change email failed: %s", detail)


_DEFAULT_STP_DOMAIN = "(default)"


def _ip_to_stp_domain(ip_str, subnet_map):
    """Return the STP domain name for ip_str by longest-prefix match.

    Empty subnet_map -> '(default)' for every IP (backward-compatible
    single-domain behavior). Configured subnet_map -> the matching
    domain name, or None if no prefix matches (caller treats None as
    'unassigned' and excludes from consensus checks).
    """
    if not subnet_map:
        return _DEFAULT_STP_DOMAIN
    try:
        addr = ipaddress.ip_address(ip_str)
    except ValueError:
        return None
    best = None
    for net, name in subnet_map.items():
        if addr in net and (best is None or net.prefixlen > best[0].prefixlen):
            best = (net, name)
    return best[1] if best else None


def _detect_root_mismatches(conn, poll_timestamp, subnet_map):
    """Find STP instances where this poll's responsive, in-service switches
    disagree on the root bridge — within each STP domain.

    Only considers rows with updated_at == poll_timestamp so stale entries
    from a previous poll can't falsify consensus. Excludes:
      - devices with link_up_count == 0 (reachable via management only).
      - devices whose IP doesn't match any [stp_domains] prefix when domains
        are configured (unassigned).

    Returns (mismatches, not_in_service_count, unassigned_count). Mismatches:
        [{'domain': 'main-campus', 'instance': 'CIST',
          'groups': [{'root_priority': 0, 'root_mac': '..',
                      'devices': [(ip, hostname), ...]}, ...]}, ...]
    Only (domain, instance) pairs with >1 distinct (priority, mac) group
    are returned.
    """
    rows = conn.execute("""
        SELECT s.instance, s.root_priority, s.root_mac, s.ip,
               COALESCE(d.hostname, s.ip) AS hostname,
               d.link_up_count
        FROM stp_root_state s
        LEFT JOIN devices d ON d.ip = s.ip
        WHERE s.updated_at = ? AND s.root_mac IS NOT NULL
        ORDER BY s.instance, s.root_priority, s.root_mac, hostname
    """, (poll_timestamp,)).fetchall()

    not_in_service = 0
    unassigned = 0
    by_domain_instance = {}
    for r in rows:
        if r["link_up_count"] is not None and r["link_up_count"] <= 0:
            not_in_service += 1
            continue
        domain = _ip_to_stp_domain(r["ip"], subnet_map)
        if domain is None:
            unassigned += 1
            continue
        key = (r["root_priority"], r["root_mac"])
        by_domain_instance.setdefault(
            (domain, r["instance"]), {}
        ).setdefault(key, []).append((r["ip"], r["hostname"]))

    mismatches = []
    for (domain, instance), groups in by_domain_instance.items():
        if len(groups) < 2:
            continue
        # Smallest group first — the minority/outlier roots are the interesting
        # signal; the majority/agreeing group sorts to the end so it can be
        # hidden or collapsed by the caller.
        ordered = sorted(groups.items(), key=lambda kv: (len(kv[1]), kv[0]))
        mismatches.append({
            "domain": domain,
            "instance": instance,
            "groups": [
                {"root_priority": k[0], "root_mac": k[1], "devices": v}
                for k, v in ordered
            ],
        })
    # Stable order: domain name, then instance.
    mismatches.sort(key=lambda m: (m["domain"], m["instance"]))
    return mismatches, not_in_service, unassigned


def _root_disagree_fingerprint(mm):
    """Stable digest of one mismatch's contending roots and their membership.

    Covers the (priority, mac) of every group and which switches sit in each,
    so a switch crossing from the minority root to the majority one counts as
    a change. Group and device order are normalized first — _detect_root_
    mismatches orders groups by size, and a tie in size would otherwise let
    dict order flip the digest with nothing actually different.
    """
    canon = sorted(
        (str(g["root_priority"]), str(g["root_mac"]),
         ",".join(sorted(ip for ip, _ in g["devices"])))
        for g in mm["groups"]
    )
    return hashlib.sha256(
        "|".join(":".join(parts) for parts in canon).encode()
    ).hexdigest()[:16]


def _sync_root_disagree_state(conn, mismatches, now):
    """Reconcile stp_root_disagree_state against this poll's mismatches.

    Returns {(domain, instance): first_seen_or_None} — None marks a pair whose
    breakdown is new or has changed since it was last logged, i.e. one the
    caller should log in full. Pairs that dropped out of disagreement are
    deleted, so a later recurrence is reported fresh rather than silently
    folded into a stale fingerprint.
    """
    prior = {
        (r["domain"], r["instance"]): (r["fingerprint"], r["first_seen"])
        for r in conn.execute(
            "SELECT domain, instance, fingerprint, first_seen "
            "FROM stp_root_disagree_state")
    }
    status = {}
    live = set()
    for mm in mismatches:
        key = (mm["domain"], mm["instance"])
        live.add(key)
        fp = _root_disagree_fingerprint(mm)
        was = prior.get(key)
        if was and was[0] == fp:
            status[key] = was[1]
            conn.execute(
                "UPDATE stp_root_disagree_state SET updated_at = ? "
                "WHERE domain = ? AND instance = ?", (now, key[0], key[1]))
        else:
            status[key] = None
            conn.execute(
                "INSERT INTO stp_root_disagree_state "
                "(domain, instance, fingerprint, first_seen, updated_at) "
                "VALUES (?, ?, ?, ?, ?) "
                "ON CONFLICT(domain, instance) DO UPDATE SET "
                "fingerprint = excluded.fingerprint, "
                "first_seen = excluded.first_seen, "
                "updated_at = excluded.updated_at",
                (key[0], key[1], fp, now, now))
    for key in prior:
        if key not in live:
            conn.execute(
                "DELETE FROM stp_root_disagree_state "
                "WHERE domain = ? AND instance = ?", (key[0], key[1]))
    conn.commit()
    return status


def handle_flap_digest(cfg, min_count=1, reset=False):
    """Email a digest of port flap counters (optionally clearing them after).

    min_count filters the report to ports with >= N flaps. reset=True clears
    all of port_flaps after a successful send; otherwise counters are kept
    so retention can be managed independently via `clear flap`.
    """
    conn = _db()
    conn.row_factory = sqlite3.Row
    # 3.8.10: garbage-collect expired flap silences first so the
    # silenced-set we apply below doesn't include stale entries.
    _gc_expired_flap_silences(conn)
    silenced = _get_flap_silenced_ports(conn)
    rows = conn.execute("""
        SELECT ip, hostname, interface, flap_count, first_seen, last_seen
        FROM port_flaps
        WHERE flap_count >= ?
        ORDER BY flap_count DESC, hostname, interface
    """, (min_count,)).fetchall()
    conn.close()
    silenced_skipped = sum(1 for r in rows
                           if (r["ip"], r["interface"]) in silenced)
    if silenced:
        rows = [r for r in rows
                if (r["ip"], r["interface"]) not in silenced]

    flux_conn = _db()
    flux_conn.row_factory = sqlite3.Row
    flux = _compute_mac_flux(flux_conn,
                               cfg.get("mac_flux_threshold_24h", 5),
                               cfg.get("mac_flux_uplink_ceiling", 24),
                               cfg.get("mac_flux_max_vlans", 2),
                               cfg.get("mac_flux_rotation_max_times_seen", 3),
                               cfg.get("mac_flux_rotation_max_span_sec", 7200))
    excluded = _get_excluded_ports(flux_conn, cfg)
    if excluded:
        flux = [p for p in flux if (p["ip"], p["interface"]) not in excluded]
    flux_conn.close()

    if not rows and not flux:
        log.info("No port flaps >= %d and no MAC flux to report.", min_count)
        return

    total_flaps = sum(r["flap_count"] for r in rows)
    subj_bits = []
    if rows:
        subj_bits.append(f"{len(rows)} flapping port(s), {total_flaps} flap(s)")
    if flux:
        subj_bits.append(f"{len(flux)} MAC-flux port(s)")
    if not subj_bits and silenced_skipped:
        # Nothing to email — but we still want the log to record that
        # silenced rows were skipped.
        log.info("No reportable flaps (silenced=%d, flux=0).", silenced_skipped)
        return
    subject = "[netops] Port flap digest — " + "; ".join(subj_bits)
    body_lines = [f"Port flap digest at {datetime.now():%Y-%m-%d %H:%M}"]
    if rows:
        sil_note = (f"  (+{silenced_skipped} silenced port(s) filtered out — "
                    f"see `netops ignore list flap`)") if silenced_skipped \
                   else ""
        body_lines += [
            f"Ports with >= {min_count} flap(s): {len(rows)} (total {total_flaps} flaps){sil_note}",
            "",
            "  FLAPS  HOSTNAME / IP                         INTERFACE         FIRST SEEN           LAST SEEN",
            "  -----  ------------------------------------  ----------------  -------------------  -------------------",
        ]
        for r in rows:
            label = f"{r['hostname'] or r['ip']} ({r['ip']})"
            body_lines.append(
                f"  {r['flap_count']:>5}  {label:<36}  {r['interface']:<16}  "
                f"{r['first_seen']:<19}  {r['last_seen']:<19}"
            )
            hint = _port_macs_digest_hint(r["ip"], r["interface"])
            if hint:
                body_lines.append(hint)
    if flux:
        thr = cfg.get("mac_flux_threshold_24h", 5)
        if rows:
            body_lines.append("")
        body_lines += [
            f"--- MAC flux (non-trunk; >= {thr} distinct MAC(s) in 24h — "
            "possible unauthorized device) ---",
            "  MACS  VLANS  HOSTNAME / IP                         INTERFACE",
            "  ----  -----  ------------------------------------  ----------------",
        ]
        for c in flux:
            label = f"{c['hostname']} ({c['ip']})"
            body_lines.append(
                f"  {c['distinct']:>4}  {c.get('vlans', 1):>5}  "
                f"{label:<36}  {c['interface']:<16}")
            hint = _port_macs_digest_hint(c["ip"], c["interface"])
            if hint:
                body_lines.append(hint)
    body = "\n".join(body_lines)

    log.info("--- Port Flap Digest ---")
    for line in body_lines[1:]:
        log.info(line)

    ok, detail = _send_email(cfg, subject, body)
    if ok:
        log.info("Flap digest email %s", detail)
        if reset:
            conn = _db()
            conn.execute("DELETE FROM port_flaps")
            conn.commit()
            conn.close()
            log.info("Flap counters reset.")
    else:
        log.error("Flap digest email failed: %s — counters NOT reset", detail)


def handle_stp_digest(cfg, detail=False, return_sections=False):
    """Email a single digest covering STP-disabled, STP-mode-mismatched
    devices, and the current root-bridge view per instance.

    Queries current state from the DB (populated by 'monitor stp'). Sends
    nothing when all three sections are empty. Does not modify any state —
    run as often as you want; designed for once-a-day cron.

    detail=True lists agreeing switches in full (the majority group per
    disagreeing instance, and instances where all switches agree). Default
    keeps the email focused on the outliers.

    return_sections=True suppresses the email and instead returns
    (counts, lines) so the weekly health digest can embed this content as
    one section of the single consolidated email (lines omit the standalone
    title; empty lines when there's nothing to report).
    """
    expected = (cfg.get("stp_expected_mode") or "mstp").lower()
    conn = _db()
    # Require sustained-disabled state before listing. stp_disabled_since
    # is set to the timestamp when stp_enabled first transitioned to 0
    # (preserved across consecutive-disabled polls) and cleared back to
    # NULL on 0→1. Filter keeps:
    #   - Devices that have been disabled for ≥ SUSTAINED_MIN (real)
    #   - Devices with NULL since (legacy rows from before tracking, OR
    #     devices that were already disabled when the column was added —
    #     conservative: show them, they're presumed real)
    # Excludes: transient false-positives that just became disabled
    # within the past SUSTAINED_MIN minutes (e.g. NC-BDF on 2026-05-11
    # where a port-flap storm produced an empty SNMP table for one poll).
    STP_DISABLED_SUSTAINED_MIN = 15
    disabled_rows = conn.execute("""
        SELECT ip, hostname, dns_name, model, stp_last_check
        FROM devices
        WHERE stp_enabled = 0
          AND status IN ('active', 'inactive')
          AND (stp_disabled_since IS NULL
               OR stp_disabled_since < datetime('now','localtime',?))
        ORDER BY hostname, ip
    """, (f"-{STP_DISABLED_SUSTAINED_MIN} minutes",)).fetchall()
    mismatch_rows = conn.execute("""
        SELECT ip, hostname, dns_name, model, stp_mode, stp_last_check
        FROM devices
        WHERE stp_mode IS NOT NULL
          AND lower(stp_mode) != ?
          AND status IN ('active', 'inactive')
        ORDER BY hostname, ip
    """, (expected,)).fetchall()
    # Exclude devices with zero data-plane links from the consensus check —
    # they're reachable via management only ('not in service') and can't
    # meaningfully be in or out of STP consensus.
    root_rows = conn.execute("""
        SELECT s.instance, s.root_priority, s.root_mac,
               s.ip, COALESCE(d.hostname, s.ip) AS hostname
        FROM stp_root_state s
        LEFT JOIN devices d ON d.ip = s.ip
        WHERE s.root_mac IS NOT NULL
          AND (d.link_up_count IS NULL OR d.link_up_count > 0)
        ORDER BY s.instance, s.root_priority, s.root_mac, hostname
    """).fetchall()
    # Recent STP port changes (last 7 d) — input for the per-port classifier
    # section. The digest is weekly, so the window matches the cadence: a 24 h
    # window would drop six days of transitions. Hostname pulled here so the
    # render loop doesn't reopen the connection.
    since_7d = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d %H:%M:%S")
    recent_rows = conn.execute("""
        SELECT c.ip, c.interface,
               COUNT(*) AS change_count,
               MAX(c.changed_at) AS latest_change,
               COALESCE(d.hostname, c.ip) AS hostname
        FROM stp_changes c
        LEFT JOIN devices d ON d.ip = c.ip
        WHERE c.changed_at > ?
        GROUP BY c.ip, c.interface
        ORDER BY change_count DESC, latest_change DESC
    """, (since_7d,)).fetchall()
    conn.close()

    # Group root view by (domain, instance) for the digest section + detect
    # disagreements. Multi-site networks legitimately have different roots
    # in different L2 domains; the [stp_domains] config tells us how to
    # split things. Devices unmatched by any prefix are excluded with a
    # count for the operator to act on.
    subnet_map = cfg.get("stp_domains") or {}
    n_unassigned = 0
    by_domain_instance = {}
    for r in root_rows:
        domain = _ip_to_stp_domain(r["ip"], subnet_map)
        if domain is None:
            n_unassigned += 1
            continue
        by_domain_instance.setdefault(
            (domain, r["instance"]), {}
        ).setdefault((r["root_priority"], r["root_mac"]), []).append(
            (r["ip"], r["hostname"]))
    disagree_keys = [k for k, g in by_domain_instance.items() if len(g) > 1]

    n_dis = len(disabled_rows)
    n_mis = len(mismatch_rows)
    n_dq = len(disagree_keys)
    n_recent = len(recent_rows)
    counts = {"disabled": n_dis, "mismatch": n_mis,
              "root_disagree": n_dq, "recent": n_recent}
    if n_dis == 0 and n_mis == 0 and n_dq == 0 and n_recent == 0:
        if return_sections:
            return counts, []
        log.info("No STP configuration issues or recent port changes to report.")
        return

    subject_bits = []
    if n_dis:    subject_bits.append(f"{n_dis} disabled")
    if n_mis:    subject_bits.append(f"{n_mis} mode-mismatch")
    if n_dq:     subject_bits.append(f"{n_dq} root-disagreement")
    if n_recent: subject_bits.append(f"{n_recent} recent port change(s)")
    subject = (f"[netops] STP digest — {', '.join(subject_bits)}"
               if subject_bits else "[netops] STP digest")

    # Count firewalls scoped out of STP so the operator sees them in
    # the digest header even though they don't appear in any section.
    cn = _db()
    fw_excluded = cn.execute(
        "SELECT COUNT(*) FROM devices WHERE status='active' "
        "AND role='firewall'").fetchone()[0]
    cn.close()

    lines = [
        f"STP configuration digest at {datetime.now():%Y-%m-%d %H:%M}",
        f"Expected mode: {expected}",
    ]
    if fw_excluded:
        lines.append(
            f"Firewalls excluded from STP scope: {fw_excluded}  "
            f"(use `netops show devices firewall` to list)")
    lines.append("")
    if n_dis:
        lines.append(f"--- STP disabled ({n_dis}) ---")
        for r in disabled_rows:
            label = r["hostname"] or r["dns_name"] or r["ip"]
            last = r["stp_last_check"] or "(never polled)"
            lines.append(f"  {label} ({r['ip']})  [{r['model'] or '?'}]  last-checked {last}")
        lines.append("")
    if n_mis:
        lines.append(f"--- STP mode mismatch (expected: {expected}) ({n_mis}) ---")
        for r in mismatch_rows:
            label = r["hostname"] or r["dns_name"] or r["ip"]
            last = r["stp_last_check"] or "(never polled)"
            lines.append(f"  {label} ({r['ip']})  running: {r['stp_mode']}  [{r['model'] or '?'}]  last-checked {last}")
        lines.append("")
    # Root-bridge section:
    #   default (no --detail): show only (domain, instance) pairs with
    #     disagreement; within those, list minority groups in full and
    #     collapse the majority group to a count-only line. Pairs where
    #     everyone agrees are omitted since the email only fires when
    #     SOMETHING is wrong.
    #   --detail: show every (domain, instance) pair and every device.
    renderable = [(k, g) for k, g in by_domain_instance.items()
                  if detail or len(g) > 1]
    if renderable:
        section_label = ("Root bridge per STP domain + instance"
                         if subnet_map else "Root bridge per STP instance")
        lines.append(f"--- {section_label} ---")
        # Disagreeing pairs first; agreeing ones only appear in --detail
        # mode and sort last.
        ordered_pairs = sorted(
            renderable, key=lambda kv: (len(kv[1]) == 1, kv[0]))
        for (domain, inst), groups in ordered_pairs:
            label = f"{domain} / {inst}" if subnet_map else inst
            if len(groups) == 1:
                (prio, mac), devs = next(iter(groups.items()))
                lines.append(f"  {label}: all {len(devs)} device(s) agree — "
                             f"priority={prio} mac={mac}")
                if detail:
                    for ip, hostname in devs:
                        lines.append(f"      {hostname} ({ip})")
            else:
                lines.append(f"  {label}: *** DISAGREEMENT — "
                             f"{len(groups)} distinct roots ***")
                ordered = sorted(groups.items(), key=lambda kv: (len(kv[1]), kv[0]))
                majority_idx = len(ordered) - 1
                for i, ((prio, mac), devs) in enumerate(ordered):
                    is_majority = (i == majority_idx)
                    suffix = " agree (pass --detail to list)" if is_majority and not detail else ""
                    lines.append(f"    priority={prio} mac={mac}  "
                                 f"({len(devs)} switch(es)){suffix}")
                    if is_majority and not detail:
                        continue
                    for ip, hostname in devs:
                        lines.append(f"      {hostname} ({ip})")
        lines.append("")

    # Recent port changes — runs investigate_stp_ports per device to get
    # BPDU/edge/link signals, then renders one classifier-tagged line per
    # port. Bounded by the number of unique devices with confirmed STP
    # changes in the last 24 h. Investigation failure → 'unknown' tag,
    # not a hard error.
    if n_recent:
        recent_classifications = {}
        ports_by_device = {}
        for r in recent_rows:
            ports_by_device.setdefault(r["ip"], []).append(r["interface"])
        # Look up the most-recent (old_role, new_role) per port so the
        # classifier can apply the role-change guard for snmp_transient.
        latest_roles_conn = _db()
        latest_roles = {}
        for r in recent_rows:
            lr = latest_roles_conn.execute(
                "SELECT old_role, new_role FROM stp_changes "
                "WHERE ip = ? AND interface = ? "
                "ORDER BY changed_at DESC LIMIT 1",
                (r["ip"], r["interface"])
            ).fetchone()
            if lr:
                latest_roles[(r["ip"], r["interface"])] = (
                    lr["old_role"], lr["new_role"])
        latest_roles_conn.close()

        for ip, ifaces in ports_by_device.items():
            try:
                inv = investigate_stp_ports(ip, ifaces, cfg)
            except Exception as e:
                log.debug("digest investigation failed for %s: %s", ip, e)
                inv = {p: {"ok": False, "parsed": {}} for p in ifaces}
            for iface, info in inv.items():
                pf, tcn_s = _stp_event_context(ip, iface)
                old_r, new_r = latest_roles.get((ip, iface), (None, None))
                label, _conf, _ev = _classify_stp_event(
                    info.get("parsed") or {}, pf, tcn_s,
                    old_role=old_r, new_role=new_r)
                recent_classifications[(ip, iface)] = label

        lines.append(f"--- Recent STP port changes (last 7d) ({n_recent}) ---")
        for r in recent_rows:
            label = recent_classifications.get(
                (r["ip"], r["interface"]), LIKELY_UNCLASSIFIED)
            tag = _STP_CLASSIFIER_SHORT_TAG.get(label, "unknown")
            lines.append(
                f"  [{tag:<14}] {r['hostname']} ({r['ip']}) {r['interface']}  "
                f"{r['change_count']} change(s) — latest @ {r['latest_change']}"
            )
        lines.append("")

    if n_unassigned:
        lines.append(f"--- Unassigned ({n_unassigned} device(s)) ---")
        lines.append("These IPs aren't matched by any [stp_domains] prefix and "
                     "were excluded from the consensus check.")
        lines.append("Add their subnets to [stp_domains] in netops.conf to include them.")
        lines.append("")

    # Reference legend — always rendered so an on-call recipient who isn't
    # familiar with the report can interpret it without out-of-band docs.
    lines.append("--- Reference ---")
    lines.append("")
    lines.append("Section meanings:")
    lines.append("  STP disabled               Switches with stp_enabled=0 for at "
                 "least 15 minutes (sustained-state filter)")
    lines.append("                             avoids transient false positives "
                 "from one-off empty SNMP polls.")
    lines.append("  STP mode mismatch          Running mode (rstp/stp/etc.) "
                 "differs from the expected mode for")
    lines.append("                             this network (configurable, "
                 "default: mstp).")
    lines.append("  STP root-bridge consensus  Per (domain, instance): root-bridge "
                 "MAC agreement vs. disagreement")
    lines.append("                             across switches that should share "
                 "a domain.")
    lines.append("  Recent STP port changes    Confirmed role/state transitions in "
                 "the last 7d, tagged by the")
    lines.append("                             heuristic classifier (see below).")
    lines.append("")
    lines.append("STP port-change classifier tags (in [brackets] on each port-change line):")
    lines.append("  [physical      ]  Physical link instability — check the cable, "
                 "SFP/optic, and port hardware.")
    lines.append("  [one_way       ]  Suspected one-way link — TX active, RX silent. "
                 "Swap optics, verify fiber pair.")
    lines.append("  [snmp_transient]  Likely SNMP/MIB-cache transient — no real STP "
                 "event observed. Operator action")
    lines.append("                    probably not required; flag for tool review "
                 "if the pattern persists.")
    lines.append("  [tc_flush      ]  Topology-change flush — a TCN propagated "
                 "through this bridge. The port itself")
    lines.append("                    is stable; brief learning state is expected "
                 "during MAC-table refresh.")
    lines.append("  [real_event    ]  Genuine STP topology change — investigate "
                 "which device became the new")
    lines.append("                    designated bridge upstream and why.")
    lines.append("  [unknown       ]  Available signals don't fit a known pattern; "
                 "manual review recommended.")

    if return_sections:
        # Drop the standalone "STP configuration digest at ..." title line;
        # the health digest supplies its own section header + timestamp.
        return counts, lines[1:]

    body = "\n".join(lines)
    log.info("--- STP Config Digest ---")
    log.info("Disabled:      %d device(s)", n_dis)
    log.info("Mismatched:    %d device(s) not running %s", n_mis, expected)
    log.info("Root disagree: %d instance(s)", n_dq)
    log.info("Recent ports:  %d (last 7d)", n_recent)
    ok, detail = _send_email(cfg, subject, body)
    if ok:
        log.info("STP digest email %s", detail)
    else:
        log.error("STP digest email failed: %s", detail)


def handle_backup_digest(cfg, detail=False, return_sections=False):
    """Email a weekly backup-status report: never-backed-up devices, stale
    backups, recent config changes, and (with detail=True) a full inventory.

    Always emits the email when there's at least one active device — this
    is a status report, not an issue alert, so recipients expect it on a
    fixed weekly cadence. Stale threshold is [monitor] backup_stale_days
    (default 7).

    return_sections=True suppresses the email and returns (counts, lines)
    so the weekly health digest can embed it as one section (lines omit the
    standalone title).
    """
    stale_days = int(cfg.get("backup_stale_days", 7))
    now_dt = datetime.now()
    now = now_dt.strftime("%Y-%m-%d %H:%M:%S")
    stale_cutoff = (now_dt - timedelta(days=stale_days)).strftime("%Y-%m-%d %H:%M:%S")
    week_cutoff  = (now_dt - timedelta(days=7)).strftime("%Y-%m-%d %H:%M:%S")

    conn = _db()
    rows = conn.execute("""
        SELECT d.ip, d.hostname, d.dns_name, d.model, d.status, d.last_seen,
               (SELECT MAX(backed_up_at) FROM backups WHERE ip = d.ip) AS last_backup,
               (SELECT MAX(backed_up_at) FROM backups
                  WHERE ip = d.ip AND changed = 1)                      AS last_change
        FROM devices d
        WHERE d.status IN ('active', 'inactive')
        ORDER BY d.hostname, d.ip
    """).fetchall()
    non_switch_count = conn.execute(
        "SELECT COUNT(*) FROM devices WHERE status = 'non-switch'"
    ).fetchone()[0]
    unknown_count = conn.execute(
        "SELECT COUNT(*) FROM devices WHERE status = 'unknown'"
    ).fetchone()[0]
    conn.close()
    if not rows:
        if return_sections:
            return {"never": 0, "stale": 0, "changed": 0,
                    "active": 0, "inactive": 0}, []
        log.info("No active/inactive devices — skipping backup digest.")
        return

    active   = [r for r in rows if r["status"] == "active"]
    inactive = [r for r in rows if r["status"] == "inactive"]
    never    = [r for r in rows if not r["last_backup"]]
    # Stale = has a backup but it's older than the cutoff. Inactive devices
    # with no recent backup attempt surface here too.
    stale    = [r for r in rows if r["last_backup"] and r["last_backup"] < stale_cutoff]
    recent_changed = [r for r in rows if r["last_change"] and r["last_change"] >= week_cutoff]

    def _label(r):
        return r["hostname"] or r["dns_name"] or r["ip"]

    def _fmt(ts, when_null="(never)"):
        return ts or when_null

    subject_bits = []
    if never:          subject_bits.append(f"{len(never)} never backed up")
    if stale:          subject_bits.append(f"{len(stale)} stale")
    if recent_changed: subject_bits.append(f"{len(recent_changed)} changed")
    subject = (f"[netops] Weekly backup digest — {', '.join(subject_bits)}"
               if subject_bits else "[netops] Weekly backup digest — all healthy")

    fleet_line = (f"Fleet: {len(active)} active, {len(inactive)} inactive")
    extras = []
    if non_switch_count:
        extras.append(f"{non_switch_count} non-switch")
    if unknown_count:
        extras.append(f"{unknown_count} unknown (no auth)")
    if extras:
        fleet_line += f", {', '.join(extras)} (excluded)"
    fleet_line += f". Stale threshold: {stale_days} day(s)."
    lines = [
        f"Weekly backup digest at {now_dt:%Y-%m-%d %H:%M}",
        fleet_line,
        "",
    ]
    if never:
        lines.append(f"--- Never backed up ({len(never)}) ---")
        for r in never:
            lines.append(f"  {_label(r)} ({r['ip']})  [{r['model'] or '?'}]  "
                         f"status={r['status']}  last-polled={_fmt(r['last_seen'])}")
        lines.append("")
    if stale:
        lines.append(f"--- Stale backup (> {stale_days} days) ({len(stale)}) ---")
        for r in stale:
            lines.append(
                f"  {_label(r)} ({r['ip']})  status={r['status']}  "
                f"last-backup={_fmt(r['last_backup'])}  "
                f"last-polled={_fmt(r['last_seen'])}")
        lines.append("")
    if recent_changed:
        lines.append(f"--- Config changes in the past 7 days ({len(recent_changed)}) ---")
        for r in recent_changed:
            lines.append(f"  {_label(r)} ({r['ip']})  last-change={r['last_change']}")
        lines.append("")

    healthy = [r for r in rows if r not in never and r not in stale]
    if detail:
        lines.append(f"--- Full inventory ({len(rows)}) ---")
        for r in rows:
            lines.append(
                f"  {_label(r)} ({r['ip']})  status={r['status']}  "
                f"backup={_fmt(r['last_backup'])}  "
                f"change={_fmt(r['last_change'], '(never)')}  "
                f"polled={_fmt(r['last_seen'])}")
    else:
        lines.append(f"--- Healthy ({len(healthy)} device(s) with backup within "
                     f"{stale_days} day(s); pass --detail to list) ---")

    if return_sections:
        # Drop the standalone title line; keep the fleet summary + sections.
        return ({"never": len(never), "stale": len(stale),
                 "changed": len(recent_changed),
                 "active": len(active), "inactive": len(inactive)},
                lines[1:])

    body = "\n".join(lines)
    log.info("--- Backup Digest ---")
    log.info("Never backed up: %d", len(never))
    log.info("Stale:           %d (> %d days)", len(stale), stale_days)
    log.info("Changed (7d):    %d", len(recent_changed))
    log.info("Healthy:         %d", len(healthy))
    ok, send_detail = _send_email(cfg, subject, body)
    if ok:
        log.info("Backup digest email %s", send_detail)
    else:
        log.error("Backup digest email failed: %s", send_detail)


def _parse_op_extra(extra_str):
    """Parse 'k=v k=v' extra column into a dict. Numeric values are
    coerced to int; everything else stays string."""
    if not extra_str:
        return {}
    out = {}
    for tok in extra_str.split():
        if "=" not in tok:
            continue
        k, _, v = tok.partition("=")
        try:
            out[k] = int(v)
        except ValueError:
            out[k] = v
    return out


def _grep_log_lines(pattern, since_dt, max_matches=2000):
    """Count + sample log lines matching a regex pattern that fall on or
    after since_dt. Returns (count, sample_first_line_or_None)."""
    log_path = LOG_FILE
    if not os.path.exists(log_path):
        return 0, None
    rx = re.compile(pattern)
    # Date prefix: '2026-05-17 17:25:16' — easy lexical compare
    since_str = since_dt.strftime("%Y-%m-%d %H:%M:%S")
    count = 0
    first = None
    try:
        # Read tail-ish (last 50 MB) to avoid pathological reads
        sz = os.path.getsize(log_path)
        with open(log_path, "rb") as f:
            if sz > 50 * 1024 * 1024:
                f.seek(-50 * 1024 * 1024, os.SEEK_END)
                f.readline()  # discard partial line
            for raw in f:
                try:
                    line = raw.decode("utf-8", errors="replace")
                except Exception:
                    continue
                if line[:19] < since_str:
                    continue
                if rx.search(line):
                    count += 1
                    if first is None:
                        first = line.strip()
                    if count >= max_matches:
                        break
    except OSError:
        pass
    return count, first


def _human_count(n):
    """Compact a large counter for digest tables: 5792940 -> '5.8M', 245066
    -> '245K', 950 -> '950'. Keeps error/discard columns narrow + scannable."""
    try:
        n = int(n)
    except (TypeError, ValueError):
        return str(n)
    if n >= 1_000_000:
        return f"{n/1_000_000:.1f}M"
    if n >= 10_000:
        return f"{n/1000:.0f}K"
    if n >= 1000:
        return f"{n/1000:.1f}K"
    return str(n)


def _flap_perday_sections(cfg, window_start):
    """Build the weekly digest's flap section: ports that flapped >= the
    per-day threshold on their WORST single day in the window (from the
    flap_events timeline), plus the non-trunk MAC-flux signal that used to
    live in the standalone flap digest. Returns (counts, lines)."""
    per_day_min = int(cfg.get("digest_flap_min_per_day", 50) or 0)
    conn = _db()
    conn.row_factory = sqlite3.Row
    rows = conn.execute("""
        SELECT q.ip, COALESCE(d.hostname, q.ip) AS hostname, q.interface,
               MAX(q.cnt) AS peak_day, SUM(q.cnt) AS total,
               MAX(q.last_flap) AS last_flap, COUNT(*) AS active_days
        FROM (
            SELECT ip, interface, date(flapped_at) AS d,
                   COUNT(*) AS cnt, MAX(flapped_at) AS last_flap
            FROM flap_events
            WHERE flapped_at >= ?
            GROUP BY ip, interface, date(flapped_at)
        ) q
        LEFT JOIN devices d ON d.ip = q.ip
        GROUP BY q.ip, q.interface
        HAVING MAX(q.cnt) >= ?
        ORDER BY peak_day DESC, total DESC
    """, (window_start, per_day_min)).fetchall()
    # MAC flux — non-trunk ports seeing many distinct MACs (rogue device).
    flux = _compute_mac_flux(conn,
                             cfg.get("mac_flux_threshold_24h", 5),
                             cfg.get("mac_flux_uplink_ceiling", 24),
                             cfg.get("mac_flux_max_vlans", 2),
                             cfg.get("mac_flux_rotation_max_times_seen", 3),
                             cfg.get("mac_flux_rotation_max_span_sec", 7200))
    excluded = _get_excluded_ports(conn, cfg)
    if excluded:
        flux = [p for p in flux if (p["ip"], p["interface"]) not in excluded]
    conn.close()

    counts = {"ports": len(rows),
              "total_flaps": sum(r["total"] for r in rows),
              "flux": len(flux)}
    lines = []
    if rows:
        lines += [
            f"Ports flapping >= {per_day_min}/day ({len(rows)}):",
            "  PEAK/DAY  TOTAL  HOST (IP)                             INTERFACE         LAST FLAP",
            "  --------  -----  ------------------------------------  ----------------  -------------------",
        ]
        for r in rows:
            label = f"{r['hostname']} ({r['ip']})"
            lines.append(
                f"  {r['peak_day']:>8}  {r['total']:>5}  {label:<36.36}  "
                f"{r['interface']:<16.16}  {r['last_flap']}")
            hint = _port_macs_digest_hint(r["ip"], r["interface"])
            if hint:
                lines.append(hint)
    if flux:
        thr = cfg.get("mac_flux_threshold_24h", 5)
        if rows:
            lines.append("")
        lines += [
            f"MAC flux (non-trunk; >= {thr} distinct MAC(s)/24h — possible "
            "unauthorized device) ({}):".format(len(flux)),
            "  MACS  VLANS  HOST (IP)                             INTERFACE",
            "  ----  -----  ------------------------------------  ----------------",
        ]
        for c in flux:
            label = f"{c['hostname']} ({c['ip']})"
            lines.append(
                f"  {c['distinct']:>4}  {c.get('vlans', 1):>5}  "
                f"{label:<36.36}  {c['interface']:<16.16}")
    return counts, lines, rows


def _port_errors_sections(cfg, window_start, top=25):
    """Build the weekly digest's interface-error section: the worst error /
    discard / CRC ports in the window (aggregated per port+type from the
    port_errors timeline). Returns (counts, lines)."""
    conn = _db()
    conn.row_factory = sqlite3.Row
    rows = conn.execute("""
        SELECT pe.ip, COALESCE(d.hostname, pe.ip) AS hostname, pe.interface,
               pe.error_type, SUM(pe.delta) AS total, COUNT(*) AS events,
               MAX(pe.at) AS last_at
        FROM port_errors pe LEFT JOIN devices d ON d.ip = pe.ip
        WHERE pe.at >= ?
        GROUP BY pe.ip, pe.interface, pe.error_type
        ORDER BY total DESC
        LIMIT ?
    """, (window_start, top)).fetchall()
    total_combos = conn.execute(
        "SELECT COUNT(*) FROM (SELECT 1 FROM port_errors WHERE at >= ? "
        "GROUP BY ip, interface, error_type)", (window_start,)).fetchone()[0]
    # Distinct PORTS with any error in the window (a port can span several
    # error types) — the digest header reports this total, while the table +
    # footnote count port/type combos.
    total_ports = conn.execute(
        "SELECT COUNT(*) FROM (SELECT 1 FROM port_errors WHERE at >= ? "
        "GROUP BY ip, interface)", (window_start,)).fetchone()[0]
    conn.close()

    ports = {(r["ip"], r["interface"]) for r in rows}
    counts = {"ports": len(ports), "total_ports": total_ports,
              "combos": total_combos, "total": sum(r["total"] for r in rows)}
    lines = []
    if rows:
        shown = (f"top {len(rows)} of {total_combos}"
                 if total_combos > len(rows) else f"{len(rows)}")
        lines += [
            f"Worst error ports ({shown} port+type combos):",
            "  ERRORS  HOST (IP)                             INTERFACE         TYPE          LAST SEEN",
            "  ------  ------------------------------------  ----------------  ------------  -------------------",
        ]
        for r in rows:
            label = f"{r['hostname']} ({r['ip']})"
            lines.append(
                f"  {_human_count(r['total']):>6}  {label:<36.36}  "
                f"{r['interface']:<16.16}  {r['error_type']:<12}  {r['last_at']}")
    return counts, lines, rows


def _render_digest_html(d):
    """Render the weekly health digest as a scannable, severity-driven HTML
    email. Design goal is COGNITIVE LOAD, not decoration: a one-glance status
    dashboard up top, then ONLY the categories that need attention expand into
    bordered cards (worst first); everything that's fine collapses to a single
    green 'all clear' strip. So a clean week is a few lines and a bad week puts
    the 1-2 things that matter front-and-centre. All styles are inlined for
    Gmail/Outlook (which strip <style> blocks). `d` is the data bundle assembled
    by handle_health_digest."""
    esc = html.escape
    GREEN_BG, GREEN_FG = "#e8f5e9", "#2e7d32"
    AMBER_BG, AMBER_FG = "#fff8e1", "#ef6c00"
    RED_BG,   RED_FG   = "#fdecea", "#c62828"
    GREY = "#5f6b7a"

    def tile(icon, value, label, bg, fg):
        return (
            f'<div style="display:inline-block;width:116px;vertical-align:top;'
            f'background:{bg};border-radius:10px;padding:12px 6px;margin:4px;'
            f'text-align:center;">'
            f'<div style="font-size:22px;line-height:1;">{icon}</div>'
            f'<div style="font-size:21px;font-weight:800;color:{fg};margin-top:5px;">{value}</div>'
            f'<div style="font-size:11px;color:{GREY};text-transform:uppercase;'
            f'letter-spacing:.5px;margin-top:3px;">{esc(label)}</div></div>')

    def card(icon, title, badge, accent, inner):
        return (
            f'<div style="background:#ffffff;border:1px solid #e3e6ea;'
            f'border-left:5px solid {accent};border-radius:10px;padding:14px 16px;'
            f'margin:14px 0;">'
            f'<div style="font-size:15px;font-weight:700;color:#1a2330;margin-bottom:10px;">'
            f'{icon}&nbsp;{esc(title)} '
            f'<span style="color:{accent};font-weight:800;">{badge}</span></div>'
            f'{inner}</div>')

    def table(headers, rows, aligns=None):
        aligns = aligns or ["left"] * len(headers)
        out = ['<table width="100%" cellspacing="0" cellpadding="0" '
               'style="border-collapse:collapse;font-size:13px;color:#2a2a2a;">']
        out.append('<tr>')
        for h, a in zip(headers, aligns):
            out.append(f'<th style="text-align:{a};padding:4px 8px;color:{GREY};'
                       f'font-size:11px;text-transform:uppercase;letter-spacing:.4px;'
                       f'border-bottom:2px solid #eceff1;">{esc(h)}</th>')
        out.append('</tr>')
        for i, cells in enumerate(rows):
            bg = "#ffffff" if i % 2 == 0 else "#f7f9fb"
            out.append(f'<tr style="background:{bg};">')
            for c, a in zip(cells, aligns):
                out.append(f'<td style="text-align:{a};padding:5px 8px;'
                           f'border-bottom:1px solid #f0f2f4;">{c}</td>')
            out.append('</tr>')
        out.append('</table>')
        return "".join(out)

    # --- status per category -> (value, severity) -----------------------
    fl, er = d["flap_counts"], d["err_counts"]
    st, bk = d["stp_counts"], d["bk_counts"]
    flagged = d["currently_flagged"]
    dark = d.get("recently_dark") or []
    stp_issues = st["disabled"] + st["mismatch"] + st["root_disagree"]
    bk_gaps = bk["never"] + bk["stale"]
    cats = {
        "flaps": (fl["ports"], "amber"),
        "errors": (er["total_ports"], "red"),
        "stp": (stp_issues, "red"),
        "backups": (bk_gaps, "amber"),
        "reach": (len(dark) or len(flagged), "red"),
    }
    sev_bg = {"amber": AMBER_BG, "red": RED_BG}
    sev_fg = {"amber": AMBER_FG, "red": RED_FG}

    # --- dashboard tiles -------------------------------------------------
    tspec = [("flaps", "⚡", "flap ports"), ("errors", "\U0001F50C", "err ports"),
             ("stp", "\U0001F332", "STP"), ("backups", "\U0001F4BE", "backups"),
             ("reach", "\U0001F4E1", "unreachable")]
    tiles = []
    for key, icon, label in tspec:
        val, sev = cats[key]
        if val:
            tiles.append(tile(icon, val, label, sev_bg[sev], sev_fg[sev]))
        else:
            tiles.append(tile(icon, "✓", label, GREEN_BG, GREEN_FG))

    # --- attention cards (worst first) ----------------------------------
    cards = []
    if er["total_ports"]:
        erows = d["err_rows"][:25]
        rows = [[esc(f"{r['hostname']} ({r['ip']})"),
                 f'<b>{esc(r["interface"])}</b>', esc(r["error_type"]),
                 f'<b style="color:{RED_FG}">{esc(_human_count(r["total"]))}</b>']
                for r in erows]
        shown = len(erows)
        more = er["combos"] - shown
        extra = (f'<div style="font-size:12px;color:{GREY};margin-top:8px;">'
                 f'{more} more port/type combos (full list in the netops console)</div>'
                 if more > 0 else "")
        badge = (f'{er["total_ports"]} ports <span style="color:{GREY};'
                 f'font-weight:600;">&middot; top {shown} by error count</span>')
        cards.append(card("\U0001F50C", "Interface errors", badge, RED_FG,
                          table(["Switch", "Port", "Type", "Errors"],
                                rows, ["left", "left", "left", "right"]) + extra))
    if fl["ports"]:
        frows = d["flap_rows"][:25]
        rows = [[esc(f"{r['hostname']} ({r['ip']})"),
                 f'<b>{esc(r["interface"])}</b>',
                 f'<b style="color:{AMBER_FG}">{r["peak_day"]}</b>',
                 str(r["total"]), esc(str(r["last_flap"])[:16])]
                for r in frows]
        shown = len(frows)
        fmore = fl["ports"] - shown
        more_note = (f'<div style="font-size:12px;color:{GREY};margin-top:8px;">'
                     f'{fmore} more port(s) (full list in the netops console)</div>'
                     if fmore > 0 else "")
        flux_note = (f'<div style="font-size:12px;color:{AMBER_FG};margin-top:8px;">'
                     f'⚠ {fl["flux"]} non-trunk MAC-flux port(s) — possible '
                     f'unauthorized device (full list in the netops console)</div>'
                     if fl["flux"] else "")
        badge = f'{fl["ports"]} ports' + (
            f' <span style="color:{GREY};font-weight:600;">&middot; top {shown} '
            f'by peak/day</span>' if fmore > 0 else "")
        cards.append(card("⚡", f"Port flaps (≥{d['per_day_min']}/day)", badge, AMBER_FG,
                          table(["Switch", "Port", "Peak/day", "Total", "Last flap"],
                                rows, ["left", "left", "right", "right", "left"])
                          + more_note + flux_note))
    if stp_issues:
        badge = ", ".join(b for b in [
            f'{st["disabled"]} disabled' if st["disabled"] else "",
            f'{st["mismatch"]} mismatch' if st["mismatch"] else "",
            f'{st["root_disagree"]} root-disagree' if st["root_disagree"] else ""] if b)
        pre = ('<pre style="margin:0;font-size:12px;color:#33404f;white-space:pre-wrap;'
               'font-family:ui-monospace,Menlo,Consolas,monospace;">'
               + esc("\n".join(d["stp_lines"])) + '</pre>')
        cards.append(card("\U0001F332", "Spanning tree", badge, RED_FG, pre))
    if dark:
        rows = [[esc(r["hostname"] or "?"), esc(r["ip"]),
                 esc(str(r["last_seen"])[:16])] for r in dark[:25]]
        inner = table(["Host", "IP", "Last seen"], rows)
        more = len(dark) - len(dark[:25])
        if more > 0:
            inner += (f'<div style="font-size:12px;color:{GREY};margin-top:8px;">'
                      f'{more} more (full list in the netops console)</div>')
        inner += (f'<div style="font-size:12px;color:{GREY};margin-top:8px;">'
                  f'{d["inactive_total"]} inactive / out of service total '
                  f'&middot; {len(flagged)} still carrying a live alert</div>')
        cards.append(card("\U0001F4E1", "Reachability — went dark this week",
                          f'{len(dark)} device(s)', RED_FG, inner))
    elif len(flagged):
        rows = [[esc(r["hostname"] or "?"), esc(r["ip"]),
                 esc(str(r["unreachable_alerted_at"])[:16])] for r in flagged[:12]]
        cards.append(card("\U0001F4E1", "Reachability",
                          f'{len(flagged)} flagged unreachable', RED_FG,
                          table(["Host", "IP", "Alerted since"], rows)))
    if bk_gaps:
        pre = ('<pre style="margin:0;font-size:12px;color:#33404f;white-space:pre-wrap;'
               'font-family:ui-monospace,Menlo,Consolas,monospace;">'
               + esc("\n".join(d["bk_lines"])) + '</pre>')
        cards.append(card("\U0001F4BE", "Backups",
                          f'{bk["never"]} never, {bk["stale"]} stale', AMBER_FG, pre))

    # --- all-clear strip -------------------------------------------------
    clear_labels = {"flaps": "port flaps", "errors": "interface errors",
                    "stp": "spanning tree", "backups": "backups",
                    "reach": "reachability"}
    clear = [clear_labels[k] for k, (v, _) in cats.items() if not v]
    if not d["traceback_count"]:
        clear.append("no log errors")
    all_clear = ""
    if clear:
        all_clear = (f'<div style="background:{GREEN_BG};border-radius:10px;'
                     f'padding:11px 15px;margin:14px 0;font-size:13px;color:{GREEN_FG};">'
                     f'✓ All clear: {esc(", ".join(clear))}</div>')

    # --- compact system footer ------------------------------------------
    tb = (f'<span style="color:{RED_FG}">{d["traceback_count"]} traceback(s)</span>'
          if d["traceback_count"] else "no tracebacks")
    sysline = (f'<b>System</b> &nbsp;&middot;&nbsp; monitor ticks {d["ticks_pct"]:.0f}% '
               f'&nbsp;&middot;&nbsp; backups {d["backup_ok"]}/{d["backup_total"]} '
               f'&nbsp;&middot;&nbsp; db {esc(d["db_size"])} '
               f'&nbsp;&middot;&nbsp; disk {d["disk_free_pct"]:.0f}% free '
               f'&nbsp;&middot;&nbsp; {tb}')
    system = (f'<div style="background:#ffffff;border:1px solid #eceff1;'
              f'border-radius:10px;padding:11px 16px;margin:14px 0;font-size:12px;'
              f'color:{GREY};">{sysline}</div>')

    active = d["fleet"].get("active", 0)
    # Optional client logo (embedded CID part — see _send_email). Only emitted
    # when a readable logo is configured; otherwise no <img> so there's no
    # broken-image box.
    logo_html = ('<img src="cid:netops-logo" alt="" style="max-height:52px;'
                 'max-width:240px;margin-bottom:10px;display:block;border:0;">'
                 if d.get("has_logo") else "")
    header = (f'<div style="padding:6px 4px 14px;">'
              + logo_html
              + f'<div style="font-size:21px;font-weight:800;color:#1a2330;">'
              f'Weekly Health Digest</div>'
              f'<div style="font-size:13px;color:{GREY};margin-top:3px;">'
              f'{esc(d["week_label"])} &nbsp;&middot;&nbsp; {d["fleet_total"]} devices, '
              f'{active} active</div></div>')
    footer = (f'<div style="font-size:11px;color:#9aa5b1;text-align:center;'
              f'padding:10px 4px;">netops v{__version__} &nbsp;&middot;&nbsp; '
              f'worst offenders first &nbsp;&middot;&nbsp; full per-section detail '
              f'in the netops console</div>')

    return (
        '<!DOCTYPE html><html><body style="margin:0;padding:0;background:#eef0f3;">'
        '<div style="max-width:680px;margin:0 auto;padding:16px;'
        'font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Arial,sans-serif;'
        'background:#eef0f3;">'
        + header
        + '<div style="text-align:center;">' + "".join(tiles) + '</div>'
        + "".join(cards)
        + all_clear
        + system
        + footer
        + '</div></body></html>')


def handle_health_digest(cfg, retain_days=35):
    """Email a weekly enterprise health digest.

    Aggregates op_events from the past 7 days, fleet inventory, backup
    health, STP/flap activity, reachability alerts, system-level metrics,
    and operator action items. Prunes op_events rows older than
    retain_days at the end of the run.
    """
    now_dt = datetime.now()
    week_ago_dt = now_dt - timedelta(days=7)
    now_str = now_dt.strftime("%Y-%m-%d %H:%M:%S")
    week_ago_str = week_ago_dt.strftime("%Y-%m-%d %H:%M:%S")
    expected_ticks = 7 * 24 * 60   # one per minute

    conn = _db()
    conn.row_factory = sqlite3.Row

    # --- Section 1: Fleet inventory --------------------------------------
    fleet = {r["status"]: r["n"] for r in conn.execute(
        "SELECT status, COUNT(*) AS n FROM devices GROUP BY status")}
    fleet_total = sum(fleet.values())
    new_this_week = conn.execute(
        "SELECT COUNT(*) FROM devices WHERE first_seen >= ?",
        (week_ago_str,)).fetchone()[0]
    currently_flagged = list(conn.execute(
        "SELECT ip, hostname, snmp_consecutive_fails, unreachable_alerted_at "
        "FROM devices WHERE unreachable_alerted_at IS NOT NULL "
        "ORDER BY unreachable_alerted_at"))
    # The unreachable-alert flag alone UNDERCOUNTS what's actually down: a
    # device demoted to 'inactive' after sustained failure drops out of it.
    # Surface the inactive/unreachable population, and especially those that
    # went dark THIS WEEK (still-recent last_seen but now inactive) — that
    # turns a silent batch outage into a visible reachability signal instead
    # of leaving it buried in stale backups.
    recently_dark = list(conn.execute(
        "SELECT ip, hostname, last_seen FROM devices "
        "WHERE status='inactive' AND last_seen >= ? "
        "ORDER BY last_seen DESC", (week_ago_str,)))
    inactive_total = sum(1 for _ in conn.execute(
        "SELECT 1 FROM devices WHERE status='inactive'"))

    # --- Section 2: Backups ---------------------------------------------
    backup_rows = list(conn.execute(
        "SELECT success, duration_ms FROM op_events "
        "WHERE op_type='backup_device' AND started_at >= ?", (week_ago_str,)))
    backup_total = len(backup_rows)
    backup_ok = sum(1 for r in backup_rows if r["success"])
    backup_fail = backup_total - backup_ok
    bk_durs = [r["duration_ms"] for r in backup_rows if r["success"]]
    bk_avg = int(sum(bk_durs) / len(bk_durs)) if bk_durs else 0
    bk_min = min(bk_durs) if bk_durs else 0
    bk_max = max(bk_durs) if bk_durs else 0
    config_changes_week = conn.execute(
        "SELECT COUNT(*) FROM backups WHERE backed_up_at >= ? AND changed = 1",
        (week_ago_str,)).fetchone()[0]
    stale_backups = list(conn.execute("""
        SELECT d.ip, d.hostname,
               (SELECT MAX(backed_up_at) FROM backups b WHERE b.ip = d.ip) AS last
        FROM devices d
        WHERE d.status = 'active'
        ORDER BY d.hostname
    """))
    stale_backups = [r for r in stale_backups
                     if r["last"] is None or r["last"] < week_ago_str]

    # --- Section 3: STP / flap activity ---------------------------------
    stp_changes_week = conn.execute(
        "SELECT COUNT(*) FROM stp_changes WHERE changed_at >= ?",
        (week_ago_str,)).fetchone()[0]
    root_changes_week = conn.execute(
        "SELECT COUNT(*) FROM stp_root_changes WHERE changed_at >= ?",
        (week_ago_str,)).fetchone()[0]
    top_changers = list(conn.execute("""
        SELECT hostname, interface, COUNT(*) AS n
        FROM stp_changes WHERE changed_at >= ?
        GROUP BY hostname, interface ORDER BY n DESC LIMIT 5
    """, (week_ago_str,)))
    # Flap events: sum extra.flap_events from monitor_stp_tick rows
    flap_total_week = 0
    tick_rows = list(conn.execute(
        "SELECT duration_ms, extra FROM op_events "
        "WHERE op_type='monitor_stp_tick' AND started_at >= ?",
        (week_ago_str,)))
    for r in tick_rows:
        fe = _parse_op_extra(r["extra"]).get("flap_events", 0)
        if isinstance(fe, int):
            flap_total_week += fe
    top_flappers = list(conn.execute("""
        SELECT hostname, interface, flap_count, last_seen FROM port_flaps
        ORDER BY flap_count DESC LIMIT 5
    """))

    # --- Section 4: Reachability events (parse log) ---------------------
    unreach_count, unreach_sample = _grep_log_lines(
        r"Unreachable: \d+ device\(s\) crossed threshold", week_ago_dt)
    recovery_count, recovery_sample = _grep_log_lines(
        r"Recovered: \d+ device\(s\)", week_ago_dt)

    # --- Section 5: System performance ----------------------------------
    ticks_total = len(tick_rows)
    tick_durs = [r["duration_ms"] for r in tick_rows]
    tick_avg = int(sum(tick_durs) / len(tick_durs)) if tick_durs else 0
    tick_min = min(tick_durs) if tick_durs else 0
    tick_max = max(tick_durs) if tick_durs else 0
    # SNMP rollup: sum snmp_count, weighted mean of snmp_avg_ms, min/max envelopes
    snmp_count_total = 0
    snmp_failed_total = 0
    snmp_min_ms = None
    snmp_max_ms = 0
    snmp_weighted_sum_ms = 0
    for r in tick_rows:
        ex = _parse_op_extra(r["extra"])
        sc = ex.get("snmp_count", 0)
        if isinstance(sc, int) and sc > 0:
            snmp_count_total += sc
            sf = ex.get("snmp_failed", 0)
            if isinstance(sf, int):
                snmp_failed_total += sf
            mn = ex.get("snmp_min_ms")
            mx = ex.get("snmp_max_ms")
            av = ex.get("snmp_avg_ms")
            if isinstance(mn, int):
                snmp_min_ms = mn if snmp_min_ms is None else min(snmp_min_ms, mn)
            if isinstance(mx, int):
                snmp_max_ms = max(snmp_max_ms, mx)
            if isinstance(av, int):
                snmp_weighted_sum_ms += av * sc
    snmp_avg_ms = int(snmp_weighted_sum_ms / snmp_count_total) if snmp_count_total else 0
    # SSH connect events by reason
    ssh_by_reason = {}
    for r in conn.execute(
        "SELECT reason, success, duration_ms FROM op_events "
        "WHERE op_type IN ('ssh_connect', 'telnet_connect') "
        "  AND started_at >= ?", (week_ago_str,)
    ):
        rs = r["reason"] or "(unknown)"
        bucket = ssh_by_reason.setdefault(rs,
                    {"total": 0, "ok": 0, "fail": 0, "durs": []})
        bucket["total"] += 1
        if r["success"]:
            bucket["ok"] += 1
            bucket["durs"].append(r["duration_ms"])
        else:
            bucket["fail"] += 1
    # Cron-skip count
    skip_count, _ = _grep_log_lines(
        r"holds the .ssh. lock", week_ago_dt)
    # Email path: count send-success/-failure log lines
    email_ok, _ = _grep_log_lines(r"email sent", week_ago_dt)
    email_fail, _ = _grep_log_lines(
        r"(?:digest|alert) email failed", week_ago_dt)
    # Tracebacks
    traceback_count, traceback_sample = _grep_log_lines(
        r"^Traceback \(most recent call last\)", week_ago_dt)

    # --- Section 6: netops-box health -----------------------------------
    db_size = os.path.getsize(DB_FILE) if os.path.exists(DB_FILE) else 0
    log_path = LOG_FILE
    log_size = os.path.getsize(log_path) if os.path.exists(log_path) else 0
    try:
        du = shutil.disk_usage(STATE_DIR)
        disk_free_pct = (du.free * 100.0) / du.total
        disk_free_gb = du.free / (1024 ** 3)
    except Exception:
        disk_free_pct, disk_free_gb = 0.0, 0.0

    # --- Section 7: Operator action items -------------------------------
    no_creds = list(conn.execute(
        "SELECT ip, hostname FROM devices "
        "WHERE status='active' AND (username IS NULL OR password_hash IS NULL) "
        "ORDER BY hostname LIMIT 10"))
    coverage = _lldp_coverage_report(conn)
    no_lldp_count = coverage["total"] - coverage["on_map"]
    # Duplicate IPs (secondary switch SVIs) that should be reachable on a
    # healthy network but failed their last reconcile probe.
    unreachable_dupes = list(conn.execute(
        "SELECT ip, hostname, duplicate_of, last_seen FROM devices "
        "WHERE status='duplicate' AND COALESCE(ssh_open,0)=0 "
        "AND COALESCE(telnet_open,0)=0 ORDER BY duplicate_of, ip"))

    conn.close()

    # ---- Render -----------------------------------------------------------
    def _fmt_size(n):
        for unit in ("B", "KB", "MB", "GB"):
            if n < 1024 or unit == "GB":
                return f"{n:.1f} {unit}" if unit != "B" else f"{int(n)} B"
            n /= 1024

    # --- Folded-in detail sections. The standalone flap/STP/backup digests
    # are retired; this consolidated email carries their content as sections.
    # Each builder opens its own short-lived read connection (the section
    # connection above is already closed). ---
    flap_counts, flap_lines, flap_rows = _flap_perday_sections(cfg, week_ago_str)
    err_counts, err_lines, err_rows = _port_errors_sections(cfg, week_ago_str)
    stp_counts, stp_lines = handle_stp_digest(cfg, return_sections=True)
    bk_counts, bk_lines = handle_backup_digest(cfg, return_sections=True)
    per_day_min = int(cfg.get("digest_flap_min_per_day", 50) or 0)
    stp_issue_total = (stp_counts["disabled"] + stp_counts["mismatch"]
                       + stp_counts["root_disagree"])

    subject_bits = []
    if flap_counts["ports"]:      subject_bits.append(f"{flap_counts['ports']} flapping")
    if err_counts["total_ports"]: subject_bits.append(f"{err_counts['total_ports']} error ports")
    if recently_dark:             subject_bits.append(f"{len(recently_dark)} went dark")
    elif currently_flagged:       subject_bits.append(f"{len(currently_flagged)} unreachable")
    if stp_issue_total:           subject_bits.append(f"{stp_issue_total} STP issue(s)")
    if bk_counts["never"] or bk_counts["stale"]:
        subject_bits.append(f"{bk_counts['never'] + bk_counts['stale']} backup gap(s)")
    if unreachable_dupes:     subject_bits.append(f"{len(unreachable_dupes)} dup IPs down")
    if traceback_count:       subject_bits.append(f"{traceback_count} tracebacks")
    subject = (f"[netops] Weekly health digest — {', '.join(subject_bits)}"
               if subject_bits else "[netops] Weekly health digest — all green")

    # AT A GLANCE: one verdict line per category so a recipient can triage in
    # seconds and only scroll into a section that's flagged.
    def _mark(bad):
        return "[!] " if bad else "[ok]"

    lines = [
        f"Weekly health digest — {week_ago_dt:%Y-%m-%d} to {now_dt:%Y-%m-%d %H:%M}",
        "",
        f"=== AT A GLANCE ===  week {week_ago_dt:%Y-%m-%d} -> {now_dt:%Y-%m-%d}",
        f"  {_mark(flap_counts['ports'])} {flap_counts['ports']} port(s) flapping "
        f">= {per_day_min}/day"
        + (f", {flap_counts['flux']} MAC-flux port(s)" if flap_counts['flux'] else ""),
        f"  {_mark(err_counts['ports'])} {err_counts['ports']} port(s) with errors "
        f"(CRC / discards / in-out errors)",
        f"  {_mark(stp_issue_total)} STP: {stp_counts['disabled']} disabled, "
        f"{stp_counts['mismatch']} mode-mismatch, {stp_counts['root_disagree']} "
        f"root-disagreement"
        + (f"; {stp_counts['recent']} recent change(s)" if stp_counts['recent'] else ""),
        f"  {_mark(bk_counts['never'] or bk_counts['stale'])} backups: "
        f"{bk_counts['never']} never, {bk_counts['stale']} stale, "
        f"{bk_counts['changed']} changed this week",
        f"  {_mark(recently_dark or currently_flagged)} reachability: "
        + (f"{len(recently_dark)} went dark this week ({inactive_total} inactive total)"
           if recently_dark
           else (f"{len(currently_flagged)} device(s) flagged down" if currently_flagged
                 else "all devices up")),
        f"  {_mark(traceback_count)} "
        + (f"{traceback_count} traceback(s) in the log" if traceback_count
           else "no tracebacks this week"),
        "",
        "=== Fleet ===",
        f"  total devices:           {fleet_total} "
        f"({fleet.get('active', 0)} active, {fleet.get('inactive', 0)} inactive, "
        f"{fleet.get('failed', 0)} failed)",
        f"  new this week:           {new_this_week}",
        f"  currently flagged down:  {len(currently_flagged)}",
    ]
    for r in currently_flagged[:8]:
        lines.append(f"    - {r['hostname'] or '?'} ({r['ip']})  "
                     f"alerted_at={r['unreachable_alerted_at']}  "
                     f"fails={r['snmp_consecutive_fails']}")
    if len(currently_flagged) > 8:
        lines.append(f"    ... and {len(currently_flagged) - 8} more")

    # === Port flaps (>= N/day) ===  (folded from the retired flap digest)
    lines += ["", f"=== Port flaps (>= {per_day_min}/day) ==="]
    lines += flap_lines or [
        f"  no port flapped >= {per_day_min} times in a single day this week"]

    # === Port errors ===  (interface error/discard/CRC counters; SNMP)
    lines += ["", "=== Port errors ==="]
    lines += err_lines or [
        "  no interface errors recorded this week "
        "(SNMP-harvested by monitor topology; empty on no-SNMP sites)"]

    # === STP ===  (folded from the retired STP digest)
    lines += ["", "=== STP ===",
              f"  weekly activity: {stp_changes_week} confirmed change(s), "
              f"{root_changes_week} root-bridge change(s), "
              f"{flap_total_week} flap event(s) across all ticks"]
    lines += stp_lines or ["  no STP config issues or recent port changes"]

    # === Backups ===  (rollup + folded detail from the retired backup digest)
    lines += [
        "",
        "=== Backups ===",
        f"  attempts this week:      {backup_total} "
        f"({backup_ok} ok, {backup_fail} failed)",
        f"  per-device duration:     avg {bk_avg/1000:.1f}s "
        f"(min {bk_min/1000:.1f}s, max {bk_max/1000:.1f}s)",
        f"  config changes detected: {config_changes_week}",
    ]
    lines += bk_lines or ["  (no active/inactive devices)"]

    lines += [
        "",
        "=== Reachability ===",
        f"  flagged unreachable (live alert): {len(currently_flagged)}",
        f"  inactive / out of service:        {inactive_total}",
        f"  WENT DARK THIS WEEK:              {len(recently_dark)} "
        f"(now inactive, last seen within 7d)",
    ]
    for r in recently_dark[:12]:
        lines.append(f"    - {r['hostname'] or '?'} ({r['ip']})  "
                     f"last_seen={r['last_seen']}")
    if len(recently_dark) > 12:
        lines.append(f"    ... and {len(recently_dark) - 12} more")
    lines += [
        f"  unreachable-alert fires (log): {unreach_count}",
        f"  recovery emails:               {recovery_count}",
    ]
    if unreach_sample:
        lines.append(f"  most recent unreachable: {unreach_sample[:160]}")

    lines += [
        "",
        "=== System performance ===",
        f"  monitor-stp ticks:       {ticks_total} of expected {expected_ticks} "
        f"({100.0 * ticks_total / expected_ticks if expected_ticks else 0:.1f}%)",
        f"  tick duration:           avg {tick_avg/1000:.1f}s "
        f"(min {tick_min/1000:.1f}s, max {tick_max/1000:.1f}s)",
        f"  SNMP queries:            {snmp_count_total:,} "
        f"({snmp_failed_total} failed)",
    ]
    if snmp_count_total:
        lines.append(f"  SNMP latency:            avg {snmp_avg_ms} ms "
                     f"(min {snmp_min_ms} ms, max {snmp_max_ms} ms)")
    if ssh_by_reason:
        lines.append("  SSH/telnet attempts by reason:")
        for rs, b in sorted(ssh_by_reason.items(),
                            key=lambda kv: -kv[1]["total"]):
            avg = (sum(b["durs"]) / len(b["durs"]) / 1000) if b["durs"] else 0
            lines.append(f"    {rs:30}  total={b['total']:>4d} "
                         f"ok={b['ok']:>4d} fail={b['fail']:>3d}  "
                         f"avg-on-success {avg:.1f}s")
    lines += [
        f"  cron-skips (flock):      {skip_count}",
        f"  email sends (ok / fail): {email_ok} / {email_fail}",
        f"  log tracebacks:          {traceback_count}",
    ]
    if traceback_sample and traceback_count:
        lines.append(f"    most recent: {traceback_sample[:160]}")

    lines += [
        "",
        "=== netops-box health ===",
        f"  netops.db size:          {_fmt_size(db_size)}",
        f"  netops.log size:         {_fmt_size(log_size)}",
        f"  filesystem free:         {disk_free_gb:.1f} GB "
        f"({disk_free_pct:.1f}%)",
    ]

    lines += [
        "",
        "=== Operator action items ===",
    ]
    if no_creds:
        lines.append(f"  devices without stored creds ({len(no_creds)}):")
        for r in no_creds[:5]:
            lines.append(f"    - {r['hostname'] or '?'} ({r['ip']})")
        if len(no_creds) > 5:
            lines.append(f"    ... and {len(no_creds) - 5} more")
    if no_lldp_count:
        lines.extend(_format_coverage_lines(coverage, indent="  ",
                                            per_bucket=10))
    if unreachable_dupes:
        lines.append(f"  duplicate IPs unreachable ({len(unreachable_dupes)}) "
                     f"— secondary switch IPs that should be up but aren't:")
        for r in unreachable_dupes[:10]:
            lines.append(f"    - {r['hostname'] or '?'} ({r['ip']})  "
                         f"dup of {r['duplicate_of']}  "
                         f"last_seen={r['last_seen'] or '(never)'}")
        if len(unreachable_dupes) > 10:
            lines.append(f"    ... and {len(unreachable_dupes) - 10} more")
    if currently_flagged:
        lines.append(f"  unreachable_alerted_at stuck on "
                     f"{len(currently_flagged)} device(s) — "
                     f"investigate or clear manually if resolved")
    if not (no_creds or no_lldp_count or currently_flagged or unreachable_dupes):
        lines.append("  nothing requires attention 🎉")

    body = "\n".join(lines)
    log.info("--- Weekly health digest ---")
    log.info("Fleet:        %d total, %d active", fleet_total, fleet.get("active", 0))
    log.info("Backups:      %d/%d ok", backup_ok, backup_total)
    log.info("STP changes:  %d (confirmed, last 7d)", stp_changes_week)
    log.info("Tracebacks:   %d", traceback_count)
    # Rich HTML rendering (multipart; plain `body` stays the fallback). Best-
    # effort — any render error falls back to plain-text only, never blocks the
    # send.
    html_body = None
    try:
        html_body = _render_digest_html({
            "week_label": f"{week_ago_dt:%Y-%m-%d} → {now_dt:%Y-%m-%d}",
            "per_day_min": per_day_min,
            "flap_counts": flap_counts, "err_counts": err_counts,
            "stp_counts": stp_counts, "bk_counts": bk_counts,
            "flap_rows": flap_rows, "err_rows": err_rows,
            "stp_lines": stp_lines, "bk_lines": bk_lines,
            "currently_flagged": currently_flagged,
            "recently_dark": recently_dark, "inactive_total": inactive_total,
            "fleet": fleet, "fleet_total": fleet_total,
            "ticks_pct": (100.0 * ticks_total / expected_ticks) if expected_ticks else 0.0,
            "backup_ok": backup_ok, "backup_total": backup_total,
            "db_size": _fmt_size(db_size), "disk_free_pct": disk_free_pct,
            "traceback_count": traceback_count,
            "has_logo": bool(_prepare_email_logo(cfg)),
        })
    except Exception as e:
        log.debug("digest HTML render failed (%s) — sending plain-text only", e)
    ok, detail = _send_email(cfg, subject, body, html_body=html_body)
    if ok:
        log.info("Weekly health digest email %s", detail)
    else:
        log.error("Weekly health digest email failed: %s", detail)

    # --- Retention: prune old op_events rows -----------------------------
    cutoff_dt = now_dt - timedelta(days=retain_days)
    cutoff_str = cutoff_dt.strftime("%Y-%m-%d %H:%M:%S")
    # FDB (port_macs) + ARP (ip_arp) keep their OWN longer, configurable window
    # — deduplicated location history worth retaining for device forensics.
    fdb_days = int(cfg.get("fdb_history_days", 365) or 365)
    fdb_cutoff_str = (now_dt - timedelta(days=fdb_days)).strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    deleted = conn.execute(
        "DELETE FROM op_events WHERE started_at < ?", (cutoff_str,))
    n_deleted = deleted.rowcount
    # port_macs FDB history keeps its OWN fdb_history_days window (default 1yr).
    # Also drop long-cleared flux-alert episodes (op_events window) so that
    # table stays bounded (active / not-yet-cleared rows are always kept).
    try:
        pm = conn.execute(
            "DELETE FROM port_macs WHERE last_seen < ?", (fdb_cutoff_str,))
        n_pm = pm.rowcount
        conn.execute(
            "DELETE FROM mac_flux_alerts "
            "WHERE cleared_at IS NOT NULL AND cleared_at < ?", (cutoff_str,))
    except sqlite3.OperationalError:
        n_pm = 0                        # pre-3.8.0 DB without the tables
    # 3.8.31 ID-loop tables. ip_arp keeps the fdb_history_days window (it's
    # device-location forensics like port_macs); dns_cache has its own much
    # shorter window (1d) since PTR records are short-lived and the cache only
    # exists to skip redundant lookups within a render and the next few.
    try:
        ia = conn.execute(
            "DELETE FROM ip_arp WHERE last_seen < ?", (fdb_cutoff_str,))
        n_ia = ia.rowcount
    except sqlite3.OperationalError:
        n_ia = 0                        # pre-3.8.31 DB without ip_arp
    # flap_events: append-only per-flap timeline with its OWN retention window
    # ([monitor] flap_history_days, default 30) — independent of op_events.
    try:
        flap_days = int(cfg.get("flap_history_days", 30) or 30)
        flap_cutoff = (now_dt - timedelta(days=flap_days)).strftime(
            "%Y-%m-%d %H:%M:%S")
        fe = conn.execute(
            "DELETE FROM flap_events WHERE flapped_at < ?", (flap_cutoff,))
        n_fe = fe.rowcount
    except sqlite3.OperationalError:
        n_fe = 0                        # pre-3.10.1 DB without flap_events
    # port_errors: append-only interface-error timeline with its OWN window
    # ([monitor] port_error_days, default 365 — long, for intermittent-fault
    # forensics). Also age out port_error_state rows for interfaces not polled
    # within the window (decommissioned ports) so the snapshot stays bounded.
    try:
        perr_days = int(cfg.get("port_error_days", 365) or 365)
        perr_cutoff = (now_dt - timedelta(days=perr_days)).strftime(
            "%Y-%m-%d %H:%M:%S")
        pe = conn.execute(
            "DELETE FROM port_errors WHERE at < ?", (perr_cutoff,))
        n_pe = pe.rowcount
        conn.execute(
            "DELETE FROM port_error_state WHERE updated_at < ?", (perr_cutoff,))
    except sqlite3.OperationalError:
        n_pe = 0                        # pre-3.11.0 DB without port_errors
    dns_cutoff = (now_dt - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S")
    try:
        dc = conn.execute(
            "DELETE FROM dns_cache WHERE looked_up_at < ?", (dns_cutoff,))
        n_dc = dc.rowcount
    except sqlite3.OperationalError:
        n_dc = 0                        # pre-3.8.31 DB without dns_cache
    conn.commit()
    conn.close()
    if n_deleted:
        log.info("Pruned %d op_events row(s) older than %d days",
                 n_deleted, retain_days)
    if n_pm:
        log.info("Pruned %d port_macs row(s) older than %d days",
                 n_pm, retain_days)
    if n_ia:
        log.info("Pruned %d ip_arp row(s) older than %d days",
                 n_ia, retain_days)
    if n_fe:
        log.info("Pruned %d flap_events row(s) older than %d days",
                 n_fe, int(cfg.get("flap_history_days", 30) or 30))
    if n_pe:
        log.info("Pruned %d port_errors row(s) older than %d days",
                 n_pe, int(cfg.get("port_error_days", 365) or 365))
    if n_dc:
        log.info("Pruned %d dns_cache row(s) older than 1 day", n_dc)


def handle_investigate_stp(cfg, ip, port):
    """Run the STP investigation for a single (ip, port) and print to stdout.

    Retrospective counterpart to the auto-attached investigation that
    monitor stp emails on confirmed changes — use this to drill down on a
    port whose alert already came and went, or to check baseline state.
    """
    results = investigate_stp_ports(ip, [port], cfg)
    info = results[port]
    # Reuse the same block formatter as the email so stdout and email look
    # identical — the operator learns one layout.
    conn = _db()
    row = conn.execute("SELECT hostname FROM devices WHERE ip = ?", (ip,)).fetchone()
    conn.close()
    ch = {"ip": ip, "interface": port,
          "hostname": (row["hostname"] if row else ip) or ip}
    for line in _format_investigation_block(ch, info):
        print(line)
    return 0 if info.get("ok") else 1


def handle_flap_reset(min_count=0, dry_run=False):
    """Clear port flap counters. With min_count>0, only clears rows at or
    above that threshold. dry_run prints what would be cleared without
    deleting.
    """
    conn = _db()
    rows = conn.execute("""
        SELECT ip, hostname, interface, flap_count
        FROM port_flaps
        WHERE flap_count >= ?
        ORDER BY flap_count DESC, hostname, interface
    """, (min_count,)).fetchall()

    if not rows:
        log.info("No port flap rows >= %d to clear.", min_count)
        conn.close()
        return

    log.info("--- Port Flap Reset%s ---", " (dry-run)" if dry_run else "")
    log.info("  FLAPS  HOSTNAME / IP                         INTERFACE")
    for r in rows:
        label = f"{r['hostname'] or r['ip']} ({r['ip']})"
        log.info(f"  {r['flap_count']:>5}  {label:<36}  {r['interface']}")

    if dry_run:
        log.info("Dry run: %d row(s) would be cleared.", len(rows))
        conn.close()
        return

    conn.execute("DELETE FROM port_flaps WHERE flap_count >= ?", (min_count,))
    conn.commit()
    conn.close()
    log.info("Cleared %d port flap row(s) with flap_count >= %d.",
             len(rows), min_count)


def handle_test_email(cfg, target):
    """Send a canned test email. target is 'all' or one configured address."""
    recipients = [a.strip() for a in cfg.get("email_to", "").split(",") if a.strip()]
    if not recipients:
        print("No recipients configured in [email] to= — nothing to test.")
        return
    if target == "all":
        send_to = recipients
    elif target in recipients:
        send_to = [target]
    else:
        print(f"test-email: '{target}' is not a configured recipient.")
        print(f"  Configured: {', '.join(recipients)}")
        print("  Use 'test-email all', or one of the addresses above.")
        return

    smtp_server = cfg.get("smtp_server") or "(unset)"
    smtp_port = cfg.get("smtp_port", 587)
    from_addr = cfg.get("email_from", "netops@localhost")
    auth = "yes" if (cfg.get("smtp_username") and cfg.get("smtp_password")) else "no"

    print("Email configuration:")
    print(f"  smtp_server: {smtp_server}")
    print(f"  smtp_port:   {smtp_port}")
    print(f"  from:        {from_addr}")
    print(f"  to:          {', '.join(send_to)}")
    print(f"  auth:        {auth}")
    print()

    cfg = dict(cfg)
    cfg["email_to"] = ", ".join(send_to)

    subject = "[netops] test email"
    body = (
        f"This is a test message from netops v{__version__}.\n"
        f"Sent at {datetime.now():%Y-%m-%d %H:%M:%S}.\n\n"
        "If you received this, your [email] configuration is working."
    )
    print("Sending test message...")
    ok, detail = _send_email(cfg, subject, body)
    if ok:
        print(f"OK — {detail}")
    else:
        print(f"FAILED — {detail}")


def _apply_cli_overrides(cfg, args):
    """Apply per-invocation subnet/username/password overrides from CLI args."""
    if getattr(args, "subnet", None):
        cfg["subnets"] = [s.strip() for s in args.subnet.split(",") if s.strip()]
    if getattr(args, "username", None):
        users = [u.strip() for u in args.username.split(",") if u.strip()]
        # SECURITY: reject smuggled ssh-option payloads at the input boundary,
        # before they ever reach _ssh_attempt's argv construction. This is
        # belt-and-braces alongside the validation inside _ssh_attempt itself.
        for u in users:
            _safe_credential_field(u, "username")
        cfg["usernames"] = users
    if getattr(args, "password", None):
        # Passwords go via child.sendline(), not argv — no flag-smuggling
        # risk — but reject newline/CR which would break pexpect's line
        # protocol and could inject a fake login response.
        if "\n" in args.password or "\r" in args.password:
            raise ValueError("password contains newline/carriage return")
        cfg["passwords"]["default"] = args.password


def _require_subnets(cfg):
    if not cfg["subnets"]:
        log.error("No subnets configured. Use -s or set subnets in netops.conf [scan].")
        sys.exit(1)


_thread_caps_warned = False


def _warn_thread_caps(cfg):
    # Emit the cap warnings AT MOST ONCE per process. A single CLI invocation
    # only reaches here once anyway, but the interactive console is one
    # long-lived process where every connection-using command (connect,
    # monitor, backup, ...) re-dispatches through here — without this guard the
    # "scan_threads reduced …" line repeats on every command during a
    # maintenance session. Shown once at first use, then suppressed.
    global _thread_caps_warned
    if _thread_caps_warned:
        return
    _thread_caps_warned = True
    caps = cfg["_thread_caps"]
    if caps["scan_capped"]:
        log.warning(
            "scan_threads reduced from %d to %d — %s",
            caps["raw_scan"], cfg["scan_threads"], caps["scan_limit_reason"],
        )
    if caps["ssh_capped"]:
        log.warning(
            "ssh_threads reduced from %d to %d — %s",
            caps["raw_ssh"], cfg["ssh_threads"],
            caps["ssh_limit_reason"],
        )
    log.debug(
        "Threads: scan=%d (cap %d), ssh=%d (cap %d), "
        "fd_limit=%d, memory=%d MB",
        cfg["scan_threads"], caps["scan_cap"],
        cfg["ssh_threads"], caps["ssh_cap"],
        caps["fd_limit"], caps["total_mem_mb"],
    )


def handle_refresh_dns():
    """Reverse-DNS every device in the DB and update the dns_name column."""
    conn = _db()
    ips = [r["ip"] for r in conn.execute(
        "SELECT ip FROM devices WHERE status IN ('active', 'inactive', 'failed', 'duplicate')"
    ).fetchall()]
    conn.close()
    if not ips:
        log.info("No devices in DB.")
        return
    log.info("Resolving reverse DNS for %d device(s)...", len(ips))
    dns_map = resolve_dns_batch(ips)

    conn = _db()
    updated = 0
    for ip, name in dns_map.items():
        conn.execute("UPDATE devices SET dns_name = ? WHERE ip = ?", (name, ip))
        if name:
            updated += 1
    conn.commit()
    conn.close()

    resolved = sum(1 for v in dns_map.values() if v)
    log.info("DNS refresh complete: %d/%d resolved to a name", resolved, len(ips))


def run_console(cfg):
    """Interactive REPL for netops. Re-parses each line as a subcommand.

    When the environment variable NETOPS_ADMIN is set (populated by the
    authorized_keys 'command=' wrapper for SSH console access), the
    prompt reflects that identity and every command is logged with it
    for audit. Access is gated entirely by SSH key auth at the OS layer —
    no in-REPL password. A local shell run (NETOPS_ADMIN unset) renders
    a plain 'netops>' prompt.
    """
    import cmd
    import shlex
    try:
        import readline
    except ImportError:
        readline = None

    admin = os.environ.get("NETOPS_ADMIN", "").strip() or None

    # Per-user history file so each admin's ~/.netops_history_<admin> is
    # isolated. Falls back to the classic path when identity isn't set
    # (local shell invocation).
    hist_suffix = f"_{admin}" if admin else ""
    history_file = os.path.expanduser(f"~/.netops_history{hist_suffix}")
    if readline:
        try:
            readline.read_history_file(history_file)
        except (FileNotFoundError, OSError):
            pass
        readline.set_history_length(1000)

    # Cache device IPs/hostnames for tab completion
    def _device_names():
        try:
            conn = _db()
            rows = conn.execute(
                "SELECT ip, hostname FROM devices WHERE status IN ('active', 'inactive')"
            ).fetchall()
            conn.close()
            names = []
            for r in rows:
                names.append(r["ip"])
                if r["hostname"]:
                    names.append(r["hostname"])
            return names
        except Exception:
            return []

    class NetopsShell(cmd.Cmd):
        _admin = admin
        # Stashed text from a help-via-? invocation. The next readline
        # prompt cycle re-inserts this so the operator's typed line
        # survives showing context help — Junos/Cisco-style.
        _stashed_prefix = None

        intro = (
            f"netops v{__version__} — interactive console. "
            "Type '?' (no Enter needed) for context-aware help, "
            "'<command> ?' for help on a command, 'exit' to leave."
            "\nDB: {db}"
            + (f"\nUser: {admin}" if admin else "")
        ).format(db=DB_FILE)

        def __init__(self):
            super().__init__()
            if readline is not None:
                # Bind '?' to "insert ? then accept-line" so the operator
                # doesn't have to press Enter to see help. GNU readline
                # re-interprets a macro's OWN output, so the plain
                # '"?": "?\r"' macro makes the inserted '?' re-fire this
                # binding -> "maximum macro execution nesting level
                # exceeded" (seen on the Linux servers). quoted-insert
                # (\C-v) inserts the next macro char literally, breaking
                # the recursion. libedit (macOS) doesn't re-interpret macro
                # output, so it keeps the simple form.
                if "libedit" in (readline.__doc__ or ""):
                    readline.parse_and_bind('"?": "?\\r"')
                else:
                    readline.parse_and_bind('"?": "\\C-v?\\r"')
                # When help fires, onecmd stashes the typed prefix;
                # this hook re-inserts it into the next prompt so the
                # operator doesn't lose their command sequence.
                readline.set_pre_input_hook(self._restore_prefix)

        def _restore_prefix(self):
            if self._stashed_prefix:
                readline.insert_text(self._stashed_prefix)
                readline.redisplay()
                self._stashed_prefix = None

        # Subset of COMMAND_SUMMARIES hidden from the console because
        # they cannot meaningfully run there. Read at class-body time —
        # the module-level CONSOLE_HIDDEN_COMMANDS constant (defined
        # near COMMAND_SUMMARIES) carries the rationale.
        _command_summaries = (
            [(c, s) for c, s in COMMAND_SUMMARIES
             if c not in CONSOLE_HIDDEN_COMMANDS]
            + [
                ("help", "Show help ('help <cmd>' or '<cmd> ?')"),
                ("exit", "Leave the console"),
                ("quit", "Leave the console"),
            ]
        )
        _commands = [c for c, _ in _command_summaries]
        _monitor_targets = MONITOR_TARGETS
        _single_flap_targets = SINGLE_FLAP_TARGETS

        # Junos-style prompt. user@netops> when NETOPS_ADMIN is set
        # (SSH console), plain netops> otherwise (local shell).
        prompt = (f"{admin}@netops> " if admin else "netops> ")

        # Console-only extras (help/exit/quit) on top of the shared HELP_DETAILS.
        _help_details = {
            **HELP_DETAILS,
            "help": [
                "Usage: help [<command>]  or  <command> ?",
                "  Without args: list all commands.",
                "  With a command: show detailed usage for that command.",
            ],
            "exit": ["Usage: exit   — leave the console."],
            "quit": ["Usage: quit   — leave the console."],
        }

        def _show_root_help(self):
            print("Commands:")
            for c, summary in self._command_summaries:
                print(f"  {c:<14} {summary}")
            print("\nType '<command> ?' for details on a specific command.")

        def _show_command_help(self, command):
            """Print context-sensitive help for a command (Cisco-style `command ?`)."""
            details = self._help_details.get(command)
            if not details:
                print(f"No help available for '{command}'")
                return
            for line in details:
                print(line)
            if command == "show":
                print("  Options:")
                print("    --csv FILE      write to CSV file instead of a table")
                print("  port-flaps filters:")
                print("    --ip IP         show only this device")
                print("    --interface IF  show only this interface (exact match)")
                print("    --min-count N   only rows with >= N flaps (default 1)")

        def onecmd(self, line):
            """Intercept trailing '?' for Cisco-style context help.

            When '?' arrives via the readline macro (no Enter), the
            line ends with '?' and we stash the prefix-without-? so
            the pre_input_hook restores it on the next prompt — the
            operator's typed command sequence isn't lost.
            """
            # Audit every console line (raw, before any legacy
            # mapping) to syslog so the security trail shows exactly
            # what the operator typed. Skip pure empty lines so the
            # log isn't full of bare-newline noise.
            stripped = line.strip()
            if stripped:
                _audit_log("console_cmd", command=line)

            if stripped.endswith("?"):
                head = stripped[:-1].strip()
                # Stash the typed prefix (with a trailing space when
                # non-empty so the cursor lands at a natural position
                # for typing the next token).
                self._stashed_prefix = (head + " ") if head else ""
                if not head:
                    self._show_root_help()
                    return False
                command = head.split()[0]
                if command in self._commands:
                    self._show_command_help(command)
                    return False
                print(f"Unknown command '{command}'. Type '?' for command list.")
                return False
            return super().onecmd(line)

        # Commands that require a positional argument. Bare invocation in the
        # REPL routes to '<cmd> ?' help instead of argparse's usage dump. When
        # the second token is present but not in the choice set, we show the
        # same help with a short error line. None = no finite choice list
        # (validation happens downstream in _dispatch or the handler).
        _required_arg = {
            "monitor": {"stp", "flap", "topology", "config"},
            "clear":   {"flap"},
            "digest":  {"flap", "stp", "backup", "health"},
            "test":    {"ssh", "snmp"},
            "connect": None,
            "show":    None,
            "configure": None,
            "test-email": None,
            "add":     None,
            "remove":  None,
            "silence": None,
            "ignore":  {"flap", "flux", "list", "delete"},
            "mark":    {"non-switch", "unknown", "switch", "firewall"},
        }
        _digest_targets = [
            ("health", "The consolidated weekly digest (the scheduled one)"),
            ("flap",   "Port flap counters"),
            ("stp",    "STP disabled + mode-mismatched devices"),
            ("backup", "Weekly backup status + stale backups + recent changes"),
        ]

        # Legacy hyphenated command forms that map to the current device-style
        # grouped verbs. Lets users paste or remember older syntax without
        # hitting argparse dumps.
        _legacy_map = {
            "monitor-stp":  ["monitor", "stp"],
            "monitor-flap": ["monitor", "flap"],
            "digest-stp":   ["digest", "stp"],
            "digest-flap":  ["digest", "flap"],
            "clear-flap":   ["clear", "flap"],
        }

        def default(self, line):
            """Route non-builtin lines through the same argparse as CLI."""
            try:
                argv = shlex.split(line)
            except ValueError as e:
                print(f"parse error: {e}")
                return
            if not argv:
                return
            # Accept 'netops <cmd>' (shell-style paste) by stripping the
            # redundant binary name — we're already inside the netops REPL.
            if argv[0] == "netops":
                argv = argv[1:]
            if not argv:
                self._show_root_help()
                return
            # Translate legacy hyphenated forms to device-style groups.
            if argv[0] in self._legacy_map:
                argv = self._legacy_map[argv[0]] + argv[1:]
            cmd = argv[0]
            # Audit log every command with the admin identity so netops.log
            # shows who ran what over the SSH console.
            log.info("repl: user=%s cmd=%s",
                     self._admin or "(local)", " ".join(argv))
            # Unknown first token — show friendly help instead of argparse's
            # usage dump + 'invalid choice' message.
            if cmd not in self._commands:
                print(f"% Unknown command '{cmd}'. Type '?' for command list.")
                return
            if cmd in self._required_arg:
                if len(argv) == 1:
                    self._show_command_help(cmd)
                    return
                choices = self._required_arg[cmd]
                if choices is not None and argv[1] not in choices:
                    print(f"% Invalid target '{argv[1]}'.")
                    self._show_command_help(cmd)
                    return
            # Rebuild args using the main parser (so all subcommands work)
            old_argv = sys.argv
            try:
                sys.argv = ["netops"] + argv
                args = parse_args()
                _dispatch(args, cfg)
            except SystemExit:
                # argparse calls sys.exit on error or --help; swallow it
                pass
            except KeyboardInterrupt:
                print("\n^C")
            except Exception as e:
                print(f"error: {e}")
            finally:
                sys.argv = old_argv

        def completenames(self, text, *ignored):
            return [c for c in self._commands if c.startswith(text)]

        def completedefault(self, text, line, begidx, endidx):
            # Tab-complete: show → categories + device names; remove → device names;
            # monitor/digest/clear → their target list.
            parts = line[:begidx].split()
            if parts and parts[0] == "show":
                cands = _complete_show(parts, text)
                # completing the pattern of `show devices ip|name <TAB>`
                if (len(parts) == 3 and parts[1] == "devices"
                        and parts[2] in ("ip", "name")):
                    cands += [n for n in _device_names() if n.startswith(text)]
                # `show devices <TAB>` also completes device hostnames —
                # the bare-selector form of the facet tree.
                if len(parts) == 2 and parts[1] == "devices":
                    cands += [n for n in _device_names() if n.startswith(text)]
                return cands
            if parts and parts[0] == "silence":
                if len(parts) == 1:
                    return ([o for o in ("name", "all", "list", "delete")
                             if o.startswith(text)]
                            + [n for n in _device_names()
                               if n.startswith(text)])
                return []
            if parts and parts[0] == "ignore":
                if len(parts) == 1:
                    return [o for o in ("flap", "flux", "list", "delete")
                            if o.startswith(text)]
                if len(parts) == 2 and parts[1] == "delete":
                    return [o for o in ("flap", "flux") if o.startswith(text)]
                return []
            if parts and parts[0] == "configure":
                if len(parts) == 1:
                    return [o for o in ("email",) if o.startswith(text)]
                if len(parts) == 2 and parts[1] == "email":
                    return [o for o in ("add", "remove", "list")
                            if o.startswith(text)]
                if (len(parts) == 3 and parts[1] == "email"
                        and parts[2] == "remove"):
                    return [r for r in _email_recipients()
                            if r.startswith(text)]
                return []
            if parts and parts[0] == "test-email" and len(parts) == 1:
                opts = ["all"] + _email_recipients()
                return [o for o in opts if o.startswith(text)]
            if parts and parts[0] in ("remove", "connect"):
                return [n for n in _device_names() if n.startswith(text)]
            if parts and parts[0] == "monitor":
                return [t for t, _ in self._monitor_targets if t.startswith(text)]
            if parts and parts[0] == "digest":
                return [t for t, _ in self._digest_targets if t.startswith(text)]
            if parts and parts[0] == "clear":
                return [t for t, _ in self._single_flap_targets if t.startswith(text)]
            if parts and parts[0] == "test":
                if len(parts) == 1:
                    return [t for t in ("ssh", "snmp") if t.startswith(text)]
                if len(parts) == 2 and parts[1] in ("ssh", "snmp"):
                    return [t for t in ("retest",) if t.startswith(text)]
                return []
            return []

        # Built-in commands
        def do_exit(self, arg):
            """Exit the console."""
            return True

        def do_quit(self, arg):
            """Exit the console."""
            return True

        def do_EOF(self, arg):
            """Ctrl-D exits."""
            print()
            return True

        def emptyline(self):
            pass  # don't repeat last command on empty enter

        def do_help(self, arg):
            """help [<command>]   — list commands, or details for one."""
            arg = (arg or "").strip()
            if arg:
                if arg in self._commands:
                    self._show_command_help(arg)
                else:
                    print(f"Unknown command '{arg}'. Type '?' for command list.")
                return
            self._show_root_help()

    _audit_log("console_start")
    try:
        NetopsShell().cmdloop()
    except KeyboardInterrupt:
        print("\n^C (use 'exit' to leave)")
        _audit_log("console_exit", reason="keyboard_interrupt")
    else:
        _audit_log("console_exit", reason="normal")
    finally:
        if readline:
            try:
                readline.write_history_file(history_file)
            except OSError:
                pass


def _dispatch(args, cfg):
    """Shared dispatch used by both main() and the console REPL."""
    cmd = args.command
    if cmd == "show":
        # Junos-style hierarchical grammar (resolved by _resolve_show):
        #   show spanning-tree [state] / show devices switch [status] /
        #   show devices ip|name <pattern> [detail] / show <flat-category>.
        kind, val = _resolve_show(args.arg)
        if kind == "error":
            print(val)
            return
        if kind == "coverage":
            return handle_lldp_coverage()
        if kind == "filter":
            field, pattern, detail = val
            return handle_devices_filter(field, pattern, detail,
                                         csv_path=args.csv)
        if kind == "mac":
            return handle_mac(val, csv_path=args.csv,
                              since=args.since, until=args.until)
        if kind == "lldp-search":
            return handle_lldp_search(val, csv_path=args.csv)
        if kind == "devices-facet":
            df_field, df_pattern, df_facet = val
            return handle_devices_facet(df_field, df_pattern, df_facet, cfg,
                                        csv_path=args.csv,
                                        since=args.since, until=args.until)
        if kind == "stp-summary":
            if args.csv:
                print("show spanning-tree: --csv applies to a single state "
                      "(e.g. show spanning-tree blocked --csv FILE); ignoring.")
            for state in _SHOW_STP_STATES:
                print(f"\n----- spanning-tree {state} -----")
                handle_list(_SHOW_STP_CATEGORY[state],
                            ip=args.ip, interface=args.interface,
                            min_count=args.min_count, show_all=False)
            return
        if kind == "category-all":
            return handle_list(val, csv_path=args.csv,
                               ip=args.ip, interface=args.interface,
                               min_count=args.min_count, show_all=True,
                               since=args.since, until=args.until,
                               etype=args.error_type)
        return handle_list(val, csv_path=args.csv,
                           ip=args.ip, interface=args.interface,
                           min_count=args.min_count, show_all=False,
                           since=args.since, until=args.until,
                           etype=args.error_type)
    if cmd == "add":
        return handle_add(args.ip)
    if cmd == "remove":
        return handle_remove(args.ip)
    if cmd == "mark":
        return handle_mark(args.target, args.ip)
    if cmd == "ignore":
        return handle_ignore(args.words, cfg)
    if cmd == "silence":
        return handle_silence(args.words)
    if cmd == "import":
        return handle_import(cfg, path=args.path, fmt=args.format)
    if cmd == "export" and getattr(args, "target", "devices") == "topology":
        return handle_export_topology(
            cfg, fmt=args.format, output=args.output,
            max_age_hours=args.max_age_hours,
            include_unknown=args.include_unknown,
            include_endpoints=args.include_endpoints,
            expand_neighbors=args.expand_neighbors)
    if cmd == "export":
        return handle_export(cfg)
    if cmd == "configure":
        return handle_configure(args.tokens)
    if cmd == "test-email":
        return handle_test_email(cfg, args.target)
    if cmd == "check-config":
        return handle_check_config(cfg)
    if cmd == "manual":
        return handle_manual()
    if cmd == "wipe-db":
        return handle_wipe_db()
    # Connection-using commands
    _warn_thread_caps(cfg)
    _apply_cli_overrides(cfg, args)
    if cmd == "scan":
        _require_subnets(cfg)
        return run_scan(cfg, dry_run=args.dry_run)
    if cmd in ("discover", "discovery"):
        _require_subnets(cfg)
        return run_discover(cfg, dry_run=args.dry_run,
                             email=getattr(args, "email", False))
    if cmd == "probe":
        return run_probe(cfg, status_filter=args.status,
                         csv_path=args.csv, timeout=args.timeout,
                         threads=args.threads)
    if cmd == "test" and args.proto == "ssh":
        return run_retest(cfg) if args.mode == "retest" else run_test(cfg)
    if cmd == "test" and args.proto == "snmp":
        return run_snmp_test(cfg, retest=(args.mode == "retest"))
    if cmd == "backup":
        return run_backup(cfg, sanitize=not args.no_sanitize)
    if cmd == "monitor" and args.target == "topology":
        return run_topology(cfg)
    if cmd == "monitor" and args.target == "config":
        return run_monitor_config(cfg, email=not getattr(args, "no_email", False))
    if cmd == "monitor":
        return run_monitor(cfg, mode=args.target,
                           detail=getattr(args, "detail", False),
                           email=not getattr(args, "no_email", False))
    if cmd == "digest" and args.target == "flap":
        return handle_flap_digest(cfg, min_count=args.min_count, reset=args.reset)
    if cmd == "digest" and args.target == "stp":
        return handle_stp_digest(cfg, detail=getattr(args, "detail", False))
    if cmd == "digest" and args.target == "backup":
        return handle_backup_digest(cfg, detail=getattr(args, "detail", False))
    if cmd == "digest" and args.target == "health":
        return handle_health_digest(cfg)
    if cmd == "clear" and args.target == "flap":
        return handle_flap_reset(min_count=args.min_count, dry_run=args.dry_run)
    if cmd == "reconcile" and args.target == "duplicates":
        return reconcile_duplicates(cfg, do_probe=not args.no_probe)
    if cmd == "investigate" and args.target == "stp":
        return handle_investigate_stp(cfg, args.ip, args.port)
    if cmd == "console":
        return run_console(cfg)
    if cmd == "refresh-dns":
        return handle_refresh_dns()
    if cmd == "daemon":
        return run_daemon(cfg)
    if cmd == "connect":
        return run_connect(cfg, args.device, check_only=args.check)


# ===========================================================================
# netopsd — privileged broker daemon (Phase 1: authorization foundation)
# ===========================================================================
# Why a daemon: only a privileged broker can BOTH keep secrets.conf out of
# operators' hands AND give them instant, password-free switch access. The
# daemon holds the credentials; a human's `netops connect <switch>` asks it
# over a Unix socket. The daemon authenticates the caller via SO_PEERCRED —
# the KERNEL-verified uid of the socket peer, which (unlike SUDO_USER or
# SSH_CONNECTION env vars) cannot be spoofed — checks a per-user/per-device
# ACL, and (Phase 2) proxies the SSH session so the plaintext never leaves
# the daemon. Phase 1 ships the socket, wire protocol, ACL, and the
# authorization decision (`connect --check`); the pty session proxy follows.

# systemd's RuntimeDirectory=netops normally provides /run/netops; override
# with NETOPS_RUNTIME_DIR for testing without root.
NETOPSD_RUNTIME_DIR = os.environ.get("NETOPS_RUNTIME_DIR", "/run/netops")
NETOPSD_SOCK = os.path.join(NETOPSD_RUNTIME_DIR, "netopsd.sock")
_NETOPSD_PROTO = 1
_NETOPSD_MAXFRAME = 1 << 16          # 64 KiB cap on a single control frame
_NETOPSD_HANDSHAKE_TIMEOUT = 30      # seconds to send the request
# Client-side ceiling for the connect handshake: opening the brokered session
# (legacy-KEX retries + credential attempts against an old switch) can take a
# while, so allow generous time — but bound it so a wedged daemon doesn't hang
# the console forever. Set explicitly on the client socket so we never inherit a
# stray process-wide default timeout (see reverse_dns).
_NETOPSD_CONNECT_TIMEOUT = 90


def _recv_exactly(sock, n):
    """Read exactly n bytes; None on clean EOF before n arrive."""
    buf = b""
    while len(buf) < n:
        chunk = sock.recv(n - len(buf))
        if not chunk:
            return None
        buf += chunk
    return buf


def _send_frame(sock, obj):
    """Send one length-prefixed JSON control frame."""
    data = json.dumps(obj).encode("utf-8")
    sock.sendall(struct.pack("!I", len(data)) + data)


def _recv_frame(sock):
    """Receive one length-prefixed JSON control frame, or None on EOF."""
    hdr = _recv_exactly(sock, 4)
    if hdr is None:
        return None
    (length,) = struct.unpack("!I", hdr)
    if length == 0 or length > _NETOPSD_MAXFRAME:
        raise ValueError(f"frame length {length} out of range")
    body = _recv_exactly(sock, length)
    if body is None:
        return None
    return json.loads(body.decode("utf-8"))


def _peer_credentials(sock):
    """(pid, uid, gid) of the connected peer via SO_PEERCRED. Linux-only by
    design — the kernel vouches for the uid, so it can't be spoofed."""
    creds = sock.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED,
                            struct.calcsize("3i"))
    pid, uid, gid = struct.unpack("3i", creds)
    return pid, uid, gid


def _peer_identity(uid):
    """Resolve uid -> (username, [group names]) for ACL matching + audit.
    Groups = primary group + every supplementary group the user belongs to."""
    username = None
    if pwd is not None:
        try:
            username = pwd.getpwuid(uid).pw_name
        except KeyError:
            pass
    groups = []
    if grp is not None and username is not None:
        # Supplementary groups first, in their OWN try: a flaky/orphan primary
        # GID lookup (deleted group, SSSD/winbind blip against AD/LDAP) must not
        # wipe out every @group rule and silently default-deny.
        try:
            groups = [g.gr_name for g in grp.getgrall() if username in g.gr_mem]
        except KeyError:
            pass
        try:
            primary = grp.getgrgid(pwd.getpwnam(username).pw_gid).gr_name
            if primary not in groups:
                groups.append(primary)
        except KeyError:
            pass
    return username, groups


def load_connect_acl():
    """Parse the [connect] ACL from netops.conf.

    Each entry maps a principal — a bare username, or @groupname — to
    comma-separated device glob patterns (matched against a device's hostname
    AND its IP). Default-deny: an absent or empty [connect] section means
    nobody may connect.

        [connect]
        sloeckle   = *
        @netops-ops = SW-*, 10.1.*
        jdoe       = SW-ADM-*
    """
    acl = {}
    cp = configparser.ConfigParser()
    cp.optionxform = str   # preserve case — Linux/AD principals are case-sensitive
    try:
        cp.read(CONFIG_FILE)
    except Exception as e:
        log.warning("netopsd: could not read [connect] ACL from %s: %s",
                    CONFIG_FILE, e)
        return acl
    if cp.has_section("connect"):
        for principal, pats in cp.items("connect"):
            patterns = [p.strip() for p in pats.split(",") if p.strip()]
            if patterns:
                acl[principal.strip()] = patterns
    return acl


def connect_authorized(acl, username, groups, device_ip, device_host):
    """Return (allowed, matched_rule). A principal '<user>' or '@<group>'
    grants its glob patterns; a pattern matches the device's hostname OR ip
    (case-insensitive fnmatch). Default-deny."""
    principals = ([username] if username else []) + ["@" + g for g in groups]
    targets = [t for t in (device_host, device_ip) if t]
    for principal in principals:
        for pat in acl.get(principal, []):
            for tgt in targets:
                if fnmatch.fnmatch(tgt.lower(), pat.lower()):
                    return True, f"{principal} -> {pat}"
    return False, None


def _resolve_connect_target(name):
    """Resolve an IP or hostname to a single device row — the ACTIVE PRIMARY,
    never a duplicate. Reuses the canonical _choose_primary (status, then
    not-already-a-duplicate, then lowest IP); if the match (e.g. an explicit
    duplicate IP from an old runbook) is flagged duplicate_of another row,
    follow that pointer to the live primary. None if unknown."""
    conn = _db()
    rows = conn.execute(
        "SELECT ip, hostname, status, duplicate_of FROM devices "
        "WHERE ip = ? OR lower(hostname) = lower(?)", (name, name)).fetchall()
    if not rows:
        conn.close()
        return None
    # If the operator named an EXACT IP, honour it literally — connect to THAT
    # management IP even if it's flagged a duplicate. This is deliberate: when a
    # stack's primary SVI is down you must be able to reach it through a working
    # secondary IP, so `connect <that-ip>` must not silently redirect to the
    # (possibly dead) primary. A hostname — ambiguous across a switch's several
    # IPs — still resolves to the primary via _choose_primary below.
    exact_ip = next((r for r in rows if r["ip"] == name), None)
    if exact_ip is not None:
        conn.close()
        return exact_ip
    chosen = _choose_primary(rows)
    if chosen["duplicate_of"]:
        primary = conn.execute(
            "SELECT ip, hostname, status, duplicate_of FROM devices WHERE ip = ?",
            (chosen["duplicate_of"],)).fetchone()
        if primary:
            chosen = primary
    conn.close()
    return chosen


# --- SSH session slot broker (Phase A of the full-daemon plan) -------------
# Replaces the whole-job advisory flock with per-SESSION admission control:
# jobs request a token per device connection; the daemon enforces
#   * one session per DEVICE (a switch tolerates ~1 netops session — the
#     config-watch-vs-stp-tick collisions that forced telnet fallbacks
#     die here),
#   * a global concurrency cap (slot_cap, default ssh_threads),
#   * priority classes with a DYNAMIC BULK ALLOWANCE: while anything
#     higher-priority is active (or was within the last 15s), bulk jobs
#     (discover/test sweeps) are granted at most 2 concurrent slots —
#     new grants shrink, in-flight sessions drain by attrition in
#     seconds, and a monitor tick slips through mid-sweep. Nothing is
#     ever killed.
# A grant's lifetime is its client CONNECTION: the acquiring socket stays
# open until the client releases or dies — EOF reaps the slot, so a
# crashed job can never leak one. Jobs fall back to the legacy flock
# when the daemon is unreachable.
_SLOT_CLASSES = {"interactive": 0, "tick": 1, "config": 2,
                 "backup": 3, "bulk": 4}
_SLOT_BULK_SHRUNK = 2          # bulk grants while higher classes are active
_SLOT_HIGH_QUIET_S = 15        # seconds of higher-class silence to restore
_SLOT_BROKER = None            # set by run_daemon
_IN_DAEMON = False             # set by run_daemon (in-process acquire path)


class _SlotBroker:
    def __init__(self, cap):
        self.cap = max(2, int(cap))
        self._lock = threading.Lock()
        self._cond = threading.Condition(self._lock)
        self._grants = {}          # token -> dict(ip, cls, user, job, at)
        self._by_ip = {}           # ip -> token
        self._waiters = []         # [(prio, seq, ip, waiter_id)]
        self._seq = 0
        self._next_token = 1
        self._last_high = 0.0      # monotonic ts of last class<=3 activity

    def _bulk_allowance(self):
        if time.monotonic() - self._last_high < _SLOT_HIGH_QUIET_S:
            return _SLOT_BULK_SHRUNK
        return self.cap

    def _active_bulk(self):
        return sum(1 for g in self._grants.values() if g["cls"] == "bulk")

    def _note_activity(self, cls):
        if _SLOT_CLASSES.get(cls, 4) <= _SLOT_CLASSES["backup"]:
            self._last_high = time.monotonic()

    def acquire(self, ip, cls, user="", job="", timeout=120):
        """Block until a slot for ip is granted, or return None on timeout."""
        prio = _SLOT_CLASSES.get(cls, 4)
        deadline = time.monotonic() + timeout
        with self._cond:
            self._note_activity(cls)
            self._seq += 1
            me = (prio, self._seq, ip, object())
            self._waiters.append(me)
            try:
                while True:
                    if self._can_grant(me):
                        self._waiters.remove(me)
                        token = str(self._next_token)
                        self._next_token += 1
                        self._grants[token] = {
                            "ip": ip, "cls": cls, "user": user, "job": job,
                            "at": datetime.now().strftime("%H:%M:%S"),
                        }
                        self._by_ip[ip] = token
                        self._note_activity(cls)
                        return token
                    remaining = deadline - time.monotonic()
                    if remaining <= 0:
                        self._waiters.remove(me)
                        return None
                    self._cond.wait(min(remaining, 5))
            except BaseException:
                if me in self._waiters:
                    self._waiters.remove(me)
                raise

    def _can_grant(self, me):
        prio, seq, ip, _ = me
        if ip in self._by_ip:
            return False
        if len(self._grants) >= self.cap:
            return False
        if prio >= _SLOT_CLASSES["bulk"] \
                and self._active_bulk() >= self._bulk_allowance():
            return False
        # Don't jump an earlier, higher-or-equal-priority waiter that
        # could also be granted right now (approximate fairness).
        for other in self._waiters:
            if other is me:
                continue
            oprio, oseq, oip, _ = other
            if (oprio, oseq) < (prio, seq) and oip not in self._by_ip:
                if not (oprio >= _SLOT_CLASSES["bulk"]
                        and self._active_bulk() >= self._bulk_allowance()):
                    return False
        return True

    def release(self, token):
        with self._cond:
            g = self._grants.pop(token, None)
            if g:
                self._by_ip.pop(g["ip"], None)
                self._note_activity(g["cls"])
                self._cond.notify_all()

    def status(self):
        with self._lock:
            waiting = {}
            for prio, _seq, _ip, _ in self._waiters:
                name = next((k for k, v in _SLOT_CLASSES.items()
                             if v == prio), str(prio))
                waiting[name] = waiting.get(name, 0) + 1
            return {
                "cap": self.cap,
                "active": len(self._grants),
                "bulk_allowance": self._bulk_allowance(),
                "grants": [dict(token=t, **g)
                           for t, g in sorted(self._grants.items())],
                "waiting": waiting,
            }


def _netopsd_handle(client, cfg):
    """Serve one client connection: authenticate via SO_PEERCRED, dispatch."""
    try:
        client.settimeout(_NETOPSD_HANDSHAKE_TIMEOUT)
        try:
            pid, uid, gid = _peer_credentials(client)
        except Exception as e:
            log.warning("netopsd: SO_PEERCRED failed: %s", e)
            _send_frame(client, {"ok": False, "reason": "peer authentication failed"})
            return
        username, groups = _peer_identity(uid)

        req = _recv_frame(client)
        if not isinstance(req, dict):
            return
        op = req.get("op")

        if op == "ping":
            _send_frame(client, {"ok": True, "proto": _NETOPSD_PROTO,
                                 "slots": _SLOT_BROKER is not None,
                                 "you": {"uid": uid, "user": username,
                                         "groups": groups}})
            return

        if op in ("slot", "slots"):
            # Internal job machinery — only the service account (the uid
            # the daemon itself runs as) or root may hold session slots.
            # "slots" (status) is also allowed for netops-group members
            # so the console's `show slots` works for operators.
            if _SLOT_BROKER is None:
                _send_frame(client, {"ok": False, "reason": "slot broker "
                                     "not initialized"})
                return
            is_service = uid in (0, os.getuid())
            if op == "slots":
                if not (is_service or "netops" in groups):
                    _send_frame(client, {"ok": False,
                                         "reason": "not authorized"})
                    return
                _send_frame(client, {"ok": True,
                                     **_SLOT_BROKER.status()})
                return
            if not is_service:
                _send_frame(client, {"ok": False, "reason": "not authorized"})
                return
            ip = (req.get("ip") or "").strip()
            cls = req.get("cls") or "backup"
            job = (req.get("job") or "")[:40]
            timeout = min(600, max(1, int(req.get("timeout") or 120)))
            if not ip:
                _send_frame(client, {"ok": False, "reason": "no ip"})
                return
            token = _SLOT_BROKER.acquire(ip, cls, user=username or "",
                                         job=job, timeout=timeout)
            if token is None:
                _send_frame(client, {"ok": False, "reason": "slot-timeout"})
                return
            try:
                _send_frame(client, {"ok": True, "token": token})
                # The grant lives as long as this connection: wait for an
                # explicit release frame or EOF (client exit/crash).
                client.settimeout(None)
                while True:
                    rel = _recv_frame(client)
                    if rel is None or (isinstance(rel, dict)
                                       and rel.get("op") == "release"):
                        break
            finally:
                _SLOT_BROKER.release(token)
            return

        if op in ("check", "connect"):
            device = (req.get("device") or "").strip()
            target = _resolve_connect_target(device)
            if target is None:
                _send_frame(client, {"ok": False,
                                     "reason": f"unknown device: {device!r}"})
                _audit_log("connect_deny", actor=username, uid=uid,
                           device=device, reason="unknown-device")
                return
            acl = load_connect_acl()
            allowed, rule = connect_authorized(
                acl, username, groups, target["ip"], target["hostname"])
            if not allowed:
                # Distinguish "no [connect] ACL configured at all" (default-deny
                # for EVERYONE — an admin simply hasn't set it up yet) from "the
                # ACL exists but has no rule covering you," so the operator knows
                # whether it's a setup gap or a deliberate permissions decision.
                if not acl:
                    reason = ("no [connect] ACL configured on this host — "
                              "netopsd default-denies until an admin adds a "
                              "[connect] section to netops.conf (see man netops)")
                    deny_why = "no-acl"
                else:
                    reason = "not authorized"
                    deny_why = "acl"
                _send_frame(client, {"ok": False, "reason": reason,
                                     "device": target["hostname"],
                                     "ip": target["ip"]})
                _audit_log("connect_deny", actor=username, uid=uid,
                           device=target["hostname"], ip=target["ip"],
                           reason=deny_why)
                return
            if op == "check":
                _send_frame(client, {"ok": True, "device": target["hostname"],
                                     "ip": target["ip"], "rule": rule,
                                     "primary_status": target["status"]})
                _audit_log("connect_check", actor=username, uid=uid,
                           device=target["hostname"], ip=target["ip"], rule=rule)
                return
            # op == "connect": authorized — open the session and proxy it.
            # child=None + ONE try/finally so the connect_close audit and child
            # cleanup ALWAYS run once a login succeeded, even if the OK frame or
            # the proxy raises (e.g. the client raced the disconnect) — never a
            # silent credentialed login with no audit footprint.
            child = None
            start = time.time()
            # Interactive sessions hold a device slot too, so a backup or
            # sweep never piles onto a switch an operator is working on.
            slot_token = None
            if _SLOT_BROKER is not None:
                slot_token = _SLOT_BROKER.acquire(
                    target["ip"], "interactive", user=username or "",
                    job="connect", timeout=30)
                if slot_token is None:
                    _send_frame(client, {
                        "ok": False,
                        "reason": "device is busy (another netops session "
                                  "holds its slot) — try again shortly"})
                    return
            try:
                try:
                    child, proto, banner = _broker_open_session(cfg, target)
                except Exception as e:
                    _send_frame(client, {"ok": False,
                                         "reason": f"login failed: {e}"})
                    _audit_log("connect_login_fail", actor=username, uid=uid,
                               device=target["hostname"], ip=target["ip"],
                               reason=str(e))
                    return
                # connect_open logged BEFORE the OK frame so the credentialed
                # login is recorded even if the client has already vanished.
                _audit_log("connect_open", actor=username, uid=uid,
                           device=target["hostname"], ip=target["ip"], rule=rule)
                _send_frame(client, {"ok": True, "device": target["hostname"],
                                     "ip": target["ip"], "rule": rule,
                                     "proto": proto})
                client.settimeout(None)
                if banner:
                    _send_stream(client, _FRAME_DATA, banner)
                _broker_proxy(client, child, req.get("rows"), req.get("cols"))
            finally:
                if slot_token is not None:
                    _SLOT_BROKER.release(slot_token)
                if child is not None:
                    try:
                        child.close(force=True)
                    except Exception:
                        pass
                    _audit_log("connect_close", actor=username, uid=uid,
                               device=target["hostname"], ip=target["ip"],
                               seconds=round(time.time() - start, 1))
            return

        _send_frame(client, {"ok": False, "reason": f"unknown op: {op!r}"})
    except Exception as e:
        log.warning("netopsd handler error: %s", e)
    finally:
        try:
            client.close()
        except Exception:
            pass


# --- Brokered-session stream (post-handshake) -----------------------------
# After the JSON handshake, the socket carries typed frames so terminal data
# and out-of-band control (window resize, close) share one channel:
#   1 type byte + 4-byte BE length + payload.
_FRAME_DATA = b"d"     # raw pty bytes (both directions)
_FRAME_WINSZ = b"w"    # client->daemon: payload = !HH rows, cols
_FRAME_CLOSE = b"q"    # either direction: session ending

_NETOPSD_PROXY_TICK = 30          # select wake-up so idle/max checks run
_NETOPSD_IDLE_TIMEOUT = 1800      # 30 min idle → drop a forgotten session
_NETOPSD_MAX_SESSION = 28800      # 8 h hard ceiling on any brokered session
_NETOPSD_MAX_WORKERS = 64         # concurrent handlers — back-pressure cap


def _broker_notice(client, why):
    """Best-effort visible notice to the operator before a daemon-side close."""
    try:
        _send_stream(client, _FRAME_DATA,
                     f"\r\n[netops] session closed — {why}.\r\n".encode())
    except Exception:
        pass


def _send_stream(sock, ftype, payload=b""):
    sock.sendall(ftype + struct.pack("!I", len(payload)) + payload)


def _recv_stream(sock):
    """Return (type_byte, payload) or (None, None) on EOF."""
    hdr = _recv_exactly(sock, 5)
    if hdr is None:
        return None, None
    ftype = hdr[0:1]
    (length,) = struct.unpack("!I", hdr[1:5])
    if length > _NETOPSD_MAXFRAME:
        raise ValueError(f"stream frame length {length} out of range")
    payload = _recv_exactly(sock, length) if length else b""
    if payload is None:
        return None, None
    return ftype, payload


def _broker_open_session(cfg, target):
    """Log in to the target with the device's stored credential and return the
    authenticated pexpect child (sitting at the device prompt). The credential
    is read here, inside the daemon — it never touches the client."""
    ip = target["ip"]
    conn = _db()
    row = conn.execute(
        "SELECT username, password_hash FROM devices WHERE ip = ?", (ip,)
    ).fetchone()
    conn.close()
    known_user = row["username"] if row else None
    known_pw = row["password_hash"] if row else None
    _login_ctx.plain = True   # connect = raw session; no login setup at all
    try:
        child, proto, _user, _pwh, _so, _to = connect_device(
            ip, cfg["usernames"], cfg["passwords"]["default"],
            password_list=cfg.get("password_list"),
            known_username=known_user, known_password_hash=known_pw,
            user_passwords=cfg.get("user_passwords"))
    finally:
        _login_ctx.plain = False
    if child is None:
        raise RuntimeError("all credentials failed")

    # pexpect's expect() during login already consumed the device banner/MOTD
    # and the first prompt off the pty into its Python-side before/after/buffer
    # — the proxy reads the raw fd and can't see them. Drain them so the
    # operator sees the prompt (not a blank screen) and any compliance banner
    # is preserved.
    def _as_bytes(x):
        if isinstance(x, bytes):
            return x
        if isinstance(x, str):
            return x.encode("utf-8", "replace")
        return b""   # pexpect EOF/TIMEOUT sentinels, None, etc.
    banner = (_as_bytes(getattr(child, "before", b"")) +
              _as_bytes(getattr(child, "after", b"")) +
              _as_bytes(getattr(child, "buffer", b"")))
    return child, proto, banner


def _broker_proxy(client, child, rows=None, cols=None):
    """Relay bytes between the client socket and the switch pty until either
    end closes. The daemon never interprets the stream — the client only ever
    reaches the switch's pty, never the daemon's context."""
    fd = child.child_fd
    if rows and cols:
        try:
            child.setwinsize(int(rows), int(cols))
        except Exception:
            pass
    client.settimeout(None)
    started = last = time.time()
    while True:
        try:
            readable, _, _ = select.select([fd, client], [], [],
                                           _NETOPSD_PROXY_TICK)
        except (OSError, ValueError):
            break
        now = time.time()
        if not readable:
            if now - last > _NETOPSD_IDLE_TIMEOUT:
                _broker_notice(client, "idle timeout")
                break
            if now - started > _NETOPSD_MAX_SESSION:
                _broker_notice(client, "max session duration reached")
                break
            continue
        last = now
        if fd in readable:
            try:
                data = os.read(fd, 4096)
            except OSError:
                data = b""
            if not data:
                break                      # switch closed the session
            _send_stream(client, _FRAME_DATA, data)
        if client in readable:
            try:
                ftype, payload = _recv_stream(client)
            except (ValueError, OSError):
                break
            if ftype is None or ftype == _FRAME_CLOSE:
                break
            if ftype == _FRAME_DATA:
                try:
                    os.write(fd, payload)
                except OSError:
                    break
            elif ftype == _FRAME_WINSZ and len(payload) >= 4:
                wr, wc = struct.unpack("!HH", payload[:4])
                try:
                    child.setwinsize(wr, wc)
                except Exception:
                    pass
    try:
        _send_stream(client, _FRAME_CLOSE)
    except Exception:
        pass


def run_daemon(cfg):
    """Run netopsd: accept Unix-socket connections, authenticate each peer via
    SO_PEERCRED, enforce the [connect] ACL. systemd-managed (netopsd.service);
    foreground by design."""
    if not hasattr(socket, "SO_PEERCRED"):
        log.error("netopsd requires Linux SO_PEERCRED; this platform lacks it.")
        return 1
    try:
        os.makedirs(NETOPSD_RUNTIME_DIR, exist_ok=True)
    except OSError as e:
        log.error("netopsd: cannot create %s: %s", NETOPSD_RUNTIME_DIR, e)
        return 1
    if os.path.exists(NETOPSD_SOCK):
        try:
            os.unlink(NETOPSD_SOCK)
        except OSError:
            pass
    srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    srv.bind(NETOPSD_SOCK)
    os.chmod(NETOPSD_SOCK, 0o660)
    srv.listen(16)

    # Phase A: SSH session slot broker. Cap defaults to ssh_threads —
    # override with [daemon] slot_cap.
    global _SLOT_BROKER, _IN_DAEMON
    _SLOT_BROKER = _SlotBroker(cfg.get("daemon_slot_cap")
                               or cfg.get("ssh_threads") or 8)
    _IN_DAEMON = True

    log.info("netopsd listening on %s (proto %d, slot cap %d)",
             NETOPSD_SOCK, _NETOPSD_PROTO, _SLOT_BROKER.cap)
    _audit_log("netopsd_start", socket=NETOPSD_SOCK)

    # Give brokered pty children a sane TERM — systemd starts the daemon with
    # TERM unset, which renders as "dumb" on the switch. This is a coarse,
    # process-wide default; honoring each client's own req["term"] per session
    # needs `env=` threaded through connect_device/ssh_connect/_spawn — a
    # follow-up (the client already sends term; the daemon reserves the field).
    os.environ.setdefault("TERM", "xterm")

    stop = threading.Event()

    def _shutdown(signum, _frame):
        log.info("netopsd: signal %d — shutting down", signum)
        stop.set()
        try:
            srv.close()
        except Exception:
            pass
    signal.signal(signal.SIGTERM, _shutdown)
    signal.signal(signal.SIGINT, _shutdown)

    workers = threading.Semaphore(_NETOPSD_MAX_WORKERS)
    try:
        while not stop.is_set():
            try:
                client, _ = srv.accept()
            except OSError:
                break
            # Bound concurrent handlers so one looping/compromised netops-group
            # account can't exhaust the daemon's fds + pexpect children.
            if not workers.acquire(blocking=False):
                try:
                    _send_frame(client, {"ok": False,
                                         "reason": "netopsd busy — too many sessions"})
                finally:
                    try:
                        client.close()
                    except Exception:
                        pass
                continue

            def _run(c=client):
                try:
                    _netopsd_handle(c, cfg)
                finally:
                    workers.release()
            threading.Thread(target=_run, daemon=True).start()
    finally:
        try:
            os.unlink(NETOPSD_SOCK)
        except OSError:
            pass
        _audit_log("netopsd_stop")
    return 0


def _terminal_winsize():
    """(rows, cols) of the controlling terminal, best-effort."""
    if fcntl is None or termios is None:
        return 24, 80
    try:
        packed = fcntl.ioctl(sys.stdin.fileno(), termios.TIOCGWINSZ, b"\0" * 8)
        rows, cols, _, _ = struct.unpack("HHHH", packed)
        return rows or 24, cols or 80
    except Exception:
        return 24, 80


def _connect_client_proxy(sock):
    """Put the local terminal in raw mode and relay it to the brokered session
    until the daemon signals close. The client only ever exchanges the switch
    pty's bytes — it never sees the credential."""
    if termios is None or not sys.stdin.isatty():
        log.error("connect requires an interactive terminal")
        return 1
    stdin_fd = sys.stdin.fileno()
    old_attrs = termios.tcgetattr(stdin_fd)

    def _on_winch(_signum, _frame):
        r, c = _terminal_winsize()
        try:
            _send_stream(sock, _FRAME_WINSZ, struct.pack("!HH", r, c))
        except Exception:
            pass

    try:
        tty.setraw(stdin_fd)
        r, c = _terminal_winsize()
        _send_stream(sock, _FRAME_WINSZ, struct.pack("!HH", r, c))
        signal.signal(signal.SIGWINCH, _on_winch)
        sock.settimeout(None)
        while True:
            try:
                readable, _, _ = select.select([stdin_fd, sock], [], [])
            except (OSError, ValueError):
                break
            if stdin_fd in readable:
                data = os.read(stdin_fd, 4096)
                if not data:
                    _send_stream(sock, _FRAME_CLOSE)
                    break
                _send_stream(sock, _FRAME_DATA, data)
            if sock in readable:
                try:
                    ftype, payload = _recv_stream(sock)
                except (ValueError, OSError):
                    break
                if ftype is None or ftype == _FRAME_CLOSE:
                    break
                if ftype == _FRAME_DATA:
                    os.write(sys.stdout.fileno(), payload)
    finally:
        termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_attrs)
        try:
            signal.signal(signal.SIGWINCH, signal.SIG_DFL)
        except Exception:
            pass
        sys.stdout.write("\r\n[netops] session closed.\r\n")
        sys.stdout.flush()
    return 0


def run_connect(cfg, device, check_only=False):
    """Client: ask netopsd to authorize (and broker) a session to <device>.
    The credential never leaves the daemon."""
    if not os.path.exists(NETOPSD_SOCK):
        log.error("netopsd socket %s not found — is netopsd running? "
                  "(sudo systemctl status netopsd)", NETOPSD_SOCK)
        return 1
    try:
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        # Explicit handshake timeout — don't inherit a stray process-wide default
        # (a DNS batch earlier in a long-lived console can leave it at ~2s). Reset
        # to blocking once the interactive proxy starts (_connect_client_proxy).
        sock.settimeout(_NETOPSD_CONNECT_TIMEOUT)
        sock.connect(NETOPSD_SOCK)
    except OSError as e:
        log.error("cannot reach netopsd: %s", e)
        return 1

    req = {"op": "check" if check_only else "connect", "device": device}
    if not check_only:
        rows, cols = _terminal_winsize()
        req.update(rows=rows, cols=cols, term=os.environ.get("TERM", ""))
    try:
        _send_frame(sock, req)
        resp = _recv_frame(sock)
    except OSError as e:
        log.error("netopsd communication error: %s", e)
        sock.close()
        return 1

    if resp is None:
        log.error("netopsd closed the connection unexpectedly")
        sock.close()
        return 1
    if not resp.get("ok"):
        print(f"DENIED: {resp.get('reason')}", file=sys.stderr)
        sock.close()
        return 1
    if check_only:
        print(f"ALLOWED  {resp.get('device')} ({resp.get('ip')})  "
              f"via [{resp.get('rule')}]  primary_status={resp.get('primary_status')}")
        sock.close()
        return 0

    # Authorized and the session is open daemon-side — go interactive.
    print(f"[netops] connected to {resp.get('device')} ({resp.get('ip')}) "
          f"via netopsd. Log out of the device normally to end the session.",
          file=sys.stderr)
    try:
        return _connect_client_proxy(sock)
    finally:
        sock.close()


def main():
    args = parse_args()
    # `manual` just prints the bundled docs — no config, DB, or logging
    # needed, so handle it before load_config() (works even on a freshly
    # installed / unconfigured box).
    if getattr(args, "command", None) == "manual":
        return handle_manual()
    cfg = load_config()
    # Shared flags use argparse.SUPPRESS default — attributes may be missing.
    dbg = getattr(args, "debug", False)
    dbgfile = getattr(args, "debug_file", None)
    logmode = getattr(args, "log_mode", "append")
    db_override = getattr(args, "db", None)

    debug_file = (os.path.abspath(os.path.expanduser(dbgfile))
                  if dbgfile else None)
    setup_logging(debug=dbg, debug_file=debug_file,
                  log_format=cfg["log_format"], log_mode=logmode)

    # --- Resolve DB file (before init_db / any DB access) ---
    if db_override:
        global DB_FILE
        DB_FILE = os.path.abspath(os.path.expanduser(db_override))

    cmd = args.command

    # wipe-db runs before init_db so we don't recreate the file we're about to delete.
    if cmd == "wipe-db":
        handle_wipe_db()
        return

    init_db()
    check_tools()
    # Audit log every CLI invocation (1 line per `netops <subcommand>` run).
    # Skipped for the console subcommand here — that's logged per-line
    # below in run_console so we capture each interactive command, not
    # just the entry into the REPL.
    if cmd != "console":
        _audit_log("cli", command=" ".join(sys.argv[1:]) or "(no args)")
    _dispatch(args, cfg)


if __name__ == "__main__":
    main()
