#!/usr/bin/env python3
"""Backup running configs from network devices via SSH."""

__version__ = "3.8.17"

import argparse
import configparser
import hashlib
import ipaddress
import itertools
import logging
import os
import re
import shutil
import socket
import sqlite3
import subprocess
import sys
import tempfile
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta

import pexpect
if sys.platform == "win32":
    from pexpect.popen_spawn import PopenSpawn

# Base SSH options — no legacy algorithms; those are added only when needed
_SSH_BASE_OPTS = [
    "-o", "StrictHostKeyChecking=no",
    "-o", "UserKnownHostsFile=/dev/null",
]

# Prompt pattern for common network devices
# Handles trailing ANSI escape sequences (cursor positioning, etc.) including
# partial sequences that arrive mid-stream from old telnet full-screen switches.
# Pattern: hostname[#>%] + any mix of whitespace / ANSI sequences + end-of-buffer.
# `%` matches the Juniper FreeBSD shell prompt seen when logging in as root.
PROMPT_RE = r"[\w\-/\.]+[#>%]\s*(?:\x1b[^\x1b\n]*)*\s*$"
_SHELL_PROMPT_RE = r"%\s*(?:\x1b[^\x1b\n]*)*\s*$"

# Tool availability (set by check_tools() at startup)
_HAS_SSH = False
_HAS_TELNET = False

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))


def _resolve_base_dir(env_var, script_dir_markers, fhs_default):
    """Resolve a base directory, preferring an in-place install.

    Priority, highest first:
      1. $<env_var> — explicit override (for testing or non-standard layouts).
      2. SCRIPT_DIR, if it already holds one of script_dir_markers. This is
         the back-compat path: existing run-in-place installs (e.g.
         /home/netops/netops on SLU/PSC/Drew) keep using files next to the
         script and are completely unaffected by packaging — even on a box
         that also happens to have the FHS directory.
      3. fhs_default, if that directory exists (a packaged install: the
         .deb's postinst creates /etc/netops and /var/lib/netops).
      4. SCRIPT_DIR — plain `git clone` / dev checkout fallback.
    """
    override = os.environ.get(env_var)
    if override:
        return os.path.abspath(os.path.expanduser(override))
    for marker in script_dir_markers:
        if os.path.exists(os.path.join(SCRIPT_DIR, marker)):
            return SCRIPT_DIR
    if os.path.isdir(fhs_default):
        return fhs_default
    return SCRIPT_DIR


# Config (read-only at runtime): netops.conf / legacy config.ini, secrets.conf.
# Packaged location: /etc/netops.  Override with NETOPS_CONFIG_DIR.
CONFIG_DIR = _resolve_base_dir(
    "NETOPS_CONFIG_DIR",
    ("netops.conf", "config.ini", "secrets.conf"),
    "/etc/netops",
)
# Mutable state: netops.db, netops.log, configs/ (+ its git repo), debug log,
# device list.  Packaged location: /var/lib/netops.  Override with
# NETOPS_STATE_DIR.  Resolved independently of CONFIG_DIR so /etc vs /var/lib
# can diverge on a packaged box while staying unified on an in-place install.
STATE_DIR = _resolve_base_dir(
    "NETOPS_STATE_DIR",
    ("netops.db",),
    "/var/lib/netops",
)

CONFIGS_DIR = os.path.join(STATE_DIR, "configs")
LOG_FILE = os.path.join(STATE_DIR, "netops.log")
_CONFIG_FILE_NEW = os.path.join(CONFIG_DIR, "netops.conf")
_CONFIG_FILE_LEGACY = os.path.join(CONFIG_DIR, "config.ini")
# Prefer netops.conf for consistency with secrets.conf. Fall back to the
# legacy config.ini name so existing deployments keep working — a one-line
# rename completes the migration when the operator is ready. load_config()
# logs a deprecation note when the legacy filename is in use.
CONFIG_FILE = _CONFIG_FILE_NEW if os.path.exists(_CONFIG_FILE_NEW) else _CONFIG_FILE_LEGACY
DB_FILE = os.path.join(STATE_DIR, "netops.db")
SECRETS_FILE = os.path.join(CONFIG_DIR, "secrets.conf")

# secrets.conf is meant to be shareable with a dedicated `netops` group
# (mode 0640, group netops) so multiple operators can run netops — manually
# or as their login shell — without each keeping a private copy of the
# credentials. The packaged install sets this up in debian/postinst.
# Security rule: the owner may have any bits, the group may *read* (0o040)
# but not write or exec, and the world gets nothing. Forbidden mask is
# therefore group-write + group-exec + every world bit.
SECRET_GROUP = "netops"
_SECRET_MODE = 0o640
_SECRET_FORBIDDEN_MODE = 0o037

log = logging.getLogger("netops")


# ---------------------------------------------------------------------------
# Operational event log — instrumentation for 'digest health'.
# Per-SNMP-query rows would explode the DB (~15M rows/week/site), so we
# aggregate via a tick-local accumulator (_snmp_timing_*) and write one
# row per monitor-stp tick / per backup-device / per ssh-connect-attempt.
# ---------------------------------------------------------------------------

_snmp_timing_lock = threading.Lock()
_snmp_timing_state = {"times": [], "failed": 0, "enabled": False}


def _snmp_timing_start():
    """Reset the SNMP timing accumulator for the upcoming scope."""
    with _snmp_timing_lock:
        _snmp_timing_state["times"] = []
        _snmp_timing_state["failed"] = 0
        _snmp_timing_state["enabled"] = True


def _snmp_timing_record(duration_ms, success):
    """Called from _snmp_run. Cheap no-op when accumulation isn't enabled."""
    if not _snmp_timing_state["enabled"]:
        return
    with _snmp_timing_lock:
        _snmp_timing_state["times"].append(duration_ms)
        if not success:
            _snmp_timing_state["failed"] += 1


def _snmp_timing_stop():
    """Stop accumulating and return a small stats dict, or None if no data."""
    with _snmp_timing_lock:
        _snmp_timing_state["enabled"] = False
        times = _snmp_timing_state["times"]
        failed = _snmp_timing_state["failed"]
        _snmp_timing_state["times"] = []
        _snmp_timing_state["failed"] = 0
    if not times:
        return None
    return {
        "snmp_count": len(times),
        "snmp_failed": failed,
        "snmp_min_ms": min(times),
        "snmp_max_ms": max(times),
        "snmp_avg_ms": int(sum(times) / len(times)),
    }


def _record_op(op_type, *, started_at=None, duration_ms=0, success=True,
               ip=None, reason=None, extra=None, error=None):
    """Insert one row into op_events. Best-effort — failures here are
    swallowed so instrumentation doesn't break the operation that fired
    it."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    if started_at is None:
        started_at = now
    if isinstance(extra, dict):
        extra = " ".join(f"{k}={v}" for k, v in extra.items())
    try:
        conn = _db()
        conn.execute(
            "INSERT INTO op_events "
            "(op_type, started_at, duration_ms, success, ip, reason, extra, error) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
            (op_type, started_at, int(duration_ms), 1 if success else 0,
             ip, reason, extra, error))
        conn.commit()
        conn.close()
    except Exception as e:
        log.debug("op_events insert failed: %s", e)


# ---------------------------------------------------------------------------
# Database
# ---------------------------------------------------------------------------

def init_db():
    """Create the database and tables if they don't exist."""
    conn = sqlite3.connect(DB_FILE, timeout=30)
    # WAL lets a long-running writer (e.g. cron 'monitor stp') coexist with
    # concurrent readers/writers instead of blocking them on the whole DB.
    conn.execute("PRAGMA journal_mode=WAL")
    # 3.8.9 rename: mac_churn_* tables -> mac_flux_*. ALTER TABLE RENAME
    # preserves data; run BEFORE CREATE TABLE IF NOT EXISTS so an existing
    # pre-3.8.9 DB isn't left with both an empty mac_flux_alerts (created
    # below) AND the original mac_churn_alerts (data stranded). On a fresh
    # install neither table exists yet and these statements are no-ops.
    for old, new in (("mac_churn_alerts",    "mac_flux_alerts"),
                     ("mac_churn_whitelist", "mac_flux_whitelist")):
        has_old = conn.execute(
            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
            (old,)).fetchone()
        has_new = conn.execute(
            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
            (new,)).fetchone()
        if has_old and not has_new:
            conn.execute(f"ALTER TABLE {old} RENAME TO {new}")
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS devices (
            ip             TEXT PRIMARY KEY,
            base_mac       TEXT,
            hostname       TEXT,
            dns_name       TEXT,
            model          TEXT,
            serial         TEXT,
            firmware       TEXT,
            hardware_info  TEXT,
            proto          TEXT,
            username       TEXT,
            password_hash  TEXT,
            ssh_open       INTEGER DEFAULT 0,
            telnet_open    INTEGER DEFAULT 0,
            platform       TEXT,
            stp_enabled    INTEGER,
            stp_mode       TEXT,
            stp_last_check TEXT,
            snmp_enabled   INTEGER,
            snmp_proto     TEXT,
            snmp_community TEXT,
            snmp_v3_user   TEXT,
            snmp_last_ok   TEXT,
            snmp_last_check TEXT,
            snmp_diag      TEXT,
            chassis_type   TEXT,
            member_count   INTEGER,
            slot_count     INTEGER,
            link_up_count  INTEGER,
            status         TEXT NOT NULL,
            fail_reason    TEXT,
            duplicate_of   TEXT,
            first_seen     TEXT NOT NULL,
            last_seen      TEXT NOT NULL
        );
        CREATE TABLE IF NOT EXISTS backups (
            id            INTEGER PRIMARY KEY AUTOINCREMENT,
            ip            TEXT NOT NULL,
            base_mac      TEXT,
            hostname      TEXT,
            filename      TEXT NOT NULL,
            config_hash   TEXT NOT NULL,
            changed       INTEGER NOT NULL DEFAULT 1,
            backed_up_at  TEXT NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_devices_mac ON devices(base_mac);
        CREATE INDEX IF NOT EXISTS idx_devices_status ON devices(status);
        CREATE INDEX IF NOT EXISTS idx_backups_ip ON backups(ip);
        CREATE INDEX IF NOT EXISTS idx_backups_mac ON backups(base_mac);

        CREATE TABLE IF NOT EXISTS stp_state (
            ip            TEXT NOT NULL,
            interface     TEXT NOT NULL,
            role          TEXT NOT NULL,
            state         TEXT NOT NULL,
            last_seen     TEXT NOT NULL,
            PRIMARY KEY (ip, interface)
        );
        CREATE TABLE IF NOT EXISTS stp_changes (
            id            INTEGER PRIMARY KEY AUTOINCREMENT,
            ip            TEXT NOT NULL,
            hostname      TEXT,
            interface     TEXT NOT NULL,
            old_role      TEXT,
            old_state     TEXT,
            new_role      TEXT NOT NULL,
            new_state     TEXT NOT NULL,
            changed_at    TEXT NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_stp_state_ip ON stp_state(ip);
        CREATE INDEX IF NOT EXISTS idx_stp_changes_ip ON stp_changes(ip);
        CREATE INDEX IF NOT EXISTS idx_stp_changes_time ON stp_changes(changed_at);

        CREATE TABLE IF NOT EXISTS stp_pending_changes (
            ip          TEXT NOT NULL,
            hostname    TEXT,
            interface   TEXT NOT NULL,
            old_role    TEXT, old_state TEXT,
            new_role    TEXT NOT NULL, new_state TEXT NOT NULL,
            first_seen  TEXT NOT NULL,
            PRIMARY KEY (ip, interface)
        );
        CREATE TABLE IF NOT EXISTS stp_root_state (
            ip                TEXT NOT NULL,
            instance          TEXT NOT NULL,   -- 'CIST' for now; future: MST1...
            root_priority     INTEGER,
            root_mac          TEXT,            -- canonical 'aa:bb:cc:dd:ee:ff'
            bridge_priority   INTEGER,
            bridge_mac        TEXT,
            is_root           INTEGER NOT NULL DEFAULT 0,
            tcn_count         INTEGER,
            last_tcn_seconds  INTEGER,
            updated_at        TEXT NOT NULL,
            PRIMARY KEY (ip, instance)
        );
        CREATE INDEX IF NOT EXISTS idx_stp_root_state_instance
            ON stp_root_state(instance, root_mac);
        CREATE TABLE IF NOT EXISTS stp_root_pending (
            ip             TEXT NOT NULL,
            instance       TEXT NOT NULL,
            old_priority   INTEGER, old_mac TEXT,
            new_priority   INTEGER, new_mac TEXT,
            first_seen     TEXT NOT NULL,
            PRIMARY KEY (ip, instance)
        );
        CREATE TABLE IF NOT EXISTS stp_root_changes (
            id            INTEGER PRIMARY KEY AUTOINCREMENT,
            ip            TEXT NOT NULL,
            hostname      TEXT,
            instance      TEXT NOT NULL,
            old_priority  INTEGER, old_mac TEXT,
            new_priority  INTEGER, new_mac TEXT,
            changed_at    TEXT NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_stp_root_changes_time
            ON stp_root_changes(changed_at);
        CREATE TABLE IF NOT EXISTS stp_port_id_cache (
            -- Junos-only: maps the JUNIPER STP port-id (the index used by
            -- JUNIPER-MIMSTP-MIB) to the physical interface name. On VC
            -- chassis (multi-FPC EX-series), this index diverges from
            -- BRIDGE-MIB's dot1dBasePort, so the SNMP fast-path needs an
            -- authoritative mapping that doesn't rely on the BRIDGE-MIB
            -- chain. Populated by `discover` from
            -- `show spanning-tree interface | no-more` Port identifier
            -- column. Aligned-indexing devices populate it identically;
            -- the cache is consulted preferentially with BRIDGE-MIB
            -- fallback on miss.
            ip            TEXT NOT NULL,
            port_id       INTEGER NOT NULL,
            interface     TEXT NOT NULL,
            last_refresh  TEXT NOT NULL,
            PRIMARY KEY (ip, port_id)
        );
        CREATE TABLE IF NOT EXISTS device_members (
            ip          TEXT NOT NULL,   -- master device IP (matches devices.ip)
            member_id   INTEGER NOT NULL,-- 0/1/2 (Junos) or 1/2/3 (Aruba/HP)
            role        TEXT,            -- Master/Backup/Conductor/Standby/Member/Commander
            serial      TEXT,
            mac         TEXT,            -- canonical 'aa:bb:cc:dd:ee:ff'
            model       TEXT,
            status      TEXT,            -- vendor status word (Prsnt/Ready/Active/Up)
            updated_at  TEXT NOT NULL,
            PRIMARY KEY (ip, member_id)
        );
        CREATE INDEX IF NOT EXISTS idx_device_members_ip ON device_members(ip);
        CREATE INDEX IF NOT EXISTS idx_device_members_serial ON device_members(serial);
        CREATE TABLE IF NOT EXISTS port_state (
            ip          TEXT NOT NULL,
            interface   TEXT NOT NULL,
            link_up     INTEGER NOT NULL,
            last_seen   TEXT NOT NULL,
            PRIMARY KEY (ip, interface)
        );
        CREATE TABLE IF NOT EXISTS port_flaps (
            ip          TEXT NOT NULL,
            hostname    TEXT,
            interface   TEXT NOT NULL,
            flap_count  INTEGER NOT NULL DEFAULT 0,
            first_seen  TEXT NOT NULL,
            last_seen   TEXT NOT NULL,
            PRIMARY KEY (ip, interface)
        );
        CREATE INDEX IF NOT EXISTS idx_port_flaps_count ON port_flaps(flap_count);
        -- Per-port LLDP neighbor snapshot, populated by 'monitor topology'.
        -- One row per (src_ip, src_port) — only the most recent neighbor
        -- per port is kept (a port shouldn't have two simultaneous LLDP
        -- neighbors in practice; trunks aggregate at the LAG level). The
        -- graph walk uses neighbor_ip when the peer is in our devices
        -- table, falling back to chassis MAC / system name otherwise.
        CREATE TABLE IF NOT EXISTS topology_edges (
            src_ip            TEXT NOT NULL,
            src_port          TEXT NOT NULL,
            neighbor_chassis  TEXT,
            neighbor_sysname  TEXT,
            neighbor_port     TEXT,
            neighbor_port_desc TEXT,
            neighbor_ip       TEXT,
            last_seen         TEXT NOT NULL,
            PRIMARY KEY (src_ip, src_port)
        );
        CREATE INDEX IF NOT EXISTS idx_topology_neighbor_ip ON topology_edges(neighbor_ip);
        CREATE INDEX IF NOT EXISTS idx_topology_src_ip ON topology_edges(src_ip);
        -- Operational event log — populated by instrumentation hooks
        -- (monitor stp ticks, backups, SSH attempts, LLDP scrape).
        -- 'digest health' aggregates these over the past 7 days. Rolled
        -- up per-tick rather than per-SNMP-query to keep the row count
        -- sane (~15M SNMP queries/week per site would blow up the DB).
        CREATE TABLE IF NOT EXISTS op_events (
            id          INTEGER PRIMARY KEY AUTOINCREMENT,
            op_type     TEXT NOT NULL,
            started_at  TEXT NOT NULL,
            duration_ms INTEGER NOT NULL,
            success     INTEGER NOT NULL,
            ip          TEXT,
            reason      TEXT,
            extra       TEXT,
            error       TEXT
        );
        CREATE INDEX IF NOT EXISTS idx_op_events_time ON op_events(started_at);
        CREATE INDEX IF NOT EXISTS idx_op_events_type_time ON op_events(op_type, started_at);
        -- Historical port<->MAC associations harvested from BRIDGE-MIB
        -- dot1dTpFdbTable (+ Q-BRIDGE dot1qTpFdbTable when VLAN-aware) on
        -- the 'monitor topology' SNMP pass (every 30 min). Lets STP-alert
        -- and flap-digest enrichment show the *last-known* MAC(s) on a port
        -- when the live FDB is empty (port flapping/down) — and feeds the
        -- non-trunk MAC-flux security signal. vlan defaults to 0 (= plain
        -- dot1d / VLAN-unaware) so the PK upsert stays deterministic.
        CREATE TABLE IF NOT EXISTS port_macs (
            ip          TEXT NOT NULL,
            interface   TEXT NOT NULL,
            mac         TEXT NOT NULL,        -- canonical 'aa:bb:cc:dd:ee:ff'
            vlan        INTEGER NOT NULL DEFAULT 0,
            oui_vendor  TEXT,
            first_seen  TEXT NOT NULL,
            last_seen   TEXT NOT NULL,
            times_seen  INTEGER NOT NULL DEFAULT 1,
            PRIMARY KEY (ip, interface, mac, vlan)
        );
        CREATE INDEX IF NOT EXISTS idx_port_macs_ip_if ON port_macs(ip, interface);
        CREATE INDEX IF NOT EXISTS idx_port_macs_last_seen ON port_macs(last_seen);
        -- Edge-trigger dedup for the non-trunk MAC-flux security notice.
        -- Mirrors the stp_pending / *_alerted_at preservation pattern: one
        -- email when a port first crosses mac_flux_threshold_24h, then
        -- suppressed until the episode clears (cleared_at set when the
        -- trailing-24h distinct-MAC count drops back under threshold).
        CREATE TABLE IF NOT EXISTS mac_flux_alerts (
            ip            TEXT NOT NULL,
            interface     TEXT NOT NULL,
            distinct_macs INTEGER NOT NULL,
            notified_at   TEXT NOT NULL,
            cleared_at    TEXT,
            PRIMARY KEY (ip, interface)
        );
        -- 3.8.8 flux-whitelist: operator-managed (via `netops whitelist
        -- add|remove|list`) list of approved (ip, interface) ports that
        -- should never alert (e.g. an approved A/V switch downstream of
        -- an access port). Unioned with the 3.8.7 cfg key
        -- [monitor] mac_flux_excluded_ports at compute time. Lives in
        -- the DB rather than netops.conf so the netops user can manage
        -- it without sudo, with an audit trail (added_at, added_by).
        CREATE TABLE IF NOT EXISTS mac_flux_whitelist (
            ip          TEXT NOT NULL,
            interface   TEXT NOT NULL,
            reason      TEXT,
            added_at    TEXT NOT NULL,
            added_by    TEXT,
            PRIMARY KEY (ip, interface)
        );
        -- 3.8.10 port-flap silence/whitelist: operator-managed (via
        -- `netops whitelist add --flap`) list of approved-or-noisy
        -- ports whose port_flaps rows should not page the operator
        -- via digest flap. Use case: a sleepy printer that link-flaps
        -- on every sleep cycle but isn't actually a network problem.
        -- Unlike mac_flux_whitelist, this table has an optional
        -- expires_at column so the operator can silence with a deadline
        -- ("until I order a new cable") rather than permanently. NULL
        -- expires_at = permanent. The reconciliation pass garbage-
        -- collects expired entries on each digest flap run.
        CREATE TABLE IF NOT EXISTS port_flap_whitelist (
            ip          TEXT NOT NULL,
            interface   TEXT NOT NULL,
            reason      TEXT,
            added_at    TEXT NOT NULL,
            added_by    TEXT,
            expires_at  TEXT,
            PRIMARY KEY (ip, interface)
        );
    """)
    # Migrate existing DBs: add columns if missing
    cols = [row[1] for row in conn.execute("PRAGMA table_info(devices)").fetchall()]
    if "firmware" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN firmware TEXT")
    if "hardware_info" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN hardware_info TEXT")
    if "password_hash" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN password_hash TEXT")
    if "ssh_open" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN ssh_open INTEGER DEFAULT 0")
    if "telnet_open" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN telnet_open INTEGER DEFAULT 0")
    if "stp_enabled" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN stp_enabled INTEGER")
    if "stp_last_check" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN stp_last_check TEXT")
    if "dns_name" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN dns_name TEXT")
    if "platform" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN platform TEXT")
    if "stp_mode" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN stp_mode TEXT")
    if "chassis_type" not in cols:
        # 'standalone' | 'stack' | 'chassis'. NULL until discovery re-runs.
        conn.execute("ALTER TABLE devices ADD COLUMN chassis_type TEXT")
    if "member_count" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN member_count INTEGER")
    if "slot_count" not in cols:
        # Mirrors member_count for every vendor we've probed; kept as its own
        # column so a future multi-blade chassis can diverge (e.g. HP 5400zl
        # with module slots per chassis member).
        conn.execute("ALTER TABLE devices ADD COLUMN slot_count INTEGER")
    if "link_up_count" not in cols:
        # Number of data-plane ports with link up, refreshed every monitor
        # stp poll. 0 means the switch is reachable via management only and
        # should be excluded from cross-switch consensus checks.
        conn.execute("ALTER TABLE devices ADD COLUMN link_up_count INTEGER")
    # SNMP columns: populated by 'discover'. snmp_enabled tri-state
    # (NULL=never probed, 1=verified working, 0=verified disabled/unreachable).
    # snmp_community / snmp_v3_user record the winning credential so future
    # polls don't have to walk the full list each time.
    if "snmp_enabled" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_enabled INTEGER")
    if "snmp_proto" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_proto TEXT")
    if "snmp_community" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_community TEXT")
    if "snmp_v3_user" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_v3_user TEXT")
    if "snmp_last_ok" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_last_ok TEXT")
    if "snmp_last_check" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_last_check TEXT")
    if "snmp_diag" not in cols:
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_diag TEXT")
    if "stp_disabled_since" not in cols:
        # Set to the timestamp when stp_enabled first transitioned 1->0
        # (or NULL/None -> 0). Cleared back to NULL on 0->1. Used by the
        # digest to require sustained-disabled state before listing, so a
        # transient empty SNMP poll (e.g. during a flap storm) doesn't
        # produce a false-positive in the next 8 AM digest.
        conn.execute("ALTER TABLE devices ADD COLUMN stp_disabled_since TEXT")
    if "snmp_consecutive_fails" not in cols:
        # Incremented on every 'monitor stp' poll that returns None (no STP
        # data via SNMP and, in auto mode, SSH fallback also failed). Reset
        # to 0 on any successful poll. Drives the unreachable-alert
        # hysteresis: alert only after N consecutive fails so a single
        # dropped UDP packet doesn't page anyone.
        conn.execute("ALTER TABLE devices ADD COLUMN snmp_consecutive_fails INTEGER DEFAULT 0")
    if "unreachable_alerted_at" not in cols:
        # Timestamp the unreachable alert email went out. NULL = no current
        # outage alert. Set when consecutive-fails crosses the threshold;
        # cleared (and a recovery email sent) on the next successful poll.
        # Ensures one email per outage, not one per minute.
        conn.execute("ALTER TABLE devices ADD COLUMN unreachable_alerted_at TEXT")
    # Migrate from old password_idx column (drop data — idx is meaningless now)
    if "password_idx" in cols:
        conn.execute("UPDATE devices SET password_hash = NULL")

    # Phantom-evidence columns: per-port state-machine activity counter
    # (jnxMIMstCistEdgeDelayWhileExpiryCount). Captured on every Junos
    # SNMP poll so a confirmed STP transition can be evaluated against
    # whether the state machine actually ran (delta>0) or the MIB just
    # flickered (delta==0). Junos-only at the moment; NULL on other
    # platforms is fine.
    stp_state_cols = [r[1] for r in conn.execute(
        "PRAGMA table_info(stp_state)").fetchall()]
    if "edge_expiry" not in stp_state_cols:
        conn.execute("ALTER TABLE stp_state ADD COLUMN edge_expiry INTEGER")
    pending_cols = [r[1] for r in conn.execute(
        "PRAGMA table_info(stp_pending_changes)").fetchall()]
    if "edge_expiry_at_first_seen" not in pending_cols:
        conn.execute("ALTER TABLE stp_pending_changes "
                     "ADD COLUMN edge_expiry_at_first_seen INTEGER")
    if "tcn_at_first_seen" not in pending_cols:
        conn.execute("ALTER TABLE stp_pending_changes "
                     "ADD COLUMN tcn_at_first_seen INTEGER")
    conn.close()


def _db():
    """Return a connection with row_factory set for dict-like access."""
    conn = sqlite3.connect(DB_FILE, timeout=30)
    conn.row_factory = sqlite3.Row
    return conn


# Held flock file descriptors — a module-level dict keeps them alive for
# the process lifetime so the kernel-owned advisory lock stays in force.
# The kernel releases the lock automatically when the process exits, so
# there's no stale-lockfile problem.
_HELD_LOCKS = {}


def _acquire_advisory_lock(name, wait=0):
    """Take an exclusive lock at STATE_DIR/.netops-<name>.lock.

    Returns True on success, False if the holder is still there at the
    deadline — the failure message names the current holder (pid + user
    + start time) so the operator knows what to wait for.

    `wait` = max seconds to block waiting for the lock (default 0 =
    non-blocking). When wait > 0, polls every 500 ms until acquired or
    the deadline expires; run_backup and run_topology use wait=90 so a
    collision with the per-minute monitor-stp tick is waited out rather
    than silently skipped.

    The lock lives in the state dir, not /tmp. The netops timers run as
    the `netops` user while ad-hoc runs may be root or a netops-group
    human; a /tmp lock file owned by one of them is unopenable by the
    others, because /tmp is sticky + world-writable and the kernel's
    fs.protected_regular hardening blocks the cross-user write-open even
    for root. STATE_DIR is netops-group-owned and not world-writable, so
    one group-writable lock file there is shareable by every netops
    runner. fcntl.lockf (POSIX record locks) is used; the kernel drops
    the lock when the holding process exits, so there is no stale-lock
    problem.
    """
    import fcntl as _fcntl
    lock_path = os.path.join(STATE_DIR, f".netops-{name}.lock")
    try:
        fh = open(lock_path, "a+")
    except OSError as e:
        log.warning("advisory lock %s open failed: %s — running without lock",
                    lock_path, e)
        return True
    # Group-writable so the netops user, root, and netops-group humans
    # can all share it (best-effort — only the creator can chmod it).
    try:
        os.fchmod(fh.fileno(), 0o664)
    except OSError:
        pass
    deadline = time.monotonic() + wait if wait > 0 else None
    while True:
        try:
            _fcntl.lockf(fh.fileno(), _fcntl.LOCK_EX | _fcntl.LOCK_NB)
            break  # acquired
        except (OSError, BlockingIOError):
            if deadline is None or time.monotonic() >= deadline:
                fh.seek(0)
                meta = (fh.read() or "").strip() or "(holder metadata unavailable)"
                fh.close()
                log.error("Another netops run holds the '%s' lock: %s", name, meta)
                log.error("Skipping this invocation. Wait for the current run, "
                          "or check %s.", lock_path)
                return False
            time.sleep(0.5)
    _HELD_LOCKS[name] = fh
    try:
        fh.seek(0); fh.truncate()
        admin = (os.environ.get("NETOPS_ADMIN")
                 or os.environ.get("USER") or "unknown")
        started = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        fh.write(f"pid={os.getpid()} user={admin} started={started} cmd={name}\n")
        fh.flush()
    except Exception:
        pass  # lock acquired; metadata write is best-effort
    return True


def upsert_device(ip, base_mac=None, hostname=None, model=None, serial=None,
                  firmware=None, hardware_info=None, proto=None, username=None,
                  password_hash=None, ssh_open=None, telnet_open=None,
                  status="active", fail_reason=None, duplicate_of=None,
                  dns_name=None, platform=None, preserve_status=False):
    """Insert or update a device record.

    Preserves first_seen on update. When updating an existing device, None
    values keep the existing data (won't overwrite with NULL).

    preserve_status: if True and the device already exists, keep its current
    status (and fail_reason) instead of overwriting. Used by import flows
    that should not reverse a device's real operational state.
    """
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    existing = conn.execute("SELECT * FROM devices WHERE ip = ?", (ip,)).fetchone()
    first_seen = existing["first_seen"] if existing else now

    # Import-style flows: don't let the caller change a device's real state.
    if existing and preserve_status:
        status = existing["status"]
        fail_reason = existing["fail_reason"]

    # Columns populated by 'monitor stp', record_snmp_result, or other
    # background paths — NEVER by upsert_device callers. Because this uses
    # INSERT OR REPLACE (i.e. DELETE + INSERT), any column we don't list
    # would be reset to its DEFAULT on every device touch. Anything written
    # outside upsert_device MUST be read here and round-tripped, or it gets
    # silently zeroed. (Pain point: this broke per-event unreachable alerts —
    # nightly backup was wiping snmp_consecutive_fails / unreachable_alerted_at
    # at 01:30 and the threshold re-fired at 01:33 every morning.)
    stp_mode = None
    snmp_enabled = None
    snmp_proto = None
    snmp_community = None
    snmp_v3_user = None
    snmp_last_ok = None
    snmp_last_check = None
    snmp_diag = None
    stp_enabled = None
    stp_last_check = None
    stp_disabled_since = None
    link_up_count = None
    snmp_consecutive_fails = 0
    unreachable_alerted_at = None

    # Preserve existing values when caller passes None
    if existing:
        keys = existing.keys()
        base_mac = base_mac if base_mac is not None else existing["base_mac"]
        hostname = hostname if hostname is not None else existing["hostname"]
        model = model if model is not None else existing["model"]
        serial = serial if serial is not None else existing["serial"]
        firmware = firmware if firmware is not None else existing["firmware"]
        hardware_info = hardware_info if hardware_info is not None else existing["hardware_info"]
        proto = proto if proto is not None else existing["proto"]
        username = username if username is not None else existing["username"]
        password_hash = password_hash if password_hash is not None else existing["password_hash"]
        ssh_open = ssh_open if ssh_open is not None else existing["ssh_open"]
        telnet_open = telnet_open if telnet_open is not None else existing["telnet_open"]
        dns_name = dns_name if dns_name is not None else (existing["dns_name"] if "dns_name" in keys else None)
        platform = platform if platform is not None else (existing["platform"] if "platform" in keys else None)
        stp_mode = existing["stp_mode"] if "stp_mode" in keys else None
        snmp_enabled  = existing["snmp_enabled"]   if "snmp_enabled"   in keys else None
        snmp_proto    = existing["snmp_proto"]     if "snmp_proto"     in keys else None
        snmp_community = existing["snmp_community"] if "snmp_community" in keys else None
        snmp_v3_user  = existing["snmp_v3_user"]   if "snmp_v3_user"   in keys else None
        snmp_last_ok  = existing["snmp_last_ok"]   if "snmp_last_ok"   in keys else None
        snmp_last_check = existing["snmp_last_check"] if "snmp_last_check" in keys else None
        snmp_diag     = existing["snmp_diag"]      if "snmp_diag"      in keys else None
        stp_enabled   = existing["stp_enabled"]    if "stp_enabled"    in keys else None
        stp_last_check = existing["stp_last_check"] if "stp_last_check" in keys else None
        stp_disabled_since = existing["stp_disabled_since"] if "stp_disabled_since" in keys else None
        link_up_count = existing["link_up_count"]  if "link_up_count"  in keys else None
        snmp_consecutive_fails = (existing["snmp_consecutive_fails"]
                                  if "snmp_consecutive_fails" in keys else 0) or 0
        unreachable_alerted_at = (existing["unreachable_alerted_at"]
                                  if "unreachable_alerted_at" in keys else None)

    # Default to 0 for new devices where ports weren't checked
    if ssh_open is None:
        ssh_open = 0
    if telnet_open is None:
        telnet_open = 0

    conn.execute("""
        INSERT OR REPLACE INTO devices
            (ip, base_mac, hostname, dns_name, model, serial, firmware, hardware_info,
             proto, username, password_hash, ssh_open, telnet_open, platform, stp_mode,
             snmp_enabled, snmp_proto, snmp_community, snmp_v3_user,
             snmp_last_ok, snmp_last_check, snmp_diag,
             stp_enabled, stp_last_check, stp_disabled_since, link_up_count,
             snmp_consecutive_fails, unreachable_alerted_at,
             status, fail_reason, duplicate_of, first_seen, last_seen)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                ?, ?, ?, ?, ?, ?, ?,
                ?, ?, ?, ?,
                ?, ?,
                ?, ?, ?, ?, ?)
    """, (ip, base_mac, hostname, dns_name, model, serial, firmware, hardware_info,
          proto, username, password_hash, ssh_open, telnet_open, platform, stp_mode,
          snmp_enabled, snmp_proto, snmp_community, snmp_v3_user,
          snmp_last_ok, snmp_last_check, snmp_diag,
          stp_enabled, stp_last_check, stp_disabled_since, link_up_count,
          snmp_consecutive_fails, unreachable_alerted_at,
          status, fail_reason, duplicate_of, first_seen, now))
    conn.commit()
    conn.close()


def record_snmp_result(ip, *, enabled, proto=None, community=None,
                        v3_user=None, diag=None, ok_now=False):
    """Update SNMP-related columns on a device row. Called after a probe.

    enabled:   1 = working, 0 = verified disabled/unreachable, None = unknown
    proto:     'v2c' or 'v3' (winning protocol; None when no cred worked)
    community: winning v2c community string, or None
    v3_user:   winning v3 user name, or None
    diag:      free-form diagnostic message (cleared when probe succeeds)
    ok_now:    bump snmp_last_ok to now when True; always bump snmp_last_check
    """
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    if ok_now:
        conn.execute(
            "UPDATE devices SET snmp_enabled=?, snmp_proto=?, snmp_community=?,"
            " snmp_v3_user=?, snmp_diag=?, snmp_last_ok=?, snmp_last_check=?"
            " WHERE ip=?",
            (enabled, proto, community, v3_user, diag, now, now, ip))
    else:
        conn.execute(
            "UPDATE devices SET snmp_enabled=?, snmp_proto=?, snmp_community=?,"
            " snmp_v3_user=?, snmp_diag=?, snmp_last_check=?"
            " WHERE ip=?",
            (enabled, proto, community, v3_user, diag, now, ip))
    conn.commit()
    conn.close()


def reverse_dns(ip, timeout=2.0):
    """Return the reverse DNS name for an IP, or empty string if unavailable."""
    try:
        old = socket.getdefaulttimeout()
        socket.setdefaulttimeout(timeout)
        try:
            name, _, _ = socket.gethostbyaddr(ip)
            return name
        finally:
            socket.setdefaulttimeout(old)
    except (socket.herror, socket.gaierror, socket.timeout, OSError):
        return ""


def resolve_dns_batch(ips, threads=50, timeout=2.0):
    """Reverse-resolve a list of IPs in parallel. Returns {ip: dns_name_or_empty}."""
    results = {}
    if not ips:
        return results
    log.debug("Reverse DNS on %d IP(s) with %d threads", len(ips), threads)
    with ThreadPoolExecutor(max_workers=min(threads, max(1, len(ips)))) as pool:
        futures = {pool.submit(reverse_dns, ip, timeout): ip for ip in ips}
        for future in as_completed(futures):
            ip = futures[future]
            results[ip] = future.result() or ""
    return results


def record_backup(ip, base_mac, hostname, filename, config_hash, changed):
    """Record a backup operation in the history."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    conn.execute("""
        INSERT INTO backups (ip, base_mac, hostname, filename, config_hash, changed, backed_up_at)
        VALUES (?, ?, ?, ?, ?, ?, ?)
    """, (ip, base_mac, hostname, filename, config_hash, int(changed), now))
    conn.commit()
    conn.close()


def _ip_sort_key(ip_str):
    """Return a sort key that orders IPs numerically (1.2.3.4 < 1.2.3.10).
    Falls back to the raw string for non-IP values so sort() never crashes.
    """
    try:
        return (0, int(ipaddress.ip_address(ip_str)))
    except (ValueError, TypeError):
        return (1, str(ip_str or ""))


def _sort_by_ip(rows, ip_field="ip"):
    """Return rows sorted numerically by their IP field."""
    return sorted(rows, key=lambda r: _ip_sort_key(r[ip_field]))


def get_devices(status=None):
    """Query devices, optionally filtered by status. Sorted numerically by IP."""
    conn = _db()
    if status:
        rows = conn.execute(
            "SELECT * FROM devices WHERE status = ?", (status,)
        ).fetchall()
        rows = _sort_by_ip(rows)
    else:
        # Sort by status first, then IP numerically within each status.
        rows = conn.execute("SELECT * FROM devices").fetchall()
        rows = sorted(rows, key=lambda r: (r["status"] or "", _ip_sort_key(r["ip"])))
    conn.close()
    return rows


def get_backups(ip=None, limit=None):
    """Query backup history, optionally filtered by IP."""
    conn = _db()
    sql = "SELECT * FROM backups"
    params = []
    if ip:
        sql += " WHERE ip = ?"
        params.append(ip)
    sql += " ORDER BY backed_up_at DESC"
    if limit:
        sql += " LIMIT ?"
        params.append(limit)
    rows = conn.execute(sql, params).fetchall()
    conn.close()
    return rows


def get_duplicates():
    """Query devices grouped by base_mac where multiple IPs share the same switch."""
    conn = _db()
    rows = conn.execute("""
        SELECT d.* FROM devices d
        WHERE d.base_mac IN (
            SELECT base_mac FROM devices
            WHERE base_mac IS NOT NULL AND base_mac != ''
            GROUP BY base_mac HAVING COUNT(*) > 1
        )
        ORDER BY d.base_mac, d.ip
    """).fetchall()
    conn.close()
    return rows


def remove_device(ip):
    """Remove a device and its backup history from the database.

    Returns True if the device existed, False otherwise.
    """
    conn = _db()
    cursor = conn.execute("DELETE FROM devices WHERE ip = ?", (ip,))
    conn.execute("DELETE FROM backups WHERE ip = ?", (ip,))
    conn.commit()
    existed = cursor.rowcount > 0
    conn.close()
    return existed


def remove_devices_by_status(status):
    """Remove all devices with the given status. Returns count removed."""
    conn = _db()
    cursor = conn.execute("DELETE FROM devices WHERE status = ?", (status,))
    count = cursor.rowcount
    conn.commit()
    conn.close()
    return count


# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------

# Memory estimate per SSH session (pexpect + system ssh overhead)
_SSH_MEM_MB = 5
# Maximum fraction of system RAM to use for concurrent SSH sessions
_SSH_MEM_FRACTION = 0.25


def _max_open_files():
    """Return the OS soft limit on open file descriptors.

    Returns a practical integer even when the OS reports 'unlimited'.
    """
    try:
        import resource
        soft = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
        # macOS can report RLIM_INFINITY; fall back to kern.maxfilesperproc
        if soft > 100_000:
            try:
                import subprocess as _sp
                out = _sp.check_output(
                    ["sysctl", "-n", "kern.maxfilesperproc"],
                    stderr=_sp.DEVNULL, text=True,
                )
                return int(out.strip())
            except Exception:
                return 1024  # safe fallback
        return soft
    except (ImportError, AttributeError):
        # Windows: no resource module; query via ctypes
        try:
            import ctypes
            kernel32 = ctypes.windll.kernel32
            # _getmaxstdio returns CRT file handle limit (default 512)
            msvcrt = ctypes.cdll.msvcrt
            return msvcrt._getmaxstdio()
        except Exception:
            return 512  # safe fallback


def _total_memory_mb():
    """Return total system memory in MB."""
    # Linux / macOS
    try:
        pages = os.sysconf("SC_PHYS_PAGES")
        page_size = os.sysconf("SC_PAGE_SIZE")
        return (pages * page_size) // (1024 * 1024)
    except (ValueError, OSError, AttributeError):
        pass
    # Windows
    try:
        import ctypes

        class MEMORYSTATUSEX(ctypes.Structure):
            _fields_ = [
                ("dwLength", ctypes.c_ulong),
                ("dwMemoryLoad", ctypes.c_ulong),
                ("ullTotalPhys", ctypes.c_ulonglong),
                ("ullAvailPhys", ctypes.c_ulonglong),
                ("ullTotalPageFile", ctypes.c_ulonglong),
                ("ullAvailPageFile", ctypes.c_ulonglong),
                ("ullTotalVirtual", ctypes.c_ulonglong),
                ("ullAvailVirtual", ctypes.c_ulonglong),
                ("ullAvailExtendedVirtual", ctypes.c_ulonglong),
            ]

        mem = MEMORYSTATUSEX()
        mem.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
        ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(mem))
        return mem.ullTotalPhys // (1024 * 1024)
    except Exception:
        pass
    return 4096  # 4 GB conservative fallback


def load_config():
    """Read netops.conf (or legacy config.ini) and return a dict of
    settings with defaults."""
    cp = configparser.ConfigParser(interpolation=None)
    # Preserve case so [user_passwords] can distinguish e.g. 'admin' vs 'Admin'
    # (network devices often treat usernames as case-sensitive).
    cp.optionxform = str
    cp.read(CONFIG_FILE, encoding="utf-8")
    # Nudge legacy filename users toward the new name for consistency
    # with secrets.conf. Soft warning; functionality unchanged.
    if CONFIG_FILE == _CONFIG_FILE_LEGACY and os.path.exists(_CONFIG_FILE_LEGACY):
        log.warning(
            "Using legacy config filename %s — rename to %s for consistency "
            "with secrets.conf. Both names are supported; the legacy fallback "
            "may be removed in a future release.",
            _CONFIG_FILE_LEGACY, _CONFIG_FILE_NEW)

    cfg = {
        "storage_mode": cp.get("general", "storage_mode", fallback="git"),
        "log_format": cp.get("general", "log_format", fallback="full"),
        "device_file": cp.get("general", "device_file", fallback="devices.txt"),
        "subnets": [
            s.strip()
            for s in cp.get("scan", "subnets", fallback="").split(",")
            if s.strip()
        ],
        "ports": [
            int(p.strip())
            for p in cp.get("scan", "ports", fallback="22,23").split(",")
            if p.strip()
        ],
        "timeout": cp.getfloat("scan", "timeout", fallback=1.0),
        "scan_threads": cp.getint("general", "scan_threads", fallback=200),
        "backup_threads": cp.getint("general", "backup_threads", fallback=20),
        "usernames": [
            u.strip()
            for u in cp.get("backup", "usernames", fallback="admin, root").split(",")
            if u.strip()
        ],
    }

    # --- Usernames: env var > secrets.conf > config.ini ---
    env_users = os.environ.get("NETOPS_USERNAMES", "").strip()
    if env_users:
        cfg["usernames"] = [u.strip() for u in env_users.split(",") if u.strip()]

    # Read secrets.conf (same dir as script, cross-platform)
    secrets = configparser.ConfigParser(interpolation=None)
    secrets.optionxform = str
    if os.path.exists(SECRETS_FILE):
        secrets.read(SECRETS_FILE, encoding="utf-8")
        # Usernames from secrets.conf override config.ini (but env var wins)
        if not env_users:
            sec_users = secrets.get("credentials", "usernames", fallback="").strip()
            if sec_users:
                cfg["usernames"] = [u.strip() for u in sec_users.split(",") if u.strip()]

    # Build passwords dict: {"default": "...", "paloalto_panos": "...", ...}
    passwords = {"default": cp.get("backup", "password", fallback="")}
    if cp.has_section("backup"):
        for key in cp.options("backup"):
            if key.startswith("password_"):
                device_type = key[len("password_"):]
                passwords[device_type] = cp.get("backup", key)
    cfg["passwords"] = passwords

    # --- Password list priority: env var > secrets.conf > config.ini ---
    env_pw = os.environ.get("NETOPS_PASSWORDS", "").strip()
    # Resolution order: env var → secrets.conf → config.ini.
    # secrets.conf is opt-in: defining the file does NOT mean opting out of
    # config.ini's [backup] passwords / password. Used to: if secrets.conf
    # existed but lacked a 'passwords' entry, password_list was set to []
    # and config.ini was ignored. That broke every call site that passed
    # password=None and relied on password_list (investigate_stp_ports,
    # monitor topology) while backup kept working because it also passes
    # the single password as a positional arg.
    def _from_config_ini():
        pw_list_raw = cp.get("backup", "passwords", fallback="")
        if pw_list_raw.strip():
            return [p.strip() for p in pw_list_raw.split(",") if p.strip()]
        return [passwords["default"]] if passwords["default"] else []

    if env_pw:
        cfg["password_list"] = [p.strip() for p in env_pw.split(",") if p.strip()]
        cfg["_pw_source"] = "env"
    elif os.path.exists(SECRETS_FILE):
        sec_pw = secrets.get("credentials", "passwords", fallback="").strip()
        if sec_pw:
            cfg["password_list"] = [p.strip() for p in sec_pw.split(",") if p.strip()]
            cfg["_pw_source"] = "secrets.conf"
        else:
            cfg["password_list"] = _from_config_ini()
            cfg["_pw_source"] = "config" if cfg["password_list"] else "none"
    else:
        cfg["password_list"] = _from_config_ini()
        cfg["_pw_source"] = "config" if cfg["password_list"] else "none"

    # Ensure default password is set for legacy callers
    if cfg["password_list"] and not passwords["default"]:
        passwords["default"] = cfg["password_list"][0]

    # --- Per-user password map (for LDAP/RADIUS-backed users and avoiding
    # lockouts from cross-product attempts). Merge config.ini then secrets.conf
    # per-key; secrets.conf wins. Values are lists of passwords for that user. ---
    user_passwords = {}
    if cp.has_section("user_passwords"):
        for user, pws in cp.items("user_passwords"):
            user_passwords[user] = [p.strip() for p in pws.split(",") if p.strip()]
    if os.path.exists(SECRETS_FILE) and secrets.has_section("user_passwords"):
        for user, pws in secrets.items("user_passwords"):
            user_passwords[user] = [p.strip() for p in pws.split(",") if p.strip()]
    cfg["user_passwords"] = user_passwords

    # --- Email config for alerts ---
    if cp.has_section("email"):
        cfg["smtp_server"] = cp.get("email", "smtp_server", fallback="")
        cfg["smtp_port"] = cp.get("email", "smtp_port", fallback="587")
        cfg["email_from"] = cp.get("email", "from", fallback="netops@localhost")
        cfg["email_to"] = cp.get("email", "to", fallback="")
        cfg["smtp_username"] = cp.get("email", "smtp_username", fallback="")
        cfg["smtp_password"] = cp.get("email", "smtp_password", fallback="")

    # Warn if config.ini contains passwords and has loose permissions
    if cfg["_pw_source"] == "config" and sys.platform != "win32":
        try:
            mode = os.stat(CONFIG_FILE).st_mode & 0o777
            if mode & _SECRET_FORBIDDEN_MODE:  # group-write or world access
                log.warning(
                    "WARNING: %s contains passwords and has permissions %04o — "
                    "consider moving passwords to secrets.conf or "
                    "'chmod 640 %s' (0640 root:%s is fine for a shared group)",
                    CONFIG_FILE, mode, CONFIG_FILE, SECRET_GROUP)
        except OSError:
            pass
    # Warn if secrets.conf has loose permissions
    if cfg["_pw_source"] == "secrets.conf" and sys.platform != "win32":
        try:
            mode = os.stat(SECRETS_FILE).st_mode & 0o777
            if mode & _SECRET_FORBIDDEN_MODE:
                log.warning(
                    "WARNING: %s has permissions %04o — group-write or "
                    "world access; consider 'chmod 640 %s' and 'chgrp %s %s'",
                    SECRETS_FILE, mode, SECRETS_FILE, SECRET_GROUP, SECRETS_FILE)
        except OSError:
            pass

    # --- Credentials must live in secrets.conf --------------------------
    # netops refuses to run if any credential material is present in
    # netops.conf (or legacy config.ini). ALL credentials — backup
    # passwords, per-user passwords, and the entire [snmp] / [snmp_v3:*]
    # configuration — belong in secrets.conf, which is 0640 root:netops so
    # a shared operator group can read it without keeping private copies.
    # This is unconditional (there is no opt-out): it guarantees an
    # incidental netops.conf edit can never expose a credential in a file
    # whose permissions are looser than secrets.conf's.
    violations = []
    if not os.path.exists(SECRETS_FILE):
        violations.append(
            f"{SECRETS_FILE} does not exist — all credentials live there. "
            f"Create it (0640 root:netops) with the credential sections.")
    elif sys.platform != "win32":
        try:
            mode = os.stat(SECRETS_FILE).st_mode & 0o777
            if mode & _SECRET_FORBIDDEN_MODE:
                violations.append(
                    f"{SECRETS_FILE} has permissions {mode:04o} (must be "
                    f"0640 or stricter — owner rw, optional group-read for "
                    f"the {SECRET_GROUP} group; no group-write or world "
                    f"access). Run 'chmod 640 {SECRETS_FILE} && chgrp "
                    f"{SECRET_GROUP} {SECRETS_FILE}'.")
        except OSError as e:
            violations.append(f"could not stat {SECRETS_FILE}: {e}")
    config_cred_keys = []
    if cp.has_section("backup"):
        for key in cp.options("backup"):
            if (key in ("password", "passwords")
                    or key.startswith("password_")) and \
                    cp.get("backup", key, fallback="").strip():
                config_cred_keys.append(f"[backup] {key}")
    if cp.has_section("user_passwords") and cp.options("user_passwords"):
        config_cred_keys.append("[user_passwords] section")
    if cp.has_section("snmp"):
        config_cred_keys.append("[snmp] section")
    for section in cp.sections():
        if section.startswith("snmp_v3:"):
            config_cred_keys.append(f"[{section}] section")
    if config_cred_keys:
        violations.append(
            f"{CONFIG_FILE} contains credential material that must move to "
            f"{SECRETS_FILE}: {', '.join(config_cred_keys)}.")
    if violations:
        log.error("credential configuration error — all credentials must "
                  "live in %s, not %s:", SECRETS_FILE, CONFIG_FILE)
        for v in violations:
            log.error("  • %s", v)
        sys.exit(2)

    # Resolve device_file relative to the state dir if not absolute (it's an
    # operator-supplied input list and any failed_devices.txt is written
    # alongside it; both belong with the mutable state, not in /usr/bin).
    if not os.path.isabs(cfg["device_file"]):
        cfg["device_file"] = os.path.join(STATE_DIR, cfg["device_file"])

    # Cap threads to OS-safe limits (file descriptors + memory)
    fd_limit = _max_open_files()
    total_mem = _total_memory_mb()

    # Scan: limited by FDs only (lightweight TCP sockets, 1 FD each)
    scan_cap = max(fd_limit - 100, 10)  # reserve 100 FDs for logging, etc.
    scan_limit_reason = (
        f"OS fd limit {fd_limit} — 100 reserved for system = {scan_cap} max"
    )

    # Backup/test: limited by FDs and memory (whichever is lower)
    # Each SSH session uses ~3 FDs and ~5 MB RAM
    fd_backup_cap = max((fd_limit - 100) // 3, 10)
    mem_backup_cap = max(int(total_mem * _SSH_MEM_FRACTION / _SSH_MEM_MB), 10)
    if fd_backup_cap <= mem_backup_cap:
        backup_cap = fd_backup_cap
        backup_limit_reason = (
            f"OS fd limit {fd_limit} — 100 reserved, "
            f"~3 fds per SSH session = {fd_backup_cap} max"
        )
    else:
        backup_cap = mem_backup_cap
        usable_mb = int(total_mem * _SSH_MEM_FRACTION)
        backup_limit_reason = (
            f"system memory {total_mem} MB — "
            f"25% usable ({usable_mb} MB) at ~{_SSH_MEM_MB} MB per SSH session "
            f"= {mem_backup_cap} max"
        )

    raw_scan = cfg["scan_threads"]
    raw_backup = cfg["backup_threads"]
    cfg["scan_threads"] = min(raw_scan, scan_cap)
    cfg["backup_threads"] = min(raw_backup, backup_cap)
    cfg["_thread_caps"] = {
        "fd_limit": fd_limit,
        "total_mem_mb": total_mem,
        "raw_scan": raw_scan,
        "raw_backup": raw_backup,
        "scan_cap": scan_cap,
        "scan_limit_reason": scan_limit_reason,
        "backup_cap": backup_cap,
        "backup_limit_reason": backup_limit_reason,
        "scan_capped": raw_scan > scan_cap,
        "backup_capped": raw_backup > backup_cap,
    }

    # Site-wide expected STP mode. Compared case-insensitively against the
    # mode parsed from each device during 'monitor stp'; mismatches surface
    # in 'show spanning-tree mismatch' and the daily 'digest stp' email.
    cfg["stp_expected_mode"] = cp.get(
        "monitor", "stp_expected_mode", fallback="mstp"
    ).strip().lower()

    # Threshold for 'digest backup' stale classification — devices whose
    # last successful backup is older than this are flagged in the email.
    cfg["backup_stale_days"] = cp.getint(
        "monitor", "backup_stale_days", fallback=7)

    # Non-trunk MAC-flux security threshold: a notice fires when a single
    # non-trunk port sees >= this many distinct MACs in a trailing 24h
    # window (rogue mini-switch / AP / MAC spoofing). 0 disables. Trunk /
    # uplink ports (LLDP switch<->switch links) are always excluded.
    # 3.8.9 rename: keys are now `mac_flux_*` (was `mac_churn_*` in
    # 3.8.0-3.8.8). _getint_legacy() reads the new name first; if that's
    # missing AND the old name is present, it reads the old name and emits
    # a one-time DEBUG hint. Existing netops.conf files with the old names
    # continue working unchanged; new installs use the new names.
    def _getint_legacy(new_key, old_key, fallback):
        if cp.has_option("monitor", new_key):
            return cp.getint("monitor", new_key)
        if cp.has_option("monitor", old_key):
            log.debug("[monitor] %s is the 3.8.9 rename of %s — both work, "
                      "but the new name is preferred", new_key, old_key)
            return cp.getint("monitor", old_key)
        return fallback
    def _get_legacy(new_key, old_key, fallback):
        if cp.has_option("monitor", new_key):
            return cp.get("monitor", new_key)
        if cp.has_option("monitor", old_key):
            log.debug("[monitor] %s is the 3.8.9 rename of %s — both work, "
                      "but the new name is preferred", new_key, old_key)
            return cp.get("monitor", old_key)
        return fallback
    cfg["mac_flux_threshold_24h"] = _getint_legacy(
        "mac_flux_threshold_24h", "mac_churn_threshold_24h", 5)
    # A non-trunk port with MORE than this many distinct MACs in 24h is
    # structurally an aggregation/uplink (not one rogue device) and is
    # excluded from the flux signal. 0 disables the ceiling check.
    cfg["mac_flux_uplink_ceiling"] = _getint_legacy(
        "mac_flux_uplink_ceiling", "mac_churn_uplink_ceiling", 24)
    # A real access port carries one data VLAN (+ maybe a voice VLAN). A
    # port whose MACs span more than this many VLANs is a trunk or a
    # virtualization host (VMware vSwitch uplink etc.) — excluded from the
    # flux signal so vMotion / many-VM ports don't false-alarm. 0 disables.
    cfg["mac_flux_max_vlans"] = _getint_legacy(
        "mac_flux_max_vlans", "mac_churn_max_vlans", 2)
    # 3.8.7 rotation filter — only MACs that look like brief passers-
    # through count toward the flux threshold. A MAC qualifies as
    # "rotating" when its times_seen <= max_times_seen AND its span
    # (last_seen - first_seen) < max_span_sec. Stable residents (busy
    # AV-room ports, multi-device desks) score 0 rotating MACs and drop
    # out cleanly. Set either to 0 to disable that part of the filter
    # and fall back to 3.8.6's "count all 24h distincts" semantics.
    cfg["mac_flux_rotation_max_times_seen"] = _getint_legacy(
        "mac_flux_rotation_max_times_seen",
        "mac_churn_rotation_max_times_seen", 3)
    cfg["mac_flux_rotation_max_span_sec"] = _getint_legacy(
        "mac_flux_rotation_max_span_sec",
        "mac_churn_rotation_max_span_sec", 7200)
    # 3.8.7 manual whitelist — approved (ip:interface) pairs that should
    # never alert (e.g. an approved A/V switch downstream of an access
    # port). Whitespace- or comma-separated list of ip:interface tokens.
    # Existing mac_flux_alerts rows for excluded ports clear on the
    # next 30-min tick because they drop out of the current flux list.
    raw = _get_legacy(
        "mac_flux_excluded_ports", "mac_churn_excluded_ports", "").strip()
    excluded = set()
    for tok in re.split(r"[\s,]+", raw):
        if ":" in tok:
            ip, _, iface = tok.partition(":")
            if ip and iface:
                excluded.add((ip.strip(), iface.strip()))
    cfg["mac_flux_excluded_ports"] = excluded

    # Per-event unreachable-alert hysteresis: 'monitor stp' emails after N
    # consecutive failed polls so a single dropped UDP packet doesn't page.
    # 3 polls at every-minute cadence = ~3 minutes of sustained failure.
    cfg["snmp_unreachable_polls"] = cp.getint(
        "monitor", "snmp_unreachable_polls", fallback=3)

    # 'monitor stp' transport selection. 'auto' uses SNMP on devices with
    # snmp_enabled=1 (cached cred from discover) and falls back to SSH
    # otherwise — the load-reducing default. 'snmp' forces SNMP and skips
    # devices that haven't been probed yet; 'ssh' keeps the legacy path
    # everywhere.
    cfg["stp_poll_mode"] = cp.get(
        "monitor", "stp_mode", fallback="auto").strip().lower()
    if cfg["stp_poll_mode"] not in ("auto", "snmp", "ssh"):
        log.warning("[monitor] stp_mode=%r invalid; using 'auto'",
                    cfg["stp_poll_mode"])
        cfg["stp_poll_mode"] = "auto"

    # Optional STP domain mapping — for multi-site networks where switches
    # in different L2 domains (separate sites, IPsec-tunneled offices, etc.)
    # legitimately have different root bridges. Keys are IP prefixes, values
    # are domain names. Longest-prefix match wins. Empty map = single
    # implicit domain (backward-compatible default for single-site setups).
    cfg["stp_domains"] = {}
    if cp.has_section("stp_domains"):
        for prefix_str, domain in cp.items("stp_domains"):
            try:
                net = ipaddress.ip_network(prefix_str.strip(), strict=False)
            except ValueError:
                log.warning("[stp_domains] invalid prefix %r — skipping",
                            prefix_str)
                continue
            cfg["stp_domains"][net] = domain.strip()

    # Optional remote git URL for pushing committed backups. Supports any
    # form git itself does: github.com SSH or HTTPS, on-prem ssh:// URL,
    # bare repo on a network share, etc. Authentication (keys, PAT, credential
    # helper) is the user's responsibility — netops doesn't store secrets.
    # When empty, backups stay local-only.
    cfg["git_remote_url"] = cp.get("backup", "git_remote_url", fallback="").strip()
    cfg["git_branch"] = cp.get("backup", "git_branch", fallback="main").strip() or "main"

    # --- SNMP credentials (optional) ---
    # Accept multiple v2c communities and multiple v3 users. The probe tries
    # the cached winner first (per-device); list-walk happens only during
    # discover. The entire [snmp] / [snmp_v3:*] config lives in secrets.conf.
    cfg["snmp"] = _load_snmp_config(secrets)

    return cfg


def _load_snmp_config(secrets):
    """Parse [snmp] + [snmp_v3:USER] sections from secrets.conf.

    SNMP credentials (v2c communities, v3 passphrases) are credentials, so
    the entire [snmp] / [snmp_v3:*] configuration lives only in secrets.conf
    — netops.conf is rejected at startup if it contains those sections.

    Returns a dict with keys:
      communities    list of v2c community strings
      v3_users       list of dicts (name, auth_proto, auth_pass, priv_proto,
                     priv_pass, security_level)
      timeout        per-request UDP timeout, seconds
      retries        retry count on no-response
      enabled        True if at least one credential is configured
    """
    out = {"communities": [], "v3_users": [], "timeout": 5, "retries": 1,
           "enabled": False}
    if not secrets:
        return out

    if secrets.has_section("snmp"):
        # 'communities' (plural, comma-list) takes precedence; fall back to
        # singular 'community' for the single-string case.
        raw = secrets.get("snmp", "communities", fallback="").strip()
        if raw:
            out["communities"] = [c.strip() for c in raw.split(",") if c.strip()]
        else:
            single = secrets.get("snmp", "community", fallback="").strip()
            if single:
                out["communities"] = [single]
        out["timeout"] = secrets.getint("snmp", "timeout", fallback=5)
        out["retries"] = secrets.getint("snmp", "retries", fallback=1)

    # v3 users: one [snmp_v3:USERNAME] section per user.
    user_keys = ("auth_proto", "auth_pass", "priv_proto", "priv_pass",
                 "security_level")
    for section in sorted(s for s in secrets.sections()
                          if s.startswith("snmp_v3:")):
        username = section.split(":", 1)[1].strip()
        if not username:
            continue
        record = {"name": username}
        for key in user_keys:
            record[key] = secrets.get(section, key, fallback="").strip()
        # security_level defaults to authPriv when auth+priv are present,
        # authNoPriv when only auth, noAuthNoPriv otherwise.
        if not record["security_level"]:
            if record["auth_pass"] and record["priv_pass"]:
                record["security_level"] = "authPriv"
            elif record["auth_pass"]:
                record["security_level"] = "authNoPriv"
            else:
                record["security_level"] = "noAuthNoPriv"
        out["v3_users"].append(record)

    out["enabled"] = bool(out["communities"]) or bool(out["v3_users"])
    return out


# ---------------------------------------------------------------------------
# Logging
# ---------------------------------------------------------------------------

def setup_logging(debug=False, debug_file=None, log_format="full", log_mode="append"):
    """Configure logging to stdout, netops.log, and optionally a separate debug file.

    debug=True       → DEBUG on console; netops.log also gets DEBUG.
    debug_file=PATH  → additionally write DEBUG to PATH (keeps netops.log as-is).
                       PATH is separate from netops.log so routine runs stay clean.
    log_mode="truncate" → open log files in write mode (fresh each run).
                          Default "append" keeps history.
    Both debug flags can be combined.
    """
    console_level = logging.DEBUG if debug else logging.INFO
    main_file_level = logging.DEBUG if debug else logging.INFO
    file_mode = "w" if log_mode == "truncate" else "a"

    if debug or log_format == "full":
        console_fmt = "%(asctime)s %(levelname)s %(message)s"
    else:
        console_fmt = "%(message)s"

    # netops.log — normal log (INFO, or DEBUG when --debug on console).
    file_handler = logging.FileHandler(LOG_FILE, mode=file_mode, encoding="utf-8")
    file_handler.setLevel(main_file_level)
    file_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))

    console_handler = logging.StreamHandler(sys.stdout)
    console_handler.setLevel(console_level)
    console_handler.setFormatter(logging.Formatter(console_fmt))

    handlers = [file_handler, console_handler]

    # Optional dedicated DEBUG file — separate from netops.log.
    if debug_file:
        debug_handler = logging.FileHandler(debug_file, mode=file_mode, encoding="utf-8")
        debug_handler.setLevel(logging.DEBUG)
        debug_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
        handlers.append(debug_handler)

    root = logging.getLogger()
    root.setLevel(logging.DEBUG)  # handlers filter per-destination
    root.handlers = handlers

    # Suppress noisy third-party loggers unless debug is going to the console.
    if not debug:
        logging.getLogger("pexpect").setLevel(logging.WARNING)


# ---------------------------------------------------------------------------
# Connection layer — pexpect + system ssh/telnet
# ---------------------------------------------------------------------------

def check_tools():
    """Check for required external tools (ssh, telnet) and set availability flags.

    ssh is required. telnet is optional but needed for devices without SSH.
    Prints install instructions if tools are missing.
    """
    global _HAS_SSH, _HAS_TELNET

    _HAS_SSH = shutil.which("ssh") is not None
    _HAS_TELNET = shutil.which("telnet") is not None

    if not _HAS_SSH:
        log.error("OpenSSH client not found. SSH is required.")
        if sys.platform == "darwin":
            log.error("  Install: brew install openssh")
        elif sys.platform == "win32":
            log.error("  Install: Settings > Apps > Optional Features > OpenSSH Client")
            log.error("  Or: Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0")
        else:
            log.error("  Install: sudo apt install openssh-client  (Debian/Ubuntu)")
            log.error("           sudo dnf install openssh-clients  (RHEL/Fedora)")
        sys.exit(1)

    if not _HAS_TELNET:
        log.warning("telnet not found — telnet fallback disabled (SSH-only mode).")
        if sys.platform == "darwin":
            log.warning("  Install: brew install telnet")
        elif sys.platform == "win32":
            log.warning("  Install: dism /online /Enable-Feature /FeatureName:TelnetClient")
            log.warning("  Or: Enable-WindowsOptionalFeature -Online -FeatureName TelnetClient")
        else:
            log.warning("  Install: sudo apt install telnet  (Debian/Ubuntu)")
            log.warning("           sudo dnf install telnet  (RHEL/Fedora)")


def _spawn(cmd, timeout=10):
    """Spawn a child process using PTY (Unix) or PopenSpawn (Windows).

    codec_errors="replace" keeps the UTF-8 decoder from raising on raw bytes
    that arrive mid-stream (e.g. telnet IAC negotiation, 0xFB / 0xFF) before
    the child wraps them into text.
    """
    if sys.platform == "win32":
        return PopenSpawn(cmd, timeout=timeout, encoding="utf-8", codec_errors="replace")
    child = pexpect.spawn(cmd, timeout=timeout, encoding="utf-8",
                          codec_errors="replace", maxread=200000)
    # Use a wide terminal so the remote device's CLI doesn't wrap long
    # commands across lines (Junos echoes '...' continuations on wrap,
    # which slip past send_command's single-line echo stripper).
    try:
        child.setwinsize(40, 300)
    except Exception:
        pass  # non-pty or platform without winsize support — harmless
    return child


def _create_askpass(password):
    """Create a temporary SSH_ASKPASS script for non-interactive password auth (Windows)."""
    fd, path = tempfile.mkstemp(suffix=".sh")
    with os.fdopen(fd, "w") as f:
        f.write(f"#!/bin/sh\necho '{password}'\n")
    os.chmod(path, 0o700)
    return path


# Algorithms we filter out even when a switch advertises them, because they
# would fail the openssh-client command-line parse or are universally
# considered too weak. Other legacy algorithms (group1-sha1, ssh-rsa,
# 3des-cbc, hmac-md5, etc.) are added on-demand when a specific device's
# negotiation error tells us they're needed — never offered upfront.
_REJECT_ALGOS = {
    "ssh-dss",  # removed in OpenSSH 10; '+' prefix can't reintroduce it
    "des",      # 56-bit cipher; OpenSSH refuses regardless
    "none",     # explicitly-disabled cipher
}


def _parse_algo_offer(output):
    """Parse SSH algorithm negotiation failures and return retry -o flags.

    OpenSSH prints messages like:
      Unable to negotiate ... no matching key exchange method found.
        Their offer: diffie-hellman-group14-sha1,diffie-hellman-group1-sha1
      Unable to negotiate ... no matching host key type found.
        Their offer: ssh-rsa,ssh-dss
      Unable to negotiate ... no matching MAC found.
        Their offer: hmac-sha1,hmac-md5

    The "Their offer:" may appear on the same line or a continuation line.

    All five negotiable layers are covered (kex, host-key, cipher, MAC,
    pubkey accepted). Anything not matched falls back to KexAlgorithms +
    HostKeyAlgorithms as a safety net.
    """
    extra_opts = []
    # Find all "Their offer: <algos>" occurrences anywhere in the output
    for m in re.finditer(r"Their offer:\s*(\S+)", output):
        raw_algos = m.group(1)
        # Filter out unsupported algorithms (see _REJECT_ALGOS for why)
        algos = ",".join(a for a in raw_algos.split(",") if a not in _REJECT_ALGOS)
        if not algos:
            continue
        # Look at text before this match to determine the algorithm category
        preceding = output[:m.start()]
        category = preceding.rsplit("no matching", 1)[-1] if "no matching" in preceding else ""
        if "key exchange" in category:
            extra_opts += ["-o", f"KexAlgorithms=+{algos}"]
        elif "host key" in category:
            extra_opts += ["-o", f"HostKeyAlgorithms=+{algos}"]
            extra_opts += ["-o", f"PubkeyAcceptedAlgorithms=+{algos}"]
        elif "cipher" in category:
            extra_opts += ["-o", f"Ciphers=+{algos}"]
        elif "MAC" in category or "mac" in category:
            extra_opts += ["-o", f"MACs=+{algos}"]
        else:
            extra_opts += ["-o", f"KexAlgorithms=+{algos}"]
            extra_opts += ["-o", f"HostKeyAlgorithms=+{algos}"]
    return extra_opts


def _ssh_attempt(ip, username, password, timeout, extra_opts=None):
    """Single SSH connection attempt. Returns child on success, None on failure.

    On algorithm negotiation failure, returns ('negotiate', output) so the
    caller can retry with appropriate options.
    """
    opts = list(_SSH_BASE_OPTS)
    if extra_opts:
        opts += extra_opts
    cmd = "ssh " + " ".join(opts) + f" -o ConnectTimeout={timeout} {username}@{ip}"
    log.debug("  SSH cmd: %s", cmd)

    askpass = None
    child = None
    try:
        if sys.platform == "win32":
            askpass = _create_askpass(password)
            env = os.environ.copy()
            env["SSH_ASKPASS"] = askpass
            env["SSH_ASKPASS_REQUIRE"] = "prefer"
            env["DISPLAY"] = ":0"
            child = PopenSpawn(cmd, timeout=timeout, encoding="utf-8",
                               codec_errors="replace", env=env)
        else:
            child = pexpect.spawn(cmd, timeout=timeout, encoding="utf-8",
                                  codec_errors="replace", maxread=200000)

        # Wait for password prompt, prompt, or failure
        idx = child.expect([
            r"[Pp]assword\s*:",          # 0 — password prompt
            PROMPT_RE,                    # 1 — already at command prompt (key auth)
            r"Permission denied",         # 2 — auth failed
            pexpect.TIMEOUT,              # 3
            pexpect.EOF,                  # 4
        ])

        if idx == 0:
            child.sendline(password)
            idx2 = child.expect([
                PROMPT_RE,                # 0 — command prompt (success)
                r"[Pp]ress any key",      # 1 — HP ProCurve post-login banner
                r"[Tt]erminal type\?",    # 2 — old JUNOS terminal type prompt
                r"[Pp]assword\s*:",       # 3 — wrong password, prompted again
                r"Permission denied",     # 4
                pexpect.TIMEOUT,          # 5
                pexpect.EOF,              # 6
            ])
            if idx2 == 1:
                # HP ProCurve "Press any key to continue" banner
                child.sendline("")
                child.expect(PROMPT_RE, timeout=timeout)
            elif idx2 == 2:
                # Old JUNOS "Terminal type? [vt100]" — accept default
                child.sendline("")
                child.expect(PROMPT_RE, timeout=timeout)
            elif idx2 != 0:
                child.close()
                return None
        elif idx == 1:
            pass  # Already at prompt
        elif idx == 4:
            # EOF — check for algorithm negotiation failure
            output = child.before or ""
            if "no matching" in output or "Bad SSH2" in output:
                child.close()
                return ("negotiate", output)
            child.close()
            return None
        else:
            child.close()
            return None

        # Detect platform from prompt and disable paging
        prompt_text = child.after or ""

        # Juniper root login lands in the FreeBSD shell (% prompt); enter JunOS CLI.
        if re.search(_SHELL_PROMPT_RE, prompt_text):
            child.sendline("cli")
            child.expect(PROMPT_RE, timeout=5)
            prompt_text = child.after or ""

        # Ruckus FastIron's `SSH@`/`telnet@` literal prefix lives JUST BEFORE
        # what PROMPT_RE captures (the regex's `[\w\-/\.]+` stops at `@`),
        # so child.after has only the suffix `hostname[#>]`. Concatenate the
        # tail of child.before so the FastIron prefix is visible to the check.
        prompt_tail = ((child.before or "")[-40:] + prompt_text)

        if re.search(r"(?:SSH|telnet)@[\w\-]+[#>]", prompt_tail):
            # Ruckus FastIron (ICX) — disable paging; never run `enable`
            # (privileged-mode entry on FastIron prompts for User Name + Password,
            # which corrupts the session if we just send the SSH password)
            child.sendline("skip-page-display")
            child.expect(PROMPT_RE, timeout=5)
        elif "@" in prompt_text:
            # Juniper — disable paging with CLI command
            child.sendline("set cli screen-length 0")
            child.expect(PROMPT_RE, timeout=5)
        else:
            # HP ProCurve / Cisco — disable paging
            child.sendline("no page")
            child.expect(PROMPT_RE, timeout=5)

            # Check if we're in unprivileged mode (> instead of #)
            if ">" in child.after:
                child.sendline("enable")
                idx = child.expect([
                    r"[Pp]assword\s*:",   # 0 — enable password
                    PROMPT_RE,            # 1 — already enabled
                    pexpect.TIMEOUT,      # 2
                ], timeout=5)
                if idx == 0:
                    child.sendline(password)
                    child.expect(PROMPT_RE, timeout=5)

        return child

    except (pexpect.ExceptionPexpect, OSError) as e:
        log.debug("  SSH failed for %s@%s: %s", username, ip, e)
        if child:
            try:
                child.close()
            except Exception:
                pass
        return None
    finally:
        if askpass:
            try:
                os.unlink(askpass)
            except Exception:
                pass


def ssh_connect(ip, username, password, timeout=10):
    """Connect to a device via SSH, modern-first with on-demand legacy fallback.

    First attempt uses openssh-client's built-in defaults — modern crypto
    only. If a specific switch fails kex/host-key/cipher/MAC negotiation,
    the device's error message is parsed and ONLY the algorithms that
    device advertises get added to a retry. This keeps modern devices
    talking modern crypto and only opts the legacy algorithms back in for
    the legacy switches that need them.

    No lockout risk in the retry loop: a kex/cipher/MAC failure happens
    before the password prompt, so it doesn't count as a failed login on
    Juniper's progressive-delay logic.
    """
    extra_opts = []
    while True:
        result = _ssh_attempt(ip, username, password, timeout, extra_opts=extra_opts or None)

        if not isinstance(result, tuple):
            return result  # child or None

        # Algorithm negotiation failed — parse and accumulate needed options.
        output = result[1]
        new_opts = _parse_algo_offer(output)
        if not new_opts:
            return None  # can't parse what's needed — give up
        extra_opts += new_opts
        log.debug("  SSH algo negotiation, retrying with: %s", extra_opts)


def telnet_connect(ip, username, password, timeout=10):
    """Connect to a device via telnet using the system telnet command.

    Handles HP ProCurve "Press any key" banner, optional username prompt,
    and password-only login. Returns a pexpect child on success, or None.
    """
    cmd = f"telnet {ip}"
    log.debug("  telnet cmd: %s", cmd)

    try:
        child = _spawn(cmd, timeout=timeout)

        # First expect: banner, username, password, or prompt
        idx = child.expect([
            r"[Pp]ress any key",          # 0 — HP ProCurve banner
            r"[Uu]ser\s*[Nn]ame\s*:",     # 1 — username prompt
            r"[Pp]assword\s*:",            # 2 — password prompt (no username)
            PROMPT_RE,                     # 3 — already at command prompt
            r"[Cc]onnection refused",      # 4
            pexpect.TIMEOUT,              # 5
            pexpect.EOF,                  # 6
        ])

        if idx == 0:
            # Press any key
            child.sendline("")
            idx = child.expect([
                r"[Uu]ser\s*[Nn]ame\s*:",  # 0 — username prompt
                r"[Pp]assword\s*:",         # 1 — password prompt
                PROMPT_RE,                  # 2 — already at prompt
                pexpect.TIMEOUT,            # 3
            ], timeout=timeout)

        if idx == 1:
            # Username prompt
            child.sendline(username)
            idx = child.expect([
                r"[Pp]assword\s*:",   # 0
                pexpect.TIMEOUT,      # 1
            ], timeout=timeout)
            if idx != 0:
                child.close()
                return None

        if idx == 2:
            # Password prompt (no username asked)
            child.sendline(password)
        elif idx == 0:
            # After username, now at password prompt
            child.sendline(password)
        elif idx == 3:
            # Already at command prompt
            pass
        else:
            child.close()
            return None

        # Wait for command prompt after login
        if idx != 3:
            idx = child.expect([
                PROMPT_RE,                # 0 — success
                r"[Pp]assword\s*:",       # 1 — wrong password
                r"[Ll]ogin incorrect",    # 2
                pexpect.TIMEOUT,          # 3
                pexpect.EOF,              # 4
            ])
            if idx != 0:
                child.close()
                return None

        # Juniper root login lands in the FreeBSD shell (% prompt); enter JunOS CLI.
        if re.search(_SHELL_PROMPT_RE, child.after or ""):
            child.sendline("cli")
            child.expect(PROMPT_RE, timeout=5)

        # Disable paging
        child.sendline("no page")
        child.expect(PROMPT_RE, timeout=5)

        # Check if we're in unprivileged mode
        if ">" in child.after:
            child.sendline("enable")
            idx = child.expect([
                r"[Pp]assword\s*:",
                PROMPT_RE,
                pexpect.TIMEOUT,
            ], timeout=5)
            if idx == 0:
                child.sendline(password)
                child.expect(PROMPT_RE, timeout=5)

        return child

    except (pexpect.ExceptionPexpect, OSError) as e:
        log.debug("  telnet failed for %s: %s", ip, e)
        try:
            child.close()
        except Exception:
            pass
        return None


def classify_fail_reason(ssh_open, telnet_open):
    """Return a specific failure reason based on observed port state.

    - Both ports closed/filtered → unreachable
    - One or both ports open but auth failed → credentials rejected
    """
    if not ssh_open and not telnet_open:
        return "unreachable (no SSH/telnet port open)"
    protos = []
    if ssh_open:
        protos.append("SSH")
    if telnet_open:
        protos.append("telnet")
    return f"credentials rejected ({'/'.join(protos)} reachable)"


def connect_device(ip, usernames, password, timeout=10, password_list=None,
                   known_username=None, known_password_hash=None,
                   user_passwords=None, reason=None):
    """Try SSH then telnet to connect to a device.

    reason: optional tag for op_events instrumentation — describes WHY
    this connection is being made ('backup', 'investigate_stp',
    'monitor_topology_fallback', 'monitor_stp_fallback', 'discover',
    etc.). Used by 'digest health' to group SSH activity by purpose.

    If password_list is provided, tries each password with each username.
    Otherwise uses the single password.

    If user_passwords is provided (dict of {user: [pw, ...]}), any user present
    in that mapping is restricted to ONLY those passwords — the bulk
    password_list is not tried for that user. Intended for LDAP/RADIUS-backed
    accounts to avoid lockouts from random-password guesses.

    Iteration is user-major: each user's passwords are fully exhausted before
    moving to the next user.

    If known_username/known_password_hash are set (from a previous successful
    connection stored in the DB) AND the matching plaintext is still in the
    current cred config, the function trusts that cached pair: it tries the
    pair once via SSH and then telnet, and if both fail it returns None
    rather than falling through to the multi-cred grid below.

    Trusting the cache is deliberate. Without it, a transient rate-limit
    refusal on a single management interface (common on ProCurve when a
    multi-IP chassis is hit in parallel) would trigger up to len(usernames)
    × len(passwords) extra SSH sessions per device per discovery, dragging
    the device deeper into rate-limit territory and amplifying the false
    negative across the rest of the run. Genuine credential rotation
    requires a manual re-test ('netops retest' after updating creds);
    that's a planned operation rather than something discovery needs to
    auto-detect on every run.

    The combo grid below only runs for devices with no cached cred at
    all (initial discovery), or where the cached hash references a
    password no longer present in the current config (we have nothing
    to "trust").

    Returns (child, proto, username, pw_hash, ssh_open, telnet_open)
    or (None, None, None, None, False, False).
    pw_hash is the SHA-256 hex digest of the working password.
    """
    # Filter Nones at the entrypoint so every downstream call site (telnet
    # password-only loop, ssh combo sweep, hash-encode lookup) sees a
    # clean list. An incomplete config entry would otherwise propagate a
    # None into ssh_connect / telnet_connect and crash deep in pexpect.
    pw_list = [p for p in (password_list if password_list else [password]) if p]
    user_passwords = {u: [p for p in (pws or []) if p]
                      for u, pws in (user_passwords or {}).items()}
    has_ssh = bool(check_ports(ip, [22], timeout=2))
    has_telnet = _HAS_TELNET and bool(check_ports(ip, [23], timeout=2))

    # op_events recording — wrap the whole attempt so the digest can see
    # per-reason latency + success rate. The helper records once and
    # returns the result tuple unchanged so call sites stay tidy.
    _cd_t0 = time.monotonic()
    _cd_started_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    def _finalize(result, error=None):
        child, proto, _user, _hash, _so, _to = result
        _record_op("ssh_connect" if not (has_telnet and not has_ssh)
                    else "telnet_connect",
                   started_at=_cd_started_at,
                   duration_ms=int((time.monotonic() - _cd_t0) * 1000),
                   success=(child is not None),
                   ip=ip, reason=reason, error=error)
        return result

    def pws_for(user):
        """Passwords to try for a given user.

        Mapped users get ONLY their listed passwords (safe for lockout-prone
        accounts). Unmapped users fall back to the bulk list. Always
        returns a None-filtered list — incomplete config (e.g. a
        placeholder entry in [user_passwords]) would otherwise propagate
        Nones into ssh_connect / hash-encode call sites and crash with
        'NoneType' object has no attribute 'encode' (observed on Drew's
        Aruba-CX fleet, which blocked all SSH-based investigation).
        """
        if user in user_passwords:
            return [p for p in user_passwords[user] if p]
        return [p for p in pw_list if p]

    # --- Try known-good credentials first (no lockout risk) ---
    if known_username is not None and known_password_hash is not None:
        # Hash may match either a user-specific or bulk-list password.
        candidates = list(pws_for(known_username)) + [p for p in pw_list if p]
        known_pw = None
        for pw in candidates:
            if hashlib.sha256(pw.encode()).hexdigest() == known_password_hash:
                known_pw = pw
                break
        if known_pw is not None:
            if has_ssh:
                log.debug("  trying known-good SSH %s on %s", known_username, ip)
                child = ssh_connect(ip, known_username, known_pw, timeout=timeout)
                if child is not None:
                    return _finalize((child, "SSH", known_username, known_password_hash, has_ssh, has_telnet))
            if has_telnet:
                log.debug("  trying known-good telnet %s on %s", known_username, ip)
                child = telnet_connect(ip, known_username, known_pw, timeout=timeout)
                if child is not None:
                    return _finalize((child, "telnet", known_username or "(no user)", known_password_hash, has_ssh, has_telnet))
            # Cached cred refused on every reachable protocol. Bail without
            # walking the combo grid — see docstring: trusting the cached
            # pair eliminates the rate-limit cascade that hit PSC-CORE's
            # 84.1 management interface during a parallel discovery.
            log.debug("  cached cred rejected on %s — not grinding combos", ip)
            return _finalize((None, None, None, None, has_ssh, has_telnet),
                             error="cached cred rejected")

    # --- Try user/password combos (user-major: exhaust user1 before user2) ---
    if has_ssh:
        for username in usernames:
            for pw in pws_for(username):
                log.debug("  trying SSH %s on %s", username, ip)
                child = ssh_connect(ip, username, pw, timeout=timeout)
                if child is not None:
                    pw_hash = hashlib.sha256(pw.encode()).hexdigest()
                    return _finalize((child, "SSH", username, pw_hash, has_ssh, has_telnet))

    if has_telnet:
        # Password-only devices (e.g. HP ProCurve) — try each bulk password
        # with no username first.
        for pw in pw_list:
            log.debug("  trying telnet (no user) on %s", ip)
            child = telnet_connect(ip, "", pw, timeout=timeout)
            if child is not None:
                pw_hash = hashlib.sha256(pw.encode()).hexdigest()
                return _finalize((child, "telnet", "(no user)", pw_hash, has_ssh, has_telnet))
        for username in usernames:
            for pw in pws_for(username):
                log.debug("  trying telnet %s on %s", username, ip)
                child = telnet_connect(ip, username, pw, timeout=timeout)
                if child is not None:
                    pw_hash = hashlib.sha256(pw.encode()).hexdigest()
                    return _finalize((child, "telnet", username, pw_hash, has_ssh, has_telnet))

    return _finalize((None, None, None, None, has_ssh, has_telnet),
                     error="all credentials exhausted")


_ANSI_RE = re.compile(
    r"\x1b"           # ESC
    r"(?:"
    r"\[[\d;?]*[a-zA-Z~]"   # CSI sequences: ESC [ params letter  (includes \x1b[?25h)
    r"|[()][0-9A-Za-z]"     # Character set selection: ESC ( B, ESC ) 0, etc.
    r"|[>=<]"               # Keypad mode: ESC > (numeric), ESC = (application)
    r"|[\x20-\x2f]+[\x30-\x7e]"  # ESC intermediate+ final (e.g. ESC # 8)
    r"|[A-Z]"               # Simple ESC-letter (cursor save/restore, etc.)
    r")"
)


_ANSI_TRUNCATED_RE = re.compile(r"\x1b\[[\d;?]*")


def _strip_ansi(text):
    """Remove ANSI escape sequences from text.

    Second pass catches truncated CSI sequences (ESC [ params with no
    terminator) — ProCurve sometimes emits 'ESC[24;' and bails, which
    the main regex can't match since it requires a final letter. Without
    this pass, the visible '[24;' fragment leaks into emails.
    """
    text = _ANSI_RE.sub("", text)
    return _ANSI_TRUNCATED_RE.sub("", text)


_PAGER_RE = r"---\(more[^)]*\)---|-- MORE --|---more---|(?:^|[\r\n])\s*--More--"


def send_command(child, command, timeout=30):
    """Send a command and return the output (stripped of echo, prompt, and ANSI codes).

    Handles pager prompts (---(more)---, -- MORE --, etc.) by sending space
    to continue until the full output is received.
    """
    child.sendline(command)
    chunks = []
    while True:
        idx = child.expect([PROMPT_RE, _PAGER_RE], timeout=timeout)
        chunks.append(child.before)
        if idx == 0:
            # Prompt matched — we have all the output
            break
        else:
            # Pager — send space to continue
            child.send(" ")
    output = _strip_ansi("".join(chunks))
    # Strip the command echo (first line)
    lines = output.splitlines()
    if lines and command in lines[0]:
        lines = lines[1:]
    return "\n".join(lines).strip()


def get_hostname(child):
    """Extract hostname from the device prompt.

    Handles both HP ProCurve ('hostname#') and Juniper ('user@hostname>').
    """
    child.sendline("")
    child.expect(PROMPT_RE, timeout=5)
    # Combine before+after so split ANSI sequences are intact for stripping
    raw = (child.before or "") + (child.after or "")
    clean = _strip_ansi(raw)
    m = re.search(r"([\w\-/\.]+)[#>]\s*$", clean)
    if not m:
        return None
    name = m.group(1)
    # Juniper prompt: user@hostname — strip the username prefix
    if "@" in clean:
        at_match = re.search(r"@([\w\-/\.]+)[#>]\s*$", clean)
        if at_match:
            name = at_match.group(1)
    return name


def disconnect(child):
    """Disconnect from a device."""
    try:
        child.sendline("exit")
        child.close()
    except Exception:
        pass


def get_device_id(output):
    """Extract unique device identifiers from device output.

    Accepts output from 'show system' (HP) or 'show version' + 'show chassis hardware' (Juniper).
    Returns dict with base_mac, serial, firmware, model (any may be absent).
    """
    info = {}

    # --- HP ProCurve / ArubaOS-Switch ---
    m = re.search(r"Base MAC Addr\s*:\s*(\S+)", output)
    if m:
        info["base_mac"] = m.group(1)
    m = re.search(r"Serial Number\s*:\s*(\S+)", output)
    if m:
        info["serial"] = m.group(1)
    m = re.search(r"Firmware revision\s*:\s*(\S+)", output)
    if m:
        info["firmware"] = m.group(1)
    if "firmware" not in info:
        m = re.search(r"Software revision\s*:\s*(\S+)", output)
        if m:
            info["firmware"] = m.group(1)

    # HP/Aruba model from 'show version' first line:
    #   "HP J9729A 2920-48G-PoE+ Switch"
    #   "Aruba JL321A 2930M-24G-PoE+ Switch"
    #   "HP J8697A E5406-44G-PoE+/4G-SFP v2 zl Switch with Premium Software"
    if "model" not in info:
        m = re.search(r"^((?:HP|Aruba)\s+\S+\s+.+?)(?:\s+[Ss]witch.*)?\s*$",
                      output, re.MULTILINE)
        if m:
            info["model"] = m.group(1).strip()

    # --- ArubaOS-CX ---
    if "base_mac" not in info:
        m = re.search(r"Base MAC Address\s*:\s*(\S+)", output)
        if m:
            info["base_mac"] = m.group(1)
    if "serial" not in info:
        m = re.search(r"Chassis Serial Nbr\s*:\s*(\S+)", output)
        if m:
            info["serial"] = m.group(1)
    if "firmware" not in info:
        m = re.search(r"ArubaOS-CX Version\s*:\s*(\S+)", output)
        if m:
            info["firmware"] = m.group(1)
    if "model" not in info:
        m = re.search(r"Product Name\s*:\s*(.+)", output)
        if m:
            info["model"] = m.group(1).strip()

    # --- Juniper JunOS ---
    # Junos version: "Junos: 23.4R2-S6.9" or "JUNOS Base OS Software Suite [12.3R12-S21]"
    if "firmware" not in info:
        m = re.search(r"Junos:\s*(\S+)", output)
        if m:
            info["firmware"] = m.group(1)
    if "firmware" not in info:
        m = re.search(r"JUNOS Base OS Software Suite \[([^\]]+)\]", output)
        if m:
            info["firmware"] = m.group(1)
    # Juniper model: "Model: srx345-dual-ac" or "Model: ex3300-48t"
    if "model" not in info:
        m = re.search(r"^Model:\s*(\S+)", output, re.MULTILINE)
        if m:
            info["model"] = m.group(1)
    # Juniper serial from show chassis hardware: first alphanumeric 6+ chars on Chassis line
    # (Part numbers have hyphens like 750-034247, serials don't: GA0214410099, DS3817AF0171)
    if "serial" not in info:
        m = re.search(r"^Chassis\b[^\n]*?\b([A-Z][A-Z0-9]{5,})\b", output, re.MULTILINE)
        if m:
            info["serial"] = m.group(1)
    # Juniper MAC from show chassis mac-addresses:
    #   older: "Public base address     28:c0:da:35:10:00"
    #   newer: "Base address    ec:38:73:5f:7a:5e"
    if "base_mac" not in info:
        m = re.search(r"(?:Public\s+)?[Bb]ase\s+address\s+([\da-fA-F:]+)", output)
        if m:
            info["base_mac"] = m.group(1).replace(":", "")

    # --- Ruckus FastIron (ICX) ---
    # show version banner format:
    #   "  HW: Stackable ICX7750-48F"   or   "  HW: ICX7450-48-HPOE"
    #   "        SW: Version 08.0.80fT201"
    #   "      Serial  #:CRH3303P005"     (first match wins — UNIT 1)
    if "model" not in info:
        m = re.search(r"^\s*HW:\s*(?:Stackable\s+)?(ICX\S+)", output, re.MULTILINE)
        if m:
            info["model"] = m.group(1)
    if "firmware" not in info:
        m = re.search(r"^\s*SW:\s*Version\s+(\S+)", output, re.MULTILINE)
        if m:
            info["firmware"] = m.group(1)
    if "serial" not in info:
        m = re.search(r"^\s*Serial\s*#\s*:\s*(\S+)", output, re.MULTILINE)
        if m:
            info["serial"] = m.group(1)

    return info


# ---------------------------------------------------------------------------
# STP state parsers
# ---------------------------------------------------------------------------

def parse_stp_junos(output):
    """Parse Juniper 'show spanning-tree interface' output.

    Returns list of dicts: [{interface, role, state}, ...].
    Only includes ports with a meaningful STP role (skips header/blank lines).
    """
    results = []
    for line in output.splitlines():
        # ge-0/0/0  128:490  128:490  32768.xxxx  200000  FWD  DESG
        m = re.match(
            r"(\S+)\s+"           # interface
            r"\d+:\d+\s+"        # port ID
            r"\d+:\d+\s+"        # designated port ID
            r"\S+\s+"            # designated bridge ID
            r"\d+\s+"            # cost
            r"(\S+)\s+"          # state (FWD/BLK)
            r"(\S+)",            # role (DESG/ROOT/ALT/DIS)
            line)
        if m:
            results.append({
                "interface": m.group(1),
                "state": m.group(2),
                "role": m.group(3),
            })
    return results


def parse_junos_stp_port_id_table(output):
    """Extract (port_id, interface) pairs from 'show spanning-tree interface'
    output for the JUNIPER STP-port-id cache.

    The same table parse_stp_junos uses, but we capture the port-id half of
    the second column ('128:490' -> 490). The port-id is the index used by
    JUNIPER-MIMSTP-MIB; on VC switches this diverges from BRIDGE-MIB's
    dot1dBasePort. Strips Junos '.0' logical-unit suffix so the interface
    name matches what the SNMP collector emits.

    Returns list of (port_id_int, interface_str) tuples.
    """
    results = []
    for line in output.splitlines():
        m = re.match(
            r"(\S+)\s+"           # interface
            r"\d+:(\d+)\s+"      # port ID — capture the port-id half
            r"\d+:\d+\s+"        # designated port ID
            r"\S+\s+"            # designated bridge ID
            r"\d+\s+"            # cost
            r"\S+\s+"            # state
            r"\S+",              # role
            line)
        if m:
            ifname = m.group(1)
            port_id = int(m.group(2))
            if "." in ifname:
                ifname = ifname.split(".", 1)[0]
            results.append((port_id, ifname))
    return results


def _upsert_junos_stp_port_id_cache(ip, port_ids):
    """Replace stp_port_id_cache rows for `ip` with the given mapping.

    `port_ids` is a list of (port_id, interface) tuples produced by
    parse_junos_stp_port_id_table. Empty input is a no-op (don't blow away
    a previous cache when the parse came back empty for a transient reason).
    """
    if not port_ids:
        return
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    try:
        # Replace strategy so a port that disappeared since the last
        # discover doesn't linger as a stale entry.
        conn.execute("DELETE FROM stp_port_id_cache WHERE ip = ?", (ip,))
        conn.executemany(
            "INSERT INTO stp_port_id_cache (ip, port_id, interface, last_refresh)"
            " VALUES (?, ?, ?, ?)",
            [(ip, pid, name, now) for pid, name in port_ids]
        )
        conn.commit()
    finally:
        conn.close()


def parse_stp_procurve(output):
    """Parse HP ProCurve 'show spanning-tree' output (MSTP format).

    Returns list of dicts: [{interface, role, state}, ...].
    ProCurve doesn't have an explicit Role column — we infer from State.
    """
    results = []
    for line in output.splitlines():
        # 1/1   10GbE-T  | 4000  128  Forwarding  | 8030e0-eb5d0a  2  Yes
        # Trk1  Trk      | 2000  128  Forwarding  | ...
        m = re.match(
            r"\s*(\S+)\s+"       # port (1/1, Trk1, etc.)
            r"\S+\s+\|\s+"      # type + separator
            r"\S+\s+"           # cost (number or Auto)
            r"\d+\s+"           # priority
            r"(\w+)",           # state (Forwarding/Blocking/Disabled/Learning)
            line)
        if m:
            port = m.group(1)
            state = m.group(2)
            if state == "Disabled":
                role = "Disabled"
            elif state == "Blocking":
                role = "Alternate"
            elif state == "Forwarding":
                role = "Designated"
            else:
                role = state
            results.append({
                "interface": port,
                "state": state,
                "role": role,
            })
    return results


def parse_stp_aruba_cx(output):
    """Parse ArubaOS-CX 'show spanning-tree' output.

    Returns list of dicts: [{interface, role, state}, ...].
    CX has explicit Role and State columns.
    """
    results = []
    for line in output.splitlines():
        # 1/1/6   Designated  Forwarding  2000  128  P2P  ...
        m = re.match(
            r"(\S+)\s+"          # port (1/1/6)
            r"(\S+)\s+"         # role (Designated/Alternate/Root/Disabled)
            r"(\S+)\s+"         # state (Forwarding/Blocking/Down)
            r"\d+\s+"           # cost
            r"\d+\s+"           # priority
            r"\S+",             # type (P2P/Shr)
            line)
        if m:
            results.append({
                "interface": m.group(1),
                "role": m.group(2),
                "state": m.group(3),
            })
    return results


def is_stp_blocking(port_info):
    """Return True if this port represents a real STP block (not just link-down)."""
    role = port_info.get("role", "")
    state = port_info.get("state", "")
    # Juniper: BLK + ALT = real block; BLK + DIS = no link
    if state == "BLK" and role == "ALT":
        return True
    # ProCurve: Blocking state = real block
    if state == "Blocking":
        return True
    # ArubaOS-CX: Alternate + Blocking = real block
    if role == "Alternate" and state == "Blocking":
        return True
    return False


def _is_stp_offline_state(role, state):
    """A port's STP state is 'offline' when it's Disabled/Down/missing (GONE)."""
    if (role, state) == ("GONE", "GONE"):
        return True
    return role in ("DIS", "Disabled") and state in ("BLK", "Disabled", "Down")


def _is_stp_transient_state(role, state):
    """Learning/Listening are convergence steps in the STP state machine, not
    steady states. A port caught in LRN/LIS is mid-transition; using it as an
    alert reference produces phantom events like 'DESG/LRN -> GONE/GONE' when
    the port later disappears before settling."""
    return state in ("LRN", "Learning", "LIS", "Listening")


def should_alert_stp_change(old, new):
    """Return True if a change from old (role, state) to new (role, state) is
    alertable (topology-relevant), as opposed to a link-flap artifact.

    Rules:
      - either side is transient (LRN/LIS): silent — wait for the port to settle.
      - old -> down/gone: alert only if the port was previously blocking
        (a suppressed redundant path going down matters; a forwarding port
        going down is a link event, not STP).
      - down/gone -> new: silent (link coming back up is a flap recovery).
      - Any other role/state transition: alert.
    """
    if _is_stp_transient_state(*old) or _is_stp_transient_state(*new):
        return False
    new_off = _is_stp_offline_state(*new)
    old_off = _is_stp_offline_state(*old)
    if new_off:
        return is_stp_blocking({"role": old[0], "state": old[1]})
    if old_off:
        return False
    return True


# Junos data-plane Ethernet prefixes — the only interfaces we count for
# link-state, flap tracking, and in-service heuristics. Excludes management
# (me/bme/vme/fxp/em) and internal virtual (dsc/gre/ipip/lo0/lsi/etc.)
# interfaces that always report 'Physical link is Up' but carry no user
# traffic.
_JUNOS_DATAPLANE_PREFIXES = ("ge-", "xe-", "et-", "fe-", "ae")


def _is_junos_dataplane(name):
    return name.startswith(_JUNOS_DATAPLANE_PREFIXES)


def parse_ifaces_junos(output):
    """Parse filtered 'show interfaces | match "^Physical|Last flapped"'.
    Returns {physical_iface: link_up_bool} restricted to data-plane ports.

    Header form: 'Physical interface: NAME, Enabled, Physical link is Up'.
    Admin-disabled interfaces show 'Administratively down' instead of
    'Enabled'. A port is up only when admin is 'Enabled' AND link is 'Up'.
    """
    results = {}
    for line in output.splitlines():
        m = re.match(
            r"^Physical interface:\s*(\S+?),\s+([^,]+?),\s+"
            r"Physical link is\s+(Up|Down)",
            line)
        if not m:
            continue
        iface = m.group(1)
        if not _is_junos_dataplane(iface):
            continue
        admin_up = "enabled" in m.group(2).lower()
        link_up = (m.group(3) == "Up")
        results[iface] = admin_up and link_up
    return results


def parse_ifaces_aruba_cx(output):
    """Parse full 'show interface' output. Returns {iface: link_up_bool}.

    Uses the 'Interface NAME is (up|down|disabled)' line as link state.
    Skips vlan/lag/loopback pseudo-interfaces.
    """
    results = {}
    for line in output.splitlines():
        m = re.match(r"^Interface\s+(\S+)\s+is\s+(up|down|disabled)", line)
        if not m:
            continue
        iface = m.group(1)
        if iface.startswith(("vlan", "lag", "loopback", "Port")):
            continue
        results[iface] = (m.group(2) == "up")
    return results


def parse_ifaces_procurve(output):
    """Parse HP ProCurve 'show interfaces brief' output. Returns {iface: link_up_bool}.

    A port is up only when Enabled=Yes AND Status=Up.
    """
    results = {}
    for line in output.splitlines():
        #   1/1          10GbE-T    | No        Yes     Up     5000FDx    MDI  off  0
        m = re.match(
            r"^\s+(\S+)\s+\S+\s+\|\s+\S+\s+(Yes|No)\s+(Up|Down)\b", line)
        if not m:
            continue
        results[m.group(1)] = (m.group(2) == "Yes" and m.group(3) == "Up")
    return results


# ---------------------------------------------------------------------------
# SNMP — credential probing and config inspection
# ---------------------------------------------------------------------------
#
# We shell out to net-snmp (snmpget) rather than pulling in pysnmp:
# 1. Matches the working POC and the existing test-server toolchain.
# 2. No new Python dependency. pysnmp v6 is asyncio-only and would force a
#    refactor of the existing ThreadPoolExecutor-based polling paths.
# 3. Easier to debug — operators can reproduce a failure with the same
#    snmpget command we ran.
#
# sysUpTime.0 (1.3.6.1.2.1.1.3.0) is the canary OID: universally supported,
# returns immediately, and a successful response confirms (a) SNMP is enabled,
# (b) our credentials are accepted, and (c) any ACL on the device permits us.

_SNMP_SYSUPTIME_OID = "1.3.6.1.2.1.1.3.0"


def _snmp_run(args, timeout):
    """Run an snmpget/snmpwalk command. Returns (rc, stdout, stderr).
    Records timing into the per-tick accumulator if one is active."""
    t0 = time.monotonic()
    try:
        proc = subprocess.run(args, capture_output=True, text=True,
                              timeout=timeout)
        rc, out, err = proc.returncode, proc.stdout or "", proc.stderr or ""
    except subprocess.TimeoutExpired:
        rc, out, err = -1, "", "subprocess timeout"
    except FileNotFoundError:
        rc, out, err = -1, "", "snmpget not found (install net-snmp / snmp tools)"
    except Exception as e:
        rc, out, err = -1, "", f"snmpget exec error: {e}"
    duration_ms = int((time.monotonic() - t0) * 1000)
    _snmp_timing_record(duration_ms, rc == 0 and "Timeout" not in (out + err))
    return rc, out, err


def _snmp_v2c_args(community, ip, oid, timeout, retries):
    return ["snmpget", "-v2c", "-c", community,
            "-t", str(timeout), "-r", str(retries),
            "-On", ip, oid]


def _snmp_v3_args(user_record, ip, oid, timeout, retries):
    """Build snmpget args for a v3 user record from cfg['snmp']['v3_users']."""
    args = ["snmpget", "-v3",
            "-l", user_record.get("security_level") or "noAuthNoPriv",
            "-u", user_record["name"],
            "-t", str(timeout), "-r", str(retries),
            "-On"]
    if user_record.get("auth_pass"):
        args += ["-a", user_record.get("auth_proto") or "SHA",
                 "-A", user_record["auth_pass"]]
    if user_record.get("priv_pass"):
        args += ["-x", user_record.get("priv_proto") or "AES",
                 "-X", user_record["priv_pass"]]
    args += [ip, oid]
    return args


def _snmp_response_ok(rc, stdout, stderr):
    """Did snmpget return a usable value? net-snmp's exit code isn't enough —
    timeouts and auth failures can still be rc=0 with an error in stdout."""
    if rc != 0:
        return False
    s = stdout.strip()
    if not s:
        return False
    bad = ("Timeout: No Response", "No Such Object", "No Such Instance",
           "authenticationFailure", "Unknown user name",
           "Authentication failure", "wrongDigest", "decryptionError")
    if any(b in s for b in bad):
        return False
    if any(b in (stderr or "") for b in bad):
        return False
    return True


def snmp_probe(ip, snmp_cfg, cached_proto=None, cached_cred=None,
               full_walk=False):
    """Probe an IP for SNMP reachability.

    Strategy (each step short-circuits on success):
      1. If a cached credential is supplied, try it. On success, return.
      2. If the cached cred failed AND full_walk=False, return failure
         without re-trying — this keeps routine polls to a single UDP
         roundtrip, even when the cached cred has drifted. Discovery
         re-walks and refreshes the cache.
      3. If no cached cred OR full_walk=True, iterate communities then
         v3 users, stopping at the first that responds. Used by 'discover'
         and by first-time polls on never-probed devices.

    Returns a dict:
      {ok: bool, proto: 'v2c'|'v3'|None, community: str|None,
       v3_user: str|None, error: str}

    The probe costs one UDP roundtrip per attempt — bounded by snmp_cfg
    timeout * (cached_attempt + len(communities) + len(v3_users)) when
    full_walk=True. With defaults (5s timeout, 1 retry, 2 communities, 1 v3
    user) that's ~25s worst-case per dead device. Don't call full_walk=True
    inside the per-minute monitor loop.
    """
    timeout = snmp_cfg.get("timeout", 5)
    retries = snmp_cfg.get("retries", 1)
    # net-snmp's per-attempt timeout × (retries+1) is the wall-clock budget;
    # add a small subprocess fence so a hung snmpget can't block forever.
    sub_timeout = (timeout * (retries + 1)) + 5

    result = {"ok": False, "proto": None, "community": None,
              "v3_user": None, "error": ""}

    def try_v2c(community):
        args = _snmp_v2c_args(community, ip, _SNMP_SYSUPTIME_OID,
                              timeout, retries)
        rc, out, err = _snmp_run(args, sub_timeout)
        if _snmp_response_ok(rc, out, err):
            return True, ""
        return False, (out.strip() or err.strip() or "no response")

    def try_v3(user_record):
        args = _snmp_v3_args(user_record, ip, _SNMP_SYSUPTIME_OID,
                             timeout, retries)
        rc, out, err = _snmp_run(args, sub_timeout)
        if _snmp_response_ok(rc, out, err):
            return True, ""
        return False, (out.strip() or err.strip() or "no response")

    # ---- 1. Cached credential first ----
    if cached_proto == "v2c" and cached_cred:
        ok, err = try_v2c(cached_cred)
        if ok:
            result.update({"ok": True, "proto": "v2c", "community": cached_cred})
            return result
        if not full_walk:
            result["error"] = err
            return result
    elif cached_proto == "v3" and cached_cred:
        match = next((u for u in snmp_cfg.get("v3_users", [])
                      if u["name"] == cached_cred), None)
        if match:
            ok, err = try_v3(match)
            if ok:
                result.update({"ok": True, "proto": "v3", "v3_user": cached_cred})
                return result
            if not full_walk:
                result["error"] = err
                return result

    # ---- 2. Full credential walk (discovery only) ----
    last_err = ""
    for community in snmp_cfg.get("communities", []):
        if cached_proto == "v2c" and cached_cred == community:
            continue  # already tried above
        ok, err = try_v2c(community)
        if ok:
            result.update({"ok": True, "proto": "v2c", "community": community})
            return result
        last_err = err
    for user_record in snmp_cfg.get("v3_users", []):
        if cached_proto == "v3" and cached_cred == user_record["name"]:
            continue
        ok, err = try_v3(user_record)
        if ok:
            result.update({"ok": True, "proto": "v3",
                           "v3_user": user_record["name"]})
            return result
        last_err = err

    result["error"] = last_err or "no credentials configured"
    return result


def _parse_snmp_config_junos(output):
    """Parse 'show configuration snmp | display set' output.

    Junos has two community forms:
      set snmp community NAME                          (read-only by default)
      set snmp community NAME authorization read-write
      set snmp community NAME clients PREFIX           (per-community ACL)
    The bare-name form has no trailing token, so the regex must accept
    end-of-line — earlier versions required \\s+ after the name and missed
    the read-only case.
    """
    communities, v3_users, acl_refs = [], [], []
    for line in output.splitlines():
        line = line.strip()
        m = re.match(r"set snmp community\s+(\S+?)(?:\s+|$)", line)
        if m and m.group(1) not in communities:
            communities.append(m.group(1))
        m = re.match(r"set snmp community\s+\S+\s+clients\s+(\S+)", line)
        if m:
            acl_refs.append(("clients-list", m.group(1)))
        m = re.match(r"set snmp v3 usm local-engine user\s+(\S+)", line)
        if m and m.group(1) not in v3_users:
            v3_users.append(m.group(1))
        m = re.match(r"set snmp client-list\s+(\S+)\s+(\S+)", line)
        if m:
            acl_refs.append((f"client-list:{m.group(1)}", m.group(2)))
    return communities, v3_users, acl_refs


def _parse_snmp_config_aruba_cx(output):
    """Parse the combined output of 'show snmp community' + 'show snmpv3 users'
    (and optionally 'show running-config') for Aruba-CX.

    Aruba-CX presents each in a fixed-width table:
      'show snmp community':
        Community            Access-level ACL Name      ACL Type
        --------             ------------ ----------    --------
        public               read-only    none          none
        netops_ro            read-only    siem-allow    extended

      'show snmpv3 users':
        User       AuthMode PrivMode Status   Context
        --------   -------- -------- -------- -------
        monitor    sha      aes      enabled

    Per-community ACLs are the de-facto SNMP source-IP filter on Aruba-CX
    when ACL Name != 'none'. 'snmp-server vrf <name>' bindings also matter
    (probe must come from a host reachable on that VRF).
    """
    communities, v3_users, acl_refs = [], [], []
    in_comm_table, in_user_table = False, False
    for line in output.splitlines():
        s = line.rstrip()
        if (re.match(r"^Community\b", s.lstrip())
                and "Access-level" in s and "ACL" in s):
            in_comm_table, in_user_table = True, False
            continue
        if (re.match(r"^User\b", s.lstrip())
                and "AuthMode" in s and "PrivMode" in s):
            in_user_table, in_comm_table = True, False
            continue
        if not s.strip():
            in_comm_table = in_user_table = False
            continue
        if s.lstrip().startswith("---"):
            continue
        if in_comm_table:
            tok = s.split()
            if len(tok) >= 1 and tok[0] not in communities:
                communities.append(tok[0])
            # Tabular ACL Name is column 3 (0-indexed 2). 'none' means no ACL.
            if len(tok) >= 3 and tok[2].lower() != "none":
                acl_refs.append(("acl", f"{tok[2]} (community {tok[0]})"))
            continue
        if in_user_table:
            tok = s.split()
            if tok and tok[0] not in v3_users:
                v3_users.append(tok[0])
            continue

    # Running-config form (no quotes on Aruba-CX, single-line).
    # We also pick up ACL entries that gate SNMP — both the per-community
    # 'access-list ipv4 NAME' bindings and inline 'permit/deny ... eq snmp'
    # entries that operators use as datapath filters. The latter aren't a
    # 'configured ACL Name' on the community itself, but they DO block
    # probes silently, so surface them.
    for line in output.splitlines():
        s = line.strip()
        m = re.match(r"snmp-server community\s+(\S+)", s)
        if m and m.group(1) not in communities:
            communities.append(m.group(1))
        m = re.match(r"snmpv3 user\s+(\S+)", s)
        if m and m.group(1) not in v3_users:
            v3_users.append(m.group(1))
        m = re.match(r"snmp-server vrf\s+(\S+)", s)
        if m:
            ref = ("vrf", m.group(1))
            if ref not in acl_refs:
                acl_refs.append(ref)
        # ACL entries referencing SNMP. Examples:
        #   80 permit udp 192.107.39.0/24 any eq snmp
        #   90 deny tcp any any eq snmp
        # We don't track the parent ACL name (would require multi-line
        # context); just recording that the device has SNMP-gating ACL
        # entries is enough to give the operator a pointer.
        m = re.match(
            r"\d+\s+(permit|deny)\s+\S+\s+(\S+)\s+\S+(?:\s+\S+)*\s+eq\s+snmp",
            s)
        if m:
            verb, src = m.group(1), m.group(2)
            ref = ("acl-entry", f"{verb} {src} eq snmp")
            if ref not in acl_refs:
                acl_refs.append(ref)
    return communities, v3_users, acl_refs


def _parse_snmp_config_procurve(output):
    """Parse ProCurve 'show snmp-server' / 'show running-config' / 'show ip
    authorized-managers' for SNMP info.

    ProCurve splits this across multiple outputs:
      - 'show snmp-server' is human-formatted (table of communities + v3
        users + trap receivers + source-IP policy). The default 'public'
        community appears here even when not explicitly in running-config.
      - 'show running-config | include snmp' shows non-default
        'snmp-server community "NAME" ...' lines plus 'access-method snmp'
        toggles. Quotes are optional but typical on newer ProCurves.
      - 'show ip authorized-managers' (and the equivalent
        'ip authorized-managers' lines in running-config) gate SNMP at the
        IP level — this is the de-facto ACL mechanism on ProCurve. When
        present and 'access-method snmp' is enabled, SNMP probes from
        outside that prefix will fail with no diagnostic.
    """
    communities, v3_users, acl_refs = [], [], []
    snmp_gated = False  # True if access-method snmp / authorized-managers active

    # ---- Tabular form from 'show snmp-server' ----
    # Community-name column is 32 chars; first whitespace-delimited token is
    # the name. We bail out of the table when we hit the next header or a
    # blank line so trap-receiver / excluded-MIB sections don't pollute.
    in_comm_table, in_user_table = False, False
    for line in output.splitlines():
        s = line.rstrip()
        if "Community Name" in s and "MIB View" in s:
            in_comm_table, in_user_table = True, False
            continue
        if "User Name" in s and "Auth Protocol" in s:
            in_user_table, in_comm_table = True, False
            continue
        if not s.strip():
            in_comm_table = in_user_table = False
            continue
        if s.lstrip().startswith("---"):
            continue
        if in_comm_table:
            tok = s.split()
            if tok and tok[0] not in communities:
                communities.append(tok[0])
        elif in_user_table:
            tok = s.split()
            if tok and tok[0] not in v3_users:
                v3_users.append(tok[0])

    # ---- Config-form lines ----
    for line in output.splitlines():
        s = line.strip()
        # Newer ProCurves quote the name; older versions don't.
        m = re.match(r"snmp-server community\s+\"?([^\"\s]+)\"?\s*(.*)$", s)
        if m:
            name = m.group(1)
            if name not in communities:
                communities.append(name)
            tail = m.group(2)
            tm = re.search(r"restricted-acl\s+(\S+)", tail)
            if tm:
                acl_refs.append(("restricted-acl", tm.group(1)))
            continue
        m = re.match(r"snmpv3 user\s+\"?([^\"\s]+)\"?", s)
        if m and m.group(1) not in v3_users:
            v3_users.append(m.group(1))
            continue
        # 'ip authorized-managers IP MASK access {manager|operator}' is the
        # real SNMP source-IP filter on ProCurve. Capture both forms — the
        # one-liner 'access manager' suffix and the multi-line block with
        # 'access-method snmp' as a child.
        m = re.match(
            r"ip authorized-managers\s+(\S+)\s+(\S+)(?:\s+access\s+(\S+))?",
            s)
        if m:
            ip_addr, mask = m.group(1), m.group(2)
            role = m.group(3) or "manager"
            acl_refs.append(("authorized-managers",
                             f"{ip_addr}/{mask} ({role})"))
            continue
        if re.match(r"access-method\s+snmp", s):
            snmp_gated = True

    # ---- Multi-line 'show ip authorized-managers' parser ----
    #   Address : 199.7.244.64
    #   Mask    : 255.255.255.192
    #   Access  : Manager
    #   Access Method : snmp        <-- this is the SNMP-relevant entry
    # An 'Access Method : all' entry also gates SNMP (along with everything
    # else); 'web' / 'ssh' / 'telnet' / 'tftp' do not.
    cur = {}
    snmp_relevant_methods = ("snmp", "all")
    for line in output.splitlines():
        s = line.strip()
        m = re.match(r"Address\s*:\s*(\S+)", s)
        if m:
            cur = {"addr": m.group(1)}
            continue
        m = re.match(r"Mask\s*:\s*(\S+)", s)
        if m and "addr" in cur:
            cur["mask"] = m.group(1)
            continue
        m = re.match(r"Access\s*:\s*(\S+)", s)
        if m and "addr" in cur:
            cur["role"] = m.group(1)
            continue
        m = re.match(r"Access Method\s*:\s*(\S+)", s)
        if m and "addr" in cur:
            method = m.group(1).lower()
            if method in snmp_relevant_methods:
                role = cur.get("role", "Manager")
                ref = f"{cur['addr']}/{cur.get('mask','?')} ({role}, {method})"
                # Avoid duplicates — same prefix may appear from running-config
                # and from 'show ip authorized-managers' both.
                if ("authorized-managers", ref) not in acl_refs:
                    acl_refs.append(("authorized-managers", ref))
            cur = {}
            continue

    if snmp_gated:
        # If only 'access-method snmp' was found without an explicit
        # authorized-managers block in this output, still note the gate so
        # operators see why the probe might be ACL-blocked.
        if not any(k == "authorized-managers" for k, _ in acl_refs):
            acl_refs.append(("authorized-managers", "configured (see device)"))
    return communities, v3_users, acl_refs


def inspect_snmp_via_ssh(child, platform):
    """Pull the SNMP-related running-config from a device and parse it.

    Used as a fallback diagnostic when snmp_probe fails — answers whether
    SNMP is even configured on the device, and if so, what credentials and
    ACLs the device expects (so the operator can see the drift).

    Returns a dict with: communities, v3_users, acl_refs, excerpt, error.
    'excerpt' is a trimmed slice of the raw output suitable for the digest
    email; never longer than ~30 lines.
    """
    out = {"communities": [], "v3_users": [], "acl_refs": [],
           "excerpt": "", "error": ""}
    try:
        if platform == "junos":
            raw = send_command(
                child,
                "show configuration snmp | display set | no-more",
                timeout=15)
            out["communities"], out["v3_users"], out["acl_refs"] = \
                _parse_snmp_config_junos(raw)
        elif platform == "aruba-cx":
            # 'show snmp-server' / running-config sectioning don't exist on
            # current Aruba-CX firmware ('Invalid input: snmp-server'). The
            # community/v3 tables come from dedicated 'show' commands; the
            # vrf bindings (which gate where SNMP listens) come from
            # running-config.
            raw_a = send_command(child, "show snmp community", timeout=15)
            raw_b = send_command(child, "show snmpv3 users", timeout=15)
            try:
                raw_c = send_command(child, "show running-config", timeout=30)
            except Exception:
                raw_c = ""
            # Trim running-config to snmp-related lines so the parser doesn't
            # spend cycles on irrelevant config. Join sections with a blank
            # line so the table-state machine resets between them — without
            # the gap, running-config 'snmp-server vrf default' lines bleed
            # into the v3-user table state and produce false-positive users.
            rc_lines = [ln for ln in raw_c.splitlines()
                        if "snmp" in ln.lower()]
            raw = raw_a + "\n\n" + raw_b + "\n\n" + "\n".join(rc_lines)
            out["communities"], out["v3_users"], out["acl_refs"] = \
                _parse_snmp_config_aruba_cx(raw)
        else:  # procurve
            raw_a = send_command(child, "show snmp-server", timeout=15)
            raw_b = send_command(child,
                                 "show running-config | include snmp",
                                 timeout=15)
            # 'ip authorized-managers' is the actual SNMP ACL mechanism
            # on ProCurve. The 'include snmp' grep above usually catches
            # it via the 'access-method snmp' child line, but querying
            # directly removes that dependency on indented-context behavior
            # which differs between K-series and older firmware.
            raw_c = send_command(child, "show ip authorized-managers",
                                 timeout=10)
            # Blank-line separator forces the table-state machine to reset
            # between the human-formatted 'show snmp-server' output and the
            # config-form lines that follow.
            raw = raw_a + "\n\n" + raw_b + "\n\n" + raw_c
            out["communities"], out["v3_users"], out["acl_refs"] = \
                _parse_snmp_config_procurve(raw)
    except Exception as e:
        out["error"] = f"ssh inspect failed: {e}"
        return out

    # Trim excerpt: keep lines that mention 'snmp' (case-insensitive) so the
    # diag email isn't bloated with full running-config.
    interesting = [ln for ln in raw.splitlines()
                   if "snmp" in ln.lower()][:30]
    out["excerpt"] = "\n".join(interesting)
    return out


def diff_snmp_config(device_info, snmp_cfg):
    """Compare what the device has configured against what we have configured.

    Returns a short human-readable reason string suitable for snmp_diag.
    Examples:
      'no SNMP configured on device'
      'cred drift: device has [old_ro], we have [new_ro]'
      'acl/clients-list restricts access (refs: clients=10.0.0.0/8)'
    """
    dev_comms = set(device_info.get("communities") or [])
    dev_users = set(device_info.get("v3_users") or [])
    our_comms = set(snmp_cfg.get("communities") or [])
    our_users = set(u["name"] for u in snmp_cfg.get("v3_users") or [])
    acl_refs = device_info.get("acl_refs") or []

    if not dev_comms and not dev_users:
        return "no SNMP configured on device"
    overlap_v2 = dev_comms & our_comms
    overlap_v3 = dev_users & our_users
    if not overlap_v2 and not overlap_v3:
        return (f"cred drift: device={sorted(dev_comms | dev_users)} "
                f"ours={sorted(our_comms | our_users)}")
    # We have a credential match but probe still failed → ACL or transport block.
    if acl_refs:
        ref_str = ", ".join(f"{k}={v}" for k, v in acl_refs[:5])
        return f"creds match but probe failed; acl/restrictions: {ref_str}"
    return "creds match but probe failed (network/firewall block?)"


# ---------------------------------------------------------------------------
# SNMP-based STP collection (replaces SSH polls for monitor stp on devices
# where snmp_enabled=1)
# ---------------------------------------------------------------------------
#
# Standards used (work on Aruba-CX and ProCurve):
#   1.3.6.1.2.1.1.3.0          sysUpTime (centiseconds since boot)
#   1.3.6.1.2.1.2.2.1.8.<idx>   ifOperStatus (1=up, 2=down)
#   1.3.6.1.2.1.2.2.1.9.<idx>   ifLastChange (centiseconds at last state change)
#   1.3.6.1.2.1.17.1.4.1.2.<bp> dot1dBasePortIfIndex (bridge-port -> ifIndex)
#   1.3.6.1.2.1.17.2.4.0        dot1dStpTopChanges (TCN counter, Counter32)
#   1.3.6.1.2.1.17.2.5.0        dot1dStpDesignatedRoot (BridgeId, 8 bytes)
#   1.3.6.1.2.1.17.1.1.0        dot1dBaseBridgeAddress (own MAC, 6 bytes)
#   1.3.6.1.2.1.17.2.7.0        dot1dStpTimeSinceTopologyChange (TimeTicks)
#   1.3.6.1.2.1.17.2.15.1.3.<bp> dot1dStpPortState (1=disabled,2=blocking,
#                                3=listening,4=learning,5=forwarding,6=broken)
#   1.3.6.1.2.1.17.2.16.0       dot1dStpVersion (0=stp,2=rstp,3=mstp) — RSTP-MIB
#   1.3.6.1.2.1.17.6.1.4.1.5.<bp> dot1dStpExtPortRole (1=root,2=designated,
#                                  3=alternate,4=backup,5=disabled) — RSTP-MIB
#   1.3.6.1.2.1.31.1.1.1.1.<idx> ifName (e.g. "1/1/1", "ge-0/0/0.0")
#
# Junos vendor MIB (JUNIPER-MIMSTP-MIB at 1.3.6.1.4.1.2636.3.46.1):
#   1.3.6.1.4.1.2636.3.46.1.1.6.1.12.<bp>  jnxMIMstCistPortState
#                                          (1=disabled,2=discarding,3=blocking,
#                                           4=learning,5=forwarding)
#   1.3.6.1.4.1.2636.3.46.1.1.6.1.35.<bp>  jnxMIMstCistCurrentPortRole
#                                          (0=disabled,1=alternate,2=backup,
#                                           3=root,4=designated)
#   1.3.6.1.4.1.2636.3.46.1.1.3.1.8.<vc>   jnxMIMstCistRoot (BridgeId hex)
#
# The Junos MSTP table is indexed by dot1dBasePort numbers (verified empirically
# on STABLES-BDF and MAD-RACK4-0; see reference_junos_snmp_stp memory entry).

# OID constants — keep grouped for readability and so the cron-time hot path
# isn't building strings repeatedly.
_OID_SYSUPTIME            = "1.3.6.1.2.1.1.3.0"
_OID_IF_OPER_STATUS       = "1.3.6.1.2.1.2.2.1.8"
_OID_IF_LAST_CHANGE       = "1.3.6.1.2.1.2.2.1.9"
_OID_IF_NAME              = "1.3.6.1.2.1.31.1.1.1.1"
_OID_DOT1D_BASE_PORT_IFIDX = "1.3.6.1.2.1.17.1.4.1.2"
# Forwarding DB (learned MAC -> bridge port). dot1d index = 6 MAC octets;
# Q-BRIDGE index = <vlan>.<6 MAC octets> (VLAN-aware). Value = dot1dBasePort.
_OID_DOT1D_TPFDB_PORT     = "1.3.6.1.2.1.17.4.3.1.2"
_OID_DOT1Q_TPFDB_PORT     = "1.3.6.1.2.1.17.7.1.2.2.1.2"
_OID_DOT1D_BRIDGE_ADDR    = "1.3.6.1.2.1.17.1.1.0"
_OID_DOT1D_STP_PRIORITY   = "1.3.6.1.2.1.17.2.2.0"
_OID_DOT1D_STP_VERSION    = "1.3.6.1.2.1.17.2.16.0"
_OID_DOT1D_STP_TOPCHANGES = "1.3.6.1.2.1.17.2.4.0"
_OID_DOT1D_STP_ROOT_COST  = "1.3.6.1.2.1.17.2.6.0"
_OID_DOT1D_STP_ROOT_PORT  = "1.3.6.1.2.1.17.2.7.0"
_OID_DOT1D_STP_TIME_SINCE = "1.3.6.1.2.1.17.2.3.0"
_OID_DOT1D_STP_DESIG_ROOT = "1.3.6.1.2.1.17.2.5.0"
_OID_DOT1D_STP_PORT_STATE = "1.3.6.1.2.1.17.2.15.1.3"
_OID_DOT1D_STP_PORT_ENABLE = "1.3.6.1.2.1.17.2.15.1.4"  # 1=enabled, 2=disabled
_OID_DOT1D_STP_PORT_DESIG = "1.3.6.1.2.1.17.2.15.1.8"  # dot1dStpPortDesignatedBridge
_OID_DOT1D_STP_EXT_ROLE   = "1.3.6.1.2.1.17.6.1.4.1.5"  # RSTP-MIB; rarely populated
_OID_JNX_MSTP_PORT_STATE  = "1.3.6.1.4.1.2636.3.46.1.1.6.1.12"
_OID_JNX_MSTP_PORT_ROLE   = "1.3.6.1.4.1.2636.3.46.1.1.6.1.35"
_OID_JNX_MSTP_ROOT        = "1.3.6.1.4.1.2636.3.46.1.1.3.1.8"
# State-machine activity counters used for phantom-event evidence
# gathering (verified empirically against CLI 'show spanning-tree
# interface ... detail' and 'show spanning-tree bridge'):
#   1.3.6.1.4.1.2636.3.46.1.1.6.1.14.<bp>  jnxMIMstCistEdgeDelayWhileExpiryCount
#                                          (port-level state-machine timer
#                                           expiries — incrementing means the
#                                           STP state machine actually ran)
#   1.3.6.1.4.1.2636.3.46.1.1.3.1.35.0     jnxMIMstCistTopChanges
#                                          (CIST topology-change count for the
#                                           VC master — incrementing means a
#                                           TC was processed by this bridge)
_OID_JNX_MSTP_EDGE_EXPIRY = "1.3.6.1.4.1.2636.3.46.1.1.6.1.14"
_OID_JNX_MSTP_TC_COUNT    = "1.3.6.1.4.1.2636.3.46.1.1.3.1.35.0"


class SnmpCollectError(Exception):
    """Raised when SNMP STP collection can't complete (timeout, no creds,
    parse failure). The poll_one path catches this to fall back or fail."""


def _resolve_snmp_cred(device_row, snmp_cfg):
    """Look up the cached cred for a device and return a callable that
    runs snmpget/snmpbulkwalk for it. Raises SnmpCollectError when the
    device has no cached cred or its cred isn't loaded in cfg."""
    keys = device_row.keys() if hasattr(device_row, "keys") else []
    proto = device_row["snmp_proto"] if "snmp_proto" in keys else None
    if proto == "v2c":
        community = device_row["snmp_community"] if "snmp_community" in keys else None
        if not community:
            raise SnmpCollectError("v2c cached but community missing")
        return ("v2c", community, None)
    elif proto == "v3":
        user_name = device_row["snmp_v3_user"] if "snmp_v3_user" in keys else None
        if not user_name:
            raise SnmpCollectError("v3 cached but user missing")
        v3 = next((u for u in snmp_cfg.get("v3_users") or []
                   if u["name"] == user_name), None)
        if not v3:
            raise SnmpCollectError(
                f"v3 user '{user_name}' cached but not in [snmp_v3:*] config")
        return ("v3", None, v3)
    raise SnmpCollectError("device not yet snmp-probed")


def _snmp_args_for(cred, ip, oid, op, timeout, retries):
    """Build snmpget or snmpbulkwalk argv for the given cred tuple.
    op = 'get' for snmpget, 'walk' for snmpbulkwalk."""
    proto, community, v3 = cred
    binary = "snmpget" if op == "get" else "snmpbulkwalk"
    if proto == "v2c":
        args = [binary, "-v2c", "-c", community,
                "-t", str(timeout), "-r", str(retries), "-On"]
        if op == "walk":
            args += ["-Cr50"]  # bulk repeats
        args += [ip, oid]
        return args
    # v3
    args = [binary, "-v3",
            "-l", v3.get("security_level") or "noAuthNoPriv",
            "-u", v3["name"],
            "-t", str(timeout), "-r", str(retries), "-On"]
    if v3.get("auth_pass"):
        args += ["-a", v3.get("auth_proto") or "SHA",
                 "-A", v3["auth_pass"]]
    if v3.get("priv_pass"):
        args += ["-x", v3.get("priv_proto") or "AES",
                 "-X", v3["priv_pass"]]
    if op == "walk":
        args += ["-Cr50"]
    args += [ip, oid]
    return args


# Output line shapes from net-snmp -On format:
#   .1.3.6.1.2.1.2.2.1.8.514 = INTEGER: 1
#   .1.3.6.1.2.1.2.2.1.8.514 = INTEGER: up(1)
#   .1.3.6.1.2.1.31.1.1.1.1.514 = STRING: ge-0/0/0.0
#   .1.3.6.1.2.1.17.2.5.0 = Hex-STRING: 00 00 02 00 00 11 12 11
#   .1.3.6.1.2.1.1.3.0 = Timeticks: (123456) 0:20:34.56
_SNMP_LINE_RE = re.compile(
    r"^\.([\d\.]+)\s*=\s*(\w[\w-]*):\s*(.*?)\s*$")


def _parse_snmp_value(type_name, raw_value):
    """Convert a net-snmp '-On' value string into a Python value, given the
    declared SNMP type. Returns None if unparseable."""
    type_name = type_name.lower()
    raw = raw_value.strip().rstrip('"').lstrip('"')
    if type_name in ("integer", "counter32", "counter64", "gauge32",
                     "unsigned32"):
        # Could be 'INTEGER: 5' or 'INTEGER: forwarding(5)' — extract trailing int
        m = re.search(r"\(?(\-?\d+)\)?\s*$", raw)
        if m:
            return int(m.group(1))
        return None
    if type_name == "timeticks":
        # 'Timeticks: (12345) ...' — extract centisecs
        m = re.search(r"\((\d+)\)", raw_value)
        return int(m.group(1)) if m else None
    if type_name == "hex-string":
        # '00 11 22 33 44 55' → bytes
        try:
            return bytes(int(b, 16) for b in raw.split())
        except ValueError:
            return None
    if type_name == "string":
        # Already a string
        return raw
    if type_name == "oid":
        return raw
    return raw  # unknown type — return raw text


def _snmp_walk_dict(cred, ip, base_oid, timeout=5, retries=1, op_timeout=15):
    """Walk a column OID and return {trailing_index: value}.

    trailing_index is everything after base_oid (typically a single integer
    for ifTable / dot1dStpPortTable). Strips the literal base_oid prefix
    plus the leading dot.
    """
    args = _snmp_args_for(cred, ip, base_oid, "walk", timeout, retries)
    rc, out, err = _snmp_run(args, op_timeout)
    if rc != 0 or not out.strip():
        raise SnmpCollectError(
            f"snmpwalk {base_oid} on {ip} failed: {(err or out).strip()[:120]}")
    if any(b in (out + err) for b in
           ("Authentication failure", "Unknown user name",
            "Timeout: No Response", "wrongDigest", "decryptionError")):
        raise SnmpCollectError(
            f"snmpwalk {base_oid} on {ip}: {(out or err).strip().splitlines()[0][:120]}")

    prefix = "." + base_oid + "."
    result = {}
    for line in out.splitlines():
        m = _SNMP_LINE_RE.match(line)
        if not m:
            continue
        full_oid, type_name, raw_value = m.groups()
        full_oid = "." + full_oid
        if not full_oid.startswith(prefix):
            continue
        idx_part = full_oid[len(prefix):]
        # Single-column tables have one trailing integer; we keep it as int.
        # Multi-element indexes (rare here) stay as the dot-string.
        if idx_part.isdigit():
            key = int(idx_part)
        else:
            key = idx_part
        val = _parse_snmp_value(type_name, raw_value)
        if val is not None:
            result[key] = val
    return result


def _snmp_get_value(cred, ip, oid, timeout=5, retries=1, op_timeout=10):
    """Get a single OID value. Returns the parsed value (int/bytes/string)
    or raises SnmpCollectError."""
    args = _snmp_args_for(cred, ip, oid, "get", timeout, retries)
    rc, out, err = _snmp_run(args, op_timeout)
    if rc != 0 or not _snmp_response_ok(rc, out, err):
        raise SnmpCollectError(
            f"snmpget {oid} on {ip} failed: {(out or err).strip()[:120]}")
    m = _SNMP_LINE_RE.match(out.strip().splitlines()[0])
    if not m:
        raise SnmpCollectError(f"snmpget {oid} on {ip}: unparseable: {out[:80]}")
    _, type_name, raw_value = m.groups()
    val = _parse_snmp_value(type_name, raw_value)
    if val is None:
        raise SnmpCollectError(f"snmpget {oid} on {ip}: parse-empty")
    return val


# ---- State / role translators per platform vocabulary ----
# Existing SSH parsers use platform-specific strings; SNMP collectors must
# emit the same so the downstream is_stp_blocking / should_alert_stp_change
# logic doesn't need any changes.

_DOT1D_STATE_TO_ARUBA = {
    1: "Disabled", 2: "Blocking", 3: "Listening",
    4: "Learning", 5: "Forwarding", 6: "Broken",
}
_DOT1D_STATE_TO_PROCURVE = _DOT1D_STATE_TO_ARUBA  # same vocabulary

_RSTP_ROLE_TO_ARUBA = {
    1: "Root", 2: "Designated", 3: "Alternate",
    4: "Backup", 5: "Disabled",
}
_RSTP_ROLE_TO_PROCURVE = {
    1: "Root", 2: "Designated", 3: "Alternate",
    4: "Backup", 5: "Disabled",
}

_JUNOS_STATE_INT_TO_STR = {
    1: "DIS", 2: "BLK", 3: "BLK", 4: "LRN", 5: "FWD",
}
_JUNOS_ROLE_INT_TO_STR = {
    0: "DIS", 1: "ALT", 2: "BACKUP", 3: "ROOT", 4: "DESG",
}

_DOT1D_STP_VERSION_TO_MODE = {0: "stp", 2: "rstp", 3: "mstp"}


def _infer_role_dot1d(bp, state_int, enable_int, root_port_bp, role_strings):
    """Derive STP role from BRIDGE-MIB fields.

    The RSTP-MIB role extension (dot1dStpExtPortRole) isn't populated on
    Aruba-CX or most ProCurves. We use dot1dStpRootPort directly: it
    identifies *which specific bridge-port* is the root port for this
    device. That's authoritative — the device's own STP machine names it.

      - root_port_bp == 0       -> this device IS the root; every active
                                   port is Designated for its segment.
      - bp == root_port_bp      -> this is the root port (Root in
                                   forwarding; Alternate if blocking
                                   during convergence — unusual).
      - any other port          -> Designated (forwarding) or Alternate
                                   (blocking).

    Avoids the VSX / cluster MAC gotcha entirely: the device tells us
    which port is the root port without us guessing from MACs.

    role_strings: (designated, root, alternate, disabled) — vendor vocab.
    """
    designated, root, alternate, disabled = role_strings
    # dot1dStpPortEnable=2 means STP is administratively disabled on this
    # port. SSH parsers report these as Role=Disabled regardless of state;
    # honour that vocabulary so cross-poll diffs don't churn.
    if state_int == 1 or enable_int == 2:
        return disabled
    if root_port_bp and bp == root_port_bp:
        # This port is named by the device as its root port.
        if state_int == 5:
            return root
        return alternate
    # Either we're root (root_port_bp == 0) or this is a non-root port.
    if state_int == 5:
        return designated
    if state_int in (2, 3):
        return alternate
    return alternate


def _bytes_to_mac(b):
    """Convert a 6-byte address into 'aa:bb:cc:dd:ee:ff' canonical form.
    Matches _normalize_stp_mac output so cross-switch consensus works."""
    if b is None or len(b) < 6:
        return None
    return ":".join(f"{x:02x}" for x in b[:6])


def _bridge_id_to_priority_mac(b):
    """A BridgeId is 2 bytes priority + 6 bytes MAC. Returns (priority, mac)."""
    if b is None or len(b) < 8:
        return None, None
    priority = (b[0] << 8) | b[1]
    return priority, _bytes_to_mac(b[2:8])


def _snmp_collect_common_iface(cred, ip):
    """Walk the standard ifTable + dot1dBasePortIfIndex once. Returned dicts
    are reused by the per-platform STP collector so we don't re-walk these
    on every call. Returns (sysup_centisec, base_to_ifidx, ifidx_to_name,
    ifidx_to_oper, ifidx_to_lastchg)."""
    sysup = _snmp_get_value(cred, ip, _OID_SYSUPTIME)
    base_to_ifidx = _snmp_walk_dict(cred, ip, _OID_DOT1D_BASE_PORT_IFIDX)
    ifidx_to_name = _snmp_walk_dict(cred, ip, _OID_IF_NAME)
    ifidx_to_oper = _snmp_walk_dict(cred, ip, _OID_IF_OPER_STATUS)
    ifidx_to_lastchg = _snmp_walk_dict(cred, ip, _OID_IF_LAST_CHANGE)
    return sysup, base_to_ifidx, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg


def _build_ifaces_and_flaps(sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg,
                             strip_unit_suffix=False):
    """Convert SNMP ifTable walks into the {iface: link_up_bool} +
    {iface: seconds_since_flap} dicts the existing pipeline expects.

    strip_unit_suffix=True drops the '.0' Junos logical-unit suffix from
    ifNames, matching what parse_ifaces_junos produces."""
    ifaces = {}
    last_flapped = {}
    for idx, oper in ifidx_to_oper.items():
        name = ifidx_to_name.get(idx)
        if not name:
            continue
        if strip_unit_suffix and "." in name:
            name = name.split(".", 1)[0]
        is_up = (oper == 1)
        ifaces[name] = is_up
        # Last-flap age only meaningful when the port is currently up — for
        # down ports the existing pipeline doesn't use the flap timestamp.
        lc = ifidx_to_lastchg.get(idx)
        if is_up and lc is not None:
            age_centi = max(0, sysup - lc)
            last_flapped[name] = age_centi // 100
    return ifaces, last_flapped


def _build_root_entries_standard(cred, ip, bridge_addr_bytes=None):
    """Build a single CIST root-entry dict from standard BRIDGE-MIB + RSTP-MIB
    fields. Used by Aruba-CX and ProCurve."""
    try:
        root_id = _snmp_get_value(cred, ip, _OID_DOT1D_STP_DESIG_ROOT)
    except SnmpCollectError:
        return []
    if bridge_addr_bytes is None:
        try:
            bridge_addr_bytes = _snmp_get_value(cred, ip, _OID_DOT1D_BRIDGE_ADDR)
        except SnmpCollectError:
            bridge_addr_bytes = None
    try:
        tcn = _snmp_get_value(cred, ip, _OID_DOT1D_STP_TOPCHANGES)
    except SnmpCollectError:
        tcn = None
    try:
        last_tcn_ticks = _snmp_get_value(cred, ip, _OID_DOT1D_STP_TIME_SINCE)
    except SnmpCollectError:
        last_tcn_ticks = None
    # dot1dStpPriority (this device's bridge priority for the CIST). Pulling
    # it explicitly closes the parity gap with the SSH path, which scraped
    # the priority from 'show spanning-tree bridge' / equivalent output.
    try:
        bridge_priority = _snmp_get_value(cred, ip, _OID_DOT1D_STP_PRIORITY)
    except SnmpCollectError:
        bridge_priority = None

    root_priority, root_mac = _bridge_id_to_priority_mac(root_id)
    bridge_mac = _bytes_to_mac(bridge_addr_bytes) if bridge_addr_bytes else None
    return [{
        "instance": "CIST",
        "root_priority": root_priority,
        "root_mac": root_mac,
        "bridge_priority": bridge_priority,
        "bridge_mac": bridge_mac,
        "tcn_count": tcn,
        "last_tcn_seconds": (last_tcn_ticks // 100) if last_tcn_ticks is not None else None,
    }]


def _snmp_collect_stp_aruba_cx(cred, ip):
    """SNMP-based STP poll for Aruba-CX. Returns (ports, ifaces, stp_mode,
    root_entries, last_flapped) — same shape as the SSH poll path.

    Aruba-CX doesn't populate the RSTP-MIB role extension. We use
    dot1dStpRootCost (0 → we are root) plus dot1dStpDesignatedRoot
    (the root bridge ID) to derive role. This avoids the VSX/cluster
    pitfall where dot1dBaseBridgeAddress reports the chassis MAC but the
    STP bridge ID uses a different virtual cluster MAC."""
    sysup, base_to_ifidx, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg = \
        _snmp_collect_common_iface(cred, ip)
    state_by_bp = _snmp_walk_dict(cred, ip, _OID_DOT1D_STP_PORT_STATE)
    try:
        enable_by_bp = _snmp_walk_dict(cred, ip, _OID_DOT1D_STP_PORT_ENABLE)
    except SnmpCollectError:
        enable_by_bp = {}
    try:
        own_mac_bytes = _snmp_get_value(cred, ip, _OID_DOT1D_BRIDGE_ADDR)
    except SnmpCollectError:
        own_mac_bytes = None
    try:
        root_port_bp = _snmp_get_value(cred, ip, _OID_DOT1D_STP_ROOT_PORT)
    except SnmpCollectError:
        root_port_bp = 0  # treat as 'we are root' if unreadable

    if not state_by_bp:
        ifaces, last_flapped = _build_ifaces_and_flaps(
            sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg)
        return [], ifaces, None, [], last_flapped

    role_strings = ("Designated", "Root", "Alternate", "Disabled")
    ports = []
    for bp, state_int in state_by_bp.items():
        ifidx = base_to_ifidx.get(bp)
        name = ifidx_to_name.get(ifidx) if ifidx else None
        if not name:
            continue
        # When the link is down, Aruba-CX 'show spanning-tree' renders the
        # port as Role=Disabled / State=Down. Mirror BOTH overrides so the
        # downstream offline-detection (_is_stp_offline_state) recognises
        # the result; otherwise transient combos like 'Disabled/Forwarding'
        # or 'Disabled/Blocking' (briefly emitted while the STP machine
        # catches up to the link drop) bypass the offline check, get
        # confirmed across two polls, and fire false-positive alerts that
        # bury the real Alternate/Blocking events you actually want to see.
        oper = ifidx_to_oper.get(ifidx) if ifidx else None
        if oper is not None and oper != 1:
            role_str = "Disabled"
            state_str = "Down"
        else:
            role_str = _infer_role_dot1d(bp, state_int,
                                          enable_by_bp.get(bp, 1),
                                          root_port_bp, role_strings)
            state_str = _DOT1D_STATE_TO_ARUBA.get(state_int, "Unknown")
        ports.append({
            "interface": name,
            "state": state_str,
            "role":  role_str,
        })

    stp_mode = None
    try:
        ver = _snmp_get_value(cred, ip, _OID_DOT1D_STP_VERSION)
        stp_mode = _DOT1D_STP_VERSION_TO_MODE.get(ver)
    except SnmpCollectError:
        pass

    ifaces, last_flapped = _build_ifaces_and_flaps(
        sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg)
    root_entries = _build_root_entries_standard(cred, ip,
                                                 bridge_addr_bytes=own_mac_bytes)
    return ports, ifaces, stp_mode, root_entries, last_flapped


def _snmp_collect_stp_procurve(cred, ip):
    """SNMP-based STP poll for HP ProCurve. Same standard MIBs as Aruba-CX.

    The SSH parser_stp_procurve infers role from state alone (no Role column
    in 'show spanning-tree' output). The SNMP path can do better with
    dot1dStpPortDesignatedBridge — distinguishes Designated vs Root/Alternate
    properly. We map back to ProCurve's vocabulary so downstream comparisons
    against the existing stp_state rows still work after the first transition
    poll."""
    sysup, base_to_ifidx, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg = \
        _snmp_collect_common_iface(cred, ip)
    state_by_bp = _snmp_walk_dict(cred, ip, _OID_DOT1D_STP_PORT_STATE)
    try:
        enable_by_bp = _snmp_walk_dict(cred, ip, _OID_DOT1D_STP_PORT_ENABLE)
    except SnmpCollectError:
        enable_by_bp = {}
    try:
        own_mac_bytes = _snmp_get_value(cred, ip, _OID_DOT1D_BRIDGE_ADDR)
    except SnmpCollectError:
        own_mac_bytes = None
    try:
        root_port_bp = _snmp_get_value(cred, ip, _OID_DOT1D_STP_ROOT_PORT)
    except SnmpCollectError:
        root_port_bp = 0

    if not state_by_bp:
        ifaces, last_flapped = _build_ifaces_and_flaps(
            sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg)
        return [], ifaces, None, [], last_flapped

    # ProCurve SSH parser only emits Designated/Alternate/Disabled — never
    # Root. Collapse Root → Designated to keep cross-poll diffs byte-stable
    # against existing stp_state rows.
    role_strings = ("Designated", "Designated", "Alternate", "Disabled")
    ports = []
    for bp, state_int in state_by_bp.items():
        ifidx = base_to_ifidx.get(bp)
        name = ifidx_to_name.get(ifidx) if ifidx else None
        if not name:
            continue
        # When the link is down, ProCurve 'show spanning-tree' shows
        # Role=Disabled / State=Down. Mirror BOTH overrides so transient
        # Disabled/Forwarding or Disabled/Blocking pseudo-states (briefly
        # emitted while STP catches up to the link drop) don't slip past
        # _is_stp_offline_state and fire false-positive alerts.
        oper = ifidx_to_oper.get(ifidx) if ifidx else None
        if oper is not None and oper != 1:
            role_str = "Disabled"
            state_str = "Down"
        else:
            role_str = _infer_role_dot1d(bp, state_int,
                                          enable_by_bp.get(bp, 1),
                                          root_port_bp, role_strings)
            state_str = _DOT1D_STATE_TO_PROCURVE.get(state_int, "Unknown")
        ports.append({"interface": name, "state": state_str, "role": role_str})

    stp_mode = None
    try:
        ver = _snmp_get_value(cred, ip, _OID_DOT1D_STP_VERSION)
        stp_mode = _DOT1D_STP_VERSION_TO_MODE.get(ver)
    except SnmpCollectError:
        pass

    ifaces, last_flapped = _build_ifaces_and_flaps(
        sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg)
    root_entries = _build_root_entries_standard(cred, ip,
                                                 bridge_addr_bytes=own_mac_bytes)
    return ports, ifaces, stp_mode, root_entries, last_flapped


def _snmp_collect_stp_junos(cred, ip):
    """SNMP-based STP poll for Junos. Uses JUNIPER-MIMSTP-MIB for state/role
    (indexed by dot1dBasePort) and standard ifTable for interface names.

    Junos MSTP port-IDs are dot1dBasePort numbers — see
    reference_junos_snmp_stp memory entry for empirical validation."""
    sysup, base_to_ifidx, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg = \
        _snmp_collect_common_iface(cred, ip)
    state_by_bp = _snmp_walk_dict(cred, ip, _OID_JNX_MSTP_PORT_STATE)
    try:
        role_by_bp = _snmp_walk_dict(cred, ip, _OID_JNX_MSTP_PORT_ROLE)
    except SnmpCollectError:
        role_by_bp = {}
    # Authoritative JUNIPER STP port-id -> interface map (Junos VC switches
    # have diverged JUNIPER vs BRIDGE-MIB indexing — the cache is populated
    # at discover time from CLI 'show spanning-tree interface'). Cache miss
    # falls back to the BRIDGE-MIB chain below; that fallback works on the
    # 76% of Junos devices where the two indexings happen to align.
    conn = _db()
    port_id_cache = {r["port_id"]: r["interface"] for r in conn.execute(
        "SELECT port_id, interface FROM stp_port_id_cache WHERE ip = ?",
        (ip,)).fetchall()}
    conn.close()
    # Per-port state-machine activity counter — used by the poll loop to
    # tell whether a state change matches a real state-machine event or
    # just an SNMP-cache flicker. None on parse/timeout failure (treated
    # as 'unknown' downstream, never counted as an event).
    try:
        edge_expiry_by_bp = _snmp_walk_dict(cred, ip, _OID_JNX_MSTP_EDGE_EXPIRY)
    except SnmpCollectError:
        edge_expiry_by_bp = {}
    # Bridge-wide CIST topology-change counter; complements per-port
    # edge_expiry — incrementing means the bridge processed a TC, which
    # legitimately drives port-level state transitions.
    try:
        tcn_count_value = _snmp_get_value(cred, ip, _OID_JNX_MSTP_TC_COUNT)
    except SnmpCollectError:
        tcn_count_value = None

    if not state_by_bp:
        ifaces, last_flapped = _build_ifaces_and_flaps(
            sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg,
            strip_unit_suffix=True)
        return [], ifaces, None, [], last_flapped

    # Reverse-map physical-iface names to ifIndex so the link-down override
    # works regardless of which path resolved the name (cache or fallback).
    # Filtered to physical entries (no '.N' suffix) because ifOperStatus is
    # most reliable on the physical interface row, not the logical unit.
    name_to_ifidx = {n: i for i, n in ifidx_to_name.items() if "." not in n}

    ports = []
    for bp, state_int in state_by_bp.items():
        # Prefer the CLI-populated cache (authoritative). Falls through to
        # the BRIDGE-MIB chain only on cache miss — this can mislabel ports
        # on partial-overlap-indexing devices, but on first-deploy the
        # cache is empty for everyone and we want to keep working until the
        # next discover populates it.
        name = port_id_cache.get(bp)
        if name is None:
            ifidx_fallback = base_to_ifidx.get(bp)
            name = ifidx_to_name.get(ifidx_fallback) if ifidx_fallback else None
            if name and "." in name:
                # Strip Junos '.0' logical-unit suffix to match parse_stp_junos.
                name = name.split(".", 1)[0]
        if not name:
            continue
        # Link-down override: when ifOperStatus is down, emit DIS/BLK so the
        # offline-detector recognises the result. Without this, a flap
        # transient (link drops, jnxMIMstCistCurrentPortRole momentarily
        # still reads 'designated' before the role machine catches up)
        # produces alert noise like 'DESG/FWD -> DESG/BLK' that buries the
        # real Alternate/Blocking topology events.
        ifidx = name_to_ifidx.get(name)
        oper = ifidx_to_oper.get(ifidx) if ifidx else None
        if oper is not None and oper != 1:
            role_str = "DIS"
            state_str = "BLK"
        else:
            role_int = role_by_bp.get(bp, 0)
            role_str = _JUNOS_ROLE_INT_TO_STR.get(role_int, "DIS")
            state_str = _JUNOS_STATE_INT_TO_STR.get(state_int, "DIS")
        ports.append({
            "interface": name,
            "state": state_str,
            "role":  role_str,
            "edge_expiry": edge_expiry_by_bp.get(bp),
        })

    # Junos root from jnxMIMstCistRoot. The table is indexed by VC-member;
    # member 0 is the master and gives the CIST root for the chassis.
    root_entries = []
    try:
        root_walk = _snmp_walk_dict(cred, ip, _OID_JNX_MSTP_ROOT)
        if root_walk:
            # Take the lowest-indexed entry (master member) as authoritative.
            first_idx = sorted(root_walk.keys())[0]
            root_id = root_walk[first_idx]
            root_priority, root_mac = _bridge_id_to_priority_mac(root_id)
            try:
                bridge_addr_bytes = _snmp_get_value(cred, ip,
                                                    _OID_DOT1D_BRIDGE_ADDR)
            except SnmpCollectError:
                bridge_addr_bytes = None
            # Junos populates standard dot1dStpPriority for the CIST too, so
            # the priority value matches what Aruba/ProCurve report.
            try:
                bridge_priority = _snmp_get_value(cred, ip,
                                                   _OID_DOT1D_STP_PRIORITY)
            except SnmpCollectError:
                bridge_priority = None
            root_entries = [{
                "instance": "CIST",
                "root_priority": root_priority,
                "root_mac": root_mac,
                "bridge_priority": bridge_priority,
                "bridge_mac": _bytes_to_mac(bridge_addr_bytes)
                              if bridge_addr_bytes else None,
                # tcn_count populated for phantom-evidence cross-check; if
                # this advances between two polls a TC was processed and
                # state-machine activity is expected. last_tcn_seconds is
                # left None because Junos exposes that as Timeticks at a
                # different OID and the alert path doesn't use it yet.
                "tcn_count": tcn_count_value,
                "last_tcn_seconds": None,
            }]
    except SnmpCollectError:
        pass

    # MSTP being populated implies stp_mode = 'mstp' on Junos. Other modes use
    # different OID trees we're not walking here, so we'd return None in that
    # case (preserving the last SSH-derived value).
    stp_mode = "mstp"

    ifaces, last_flapped = _build_ifaces_and_flaps(
        sysup, ifidx_to_name, ifidx_to_oper, ifidx_to_lastchg,
        strip_unit_suffix=True)
    return ports, ifaces, stp_mode, root_entries, last_flapped


def snmp_collect_stp(ip, cred, platform):
    """Dispatch SNMP STP collection by platform. Returns the same tuple as the
    SSH poll_one path: (ports, ifaces, stp_mode, root_entries, last_flapped).
    Raises SnmpCollectError on hard failure (caller handles fallback)."""
    if platform == "junos":
        return _snmp_collect_stp_junos(cred, ip)
    if platform == "aruba-cx":
        return _snmp_collect_stp_aruba_cx(cred, ip)
    if platform == "procurve":
        return _snmp_collect_stp_procurve(cred, ip)
    raise SnmpCollectError(f"unknown platform {platform!r}")


def parse_stp_mode_junos(output):
    """Extract STP mode from 'show spanning-tree bridge' on Junos.

    Returns a lowercased mode string ('mstp'/'rstp'/'vstp') or None if
    the output doesn't contain a recognizable 'Enabled protocol' line.
    """
    m = re.search(r"Enabled protocol\s*:\s*(\S+)", output)
    return m.group(1).strip().lower() if m else None


def parse_stp_mode_aruba_cx(output):
    """Extract STP mode from 'show spanning-tree' on ArubaOS-CX.

    Header line looks like: 'Spanning tree status      : Enabled Protocol: MSTP'
    """
    m = re.search(r"Protocol:\s*(\S+)", output)
    return m.group(1).strip().lower() if m else None


def parse_stp_mode_procurve(output):
    """Extract STP mode from 'show spanning-tree' on HP ProCurve.

    Line looks like: 'Force Version : MSTP-operation'. The '-operation'
    suffix is stripped so procurve, aruba-cx, and junos all normalize to
    the same mode tokens ('mstp'/'rstp'/etc.).
    """
    m = re.search(r"Force Version\s*:\s*(\S+)", output)
    if not m:
        return None
    return m.group(1).strip().lower().split("-")[0]


# ---------------------------------------------------------------------------
# STP root-bridge parsers
#
# Return list[dict] (one per STP instance). The first MVP only tracks the
# CIST/MST0 instance — that's what carries the cross-region root identity
# and is what every switch in a correctly-configured network agrees on.
# Adding per-MSTI tracking later is additive (more instances in the list).
#
# Canonical MAC form used across all parsers: 'aa:bb:cc:dd:ee:ff' lowercase,
# so outputs from Junos ('xx:xx:...'), ProCurve ('aabbcc-ddeeff'), and
# ArubaOS-CX ('xx:xx:...') compare byte-for-byte.
# ---------------------------------------------------------------------------

def _normalize_stp_mac(s):
    """Normalize an STP bridge MAC to 'aa:bb:cc:dd:ee:ff' lowercase.

    Accepts colon-separated ('02:00:00:11:12:11'), ProCurve two-halves
    ('020000-111211'), and Cisco dotted ('0200.0011.1211'). Returns None
    if the input doesn't contain 12 hex digits.
    """
    if not s:
        return None
    hexchars = re.sub(r"[^0-9a-fA-F]", "", s)
    if len(hexchars) != 12:
        return None
    return ":".join(hexchars[i:i+2] for i in range(0, 12, 2)).lower()


def parse_stp_root_junos(output):
    """Parse CIST root-bridge info from 'show spanning-tree bridge'.

    Expected lines under 'STP bridge parameters for CIST':
        Root ID                           : 0.02:00:00:11:12:11
        Bridge ID                         : 32768.28:c0:da:44:c5:01
        Number of topology changes        : 92
        Time since last topology change   : 263520 seconds
    """
    def _prio_mac(s):
        # '0.02:00:00:11:12:11' -> (0, '02:00:00:11:12:11')
        m = re.match(r"\s*(\d+)\.([0-9a-fA-F:]+)", s)
        if not m:
            return (None, None)
        return (int(m.group(1)), _normalize_stp_mac(m.group(2)))

    entry = {"instance": "CIST"}
    m = re.search(r"^\s*Root ID\s*:\s*(\S+)", output, re.MULTILINE)
    if m:
        entry["root_priority"], entry["root_mac"] = _prio_mac(m.group(1))
    m = re.search(r"^\s*Bridge ID\s*:\s*(\S+)", output, re.MULTILINE)
    if m:
        entry["bridge_priority"], entry["bridge_mac"] = _prio_mac(m.group(1))
    m = re.search(r"Number of topology changes\s*:\s*(\d+)", output)
    if m:
        entry["tcn_count"] = int(m.group(1))
    m = re.search(r"Time since last topology change\s*:\s*(\d+)", output)
    if m:
        entry["last_tcn_seconds"] = int(m.group(1))
    if entry.get("root_mac") is None:
        return []
    return [entry]


def parse_stp_root_procurve(output):
    """Parse CIST root-bridge info from ProCurve 'show spanning-tree'.

    'CST Root' fields are the universal CIST root (compare across switches).
    'Switch MAC Address'/'Switch Priority' are this bridge. ProCurve's
    'Time Since Last Change' is imprecise ('3 days') so we don't record it.
    """
    entry = {"instance": "CIST"}
    m = re.search(r"CST Root MAC Address\s*:\s*(\S+)", output)
    if m:
        entry["root_mac"] = _normalize_stp_mac(m.group(1))
    m = re.search(r"CST Root Priority\s*:\s*(\d+)", output)
    if m:
        entry["root_priority"] = int(m.group(1))
    m = re.search(r"Switch MAC Address\s*:\s*(\S+)", output)
    if m:
        entry["bridge_mac"] = _normalize_stp_mac(m.group(1))
    m = re.search(r"Switch Priority\s*:\s*(\d+)", output)
    if m:
        entry["bridge_priority"] = int(m.group(1))
    m = re.search(r"Topology Change Count\s*:\s*(\d+)", output)
    if m:
        entry["tcn_count"] = int(m.group(1))
    if entry.get("root_mac") is None:
        return []
    return [entry]


def parse_stp_root_aruba_cx(output):
    """Parse CIST root-bridge info from ArubaOS-CX 'show spanning-tree'.

    Expected block (first MAC-Address line after each 'Priority' header):
        MST0
          Root ID    Priority   : 0
                     MAC-Address: 02:00:00:11:12:11
          ...
          Bridge ID  Priority  : 32768
                     MAC-Address: 00:02:00:00:00:01
        Number of topology changes    : 335
        Last topology change occurred : 263567 seconds ago
    """
    entry = {"instance": "CIST"}
    # Split at MST0 so we don't accidentally pick up MST1 fields.
    section = output
    m = re.search(r"^MST0\b", output, re.MULTILINE)
    if m:
        end = re.search(r"^MST[1-9]\d*\b", output[m.end():], re.MULTILINE)
        section = output[m.start():m.start() + (end.start() if end else len(output))]

    m = re.search(
        r"Root ID\s*Priority\s*:\s*(\d+)\s*\n\s*MAC-Address\s*:\s*(\S+)", section)
    if m:
        entry["root_priority"] = int(m.group(1))
        entry["root_mac"] = _normalize_stp_mac(m.group(2))
    m = re.search(
        r"Bridge ID\s*Priority\s*:\s*(\d+)\s*\n\s*MAC-Address\s*:\s*(\S+)", section)
    if m:
        entry["bridge_priority"] = int(m.group(1))
        entry["bridge_mac"] = _normalize_stp_mac(m.group(2))
    m = re.search(r"Number of topology changes\s*:\s*(\d+)", section)
    if m:
        entry["tcn_count"] = int(m.group(1))
    m = re.search(r"Last topology change occurred\s*:\s*(\d+)\s*seconds", section)
    if m:
        entry["last_tcn_seconds"] = int(m.group(1))
    if entry.get("root_mac") is None:
        return []
    return [entry]


# ---------------------------------------------------------------------------
# Hardware / stacking / chassis parsers
#
# Each parser returns a list of member dicts:
#   {member_id: int, role: str|None, serial: str|None,
#    mac: str|None (canonical), model: str|None, status: str|None}
#
# 'standalone' devices still return a single-element list so the caller
# can uniformly upsert into device_members.
# ---------------------------------------------------------------------------

def parse_junos_vc(output):
    """Parse 'show virtual-chassis'. Member rows look like:
        0 (FPC 0)  Prsnt    BP0211020542 ex4200-48t 128  Master*      ...
        1 (FPC 1)  Prsnt    BP0211020034 ex4200-48t 128  Backup       ...
    Standalone devices still report one member (role 'Master*').
    Junos does not expose per-member MAC here — left None.
    """
    members = []
    line_re = re.compile(
        r"^\s*(\d+)\s+\(FPC\s+\d+\)\s+"   # member id
        r"(\S+)\s+"                        # status (Prsnt, NotPrsnt, etc.)
        r"(\S+)\s+"                        # serial
        r"(\S+)\s+"                        # model
        r"\d+\s+"                          # mstr priority
        r"(\w+)\*?")                       # role (Master*/Backup/Linecard)
    for line in output.splitlines():
        m = line_re.match(line)
        if not m:
            continue
        members.append({
            "member_id": int(m.group(1)),
            "status":    m.group(2),
            "serial":    m.group(3),
            "model":     m.group(4),
            "role":      m.group(5),
            "mac":       None,
        })
    return members


def parse_aruba_cx_vsf(vsf_output, module_output):
    """Parse ArubaOS-CX VSF stack. Combines 'show vsf' (gives member_id,
    MAC, model token, role) with 'show module' (gives per-member serial).
    """
    members = {}
    # 'show vsf' member table:
    #   1   9c:37:08:2d:a1:00   JL659A         Conductor
    vsf_re = re.compile(
        r"^\s*(\d+)\s+([0-9a-fA-F:]{17})\s+(\S+)\s+(\S+)")
    for line in vsf_output.splitlines():
        m = vsf_re.match(line)
        if not m:
            continue
        mid = int(m.group(1))
        members[mid] = {
            "member_id": mid,
            "mac":       _normalize_stp_mac(m.group(2)),
            "model":     m.group(3),
            "role":      m.group(4),
            "serial":    None,
            "status":    None,
        }
    # 'show module' line modules — 'name' col is '<member>/<slot>':
    #   1/1  JL659A  6300M 48SR5 CL6 PoE 4SFP56 Swch        SG47LMQ0WC Active (local)
    # Multi-word description forces us to anchor on the status keyword at
    # the end (Active/Ready/Standby/Down) to locate the serial before it.
    mod_re = re.compile(
        r"^\s*(\d+)/\d+\s+"
        r"\S+\s+"
        r".+?\s+"
        r"([A-Z0-9]{8,})\s+"                          # serial (e.g. SG47LMQ0WC)
        r"(Active|Ready|Standby|Down)(?:\s+\(.*?\))?\s*$")
    # Only read the 'Line Modules' section — Management Modules usually
    # have matching serials but we want the switch serial specifically.
    in_lines = False
    for line in module_output.splitlines():
        if re.match(r"\s*Line Modules", line):
            in_lines = True; continue
        if not in_lines:
            continue
        m = mod_re.match(line)
        if not m:
            continue
        mid = int(m.group(1))
        if mid in members and members[mid].get("serial") is None:
            members[mid]["serial"] = m.group(2)
            members[mid]["status"] = m.group(3)
    return [members[mid] for mid in sorted(members)]


def parse_aruba_cx_standalone(system_output, module_output):
    """Single-member record for a non-VSF ArubaOS-CX device. Uses
    'show system' (chassis serial + base MAC + product name).
    """
    serial = None; mac = None; model = None
    m = re.search(r"Chassis Serial Nbr\s*:\s*(\S+)", system_output)
    if m: serial = m.group(1)
    m = re.search(r"Base MAC Address\s*:\s*(\S+)", system_output)
    if m: mac = _normalize_stp_mac(m.group(1))
    m = re.search(r"Product Name\s*:\s*(\S.+?)\s*$", system_output, re.MULTILINE)
    if m: model = m.group(1).strip()
    if serial is None and mac is None:
        return []
    return [{
        "member_id": 1, "role": None,
        "serial": serial, "mac": mac, "model": model,
        "status": "Active",
    }]


def parse_procurve_stack(stacking_output, system_output):
    """Parse a ProCurve/Aruba 29xx/38xx stack. 'show stacking' gives
    member_id, MAC, (truncated) model, priority, status. 'show system'
    per-member sections add serial numbers.
    """
    members = {}
    # show stacking row:
    #   1  8030e0-eb7d80     Aruba R0M67A 2930M-...           255 Standby
    stk_re = re.compile(
        r"^\s+(\d+)\s+"                # member id
        r"([0-9a-fA-F-]{13})\s+"       # MAC (ProCurve 'aabbcc-ddeeff')
        r"(.+?)\s+"                    # model (greedy-stopped by priority)
        r"(\d+)\s+"                    # priority
        r"(\S+)\s*$")                  # status
    for line in stacking_output.splitlines():
        m = stk_re.match(line)
        if not m:
            continue
        mid = int(m.group(1))
        members[mid] = {
            "member_id": mid,
            "mac":       _normalize_stp_mac(m.group(2)),
            "model":     m.group(3).strip().rstrip("."),
            "role":      m.group(5),    # Commander/Standby/Member
            "status":    m.group(5),
            "serial":    None,
        }
    # 'show system' per-member blocks:
    #   Member :1
    #     MAC Addr           : 8030e0-eb7d80
    #     Serial Number      : SG8BKJS0V1
    current_id = None
    for line in system_output.splitlines():
        m = re.match(r"\s*Member\s*:(\d+)", line)
        if m:
            current_id = int(m.group(1)); continue
        if current_id is None:
            continue
        m = re.search(r"Serial Number\s*:\s*(\S+)", line)
        if m and current_id in members:
            members[current_id]["serial"] = m.group(1)
    return [members[mid] for mid in sorted(members)]


# Junos 'Last flapped : TS (X ago)' relative forms seen in the wild:
#   (00:00:41 ago)     — HH:MM:SS
#   (11:06:53 ago)     — HH:MM:SS
#   (1d 02:48 ago)     — Nd HH:MM  (no SS)
#   (3d 08:11 ago)     — Nd HH:MM
#   (5w2d 11:32 ago)   — NwNd HH:MM
#   (36w1d 22:12 ago)  — NwNd HH:MM
#   (216w1d 02:12 ago) — NwNd HH:MM
# The seconds portion is optional (present only on short durations).
_JUNOS_LAST_FLAPPED_RE = re.compile(
    r"Last flapped\s*:.*?\(\s*"
    r"(?:(\d+)w)?\s*(?:(\d+)d)?\s*"
    r"(\d+):(\d+)(?::(\d+))?\s+ago\s*\)"
)


def parse_last_flapped_junos(iface_output):
    """Extract seconds-since-last-link-flap from Junos 'show interfaces <port>'.

    Returns an int (seconds) or None if the field is absent / 'Never' /
    unparseable. Handles all seen relative-time formats, including long
    durations that lack the seconds field ('3d 08:11 ago').
    """
    m = _JUNOS_LAST_FLAPPED_RE.search(iface_output)
    if not m:
        return None
    w = int(m.group(1) or 0)
    d = int(m.group(2) or 0)
    h, mi = int(m.group(3)), int(m.group(4))
    s = int(m.group(5) or 0)
    return ((w * 7 + d) * 24 + h) * 3600 + mi * 60 + s


def parse_junos_iface_flaps(output):
    """Parse filtered 'show interfaces | match "^Physical|Last flapped"' and
    return {physical_iface: seconds_since_flap_or_None}, restricted to
    data-plane ports. Entries marked 'Never' become None. Used every poll
    to detect sub-poll flaps via gap-vs-timestamp comparison.
    """
    results = {}
    current = None
    for line in output.splitlines():
        m = re.match(r"^Physical interface:\s*(\S+?),", line)
        if m:
            name = m.group(1)
            if not _is_junos_dataplane(name):
                current = None
                continue
            current = name
            results[current] = None
            continue
        if current is None:
            continue
        if re.match(r"^\s*Last flapped\s*:\s*Never", line):
            results[current] = None
            current = None
            continue
        m = _JUNOS_LAST_FLAPPED_RE.search(line)
        if m:
            w = int(m.group(1) or 0)
            d = int(m.group(2) or 0)
            h, mi = int(m.group(3)), int(m.group(4))
            s = int(m.group(5) or 0)
            results[current] = ((w * 7 + d) * 24 + h) * 3600 + mi * 60 + s
            current = None
    return results


_ARUBA_UP_FOR_UNIT = {
    "second": 1, "seconds": 1,
    "minute": 60, "minutes": 60,
    "hour": 3600, "hours": 3600,
    "day": 86400, "days": 86400,
    "week": 7 * 86400, "weeks": 7 * 86400,
    "month": 30 * 86400, "months": 30 * 86400,
    "year": 365 * 86400, "years": 365 * 86400,
}


def _aruba_up_for_seconds(n, unit):
    return n * _ARUBA_UP_FOR_UNIT.get(unit.lower(), 365 * 86400)


def parse_last_flapped_aruba_cx(iface_output):
    """Extract seconds-since-last-link-up from ArubaOS-CX 'show interface'.

    Parses 'Link state: up for N <unit>'. Returns None if the port is
    currently down or the format doesn't match.
    """
    m = re.search(r"Link state:\s*up for\s+(\d+)\s+(\w+)", iface_output)
    if not m:
        return None
    return _aruba_up_for_seconds(int(m.group(1)), m.group(2))


def parse_aruba_cx_iface_flaps(output):
    """Parse full 'show interface' output on ArubaOS-CX and return
    {physical_iface: seconds_since_flap_or_None}. 'down' ports stay None
    (we don't know when they went down from this field). Used to detect
    sub-poll flaps every poll.
    """
    results = {}
    current = None
    for line in output.splitlines():
        m = re.match(r"^Interface\s+(\S+)\s+is\s+(up|down|disabled)", line)
        if m:
            current = m.group(1)
            results[current] = None
            continue
        if current is None:
            continue
        m = re.match(r"\s*Link state:\s*(up|down)\s+for\s+(\d+)\s+(\w+)", line)
        if m and m.group(1) == "up":
            results[current] = _aruba_up_for_seconds(int(m.group(2)), m.group(3))
            current = None
    return results


def parse_procurve_standalone(system_output):
    """Single-member record for a non-stacked ProCurve. 'show system'
    on a standalone has top-level MAC + Serial (no 'Member :N' sections).
    """
    serial = None; mac = None
    m = re.search(r"Base MAC Addr\s*:\s*(\S+)", system_output)
    if m: mac = _normalize_stp_mac(m.group(1))
    m = re.search(r"Serial Number\s*:\s*(\S+)", system_output)
    if m: serial = m.group(1)
    if serial is None and mac is None:
        return []
    return [{
        "member_id": 1, "role": None,
        "serial": serial, "mac": mac, "model": None,
        "status": "Up",
    }]


def collect_hardware(child, platform):
    """Run vendor-specific hardware/stacking commands and return
    (chassis_type, members). chassis_type is 'standalone' or 'stack'.
    Returns (None, []) on error or unsupported platform — callers should
    treat that as 'leave existing rows alone'.
    """
    try:
        if platform == "junos":
            out = send_command(child, "show virtual-chassis | no-more", timeout=20)
            members = parse_junos_vc(out)
            return ("stack" if len(members) > 1 else "standalone"), members
        if platform == "aruba-cx":
            vsf_out = send_command(child, "show vsf", timeout=15)
            mod_out = send_command(child, "show module", timeout=15)
            if "Invalid input" in vsf_out or not vsf_out.strip():
                sys_out = send_command(child, "show system", timeout=15)
                return "standalone", parse_aruba_cx_standalone(sys_out, mod_out)
            members = parse_aruba_cx_vsf(vsf_out, mod_out)
            return ("stack" if len(members) > 1 else "standalone"), members
        if platform == "procurve":
            stk_out = send_command(child, "show stacking", timeout=15)
            sys_out = send_command(child, "show system", timeout=15)
            members = parse_procurve_stack(stk_out, sys_out)
            if not members:
                return "standalone", parse_procurve_standalone(sys_out)
            return ("stack" if len(members) > 1 else "standalone"), members
    except Exception as e:
        log.debug("collect_hardware: %s — failed: %s", platform, e)
    return None, []


def _upsert_device_members(ip, chassis_type, members):
    """Replace device_members rows for this IP and set chassis_type +
    member_count + slot_count on the master devices row. Called from
    discover/retest once hardware has been collected.
    """
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    conn.execute("DELETE FROM device_members WHERE ip = ?", (ip,))
    for mm in members:
        conn.execute("""
            INSERT INTO device_members
                (ip, member_id, role, serial, mac, model, status, updated_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (ip, mm["member_id"], mm.get("role"), mm.get("serial"),
              mm.get("mac"), mm.get("model"), mm.get("status"), now))
    count = len(members) if members else None
    conn.execute(
        "UPDATE devices SET chassis_type = ?, member_count = ?, slot_count = ? "
        "WHERE ip = ?",
        (chassis_type, count, count, ip))
    conn.commit()
    conn.close()


# ---------------------------------------------------------------------------
# Subnet scanner (merged from scan_switches.py)
# ---------------------------------------------------------------------------

def check_ports(ip_str, ports, timeout):
    """Return a list of open TCP ports on the host (subset of ports).

    DEBUG logs one line per *responsive* host (open ports, or explicit refusal).
    Silent timeouts are intentionally not logged — on a /16 that would be 65k
    noise lines. Seeing "refused" vs nothing at all distinguishes "host alive
    but no service" from "nothing there / filtered".
    """
    open_ports = []
    refused = []
    for port in ports:
        try:
            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
                s.settimeout(timeout)
                s.connect((ip_str, port))
            open_ports.append(port)
        except socket.timeout:
            continue
        except ConnectionRefusedError:
            refused.append(port)
        except OSError:
            continue
    if open_ports:
        log.debug("scan %s: open %s", ip_str, ",".join(str(p) for p in open_ports))
    elif refused:
        log.debug("scan %s: host alive, refused %s",
                  ip_str, ",".join(str(p) for p in refused))
    return open_ports


def scan_subnet(subnet, ports, timeout, threads, on_responder=None):
    """Return a dict of {ip: [open_ports]} for hosts with any open ports in the subnet.

    on_responder, if given, is called as on_responder(ip, open_ports) the
    instant each responder is found — used to persist responders to the DB
    in flight, so a crash or SSH-drop mid-/8 doesn't lose hours of scan
    work. The callback runs on the main thread (the as_completed loop), so
    it's serial; wrap any DB work in its own try/except.
    """
    found = {}
    network = ipaddress.ip_network(subnet, strict=False)
    total = network.num_addresses - 2 if network.prefixlen < 31 else network.num_addresses
    done = 0

    # Process in batches to avoid creating millions of futures at once
    # (a /8 has 16M+ hosts — submitting all at once exhausts memory)
    batch_size = threads * 4
    host_iter = (str(ip) for ip in network.hosts())

    with ThreadPoolExecutor(max_workers=threads) as pool:
        while True:
            batch = list(itertools.islice(host_iter, batch_size))
            if not batch:
                break
            futures = {pool.submit(check_ports, ip, ports, timeout): ip
                       for ip in batch}
            for future in as_completed(futures):
                done += 1
                ip = futures[future]
                pct = done * 100 // total
                sys.stdout.write(f"\rScanning {subnet}... ({done}/{total}) {pct}%  ")
                sys.stdout.flush()
                open_ports = future.result()
                if open_ports:
                    found[ip] = open_ports
                    if on_responder is not None:
                        try:
                            on_responder(ip, open_ports)
                        except Exception as e:
                            log.debug("scan_subnet on_responder(%s) failed: %s",
                                      ip, e)

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()
    return found


def write_devices_file(filepath, ips, identity=None):
    """Write discovered IPs to the device list file, preserving any commented lines.

    identity is an optional dict {ip: "label (mac)"} for inline comments.
    """
    comments = []
    if os.path.exists(filepath):
        with open(filepath, encoding="utf-8") as f:
            for line in f:
                if line.strip().startswith("#"):
                    comments.append(line)

    with open(filepath, "w", encoding="utf-8") as f:
        if comments:
            for c in comments:
                f.write(c)
        else:
            f.write("# One device IP per line\n")
        for ip in ips:
            if identity and ip in identity:
                f.write(f"{ip}  # {identity[ip]}\n")
            else:
                f.write(ip + "\n")


def write_failed_file(filepath, failures):
    """Write failed devices to a file with reasons.

    failures is a list of (ip, reason) tuples.
    """
    with open(filepath, "w", encoding="utf-8") as f:
        f.write("# Devices that failed credential check — re-test with -r after fixing\n")
        for ip, reason in failures:
            f.write(f"{ip}  # {reason}\n")


def write_telnet_report(filepath, telnet_only_ips, telnet_and_ssh_ips):
    """Write a report of devices with telnet exposure for security review."""
    if not telnet_only_ips and not telnet_and_ssh_ips:
        # Clean up stale report if no telnet devices found
        if os.path.exists(filepath):
            os.remove(filepath)
        return
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(f"# Telnet security report — updated {datetime.now():%Y-%m-%d %H:%M}\n")
        f.write("#\n")
        total = len(telnet_only_ips) + len(telnet_and_ssh_ips)
        f.write(f"# {total} device(s) with telnet exposure\n\n")
        if telnet_only_ips:
            f.write("# TELNET ONLY — enable SSH on these devices\n")
            for ip in telnet_only_ips:
                f.write(f"{ip}\n")
            f.write("\n")
        if telnet_and_ssh_ips:
            f.write("# SSH + TELNET — consider disabling telnet\n")
            for ip in telnet_and_ssh_ips:
                f.write(f"{ip}\n")


def prune_telnet_report(filepath, ssh_passed_ips):
    """Remove IPs from the telnet report that now connect via SSH."""
    if not ssh_passed_ips or not os.path.exists(filepath):
        return
    remove = set(ssh_passed_ips)
    telnet_only = []
    telnet_and_ssh = []
    current_section = None
    with open(filepath, encoding="utf-8") as f:
        for line in f:
            stripped = line.strip()
            if "TELNET ONLY" in stripped:
                current_section = "only"
            elif "SSH + TELNET" in stripped:
                current_section = "both"
            elif stripped and not stripped.startswith("#"):
                ip = stripped.split()[0]
                if ip not in remove:
                    if current_section == "only":
                        telnet_only.append(ip)
                    elif current_section == "both":
                        telnet_and_ssh.append(ip)
    removed = len(remove) - len(remove - set(telnet_only) - set(telnet_and_ssh))
    write_telnet_report(filepath, telnet_only, telnet_and_ssh)
    if removed:
        log.info("Removed %d device(s) from telnet report (now using SSH)", removed)


def run_scan(cfg, dry_run=False):
    """Scan all configured subnets and update the device list file. Returns the list of found IPs.

    When dry_run is True, found IPs are printed but the file is not written.
    """
    # Collect results: {ip: [open_ports]}
    all_results = {}
    port_names = "/".join(str(p) for p in cfg["ports"])

    def _persist_responder(ip, ports):
        """Upsert each responder the instant scan_subnet finds it, so a
        crash mid-/8 doesn't lose hours of work. status='active' matches
        the final-pass behavior; the final pass still runs to add DNS."""
        proto = "SSH" if 22 in ports else ("telnet" if 23 in ports else None)
        try:
            upsert_device(ip, proto=proto,
                          ssh_open=1 if 22 in ports else 0,
                          telnet_open=1 if 23 in ports else 0,
                          status="active")
        except Exception as e:
            log.debug("scan persistence: %s upsert failed: %s", ip, e)

    for subnet in cfg["subnets"]:
        log.info("Scanning %s for open ports (%s)...", subnet, port_names)
        results = scan_subnet(subnet, cfg["ports"], cfg["timeout"],
                              cfg["scan_threads"],
                              on_responder=(None if dry_run else _persist_responder))
        log.info("Found %d host(s) in %s", len(results), subnet)
        for ip in sorted(results, key=ipaddress.ip_address):
            log.debug("  %s (ports: %s)", ip, ", ".join(str(p) for p in results[ip]))
        all_results.update(results)

    # De-duplicate and sort
    all_found = sorted(all_results.keys(), key=ipaddress.ip_address)

    # Warn about telnet
    telnet_only = []
    telnet_and_ssh = []
    for ip in all_found:
        ports = all_results[ip]
        has_ssh = 22 in ports
        has_telnet = 23 in ports
        if has_telnet and has_ssh:
            telnet_and_ssh.append(ip)
        elif has_telnet and not has_ssh:
            telnet_only.append(ip)

    # Compare against existing device list
    device_file = cfg["device_file"]
    if os.path.exists(device_file):
        existing = set(parse_devices(device_file))
    else:
        existing = set()
    scanned = set(all_found)

    added = sorted(scanned - existing, key=ipaddress.ip_address)
    removed = sorted(existing - scanned, key=ipaddress.ip_address)
    unchanged = len(scanned & existing)

    if added or removed:
        if added:
            log.info("New devices (%d):", len(added))
            for ip in added:
                log.info("  + %s", ip)
        if removed:
            log.info("Removed devices (%d):", len(removed))
            for ip in removed:
                log.info("  - %s", ip)
        if unchanged:
            log.info("Unchanged: %d device(s)", unchanged)
    elif scanned:
        log.info("Device list unchanged (%d device(s))", len(scanned))

    if all_found:
        if dry_run:
            log.info("Dry run — found %d device(s) total", len(all_found))
        else:
            log.info("Resolving reverse DNS for %d device(s)...", len(all_found))
            dns_map = resolve_dns_batch(all_found)
            for ip in all_found:
                ports = all_results[ip]
                proto = "SSH" if 22 in ports else ("telnet" if 23 in ports else None)
                upsert_device(ip, proto=proto,
                              ssh_open=1 if 22 in ports else 0,
                              telnet_open=1 if 23 in ports else 0,
                              dns_name=dns_map.get(ip, ""),
                              status="active")
            log.info("Updated database with %d scanned device(s)", len(all_found))
    else:
        log.info("No hosts found across all subnets.")

    # --- Post-scan warnings ---
    if telnet_only or telnet_and_ssh:
        log.info("--- Telnet Warnings ---")
        if telnet_only:
            log.warning("Telnet only — enable SSH on these devices:")
            for ip in telnet_only:
                log.warning("  ! %s", ip)
        if telnet_and_ssh:
            log.warning("SSH + Telnet — consider disabling telnet:")
            for ip in telnet_and_ssh:
                log.warning("  ! %s", ip)

    return all_found


def run_discover(cfg, dry_run=False, email=False):
    """Scan subnets, test credentials, and store results in the database.

    When dry_run is True, results are displayed but the database is not updated.
    When email is True, a diff of devices that appeared / became unreachable /
    had hardware/firmware/hostname changes since the pre-run state is emailed.
    Intended for cron-scheduled weekly runs; manual invocations omit the flag
    so they don't trigger alerts.
    """
    if not _acquire_advisory_lock("ssh"):
        return
    # Snapshot every device BEFORE discovery modifies them so we can compute
    # a clean diff when run with --email. We include all statuses so status
    # transitions (active -> failed / duplicate) surface in the digest with
    # their real end-state, not just "disappeared".
    pre_snapshot = {}
    if email and not dry_run:
        pre_snapshot = {r["ip"]: {
            "hostname": r["hostname"] or "",
            "model":    r["model"] or "",
            "serial":   r["serial"] or "",
            "firmware": r["firmware"] or "",
            "status":   r["status"],
            "snmp_enabled": r["snmp_enabled"] if "snmp_enabled" in r.keys() else None,
            "snmp_diag":    r["snmp_diag"]    if "snmp_diag"    in r.keys() else None,
        } for r in get_devices()}

    # --- Scan phase ---
    all_results = {}
    port_names = "/".join(str(p) for p in cfg["ports"])

    def _persist_responder(ip, ports):
        """Persist each responder the instant scan_subnet finds it, so a
        long /8 scan or a credential-phase crash doesn't lose hours of
        work. preserve_status keeps any prior real state (e.g. 'active')
        intact on already-known devices; brand-new responders get
        status='scanned' — neutral, excluded from monitor-stp's
        WHERE status='active' — until the final write resolves them."""
        proto = "SSH" if 22 in ports else ("telnet" if 23 in ports else None)
        try:
            upsert_device(ip, proto=proto,
                          ssh_open=1 if 22 in ports else 0,
                          telnet_open=1 if 23 in ports else 0,
                          status="scanned", preserve_status=True)
        except Exception as e:
            log.debug("scan persistence: %s upsert failed: %s", ip, e)

    for subnet in cfg["subnets"]:
        log.info("Scanning %s for open ports (%s)...", subnet, port_names)
        results = scan_subnet(subnet, cfg["ports"], cfg["timeout"],
                              cfg["scan_threads"],
                              on_responder=(None if dry_run else _persist_responder))
        log.info("Found %d host(s) in %s", len(results), subnet)
        for ip in sorted(results, key=ipaddress.ip_address):
            log.debug("  %s (ports: %s)", ip, ", ".join(str(p) for p in results[ip]))
        all_results.update(results)

    all_found = sorted(all_results.keys(), key=ipaddress.ip_address)

    if not all_found:
        log.info("No hosts found across all subnets.")
        return

    # Separate SSH-capable from telnet-only
    ssh_hosts = []
    telnet_only = []
    telnet_and_ssh = []
    for ip in all_found:
        ports = all_results[ip]
        has_ssh = 22 in ports
        has_telnet = 23 in ports
        if has_ssh:
            ssh_hosts.append(ip)
            if has_telnet:
                telnet_and_ssh.append(ip)
        elif has_telnet:
            telnet_only.append(ip)

    log.info("Found %d host(s) total — %d with SSH, %d telnet-only",
             len(all_found), len(ssh_hosts), len(telnet_only))

    # --- Credential test phase ---
    # Test all hosts: SSH-capable via SSH (+ telnet fallback), telnet-only via telnet
    test_hosts = ssh_hosts + telnet_only
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        return

    passed = []
    failed = []

    snmp_cfg = cfg.get("snmp") or {}

    def test_one(ip):
        log.debug("test_one: %s — starting credential test", ip)
        # Look up known-good credentials from DB to avoid lockouts. Also pull
        # cached SNMP creds so the SNMP probe can fast-path the cred we last
        # saw working on this device.
        conn = _db()
        row = conn.execute(
            "SELECT username, password_hash, snmp_proto, snmp_community, snmp_v3_user"
            " FROM devices WHERE ip = ? AND status = 'active'",
            (ip,)).fetchone()
        conn.close()
        known_user = row["username"] if row else None
        known_pw_hash = row["password_hash"] if row else None
        cached_snmp_proto = row["snmp_proto"] if row else None
        cached_snmp_cred = None
        if row:
            if cached_snmp_proto == "v2c":
                cached_snmp_cred = row["snmp_community"]
            elif cached_snmp_proto == "v3":
                cached_snmp_cred = row["snmp_v3_user"]
        if known_user:
            log.debug("test_one: %s — trying known-good user=%s first", ip, known_user)

        child, proto, username, pw_hash, ssh_open, telnet_open = connect_device(
            ip, cfg["usernames"], passwords["default"],
            password_list=cfg.get("password_list"),
            known_username=known_user, known_password_hash=known_pw_hash,
            user_passwords=cfg.get("user_passwords"),
        )
        if child is not None:
            log.debug("test_one: %s — connected via %s as %s", ip, proto, username)
            try:
                platform = _detect_platform(child)
                hostname = get_hostname(child)
            except Exception as e:
                # Session damaged (e.g. Ruckus FastIron `enable` reauth
                # left it in User Name:/Password: state that doesn't
                # match PROMPT_RE). Credentials WORKED — return PASS
                # with empty identification rather than crash the whole
                # discover run on one misbehaving device.
                log.debug("test_one: %s — platform/hostname detect failed: %s",
                          ip, e)
                platform = None
                hostname = ""
            base_mac = ""
            serial = ""
            firmware = ""
            model = ""
            hardware_info = ""
            snmp_result = {"ok": False, "proto": None, "community": None,
                           "v3_user": None, "diag": ""}
            try:
                if platform == "junos":
                    ver = send_command(child, "show version | no-more", timeout=15)
                    hw = send_command(child, "show chassis hardware | no-more", timeout=15)
                    mac_out = send_command(child, "show chassis mac-addresses | no-more", timeout=15)
                    id_output = ver + "\n" + hw + "\n" + mac_out
                    hardware_info = hw
                elif platform == "fastiron":
                    ver_out = send_command(child, "show version", timeout=15)
                    id_output = ver_out
                    hardware_info = ver_out
                else:
                    sys_out = send_command(child, "show system", timeout=10)
                    ver_out = send_command(child, "show version", timeout=10)
                    id_output = sys_out + "\n" + ver_out
                    hardware_info = sys_out
                dev_id = get_device_id(id_output)
                base_mac = dev_id.get("base_mac", "")
                serial = dev_id.get("serial", "")
                firmware = dev_id.get("firmware", "")
                model = dev_id.get("model", "")
                if not model and platform not in ("junos", "fastiron"):
                    for sys_line in id_output.splitlines():
                        sys_line = sys_line.strip()
                        if "system name" in sys_line.lower():
                            continue
                        if any(kw in sys_line.lower() for kw in
                               ["procurve", "aruba", "cisco", "juniper",
                                "palo", "switch", "router", "firewall"]):
                            model = sys_line.strip("; ")
                            break
                chassis_type, members = collect_hardware(child, platform)
            except Exception as e:
                log.debug("test_one: %s — identification failed: %s", ip, e)
                chassis_type, members = None, []

            # Junos STP port-id -> interface mapping cache. Captured here
            # while the SSH session is open so we don't pay a second connect
            # at monitor time. On VC switches the JUNIPER STP port-id can
            # diverge from BRIDGE-MIB's dot1dBasePort; the SNMP fast-path
            # consults stp_port_id_cache as authoritative.
            junos_stp_port_ids = []
            if platform == "junos":
                try:
                    stp_out = send_command(child,
                                           "show spanning-tree interface | no-more",
                                           timeout=15)
                    junos_stp_port_ids = parse_junos_stp_port_id_table(stp_out)
                    log.debug("test_one: %s — captured %d STP port-id entries",
                              ip, len(junos_stp_port_ids))
                except Exception as e:
                    log.debug("test_one: %s — STP port-id capture failed: %s",
                              ip, e)
            # SNMP probe + (on failure) SSH-side config inspection. Done
            # while the SSH session is still open so we don't pay a second
            # connect to diagnose. Skipped entirely when [snmp] is empty.
            if snmp_cfg.get("enabled"):
                probe = snmp_probe(ip, snmp_cfg,
                                   cached_proto=cached_snmp_proto,
                                   cached_cred=cached_snmp_cred,
                                   full_walk=True)
                snmp_result.update({
                    "ok": probe["ok"],
                    "proto": probe.get("proto"),
                    "community": probe.get("community"),
                    "v3_user": probe.get("v3_user"),
                })
                if not probe["ok"] and platform:
                    try:
                        dev_snmp = inspect_snmp_via_ssh(child, platform)
                        snmp_result["diag"] = diff_snmp_config(dev_snmp, snmp_cfg)
                    except Exception as e:
                        snmp_result["diag"] = f"snmp probe failed; ssh inspect error: {e}"
                elif not probe["ok"]:
                    snmp_result["diag"] = (probe.get("error")
                                           or "snmp probe failed")
                log.debug("test_one: %s — snmp ok=%s proto=%s diag=%s",
                          ip, probe["ok"], probe.get("proto"),
                          snmp_result["diag"])
            disconnect(child)
            log.debug("test_one: %s — detected hostname=%s model=%s firmware=%s mac=%s serial=%s chassis=%s members=%d",
                      ip, hostname or "?", model or "?", firmware or "?",
                      base_mac or "?", serial or "?",
                      chassis_type or "?", len(members))
            label = hostname or ip
            if model:
                label = f"{label} ({model})"
            mac_info = f", mac={base_mac}" if base_mac else ""
            info = {"hostname": hostname, "base_mac": base_mac, "serial": serial,
                    "firmware": firmware, "model": model,
                    "hardware_info": hardware_info, "proto": proto,
                    "username": username, "password_hash": pw_hash,
                    "ssh_open": ssh_open, "telnet_open": telnet_open,
                    "chassis_type": chassis_type, "members": members,
                    "snmp": snmp_result,
                    "junos_stp_port_ids": junos_stp_port_ids}
            log.debug("test_one: %s — PASS", ip)
            return (ip, f"PASS proto={proto}, user={username}"
                        f"{mac_info}, host={label}", True, info)
        else:
            reason = classify_fail_reason(ssh_open, telnet_open)
            log.debug("test_one: %s — FAIL (%s)", ip, reason)
            return (ip, f"FAIL — {reason}", False,
                    {"hostname": None, "base_mac": "", "serial": "",
                     "firmware": "", "model": "", "hardware_info": "",
                     "proto": None, "username": None, "password_hash": None,
                     "ssh_open": ssh_open, "telnet_open": telnet_open,
                     "chassis_type": None, "members": [],
                     "snmp": None,
                     "fail_reason": reason})

    if test_hosts:
        test_results = []
        total = len(test_hosts)
        done = 0
        with ThreadPoolExecutor(max_workers=cfg["backup_threads"]) as pool:
            futures = {pool.submit(test_one, ip): ip for ip in test_hosts}
            for future in as_completed(futures):
                done += 1
                pct = done * 100 // total
                sys.stdout.write(f"\rTesting credentials... ({done}/{total}) {pct}%  ")
                sys.stdout.flush()
                ip_for_future = futures[future]
                try:
                    result = future.result()
                except Exception as e:
                    log.error("test_one(%s) raised unhandled %s: %s",
                              ip_for_future, type(e).__name__, e)
                    result = (ip_for_future,
                              f"FAIL — internal error: {type(e).__name__}",
                              False,
                              {"hostname": None, "base_mac": "", "serial": "",
                               "firmware": "", "model": "", "hardware_info": "",
                               "proto": None, "username": None, "password_hash": None,
                               "ssh_open": None, "telnet_open": None,
                               "chassis_type": None, "members": [],
                               "snmp": None,
                               "fail_reason": f"internal error: {type(e).__name__}"})
                test_results.append(result)

        sys.stdout.write("\r" + " " * 120 + "\r")
        sys.stdout.flush()

        test_results.sort(key=lambda r: ipaddress.ip_address(r[0]))

        # Group passing IPs by Base MAC for dedup
        mac_groups = {}  # mac -> [(ip, info), ...]
        info_map = {}    # ip -> info
        for ip, result, ok, info in test_results:
            print(f"  {ip}: {result}")
            info_map[ip] = info
            if ok:
                base_mac = info["base_mac"]
                if base_mac:
                    mac_groups.setdefault(base_mac, []).append(ip)
                else:
                    passed.append(ip)  # no MAC — can't dedup, keep it
            else:
                failed.append((ip, info.get("fail_reason")
                               or "all connection attempts failed"))

        # Dedup: keep first IP per MAC (sorted), skip duplicates
        dedup_skipped = []
        for mac, ips_for_mac in mac_groups.items():
            passed.append(ips_for_mac[0])
            if len(ips_for_mac) > 1:
                for dup_ip in ips_for_mac[1:]:
                    dedup_skipped.append((dup_ip, ips_for_mac[0], mac))
                    failed.append((dup_ip, f"duplicate of {ips_for_mac[0]} ({mac})"))

        if dedup_skipped:
            log.info("--- Deduplication ---")
            grouped = {}
            for dup_ip, kept_ip, mac in dedup_skipped:
                grouped.setdefault((kept_ip, mac), []).append(dup_ip)
            for (kept_ip, mac), dup_ips in grouped.items():
                log.info("  %s (%s): keeping %s, skipping %s",
                         mac, kept_ip, kept_ip, ", ".join(dup_ips))

    # --- Write to database ---
    if not dry_run and test_hosts:
        # Reverse DNS for every host we touched (passed + failed) so unreachable
        # devices still get names for identification.
        all_touched = list(set([ip for ip, _ in failed] + list(passed)))
        log.info("Resolving reverse DNS for %d device(s)...", len(all_touched))
        dns_map = resolve_dns_batch(all_touched)

        for ip in passed:
            info = info_map[ip]
            upsert_device(ip, base_mac=info["base_mac"], hostname=info["hostname"],
                          dns_name=dns_map.get(ip, ""),
                          model=info.get("model"), serial=info.get("serial"),
                          firmware=info.get("firmware"),
                          hardware_info=info.get("hardware_info"),
                          proto=info["proto"], username=info["username"],
                          password_hash=info.get("password_hash"),
                          ssh_open=info.get("ssh_open"),
                          telnet_open=info.get("telnet_open"), status="active")
            if info.get("members"):
                _upsert_device_members(ip, info.get("chassis_type"), info["members"])
            if info.get("junos_stp_port_ids"):
                _upsert_junos_stp_port_id_cache(ip, info["junos_stp_port_ids"])
            snmp_info = info.get("snmp")
            if snmp_info is not None and snmp_cfg.get("enabled"):
                record_snmp_result(
                    ip,
                    enabled=1 if snmp_info["ok"] else 0,
                    proto=snmp_info.get("proto"),
                    community=snmp_info.get("community"),
                    v3_user=snmp_info.get("v3_user"),
                    diag=snmp_info.get("diag") or None,
                    ok_now=bool(snmp_info["ok"]),
                )
        for ip, reason in failed:
            info = info_map.get(ip, {})
            if "duplicate of" in reason:
                kept_ip = reason.split("duplicate of ")[1].split(" (")[0]
                upsert_device(ip, base_mac=info.get("base_mac"),
                              hostname=info.get("hostname"),
                              dns_name=dns_map.get(ip, ""),
                              model=info.get("model"), serial=info.get("serial"),
                              firmware=info.get("firmware"),
                              hardware_info=info.get("hardware_info"),
                              proto=info.get("proto"), username=info.get("username"),
                              password_hash=info.get("password_hash"),
                              ssh_open=info.get("ssh_open"),
                              telnet_open=info.get("telnet_open"),
                              status="duplicate", duplicate_of=kept_ip)
            else:
                upsert_device(ip, dns_name=dns_map.get(ip, ""),
                              status="failed", fail_reason=reason)
        log.info("Updated database: %d active, %d failed/duplicate",
                 len(passed), len(failed))

    # --- Post-discover warnings ---
    # Warn about telnet-only devices that passed (they work but should get SSH)
    telnet_passed = [ip for ip in telnet_only if ip in passed]
    if telnet_passed or telnet_and_ssh:
        log.info("--- Telnet Warnings ---")
        if telnet_passed:
            log.warning("Telnet only — enable SSH on these devices:")
            for ip in telnet_passed:
                log.warning("  ! %s", ip)
        if telnet_and_ssh:
            log.warning("SSH + Telnet — consider disabling telnet:")
            for ip in telnet_and_ssh:
                log.warning("  ! %s", ip)

    # --- Summary ---
    log.info("--- Discovery Summary ---")
    log.info("Passed:      %d", len(passed))
    log.info("Failed:      %d", len(failed))

    # --- Diff email (cron-only; --email flag gates this) ---
    if email and not dry_run:
        _send_discovery_digest(cfg, pre_snapshot)


def _send_discovery_digest(cfg, pre_snapshot):
    """Compare the post-discovery DB state against pre_snapshot and email
    a diff. Always sends when invoked (no-change weeks still report 'clean'
    so the recipient knows the weekly run completed)."""
    if not cfg.get("smtp_server"):
        log.debug("No [email] config — skipping discovery digest email")
        return

    post = {r["ip"]: {
        "hostname": r["hostname"] or "",
        "model":    r["model"] or "",
        "serial":   r["serial"] or "",
        "firmware": r["firmware"] or "",
        "status":   r["status"],
        "snmp_enabled": r["snmp_enabled"] if "snmp_enabled" in r.keys() else None,
        "snmp_diag":    r["snmp_diag"]    if "snmp_diag"    in r.keys() else None,
    } for r in get_devices()}

    def was_active(info):
        return info and info.get("status") == "active"

    new_devices = []      # appeared (new IP or non-active -> active)
    gone_devices = []     # was active, no longer active
    hostname_changes = [] # (ip, old_hostname, new_hostname)
    hardware_changes = [] # (ip, hostname, old_model, new_model, old_serial, new_serial)
    firmware_changes = [] # (ip, hostname, old_firmware, new_firmware)
    snmp_broken_now = []  # active devices where SNMP is currently broken
    snmp_newly_broken = [] # SNMP transitioned from working/unknown to broken
    snmp_newly_fixed  = [] # SNMP transitioned from broken to working

    for ip, after in post.items():
        before = pre_snapshot.get(ip)
        if after["status"] == "active" and not was_active(before):
            new_devices.append((ip, after["hostname"] or ip, after["model"], after["serial"]))
            continue
        if before and was_active(before) and after["status"] != "active":
            gone_devices.append((ip, before["hostname"] or ip, after["status"]))
            continue
        if before and was_active(before) and after["status"] == "active":
            if before["hostname"] != after["hostname"]:
                hostname_changes.append((ip, before["hostname"] or "(none)",
                                         after["hostname"] or "(none)"))
            if (before["model"] != after["model"]
                    or before["serial"] != after["serial"]):
                hardware_changes.append((ip, after["hostname"] or ip,
                                         before["model"], after["model"],
                                         before["serial"], after["serial"]))
            if before["firmware"] != after["firmware"]:
                firmware_changes.append((ip, after["hostname"] or ip,
                                         before["firmware"] or "(none)",
                                         after["firmware"] or "(none)"))
        # SNMP status — only meaningful for active devices.
        if after["status"] == "active" and after["snmp_enabled"] == 0:
            snmp_broken_now.append((ip, after["hostname"] or ip,
                                    after["snmp_diag"] or "snmp probe failed"))
            if before and before.get("snmp_enabled") in (1, None):
                snmp_newly_broken.append(ip)
        elif (after["status"] == "active" and after["snmp_enabled"] == 1
              and before and before.get("snmp_enabled") == 0):
            snmp_newly_fixed.append((ip, after["hostname"] or ip))
    # IPs in pre_snapshot that are no longer in post at all (status cleared) —
    # rare; counts as 'gone' for symmetry.
    for ip, before in pre_snapshot.items():
        if ip not in post and was_active(before):
            gone_devices.append((ip, before["hostname"] or ip, "(removed)"))

    subject_bits = []
    if new_devices:      subject_bits.append(f"{len(new_devices)} new")
    if gone_devices:     subject_bits.append(f"{len(gone_devices)} unreachable")
    if hostname_changes: subject_bits.append(f"{len(hostname_changes)} hostname")
    if hardware_changes: subject_bits.append(f"{len(hardware_changes)} hardware")
    if firmware_changes: subject_bits.append(f"{len(firmware_changes)} firmware")
    if snmp_broken_now:  subject_bits.append(f"{len(snmp_broken_now)} snmp")
    subject = (f"[netops] Discovery digest — {', '.join(subject_bits)}"
               if subject_bits else "[netops] Discovery digest — no changes")

    pre_active  = sum(1 for v in pre_snapshot.values() if v.get("status") == "active")
    post_active = sum(1 for v in post.values()          if v.get("status") == "active")
    lines = [f"Discovery digest at {datetime.now():%Y-%m-%d %H:%M}",
             f"Pre-run: {pre_active} active of {len(pre_snapshot)} total.  "
             f"Post-run: {post_active} active of {len(post)} total.", ""]
    if new_devices:
        lines.append(f"--- New devices ({len(new_devices)}) ---")
        for ip, name, model, serial in sorted(new_devices):
            lines.append(f"  {name} ({ip})  [{model or '?'}]  serial={serial or '?'}")
        lines.append("")
    if gone_devices:
        lines.append(f"--- Now unreachable ({len(gone_devices)}) ---")
        for ip, name, status in sorted(gone_devices):
            lines.append(f"  {name} ({ip})  status={status}")
        lines.append("")
    if hostname_changes:
        lines.append(f"--- Hostname changes ({len(hostname_changes)}) ---")
        for ip, old, new in sorted(hostname_changes):
            lines.append(f"  {ip}  {old} -> {new}")
        lines.append("")
    if hardware_changes:
        lines.append(f"--- Hardware changes ({len(hardware_changes)}) ---")
        for ip, name, om, nm, os_, ns in sorted(hardware_changes):
            lines.append(f"  {name} ({ip})")
            if om != nm:
                lines.append(f"    model:  {om or '(none)'} -> {nm or '(none)'}")
            if os_ != ns:
                lines.append(f"    serial: {os_ or '(none)'} -> {ns or '(none)'}")
        lines.append("")
    if firmware_changes:
        lines.append(f"--- Firmware changes ({len(firmware_changes)}) ---")
        for ip, name, of, nf in sorted(firmware_changes):
            lines.append(f"  {name} ({ip})  {of} -> {nf}")
        lines.append("")
    if snmp_broken_now:
        # Group by reason for compactness — the diag string is the dimension
        # operators want to triage on (acl drift vs cred drift vs no-snmp).
        by_reason = {}
        for ip, name, diag in snmp_broken_now:
            by_reason.setdefault(diag, []).append((ip, name))
        lines.append(f"--- SNMP issues ({len(snmp_broken_now)}) ---")
        if snmp_newly_broken:
            lines.append(f"  ({len(snmp_newly_broken)} newly broken since last run)")
        if snmp_newly_fixed:
            lines.append(f"  ({len(snmp_newly_fixed)} newly fixed)")
            for ip, name in sorted(snmp_newly_fixed):
                lines.append(f"    + {name} ({ip})  now reachable")
        for reason in sorted(by_reason):
            entries = sorted(by_reason[reason])
            lines.append(f"  {reason}  ({len(entries)})")
            for ip, name in entries:
                lines.append(f"    {name} ({ip})")
        lines.append("")
    if not subject_bits:
        lines.append("No changes detected since the previous discovery run.")

    log.info("--- Discovery Digest ---")
    log.info("New:          %d", len(new_devices))
    log.info("Unreachable:  %d", len(gone_devices))
    log.info("Hostname:     %d", len(hostname_changes))
    log.info("Hardware:     %d", len(hardware_changes))
    log.info("Firmware:     %d", len(firmware_changes))
    log.info("SNMP broken:  %d (%d newly)", len(snmp_broken_now), len(snmp_newly_broken))
    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("Discovery digest email %s", detail)
    else:
        log.error("Discovery digest email failed: %s", detail)


def run_retest(cfg):
    """Re-test failed devices from the database and promote passing ones."""
    if not _acquire_advisory_lock("ssh"):
        return
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        return

    # Read failed + inactive from DB; fall back to failed_devices.txt
    db_failed = get_devices(status="failed") + get_devices(status="inactive")
    if db_failed:
        ips = [r["ip"] for r in db_failed]
    else:
        device_file = cfg["device_file"]
        failed_file = os.path.join(os.path.dirname(device_file), "failed_devices.txt")
        if os.path.exists(failed_file):
            ips = parse_failed_file(failed_file)
            if ips:
                log.info("No failed devices in DB — reading from %s", failed_file)
        else:
            ips = []

    if not ips:
        log.info("No failed devices to re-test.")
        return

    log.info("Re-testing %d failed device(s)...", len(ips))

    passed = []
    still_failed = []

    def test_one(ip):
        child, proto, username, pw_hash, ssh_open, telnet_open = connect_device(
            ip, cfg["usernames"], passwords["default"],
            password_list=cfg.get("password_list"),
            user_passwords=cfg.get("user_passwords"),
        )
        if child is not None:
            try:
                platform = _detect_platform(child)
                hostname = get_hostname(child)
            except Exception as e:
                # Session damaged (e.g. Ruckus FastIron `enable` reauth
                # left it in User Name:/Password: state). Credentials
                # WORKED — return PASS with empty identification rather
                # than crash the whole retest run.
                log.debug("test_one: %s — platform/hostname detect failed: %s",
                          ip, e)
                platform = None
                hostname = ""
            base_mac = ""
            serial = ""
            firmware = ""
            model = ""
            hardware_info = ""
            try:
                if platform == "junos":
                    ver = send_command(child, "show version | no-more", timeout=15)
                    hw = send_command(child, "show chassis hardware | no-more", timeout=15)
                    mac_out = send_command(child, "show chassis mac-addresses | no-more", timeout=15)
                    id_output = ver + "\n" + hw + "\n" + mac_out
                    hardware_info = hw
                elif platform == "fastiron":
                    ver_out = send_command(child, "show version", timeout=15)
                    id_output = ver_out
                    hardware_info = ver_out
                else:
                    sys_out = send_command(child, "show system", timeout=10)
                    ver_out = send_command(child, "show version", timeout=10)
                    id_output = sys_out + "\n" + ver_out
                    hardware_info = sys_out
                dev_id = get_device_id(id_output)
                base_mac = dev_id.get("base_mac", "")
                serial = dev_id.get("serial", "")
                firmware = dev_id.get("firmware", "")
                model = dev_id.get("model", "")
                if not model and platform not in ("junos", "fastiron"):
                    for sys_line in id_output.splitlines():
                        sys_line = sys_line.strip()
                        if "system name" in sys_line.lower():
                            continue
                        if any(kw in sys_line.lower() for kw in
                               ["procurve", "aruba", "cisco", "juniper",
                                "palo", "switch", "router", "firewall"]):
                            model = sys_line.strip("; ")
                            break
                chassis_type, members = collect_hardware(child, platform)
            except Exception:
                chassis_type, members = None, []
            disconnect(child)
            label = hostname or ip
            if model:
                label = f"{label} ({model})"
            mac_info = f", mac={base_mac}" if base_mac else ""
            info = {"hostname": hostname, "base_mac": base_mac, "serial": serial,
                    "firmware": firmware, "model": model,
                    "hardware_info": hardware_info, "proto": proto,
                    "username": username, "password_hash": pw_hash,
                    "ssh_open": ssh_open, "telnet_open": telnet_open,
                    "chassis_type": chassis_type, "members": members}
            return (ip, f"PASS proto={proto}, user={username}"
                        f"{mac_info}, host={label}", True, info)
        else:
            reason = classify_fail_reason(ssh_open, telnet_open)
            return (ip, f"FAIL — {reason}", False,
                    {"hostname": None, "base_mac": "", "serial": "",
                     "firmware": "", "model": "", "hardware_info": "",
                     "proto": None, "username": None, "password_hash": None,
                     "ssh_open": ssh_open, "telnet_open": telnet_open,
                     "chassis_type": None, "members": [],
                     "fail_reason": reason})

    results = []
    total = len(ips)
    done = 0
    with ThreadPoolExecutor(max_workers=cfg["backup_threads"]) as pool:
        futures = {pool.submit(test_one, ip): ip for ip in ips}
        for future in as_completed(futures):
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rRe-testing credentials... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()
            ip_for_future = futures[future]
            try:
                result = future.result()
            except Exception as e:
                log.error("test_one(%s) raised unhandled %s: %s",
                          ip_for_future, type(e).__name__, e)
                result = (ip_for_future,
                          f"FAIL — internal error: {type(e).__name__}",
                          False,
                          {"hostname": None, "base_mac": "", "serial": "",
                           "firmware": "", "model": "", "hardware_info": "",
                           "proto": None, "username": None, "password_hash": None,
                           "ssh_open": None, "telnet_open": None,
                           "chassis_type": None, "members": [],
                           "fail_reason": f"internal error: {type(e).__name__}"})
            results.append(result)

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    results.sort(key=lambda r: ipaddress.ip_address(r[0]))

    info_map = {}
    mac_groups = {}
    for ip, result, ok, info in results:
        print(f"  {ip}: {result}")
        info_map[ip] = info
        if ok:
            base_mac = info["base_mac"]
            if base_mac:
                mac_groups.setdefault(base_mac, []).append(ip)
            else:
                passed.append(ip)
        else:
            still_failed.append((ip, info.get("fail_reason")
                                 or "all connection attempts failed"))

    # Dedup: keep first IP per MAC, skip duplicates
    dedup_skipped = []
    for mac, ips_for_mac in mac_groups.items():
        passed.append(ips_for_mac[0])
        if len(ips_for_mac) > 1:
            for dup_ip in ips_for_mac[1:]:
                dedup_skipped.append((dup_ip, ips_for_mac[0], mac))
                still_failed.append((dup_ip, f"duplicate of {ips_for_mac[0]} ({mac})"))

    if dedup_skipped:
        log.info("--- Deduplication ---")
        grouped = {}
        for dup_ip, kept_ip, mac in dedup_skipped:
            grouped.setdefault((kept_ip, mac), []).append(dup_ip)
        for (kept_ip, mac), dup_ips in grouped.items():
            log.info("  %s (%s): keeping %s, skipping %s",
                     mac, kept_ip, kept_ip, ", ".join(dup_ips))

    # --- Write to database ---
    for ip in passed:
        info = info_map[ip]
        upsert_device(ip, base_mac=info["base_mac"], hostname=info["hostname"],
                      model=info.get("model"), serial=info.get("serial"),
                      firmware=info.get("firmware"),
                      hardware_info=info.get("hardware_info"),
                      proto=info["proto"], username=info["username"],
                      password_hash=info.get("password_hash"),
                          ssh_open=info.get("ssh_open"),
                          telnet_open=info.get("telnet_open"), status="active")
        if info.get("members"):
            _upsert_device_members(ip, info.get("chassis_type"), info["members"])
    for ip, reason in still_failed:
        info = info_map.get(ip, {})
        if "duplicate of" in reason:
            kept_ip = reason.split("duplicate of ")[1].split(" (")[0]
            upsert_device(ip, base_mac=info.get("base_mac"),
                          hostname=info.get("hostname"),
                          model=info.get("model"), serial=info.get("serial"),
                          firmware=info.get("firmware"),
                          hardware_info=info.get("hardware_info"),
                          proto=info.get("proto"), username=info.get("username"),
                          password_hash=info.get("password_hash"),
                          status="duplicate", duplicate_of=kept_ip)
        else:
            upsert_device(ip, status="failed", fail_reason=reason)

    log.info("--- Retest Summary ---")
    log.info("Promoted:      %d", len(passed))
    log.info("Still failing: %d", len(still_failed))


# ---------------------------------------------------------------------------
# Device backup helpers
# ---------------------------------------------------------------------------

def parse_devices(filepath):
    """Parse the device list file and return a list of IP addresses.

    Strips inline comments (e.g., '10.1.1.1  # hostname (mac)' -> '10.1.1.1').
    """
    ips = []
    with open(filepath, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            ip = line.split()[0]  # strip inline comments
            ips.append(ip)
    return ips


def parse_failed_file(filepath):
    """Parse failed_devices.txt, returning list of IPs (strips inline comments)."""
    ips = []
    with open(filepath, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            ip = line.split()[0]
            ips.append(ip)
    return ips






# Compiled regex patterns for secret sanitization.
# Each tuple is (pattern, replacement). Covers HP ProCurve/ArubaOS-Switch,
# Cisco IOS/IOS-XE, Palo Alto PAN-OS, and Juniper JunOS.
# Value matching uses (?:"[^"]*"|\S+) to handle both quoted and unquoted values.
_SANITIZE_PATTERNS = [
    # --- HP ProCurve / ArubaOS-Switch ---
    # password manager/operator [user-name "..."] plaintext/sha1/sha256 VALUE
    (re.compile(r'(password\s+(?:manager|operator)\s+(?:user-name\s+"[^"]*"\s+)?(?:plaintext|sha1|sha256)\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # encrypted-password manager/operator [user-name "..."] "VALUE"
    (re.compile(r'(encrypted-password\s+(?:manager|operator)\s+(?:user-name\s+"[^"]*"\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # radius-server [host IP] [encrypted-]key [TYPE] VALUE
    (re.compile(r'(radius-server\s+(?:host\s+\S+\s+)?(?:encrypted-)?key\s+)(?:\d\s+)?(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # tacacs-server [host IP] [encrypted-]key [plaintext|7] VALUE
    (re.compile(r'(tacacs-server\s+(?:host\s+\S+\s+)?(?:encrypted-)?key\s+(?:plaintext\s+|7\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # snmpv3 user NAME auth md5/sha "HASH" priv [aes/des] "HASH"
    (re.compile(r'(snmpv3\s+user\s+\S+\s+auth\s+(?:md5|sha)\s+)(?:"[^"]*"|\S+)(\s+priv\s+(?:(?:des|aes)\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***\2***'),
    # snmp-server community VALUE ...
    (re.compile(r'(snmp-server\s+community\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # snmp-server host IP COMMUNITY ...
    (re.compile(r'(snmp-server\s+host\s+\S+\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # sntp/ntp authentication key-id N key-value VALUE
    (re.compile(r'((?:sntp|ntp)\s+authentication\s+key-id\s+\d+\s+key-value\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # encrypt-credentials pre-shared-key plaintext/hex VALUE
    (re.compile(r'(encrypt-credentials\s+pre-shared-key\s+(?:plaintext|hex)\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # wpa-passphrase / wpa-preshared-key
    (re.compile(r'(wpa-(?:passphrase|preshared-key)\s+(?:(?:ascii|hex)\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),

    # --- ArubaOS-CX ---
    # user NAME [group GRP] password ciphertext/plaintext VALUE
    (re.compile(r'(password\s+(?:ciphertext|plaintext)\s+)\S+', re.IGNORECASE), r'\1***'),
    # radius-server/tacacs-server key ciphertext/plaintext VALUE
    (re.compile(r'((?:radius-server|tacacs-server)\s+.*?key\s+(?:ciphertext|plaintext)\s+)\S+', re.IGNORECASE), r'\1***'),

    # --- Cisco IOS / IOS-XE ---
    # enable secret/password [TYPE] VALUE
    (re.compile(r'(enable\s+(?:secret|password)\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # username NAME [privilege N] secret/password [TYPE] VALUE
    (re.compile(r'(username\s+\S+\s+(?:privilege\s+\d+\s+)?(?:secret|password)\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # crypto isakmp key [TYPE] VALUE
    (re.compile(r'(crypto\s+isakmp\s+key\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # pre-shared-key local/remote [TYPE] VALUE (Cisco IKEv2 keyring)
    (re.compile(r'(pre-shared-key\s+(?:local|remote)\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # pre-shared-key address IP key [TYPE] VALUE (Cisco IKEv1 keyring)
    (re.compile(r'(pre-shared-key\s+address\s+\S+\s+key\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # pre-shared-key ascii-text/hexadecimal VALUE (Juniper)
    (re.compile(r'(pre-shared-key\s+(?:ascii-text|hexadecimal)\s+)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # neighbor IP password [TYPE] VALUE (BGP)
    (re.compile(r'(neighbor\s+\S+\s+password\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # server-private IP key [TYPE] VALUE (RADIUS/TACACS new-style)
    (re.compile(r'(server-private\s+\S+\s+.*?key\s+)(?:\d\s+)?\S+', re.IGNORECASE), r'\1***'),
    # ntp authentication-key N md5 VALUE
    (re.compile(r'(ntp\s+authentication-key\s+\d+\s+md5\s+)\S+', re.IGNORECASE), r'\1***'),
    # standby N authentication [md5 key-string [TYPE]] VALUE
    (re.compile(r'(standby\s+\d+\s+authentication\s+(?:md5\s+key-string\s+(?:\d\s+)?)?)\S+', re.IGNORECASE), r'\1***'),

    # --- Shared: Cisco / HP / general ---
    # line password [TYPE] VALUE (skip "password manager/operator" already handled above)
    (re.compile(r'(^\s*password\s+)(?!manager\b|operator\b)(?:\d\s+)?\S+', re.IGNORECASE | re.MULTILINE), r'\1***'),
    # key-string [plaintext|ciphertext|TYPE] VALUE
    (re.compile(r'(key-string\s+(?:(?:plaintext|ciphertext|0|7)\s+)?)(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # key [TYPE] VALUE (inside radius/tacacs server block — Cisco new-style)
    (re.compile(r'(^\s*key\s+)(?:\d\s+)?\S+', re.IGNORECASE | re.MULTILINE), r'\1***'),
    # ip ospf authentication-key [TYPE] VALUE
    (re.compile(r'(ip\s+ospf\s+authentication-key\s+)(?:\d\s+)?(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),
    # ip ospf message-digest-key N md5 [TYPE] VALUE
    (re.compile(r'(ip\s+ospf\s+message-digest-key\s+\d+\s+md5\s+)(?:\d\s+)?(?:"[^"]*"|\S+)', re.IGNORECASE), r'\1***'),

    # --- Palo Alto PAN-OS (XML format) ---
    (re.compile(r'(<(?:phash|password|secret|bind-password|pre-shared-key|private-key|key|auth-key|wmi-password|passphrase|snmp-community-string)>)[^<]+(</)', re.IGNORECASE), r'\1***\2'),
    # PAN-OS set format: keyword VALUE at end of line
    (re.compile(r'((?:phash|bind-password|wmi-password)\s+)\S+', re.IGNORECASE), r'\1***'),

    # --- Juniper JunOS ---
    # Quoted value followed by ## SECRET-DATA
    (re.compile(r'(\S+\s+)"[^"]*"(;\s*##\s*SECRET-DATA)', re.IGNORECASE), r'\1"***"\2'),
    # set format: encrypted-password / secret / auth keys "VALUE"
    (re.compile(r'((?:encrypted-password|secret|authentication-key|authentication-password|privacy-password|simple-password)\s+)"[^"]*"', re.IGNORECASE), r'\1"***"'),
]


def sanitize_config(config_text):
    """Remove passwords, keys, and secrets from device config text.

    Preserves usernames and config structure; replaces secret values with ***.
    """
    lines = config_text.splitlines(keepends=True)
    sanitized = []
    for line in lines:
        for pattern, replacement in _SANITIZE_PATTERNS:
            line = pattern.sub(replacement, line)
        sanitized.append(line)
    return "".join(sanitized)


def _detect_platform(child):
    """Detect device platform from prompt format.

    Returns 'fastiron' for Ruckus (SSH@hostname> / telnet@hostname>),
    'junos' for Juniper (user@hostname>), or 'procurve' for HP/default.
    """
    child.sendline("")
    child.expect(PROMPT_RE, timeout=5)
    raw = (child.before or "") + (child.after or "")
    clean = _strip_ansi(raw)
    if re.search(r"(?:SSH|telnet)@[\w\-]+[#>]", clean):
        log.debug("  platform=fastiron  prompt=%r", clean.strip()[-60:])
        return "fastiron"
    if "@" in clean and re.search(r"\w+@[\w\-/\.]+[#>]", clean):
        log.debug("  platform=junos  prompt=%r", clean.strip()[-60:])
        return "junos"
    log.debug("  platform=procurve  prompt=%r", clean.strip()[-60:])
    return "procurve"


def _classify_platform(child):
    """Classify platform to the level monitoring needs:
    junos | aruba-cx | fastiron | procurve.

    Extends _detect_platform by running `show version` on non-junos / non-fastiron
    devices — ArubaOS-CX prints "ArubaOS-CX" in its version banner; ProCurve does
    not. Also catches Ruckus FastIron via banner content as a fallback when prompt
    detection missed (older firmwares with non-standard prompt formats).
    """
    base = _detect_platform(child)
    if base in ("junos", "fastiron"):
        return base
    try:
        out = send_command(child, "show version", timeout=10)
    except Exception as e:
        log.debug("  show version failed (%s) — defaulting to procurve", e)
        return "procurve"
    if "ArubaOS-CX" in out:
        return "aruba-cx"
    if "Ruckus" in out or "Brocade Communications" in out or re.search(r"\bICX\d", out):
        return "fastiron"
    return "procurve"


def backup_device(ip, password, usernames, sanitize=True, password_list=None,
                  known_username=None, known_password_hash=None,
                  user_passwords=None):
    """Detect, connect, and back up a single device.

    Returns (filename, config, info) where info is a dict with
    hostname, base_mac, serial, firmware, model, proto, username, password_hash.
    """
    child, proto, username, pw_hash, ssh_open, telnet_open = connect_device(
        ip, usernames, password, password_list=password_list,
        known_username=known_username, known_password_hash=known_password_hash,
        user_passwords=user_passwords, reason="backup")
    if child is None:
        raise ConnectionError(f"All connection attempts failed for {ip}")

    try:
        platform = _detect_platform(child)
        hostname = get_hostname(child)

        if platform == "junos":
            config = send_command(child, "show configuration | no-more", timeout=60)
            # Also get set-format config (can be pasted into a replacement device)
            config_set = send_command(child, "show configuration | display set | no-more", timeout=60)
            id_output = ""
            hardware_info = ""
            try:
                ver = send_command(child, "show version | no-more", timeout=15)
                hw = send_command(child, "show chassis hardware | no-more", timeout=15)
                mac_out = send_command(child, "show chassis mac-addresses | no-more", timeout=15)
                id_output = ver + "\n" + hw + "\n" + mac_out
                hardware_info = hw
            except Exception:
                pass
        elif platform == "fastiron":
            config = send_command(child, "show running-config", timeout=60)
            config_set = None
            id_output = ""
            hardware_info = ""
            try:
                ver_out = send_command(child, "show version", timeout=15)
                id_output = ver_out
                hardware_info = ver_out
            except Exception:
                pass
        else:
            config = send_command(child, "show running-config", timeout=60)
            config_set = None
            id_output = ""
            hardware_info = ""
            try:
                sys_out = send_command(child, "show system", timeout=10)
                ver_out = send_command(child, "show version", timeout=10)
                id_output = sys_out + "\n" + ver_out
                hardware_info = sys_out
            except Exception:
                pass

        dev_id = get_device_id(id_output)
    finally:
        disconnect(child)

    # Use None (not "") so upsert_device preserves existing values if
    # identification came up empty for any field.
    base_mac = dev_id.get("base_mac") or None
    serial = dev_id.get("serial") or None
    firmware = dev_id.get("firmware") or None
    model = dev_id.get("model") or None

    if sanitize:
        config = sanitize_config(config)
        if config_set:
            config_set = sanitize_config(config_set)

    if hostname:
        filename = f"{hostname}_{ip}.cfg"
    else:
        filename = f"{ip}.cfg"

    info = {"hostname": hostname, "base_mac": base_mac, "serial": serial,
            "firmware": firmware, "model": model,
            "hardware_info": hardware_info, "proto": proto,
            "username": username, "password_hash": pw_hash,
            "ssh_open": ssh_open, "telnet_open": telnet_open,
            "config_set": config_set}
    # DEBUG-level so it doesn't collide with the \r progress bar during
    # multi-threaded backup runs. Still captured by --debug-file.
    log.debug("  detected: user=%s, proto=%s, hostname=%s, mac=%s",
              username, proto, hostname or "unknown", base_mac or "unknown")
    return filename, config, info


# ---------------------------------------------------------------------------
# Storage modes
# ---------------------------------------------------------------------------

def save_config_local(filename, config_text):
    """Save config using local-history mode (current/ + history/ when changed).

    Returns True if the config changed (or is new), False if identical.
    """
    current_dir = os.path.join(CONFIGS_DIR, "current")
    history_dir = os.path.join(CONFIGS_DIR, "history")
    os.makedirs(current_dir, exist_ok=True)

    current_path = os.path.join(current_dir, filename)
    stem = os.path.splitext(filename)[0]

    if os.path.exists(current_path):
        with open(current_path, encoding="utf-8") as f:
            old_config = f.read()

        if old_config == config_text:
            return False

        # Config changed — archive the old version
        device_history_dir = os.path.join(history_dir, stem)
        os.makedirs(device_history_dir, exist_ok=True)
        timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        archive_path = os.path.join(device_history_dir, f"{timestamp}.cfg")
        shutil.copy2(current_path, archive_path)
        log.info("  archived previous config to %s", archive_path)

    # Write new/updated config
    with open(current_path, "w", encoding="utf-8") as f:
        f.write(config_text)

    return True


def git_commit_changes(changed_labels, remote_url="", branch="main"):
    """Stage configs/ and commit if there are changes.

    When remote_url is set, also push to that remote on branch. Push
    failures are logged as warnings but don't raise — the local commit is
    authoritative and the next backup will retry the push.
    """
    # Setting storage_mode=git is an explicit opt-in to track configs/ —
    # if .gitignore happens to list it (auto-generated boilerplate, copied
    # template, etc.), 'git add' would refuse silently. Use -f so the
    # configured intent wins; warn if the .gitignore entry exists so the
    # operator knows what's happening.
    ignored = subprocess.run(
        ["git", "check-ignore", "-q", "configs/"],
        cwd=STATE_DIR, capture_output=True,
    )
    if ignored.returncode == 0:
        log.warning("configs/ is excluded by .gitignore but storage_mode=git "
                    "is set — using 'git add -f' to override. Remove configs/ "
                    "from .gitignore (or set storage_mode=local) to clean up.")
    subprocess.run(["git", "add", "-f", "configs/"], cwd=STATE_DIR, check=True)

    result = subprocess.run(
        ["git", "diff", "--cached", "--quiet"],
        cwd=STATE_DIR,
    )
    if result.returncode == 0:
        log.info("No config changes detected — skipping git commit.")
        return

    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    message = f"Config backup {timestamp}\n\nChanged devices:\n"
    for label in changed_labels:
        message += f"  - {label}\n"

    subprocess.run(["git", "commit", "-m", message], cwd=STATE_DIR, check=True)
    log.info("Git commit created.")

    if remote_url:
        _git_push_remote(remote_url, branch)


def _git_push_remote(remote_url, branch):
    """Ensure 'origin' points at remote_url and push the branch.

    Idempotent: adds origin if missing, updates it if the URL changed, and
    always uses -u to keep upstream tracking current. Non-fatal on failure.
    """
    try:
        existing = subprocess.run(
            ["git", "remote", "get-url", "origin"],
            cwd=STATE_DIR, capture_output=True, text=True)
        if existing.returncode != 0:
            # No 'origin' yet — add it.
            subprocess.run(["git", "remote", "add", "origin", remote_url],
                           cwd=STATE_DIR, check=True)
            log.info("Added git remote 'origin' -> %s", remote_url)
        elif existing.stdout.strip() != remote_url:
            subprocess.run(["git", "remote", "set-url", "origin", remote_url],
                           cwd=STATE_DIR, check=True)
            log.info("Updated git remote 'origin' -> %s", remote_url)
        push = subprocess.run(
            ["git", "push", "-u", "origin", branch],
            cwd=STATE_DIR, capture_output=True, text=True)
        if push.returncode == 0:
            log.info("Pushed backup commit to origin/%s", branch)
        else:
            log.warning(
                "Git push to origin/%s failed (exit %d): %s — local commit "
                "kept; next backup will retry.",
                branch, push.returncode,
                (push.stderr or push.stdout).strip())
    except Exception as e:
        log.warning("Git push skipped: %s", e)


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

# Shared help content used by both the direct-CLI path (on missing/invalid
# args, via _DeviceHelpParser below) and the console REPL (via '<cmd> ?' and
# 'help <cmd>'). Same text in both places.

COMMAND_SUMMARIES = [
    ("backup",          "Back up configs for all active devices"),
    ("discover",        "Scan subnets + test credentials"),
    ("scan",            "Port-scan only (no credential test)"),
    ("test",            "Test credentials against all devices"),
    ("retest",          "Re-test failed devices"),
    ("probe",           "TCP reachability check (no login)"),
    ("show",            "Show a category listing or device detail by IP"),
    ("add",             "Add device IP(s) to the DB"),
    ("remove",          "Remove device IP(s) from the DB"),
    ("import",          "Import devices from a file (list or CSV)"),
    ("export",          "Export device lists from the DB to flat files"),
    ("monitor",         "Poll device state (monitor stp | monitor flap)"),
    ("digest",          "Email a digest (digest flap)"),
    ("clear",           "Clear stored counters (clear flap)"),
    ("investigate",     "Gather context for an STP change (investigate stp <ip> <port>)"),
    ("refresh-dns",     "Reverse-DNS all devices in the DB"),
    ("test-email",      "Send a test email (test-email all | <address>)"),
    ("configure",       "Change settings (configure email add|remove|list)"),
    ("check-config",    "Show system limits and effective config"),
]

MONITOR_TARGETS = [
    ("stp",  "Poll STP + port state; confirm STP changes; email alerts"),
    ("flap", "Poll port state only; count flaps; no STP analysis"),
]

SINGLE_FLAP_TARGETS = [("flap", "Port flap counters")]

HELP_DETAILS = {
    "backup": [
        "Usage: backup [-s SUBNET] [-u USER] [-p PW] [--no-sanitize]",
        "  Connects to every active device and saves running config.",
        "  -s SUBNET       override subnet(s) to scan (comma-separated)",
        "  -u USER         override username(s)",
        "  -p PW           override default password",
        "  --no-sanitize   save raw configs without redacting secrets",
    ],
    "discover": [
        "Usage: discover [-s SUBNET] [-u USER] [-p PW] [--dry-run]",
        "  Scan subnets for open 22/23 and test credentials.",
        "  --dry-run       show results without writing to DB",
    ],
    "scan": [
        "Usage: scan [-s SUBNET] [--dry-run]",
        "  Port-scan only. Writes responsive IPs + open-port flags to DB.",
    ],
    "test": [
        "Usage: test [-u USER] [-p PW]",
        "  Re-test credentials against every device currently in the DB.",
    ],
    "retest": [
        "Usage: retest [-u USER] [-p PW]",
        "  Re-test only failed devices; promote passing ones to active.",
    ],
    "probe": [
        "Usage: probe [--status {active,inactive,failed,all}] [--csv FILE]",
        "       [--timeout SEC] [--threads N]",
        "  TCP reachability check for devices already in the DB. No login",
        "  attempts — just checks if ports 22/23 respond. Updates ssh_open/",
        "  telnet_open columns; does NOT change status.",
        "  Default filter: all (except duplicates).",
    ],
    "show": [
        "Usage: show <TARGET> [--csv FILE]",
        "  show spanning-tree [blocked|disabled|mismatch|root]   (alias: stp)",
        "      bare 'show spanning-tree' prints all four states",
        "  show devices switch [active|all|failed|inactive]",
        "  show devices ip <pattern> [detail]     match devices by IP",
        "  show devices name <pattern> [detail]   match by switch/DNS name",
        "  show backups | duplicates | telnet | stack-members",
        "  show port-flaps | mac-flux",
    ],
    "add": [
        "Usage: add <IP>",
        "  Add device IP(s) to the DB. Comma-separated for multiple.",
    ],
    "remove": [
        "Usage: remove <IP_OR_KEYWORD>",
        "  Remove device(s) from the DB.",
        "  Use 'failed' or 'duplicates' as keywords to bulk-remove by status.",
    ],
    "import": [
        "Usage: import [PATH] [--format {auto,list,csv}]",
        "  Import devices from a file. With no PATH, uses device_file from config.",
        "  CSV header columns (case-insensitive, only 'ip' required):",
        "    ip, hostname, dns_name, model, status",
        "  Existing devices: status/fail_reason preserved; other fields merged.",
    ],
    "export": [
        "Usage: export",
        "  Write active devices, failed devices, and telnet report to flat files.",
    ],
    "monitor": [
        "Usage: monitor <stp|flap>",
        "  stp   Poll STP + port state across all active devices,",
        "        confirm changes over two polls, email alerts, count",
        "        port flaps. Concurrency: [general] backup_threads.",
        "  flap  Poll port state only; count flaps. Skips STP commands",
        "        and does not touch STP state or alerts.",
    ],
    "digest": [
        "Usage: digest <flap|stp|backup|health>",
        "  flap [--min-count N] [--reset]",
        "    Email a digest of port flap counters accumulated since the",
        "    last reset. By default, counters are preserved — manage",
        "    retention separately with 'clear flap'.",
        "    --min-count N   only report ports with >= N flaps (default 1)",
        "    --reset         also clear all counters after a successful send",
        "  stp [--detail]",
        "    Email a combined report of STP-disabled devices, STP-mode-",
        "    mismatched devices, and the current root-bridge view per",
        "    instance (flagged when switches disagree). Expected mode is",
        "    configurable via [monitor] stp_expected_mode (default: mstp).",
        "    No email when everything is clean; safe to run on a fixed schedule.",
        "  backup [--detail]",
        "    Weekly status report covering never-backed-up devices, stale",
        "    backups, and config changes in the past 7 days. Stale threshold",
        "    is [monitor] backup_stale_days (default: 7). Always emails when",
        "    there is at least one active device.",
        "    --detail        also list every device in a full inventory table",
        "  health",
        "    Weekly enterprise-health digest: fleet inventory, backup",
        "    success rate, STP/flap activity, reachability alerts fired,",
        "    monitor-stp tick health, SNMP/SSH latency stats by reason,",
        "    netops-box disk/log/db size, and operator action items.",
        "    Also prunes op_events rows older than 35 days.",
    ],
    "clear": [
        "Usage: clear flap [--min-count N] [--dry-run]",
        "  Clear port flap counters. Run on the cadence that matches",
        "  your reporting window (e.g. daily, weekly).",
        "  --min-count N   only clear rows with >= N flaps (default 0 = all)",
        "  --dry-run       show what would be cleared without deleting",
    ],
    "refresh-dns": [
        "Usage: refresh-dns",
        "  Reverse-DNS every device in the DB and update dns_name.",
    ],
    "test-email": [
        "Usage: test-email all | <address>",
        "  Send a canned test message to verify [email] delivery.",
        "  'all' sends to every configured recipient; or name one",
        "  configured address to test just that one.",
    ],
    "configure": [
        "Usage: configure email add <address>",
        "       configure email remove <address>",
        "       configure email list",
        "  Change settings in netops.conf. Currently the [email] 'to'",
        "  recipient list for alert and digest delivery; 'list' prints the",
        "  current recipients. Passwords are NOT configured here — edit",
        "  secrets.conf directly.",
    ],
    "check-config": [
        "Usage: check-config",
        "  Show system limits, thread caps, effective config values.",
    ],
}


def print_root_help():
    """Device-style top-level help — summary line per command."""
    print("Commands:")
    for c, summary in COMMAND_SUMMARIES:
        print(f"  {c:<14} {summary}")
    print("\nType '<command> ?' or '<command> --help' for details on a command.")


def print_command_help(command):
    """Device-style per-command help. Same output for CLI and REPL."""
    details = HELP_DETAILS.get(command)
    if not details:
        print(f"No help available for '{command}'")
        return
    for line in details:
        print(line)
    if command == "show":
        print("  Options:")
        print("    --csv FILE      write to CSV file instead of a table")
        print("  port-flaps filters:")
        print("    --ip IP         show only this device")
        print("    --interface IF  show only this interface (exact match)")
        print("    --min-count N   only rows with >= N flaps (default 1)")


# Subparser class that replaces argparse's usage dump with device-style help
# for commands whose failure mode is "missing required arg" or "bad choice".
# Triggered for the short list of commands below; other subparsers keep the
# default argparse behavior.
_DEVICE_HELP_COMMANDS = {"monitor", "clear", "digest", "show", "configure",
                         "test-email", "add", "remove"}


class _DeviceHelpParser(argparse.ArgumentParser):
    def error(self, message):
        cmd = self.prog.rsplit(" ", 1)[-1] if self.prog else ""
        if cmd in _DEVICE_HELP_COMMANDS:
            # Bad-choice errors get a short '% Invalid target' prefix so the
            # user knows *why* they're looking at help again.
            if "invalid choice" in message:
                match = re.search(r"invalid choice:\s*'([^']+)'", message)
                bad = match.group(1) if match else "?"
                print(f"% Invalid target '{bad}'.")
            print_command_help(cmd)
            self.exit(2)
        super().error(message)


def parse_args():
    # Shared options inherited by every subcommand so they can appear
    # either before or after the subcommand name on the command line.
    # Use argparse.SUPPRESS as default so the subparser doesn't overwrite the
    # parent parser's value when --db (or other shared flags) appear before
    # the subcommand name. This lets `netops --db X foo` and `netops foo --db X`
    # both work.
    _shared = argparse.ArgumentParser(add_help=False)
    _shared.add_argument(
        "--db", type=str, metavar="PATH", default=argparse.SUPPRESS,
        help="Use an alternate database file (created if missing). "
             f"Default: {DB_FILE}",
    )
    _shared.add_argument(
        "--debug", action="store_true", default=argparse.SUPPRESS,
        help="Enable verbose debug logging on console and in the log file",
    )
    _shared.add_argument(
        "--debug-file", nargs="?",
        const=os.path.join(STATE_DIR, "netops-debug.log"),
        default=argparse.SUPPRESS, metavar="PATH",
        help="Write DEBUG logging to a separate file (netops.log unchanged). "
             "With no PATH, defaults to netops-debug.log in the state dir.",
    )
    _shared.add_argument(
        "--log-mode", choices=["append", "truncate"], default=argparse.SUPPRESS,
        help="Log file open mode: 'append' (default) or 'truncate' (fresh each run).",
    )

    parser = argparse.ArgumentParser(
        prog="netops", parents=[_shared],
        description=f"netops v{__version__} — Network device backup, discovery, and monitoring.",
    )
    parser.add_argument(
        "--version", action="version",
        version=f"%(prog)s {__version__}",
    )

    sub = parser.add_subparsers(dest="command", metavar="COMMAND",
                                parser_class=_DeviceHelpParser)

    # --- Credential-consumer helpers (shared arg shape) ---
    def _add_cred_overrides(p):
        p.add_argument("-u", "--username", type=str,
                       help="Override username(s) (comma-separated)")
        p.add_argument("-p", "--password", type=str,
                       help="Override default password")

    def _add_subnet_override(p):
        p.add_argument("-s", "--subnet", type=str,
                       help="Override subnet(s) to scan (comma-separated)")

    # --- Connection / workflow commands ---
    p_backup = sub.add_parser("backup", parents=[_shared],
                              help="Back up configs for all active devices")
    _add_subnet_override(p_backup)
    _add_cred_overrides(p_backup)
    p_backup.add_argument("--no-sanitize", action="store_true",
                          help="Save raw configs without redacting secrets")

    p_discover = sub.add_parser("discover", parents=[_shared],
                                aliases=["discovery"],
                                help="Scan subnets + test credentials, store passing devices")
    _add_subnet_override(p_discover)
    _add_cred_overrides(p_discover)
    p_discover.add_argument("--dry-run", action="store_true",
                            help="Show what would be found/stored without writing")
    p_discover.add_argument("--email", action="store_true",
                            help="Email a diff (new/removed/changed devices) against "
                                 "the pre-run state. Intended for weekly cron; omit "
                                 "when running manually so you don't trigger alerts.")

    p_scan = sub.add_parser("scan", parents=[_shared],
                            help="Port-scan only (no credential test). Stores open-port IPs.")
    _add_subnet_override(p_scan)
    p_scan.add_argument("--dry-run", action="store_true",
                        help="Show results without writing to the DB")

    p_test = sub.add_parser("test", parents=[_shared],
                            help="Test credentials against all devices (no backup)")
    _add_cred_overrides(p_test)

    p_retest = sub.add_parser("retest", parents=[_shared],
                              help="Re-test failed devices; promote passing ones to active")
    _add_cred_overrides(p_retest)

    p_probe = sub.add_parser("probe", parents=[_shared],
                             help="TCP reachability check for devices already in the DB")
    p_probe.add_argument(
        "--status", choices=["active", "inactive", "failed", "duplicate", "all"],
        default="all",
        help="Filter which devices to probe (default: all except duplicates)",
    )
    p_probe.add_argument(
        "--csv", type=str, metavar="FILE",
        help="Write reachability results to a CSV file",
    )
    p_probe.add_argument(
        "--timeout", type=float, default=2.0,
        help="Per-port TCP connect timeout in seconds (default 2)",
    )
    p_probe.add_argument(
        "--threads", type=int, default=100,
        help="Max concurrent probes (default 100)",
    )

    # --- DB read-only queries ---
    # `show` is overloaded: a category name lists that category,
    # a bare IPv4 address shows full device detail.
    p_show = sub.add_parser("show", parents=[_shared],
                            help="Show a grouped listing, a category, or device detail by IP")
    p_show.add_argument("arg", nargs="+", metavar="TARGET",
                        help="e.g. 'spanning-tree blocked', 'devices switch active', "
                             "'devices name core', 'backups'")
    p_show.add_argument(
        "--csv", type=str, metavar="FILE",
        help="Write the result to a CSV file instead of printing a table",
    )
    p_show.add_argument(
        "--ip", type=str, default=None,
        help="Filter port-flaps to a specific device IP",
    )
    p_show.add_argument(
        "--interface", type=str, default=None,
        help="Filter port-flaps to a specific interface (exact match)",
    )
    p_show.add_argument(
        "--min-count", type=int, default=1,
        help="Filter port-flaps to rows with at least N flaps (default 1)",
    )

    # cat-config — print one device's most recent saved config to stdout.
    # Designed for 'ssh netops@host netops cat-config <dev> > local.cfg'
    # workflows where the operator wants to grab a config without
    # arranging for scp/sftp access.
    p_catcfg = sub.add_parser("cat-config", parents=[_shared],
        help="Print the latest saved config for one device to stdout")
    p_catcfg.add_argument("device", metavar="IP_OR_HOSTNAME",
        help="device to print the saved config for (IPv4 or hostname)")
    p_catcfg.add_argument("--set", action="store_true",
        help="(Junos only) print the 'set' format from <hostname>_<ip>_set.cfg "
             "instead of the hierarchical format")

    # --- DB mutations ---
    sub.add_parser("add", parents=[_shared],
                   help="Add device IP(s) to the DB").add_argument(
        "ip", metavar="IP",
        help="IP address (comma-separated for multiple)",
    )
    sub.add_parser("remove", parents=[_shared],
                   help="Remove device IP(s) from the DB").add_argument(
        "ip", metavar="IP_OR_KEYWORD",
        help="IP (comma-separated), or 'failed'/'duplicates' to bulk-remove by status",
    )
    p_import = sub.add_parser("import", parents=[_shared],
                              help="Import devices from a file into the DB")
    p_import.add_argument(
        "path", nargs="?", default=None,
        help="Path to import (plain list or CSV). "
             "Default: the configured device_file (devices.txt).",
    )
    p_import.add_argument(
        "--format", choices=["auto", "list", "csv"], default="auto",
        help="File format (default: auto — detect by extension/header)",
    )
    sub.add_parser("export", parents=[_shared],
                   help="Export device lists from the DB to flat files")
    sub.add_parser("wipe-db", parents=[_shared],
                   help="Delete the active DB (prompts for confirmation)")

    # --- Monitoring (device-style grouped verbs) ---
    p_monitor = sub.add_parser("monitor", parents=[_shared],
                               help="Poll device state (monitor stp | monitor flap | monitor topology)")
    p_monitor.add_argument("target", choices=["stp", "flap", "topology"],
                           help="stp = STP + port state + flap counters; "
                                "flap = port state + flap counters only; "
                                "topology = fleet-wide LLDP scrape into topology_edges")
    p_monitor.add_argument("--detail", action="store_true",
                           help="(stp only) also log the majority/agreeing switches "
                                "in a root-bridge disagreement (default: suppressed)")
    _add_cred_overrides(p_monitor)

    p_digest = sub.add_parser("digest", parents=[_shared],
                              help="Email a digest (digest flap | digest stp | digest backup)")
    p_digest.add_argument("target", choices=["flap", "stp", "backup", "health"])
    p_digest.add_argument("--min-count", type=int, default=1,
                          help="(flap only) only include ports with >= N flaps (default 1)")
    p_digest.add_argument("--reset", action="store_true",
                          help="(flap only) also clear counters after a successful send")
    p_digest.add_argument("--detail", action="store_true",
                          help="(stp/backup) also list agreeing/healthy devices in full "
                               "(default: collapsed)")

    p_clear = sub.add_parser("clear", parents=[_shared],
                             help="Clear stored counters (clear flap)")
    p_clear.add_argument("target", choices=["flap"])
    p_clear.add_argument("--min-count", type=int, default=0,
                         help="Only clear ports with at least N flaps (default 0 = all)")
    p_clear.add_argument("--dry-run", action="store_true",
                         help="Show what would be cleared without deleting")

    p_investigate = sub.add_parser("investigate", parents=[_shared],
                                   help="Gather context for an STP change (investigate stp <ip> <port>)")
    p_investigate.add_argument("target", choices=["stp"])
    p_investigate.add_argument("ip", help="Device IP address")
    p_investigate.add_argument("port", help="Interface name (e.g. ge-1/0/11.0, 1/1/24, lag4)")
    _add_cred_overrides(p_investigate)

    p_whitelist = sub.add_parser("whitelist", parents=[_shared],
        help="Manage MAC-flux and port-flap exclusion lists "
             "(whitelist add|remove|list [--flap|--flux|--all])")
    p_whitelist.add_argument("action", choices=["add", "remove", "list"])
    p_whitelist.add_argument("ip", nargs="?",
        help="Device IP (required for add/remove)")
    p_whitelist.add_argument("interface", nargs="?",
        help="Port interface name (required for add/remove)")
    p_whitelist.add_argument("-r", "--reason",
        help="(add only) free-text annotation stored with the entry")
    # 3.8.10 category selectors. Defaults to --flux when none set
    # (preserves 3.8.8 behavior). --all is only meaningful for `list`.
    _wl_cat = p_whitelist.add_mutually_exclusive_group()
    _wl_cat.add_argument("--flap", action="store_true",
        help="Operate on the port-flap silence list "
             "(port_flap_whitelist) — for sleepy printers etc.")
    _wl_cat.add_argument("--flux", action="store_true",
        help="Operate on the MAC-flux whitelist (mac_flux_whitelist) "
             "— this is the default when no category is given")
    _wl_cat.add_argument("--all", action="store_true",
        help="(list only) show both categories")
    p_whitelist.add_argument("--until", metavar="YYYY-MM-DD",
        help="(--flap add only) expiry date for a time-bounded silence; "
             "the entry is auto-removed on first digest flap after that date. "
             "Accepts YYYY-MM-DD (rounds to end-of-day) or "
             "'YYYY-MM-DD HH:MM:SS'")

    # --- Introspection / setup ---
    sub.add_parser("check-config", parents=[_shared],
                   help="Show system limits, config values, and effective settings")
    p_configure = sub.add_parser("configure", parents=[_shared],
                                 help="Change settings (configure email add|remove|list)")
    p_configure.add_argument("tokens", nargs="+", metavar="...",
                             help="e.g. 'email add ops@example.com', "
                                  "'email remove old@example.com', 'email list'")
    p_testemail = sub.add_parser("test-email", parents=[_shared],
                                 help="Send a test email (test-email all | <address>)")
    p_testemail.add_argument("target", metavar="all|ADDRESS",
                             help="'all' for every configured recipient, or "
                                  "one configured email address")
    sub.add_parser("console", parents=[_shared],
                   help="Drop into an interactive netops shell")
    sub.add_parser("refresh-dns", parents=[_shared],
                   help="Reverse-DNS all devices in the DB and update dns_name")

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(0)

    args = parser.parse_args()
    if args.command is None:
        parser.print_help()
        sys.exit(0)
    return args


# ---------------------------------------------------------------------------
# CLI query and export handlers
# ---------------------------------------------------------------------------

_COLUMN_MAX_WIDTH = {"model": 20, "dns_name": 35}


def _print_table(headers, rows):
    """Print aligned columns from a list of header names and row dicts/tuples."""
    if not rows:
        return
    # Calculate column widths
    widths = [len(h) for h in headers]
    str_rows = []
    for row in rows:
        # Preserve 0 / False as their string form; only None/"" render blank.
        cells = [("" if row[h] is None else str(row[h])) for h in headers]
        for i, h in enumerate(headers):
            cap = _COLUMN_MAX_WIDTH.get(h)
            if cap and len(cells[i]) > cap:
                cells[i] = cells[i][:cap - 1] + "…"
        str_rows.append(cells)
        for i, cell in enumerate(cells):
            widths[i] = max(widths[i], len(cell))

    # Header
    header_line = "  ".join(h.upper().ljust(widths[i]) for i, h in enumerate(headers))
    print(header_line)
    print("  ".join("-" * w for w in widths))
    for cells in str_rows:
        print("  ".join(cells[i].ljust(widths[i]) for i in range(len(headers))))


def _write_csv(path, headers, rows):
    """Write rows to a CSV file. Uses full (untruncated) values."""
    import csv
    path = os.path.abspath(os.path.expanduser(path))
    with open(path, "w", encoding="utf-8", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(headers)
        for row in rows:
            writer.writerow([("" if row[h] is None else str(row[h])) for h in headers])
    print(f"Wrote {len(rows)} row(s) to {path}")


def _output(headers, rows, csv_path=None):
    """Dispatcher: write CSV if csv_path set, else print table."""
    if csv_path:
        _write_csv(csv_path, headers, rows)
    else:
        _print_table(headers, rows)


# --- `show` command grammar -------------------------------------------
# Junos-style hierarchical listing. Single source of truth for dispatch,
# help, and REPL tab-completion. The leaf values map to the internal
# category keys that handle_list() dispatches on.
_SHOW_STP_STATES = ("blocked", "disabled", "mismatch", "root")
_SHOW_DEVICE_STATUSES = ("active", "all", "failed", "inactive")
_SHOW_FLAT = ("backups", "duplicates", "telnet", "stack-members",
              "port-flaps", "mac-flux")
_SHOW_TOP = ("spanning-tree", "stp", "devices") + _SHOW_FLAT

_SHOW_STP_CATEGORY = {"blocked": "stp-blocked", "disabled": "stp-disabled",
                      "mismatch": "stp-mismatch", "root": "stp-root"}
_SHOW_DEVICE_CATEGORY = {"active": "devices", "all": "all",
                         "failed": "failed", "inactive": "inactive"}


def _resolve_show(words):
    """Resolve a `show` TARGET word-list to a (kind, value) dispatch action.

      ("category", <internal handle_list category>)
      ("stp-summary", None)                  -- bare `show spanning-tree`
      ("filter", (field, pattern, detail))   -- show devices ip|name <pat>
      ("error", <message to print>)
    """
    head, rest = words[0], words[1:]

    if head in ("spanning-tree", "stp"):
        if not rest:
            return ("stp-summary", None)
        if len(rest) == 1 and rest[0] in _SHOW_STP_CATEGORY:
            return ("category", _SHOW_STP_CATEGORY[rest[0]])
        return ("error", "show spanning-tree: expected one of "
                + ", ".join(_SHOW_STP_STATES))

    if head == "devices":
        if not rest:
            return ("category", "devices")
        if rest[0] in ("ip", "name"):
            field = rest[0]
            pat = rest[1:]
            detail = bool(pat) and pat[-1] == "detail"
            if detail:
                pat = pat[:-1]
            if len(pat) != 1:
                kindword = "partial-IP" if field == "ip" else "name"
                return ("error", f"show devices {field}: expected one "
                                 f"{kindword} pattern, optionally followed "
                                 "by 'detail'")
            return ("filter", (field, pat[0], detail))
        if rest[0] != "switch":
            return ("error", f"show devices: unknown keyword '{rest[0]}' "
                              "(expected 'switch', 'ip', or 'name')")
        status = rest[1:]
        if not status:
            return ("category", "devices")
        if len(status) == 1 and status[0] in _SHOW_DEVICE_CATEGORY:
            return ("category", _SHOW_DEVICE_CATEGORY[status[0]])
        return ("error", "show devices switch: expected one of "
                + ", ".join(_SHOW_DEVICE_STATUSES))

    if head in _SHOW_FLAT:
        if not rest:
            return ("category", head)
        if rest == ["all"] and head in ("mac-flux", "port-flaps"):
            return ("category-all", head)
        if head in ("mac-flux", "port-flaps"):
            return ("error", f"show {head}: expected nothing, or 'all'")
        return ("error", f"show {head}: takes no further keywords")

    return ("error", f"show: unrecognized target '{' '.join(words)}' — "
                      "see 'show' in the help (or 'show ?' in the console).")


def _complete_show(parts, text):
    """Keyword tab-completion candidates for `show` in the REPL.

    `parts` is the whitespace-split line before the cursor (including the
    leading 'show'); `text` is the fragment being completed. Returns the
    keyword candidates for the current cursor position — readline lists
    them on a double-Tab when more than one matches. Device-name
    completion for the first word is added by the caller.
    """
    if len(parts) <= 1:
        opts = _SHOW_TOP
    elif parts[1] in ("spanning-tree", "stp") and len(parts) == 2:
        opts = _SHOW_STP_STATES
    elif parts[1] == "devices" and len(parts) == 2:
        opts = ("switch", "ip", "name")
    elif parts[1] == "devices" and len(parts) == 3 and parts[2] == "switch":
        opts = _SHOW_DEVICE_STATUSES
    elif (parts[1] == "devices" and len(parts) == 4
          and parts[2] in ("ip", "name")):
        opts = ("detail",)
    elif len(parts) == 2 and parts[1] in ("mac-flux", "port-flaps"):
        opts = ("all",)
    else:
        opts = ()
    return [o for o in opts if o.startswith(text)]


def handle_list(category, csv_path=None, ip=None, interface=None, min_count=1,
                show_all=False):
    """Handle `show <category>` queries. If csv_path is set, write CSV instead of a table.

    ip/interface/min_count are only consulted for the 'port-flaps' category.
    """
    if category == "devices":
        rows = get_devices(status="active")
        if not rows:
            print("No active devices in database.")
            return
        _output(["ip", "hostname", "dns_name", "model", "base_mac", "firmware", "proto", "username", "last_seen"], rows, csv_path)
        print(f"\n{len(rows)} active device(s)")

    elif category == "failed":
        rows = get_devices(status="failed")
        if not rows:
            print("No failed devices in database.")
            return
        _output(["ip", "dns_name", "fail_reason", "last_seen"], rows, csv_path)
        print(f"\n{len(rows)} failed device(s)")

    elif category == "inactive":
        rows = get_devices(status="inactive")
        if not rows:
            print("No inactive devices in database.")
            return
        _output(["ip", "hostname", "base_mac", "fail_reason", "last_seen"], rows, csv_path)
        print(f"\n{len(rows)} inactive device(s)")

    elif category == "backups":
        conn = _db()
        rows = conn.execute("""
            SELECT d.ip, d.hostname, d.base_mac,
                   b.backed_up_at, b.changed, b.config_hash
            FROM devices d
            LEFT JOIN backups b ON d.ip = b.ip
                AND b.id = (SELECT MAX(id) FROM backups WHERE ip = d.ip)
            WHERE d.status = 'active'
        """).fetchall()
        conn.close()
        rows = _sort_by_ip(rows)
        if not rows:
            print("No backup data in database.")
            return
        # Format for display
        display = []
        backed_up = 0
        for r in rows:
            status = "never"
            if r["backed_up_at"]:
                backed_up += 1
                status = "changed" if r["changed"] else "unchanged"
            display.append({
                "ip": r["ip"],
                "hostname": r["hostname"] or "",
                "base_mac": r["base_mac"] or "",
                "last_backup": r["backed_up_at"] or "never",
                "status": status,
            })
        _output(["ip", "hostname", "base_mac", "last_backup", "status"], display, csv_path)
        print(f"\n{backed_up}/{len(rows)} device(s) backed up")

    elif category == "duplicates":
        rows = get_duplicates()
        if not rows:
            print("No duplicate IPs detected.")
            return
        # Group by MAC for display
        current_mac = None
        for r in rows:
            if r["base_mac"] != current_mac:
                current_mac = r["base_mac"]
                print(f"\n{r['base_mac']} ({r['hostname'] or 'unknown'}):")
            marker = " *" if r["status"] == "active" else "  "
            print(f"  {marker} {r['ip']}  ({r['status']})")
        dup_count = len(set(r["base_mac"] for r in rows))
        print(f"\n{dup_count} physical switch(es) with multiple IPs")

    elif category == "telnet":
        conn = _db()
        rows = conn.execute("""
            SELECT * FROM devices
            WHERE telnet_open = 1 AND status IN ('active', 'inactive', 'duplicate')
        """).fetchall()
        conn.close()
        rows = _sort_by_ip(rows)
        if not rows:
            print("No devices with telnet open in database.")
            return
        # Show proto and ssh_open so user can see SSH+telnet vs telnet-only
        display = []
        for r in rows:
            display.append({
                "ip": r["ip"],
                "hostname": r["hostname"] or "",
                "base_mac": r["base_mac"] or "",
                "proto": r["proto"] or "",
                "ssh": "yes" if r["ssh_open"] else "no",
                "telnet": "yes" if r["telnet_open"] else "no",
                "status": r["status"],
                "last_seen": r["last_seen"],
            })
        _output(["ip", "hostname", "base_mac", "proto", "ssh", "telnet", "status", "last_seen"], display, csv_path)
        ssh_and_telnet = sum(1 for r in rows if r["ssh_open"])
        telnet_only = len(rows) - ssh_and_telnet
        print(f"\n{len(rows)} device(s) with telnet open: {telnet_only} telnet-only, {ssh_and_telnet} SSH+telnet")

    elif category == "stp-blocked":
        conn = _db()
        # Real STP blocks only: require role=Alternate (ArubaOS-CX and our
        # inferred ProCurve role) or Juniper's BLK+ALT combo. Ignores
        # role=Disabled ports that happen to be in a Blocking state in
        # some MST instance — those are link-down noise.
        rows = conn.execute("""
            SELECT s.ip, d.hostname, s.interface, s.role, s.state, s.last_seen
            FROM stp_state s
            LEFT JOIN devices d ON s.ip = d.ip
            WHERE
              (s.role = 'ALT' AND s.state = 'BLK')                  -- Juniper
              OR (s.role = 'Alternate' AND s.state = 'Blocking')    -- ArubaOS-CX / ProCurve
        """).fetchall()
        conn.close()
        # Sort by IP numerically, then interface
        rows = sorted(rows, key=lambda r: (_ip_sort_key(r["ip"]), r["interface"] or ""))
        if not rows:
            print("No ports currently in STP blocking state.")
            return
        display = [{
            "ip": r["ip"],
            "hostname": r["hostname"] or "",
            "interface": r["interface"],
            "role": r["role"],
            "state": r["state"],
            "last_seen": r["last_seen"],
        } for r in rows]
        _output(["ip", "hostname", "interface", "role", "state", "last_seen"], display, csv_path)
        devices = len({r["ip"] for r in rows})
        print(f"\n{len(rows)} blocked port(s) across {devices} device(s)")

    elif category == "port-flaps":
        conn = _db()
        conn.row_factory = sqlite3.Row
        sql = """
            SELECT ip, hostname, interface, flap_count, first_seen, last_seen
            FROM port_flaps
            WHERE flap_count >= ?
        """
        params = [min_count]
        if ip:
            sql += " AND ip = ?"
            params.append(ip)
        if interface:
            sql += " AND interface = ?"
            params.append(interface)
        sql += " ORDER BY flap_count DESC, hostname, interface"
        rows = conn.execute(sql, params).fetchall()
        # 3.8.10: annotate silenced rows (the user's call: keep them
        # visible so operators stay aware they're still flapping; just
        # not paging anyone). The digest is what filters them out.
        silenced = _get_flap_silenced_ports(conn)
        conn.close()
        if not rows:
            filt = []
            if ip: filt.append(f"ip={ip}")
            if interface: filt.append(f"interface={interface}")
            if min_count > 0: filt.append(f"flap_count>={min_count}")
            suffix = f" (filters: {', '.join(filt)})" if filt else ""
            print(f"No port flap rows in database{suffix}.")
            return
        display = [{
            "ip": r["ip"],
            "hostname": (r["hostname"] or "")
                + (" (silenced)" if (r["ip"], r["interface"]) in silenced
                   else ""),
            "interface": r["interface"],
            "flap_count": r["flap_count"],
            "first_seen": r["first_seen"],
            "last_seen": r["last_seen"],
        } for r in rows]
        _output(["ip", "hostname", "interface", "flap_count",
                 "first_seen", "last_seen"], display, csv_path)
        devices = len({r["ip"] for r in rows})
        total = sum(r["flap_count"] for r in rows)
        silenced_count = sum(1 for r in rows
                             if (r["ip"], r["interface"]) in silenced)
        sil_suffix = (f" ({silenced_count} silenced — filtered from digest)"
                      if silenced_count else "")
        print(f"\n{len(rows)} port(s) across {devices} device(s), "
              f"{total} total flap(s){sil_suffix}")

    elif category == "stp-disabled":
        conn = _db()
        rows = conn.execute("""
            SELECT ip, hostname, dns_name, model, proto, username, stp_last_check AS last_checked
            FROM devices
            WHERE stp_enabled = 0 AND status IN ('active', 'inactive')
        """).fetchall()
        conn.close()
        rows = _sort_by_ip(rows)
        if not rows:
            print("No devices with STP disabled (run 'monitor stp' first).")
            return
        _output(["ip", "hostname", "dns_name", "model", "proto", "username", "last_checked"], rows, csv_path)
        print(f"\n{len(rows)} device(s) with STP disabled")

    elif category == "stp-mismatch":
        cfg = load_config()
        expected = (cfg.get("stp_expected_mode") or "mstp").lower()
        conn = _db()
        rows = conn.execute("""
            SELECT ip, hostname, dns_name, model, stp_mode, stp_last_check AS last_checked
            FROM devices
            WHERE stp_mode IS NOT NULL
              AND lower(stp_mode) != ?
              AND status IN ('active', 'inactive')
        """, (expected,)).fetchall()
        conn.close()
        rows = _sort_by_ip(rows)
        if not rows:
            print(f"No STP mode mismatches — all devices match '{expected}'.")
            return
        _output(["ip", "hostname", "dns_name", "model", "stp_mode", "last_checked"], rows, csv_path)
        print(f"\n{len(rows)} device(s) not running {expected}")

    elif category == "stp-root":
        conn = _db()
        rows = conn.execute("""
            SELECT s.ip, COALESCE(d.hostname, s.ip) AS hostname,
                   s.instance, s.root_priority, s.root_mac,
                   s.bridge_priority, s.bridge_mac, s.is_root,
                   s.tcn_count, s.last_tcn_seconds, s.updated_at
            FROM stp_root_state s
            LEFT JOIN devices d ON d.ip = s.ip
            ORDER BY s.instance, s.root_priority, s.root_mac, hostname
        """).fetchall()
        conn.close()
        if not rows:
            print("No root-bridge state recorded (run 'monitor stp' first).")
            return
        display = [{
            "ip": r["ip"],
            "hostname": r["hostname"],
            "instance": r["instance"],
            "root_priority": r["root_priority"],
            "root_mac": r["root_mac"],
            "bridge_priority": r["bridge_priority"],
            "bridge_mac": r["bridge_mac"],
            "is_root": "yes" if r["is_root"] else "",
            "tcn_count": r["tcn_count"],
            "last_tcn_s": r["last_tcn_seconds"],
            "updated_at": r["updated_at"],
        } for r in rows]
        _output(["ip", "hostname", "instance", "root_priority", "root_mac",
                 "bridge_priority", "bridge_mac", "is_root",
                 "tcn_count", "last_tcn_s", "updated_at"], display, csv_path)
        # Quick agreement summary so the operator knows at a glance whether
        # the network is converged.
        instances = {}
        for r in rows:
            instances.setdefault(r["instance"], set()).add(
                (r["root_priority"], r["root_mac"]))
        print()
        for inst, roots in sorted(instances.items()):
            if len(roots) == 1:
                prio, mac = next(iter(roots))
                print(f"{inst}: all {len(rows)} device(s) agree — root priority={prio} mac={mac}")
            else:
                print(f"{inst}: *** {len(roots)} distinct roots seen — see 'digest stp' or run 'monitor stp' for alert details ***")

    elif category == "stack-members":
        conn = _db()
        rows = conn.execute("""
            SELECT dm.ip, COALESCE(d.hostname, dm.ip) AS hostname,
                   d.chassis_type, dm.member_id, dm.role,
                   dm.serial, dm.mac, dm.model, dm.status, dm.updated_at
            FROM device_members dm
            LEFT JOIN devices d ON d.ip = dm.ip
            ORDER BY hostname, dm.member_id
        """).fetchall()
        conn.close()
        if not rows:
            print("No stack/chassis members recorded (run 'discover' or "
                  "'retest' to populate).")
            return
        display = [{
            "ip": r["ip"],
            "hostname": r["hostname"],
            "chassis_type": r["chassis_type"],
            "member_id": r["member_id"],
            "role": r["role"],
            "serial": r["serial"],
            "mac": r["mac"],
            "model": r["model"],
            "status": r["status"],
            "updated_at": r["updated_at"],
        } for r in rows]
        _output(["ip", "hostname", "chassis_type", "member_id", "role",
                 "serial", "mac", "model", "status", "updated_at"],
                display, csv_path)
        # Summary: count devices classified as stacks vs standalone-with-members.
        stacks = len({r["ip"] for r in rows if r["chassis_type"] == "stack"})
        standalones = len({r["ip"] for r in rows if r["chassis_type"] == "standalone"})
        print(f"\n{len(rows)} member(s) across {stacks} stack(s) + {standalones} "
              f"standalone device(s)")

    elif category == "mac-flux":
        # 3.8.9: surface mac_flux_alerts to operators directly. Default
        # shows ACTIVE alerts only ("what's firing right now?"); --all
        # includes cleared episodes for forensics. (3.8.10 fix: the
        # `--all` flag is now a proper argparse kwarg threaded through
        # handle_list — the 3.8.9 `bool(ip)` hack didn't work because
        # argparse rejected the bare positional with "unrecognized
        # arguments: all".)
        conn = _db(); conn.row_factory = sqlite3.Row
        try:
            sql = ("SELECT a.ip, COALESCE(d.hostname, a.ip) AS hostname, "
                   "       a.interface, a.distinct_macs, a.notified_at, "
                   "       COALESCE(a.cleared_at, '(active)') AS state "
                   "FROM mac_flux_alerts a "
                   "LEFT JOIN devices d ON d.ip = a.ip ")
            if not show_all:
                sql += "WHERE a.cleared_at IS NULL "
            sql += "ORDER BY a.cleared_at IS NULL DESC, a.notified_at DESC"
            rows = conn.execute(sql).fetchall()
        except sqlite3.OperationalError:
            print("(no mac_flux_alerts table — pre-3.8.0 DB?)")
            return
        if not rows:
            print("No active MAC-flux alerts.")
            if not show_all:
                print("(Use `show mac-flux all` to include cleared episodes.)")
            return
        _output(["ip", "hostname", "interface", "distinct_macs",
                 "notified_at", "state"], rows, csv_path)
        active = sum(1 for r in rows if r["state"] == "(active)")
        print(f"\n{len(rows)} alert(s) — {active} active, "
              f"{len(rows) - active} cleared")

    elif category == "all":
        rows = get_devices()
        if not rows:
            print("Database is empty.")
            return
        # list all sorts purely by IP (not status-grouped), for reports/CSV.
        rows = _sort_by_ip(rows)
        _output(["ip", "hostname", "dns_name", "base_mac", "firmware", "proto", "status", "fail_reason", "last_seen"], rows, csv_path)
        active = sum(1 for r in rows if r["status"] == "active")
        inactive = sum(1 for r in rows if r["status"] == "inactive")
        failed = sum(1 for r in rows if r["status"] == "failed")
        dupes = sum(1 for r in rows if r["status"] == "duplicate")
        print(f"\n{len(rows)} total: {active} active, {inactive} inactive, {failed} failed, {dupes} duplicate(s)")


_EXPORT_BANNER = (
    "# *** AUTO-GENERATED from database — do not edit manually ***\n"
    "# Changes here will be overwritten on next export.\n"
    "# Use the database (--list, -d, -r) to manage devices.\n"
    "#\n"
)


def handle_export(cfg):
    """Export device data from DB to flat files."""
    device_file = cfg["device_file"]
    base_dir = os.path.dirname(device_file)

    # Export active devices to devices.txt
    active = get_devices(status="active")
    if active:
        identity = {}
        for r in active:
            parts = []
            if r["hostname"]:
                parts.append(r["hostname"])
            if r["base_mac"]:
                parts.append(f"({r['base_mac']})")
            if parts:
                identity[r["ip"]] = " ".join(parts)
        ips = [r["ip"] for r in active]
        write_devices_file(device_file, ips, identity=identity)
        # Prepend export banner
        _prepend_export_banner(device_file)
        log.info("Exported %d active device(s) to %s", len(ips), device_file)
    else:
        log.info("No active devices to export.")

    # Export failed devices to failed_devices.txt
    failed = get_devices(status="failed")
    failed_file = os.path.join(base_dir, "failed_devices.txt")
    if failed:
        failures = [(r["ip"], r["fail_reason"] or "unknown") for r in failed]
        write_failed_file(failed_file, failures)
        _prepend_export_banner(failed_file)
        log.info("Exported %d failed device(s) to %s", len(failures), failed_file)

    # Export telnet report
    conn = _db()
    telnet_only = sorted(
        [r["ip"] for r in conn.execute(
            "SELECT ip FROM devices WHERE telnet_open=1 AND ssh_open=0 AND status='active'"
        ).fetchall()],
        key=_ip_sort_key,
    )
    telnet_and_ssh = sorted(
        [r["ip"] for r in conn.execute(
            "SELECT ip FROM devices WHERE telnet_open=1 AND ssh_open=1 AND status='active'"
        ).fetchall()],
        key=_ip_sort_key,
    )
    conn.close()
    telnet_report = os.path.join(base_dir, "telnet_devices.txt")
    write_telnet_report(telnet_report, telnet_only, telnet_and_ssh)
    if telnet_only or telnet_and_ssh:
        _prepend_export_banner(telnet_report)
        log.info("Exported telnet report to %s", telnet_report)


def _prepend_export_banner(filepath):
    """Prepend the auto-generated warning banner to an exported file."""
    with open(filepath, encoding="utf-8") as f:
        content = f.read()
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(_EXPORT_BANNER)
        f.write(content)


def handle_devices_filter(field, pattern, detail=False, csv_path=None):
    """`show devices ip|name <pattern>` — substring filter over the DB.

    field 'ip' matches the ip column; 'name' matches the switch hostname
    or the resolved DNS name (case-insensitive). detail=True prints full
    per-device detail instead of a listing table.
    """
    pat = pattern.lower()
    matches = []
    for r in get_devices():
        if field == "ip":
            if pattern in (r["ip"] or ""):
                matches.append(r)
        else:
            hn = (r["hostname"] or "").lower()
            dn = (r["dns_name"] if "dns_name" in r.keys() else "") or ""
            if pat in hn or pat in dn.lower():
                matches.append(r)
    if not matches:
        print(f"No devices match {field} '{pattern}'.")
        return
    if detail:
        for i, r in enumerate(matches):
            if i:
                print()
            handle_show(r["ip"])
        return
    _output(["ip", "hostname", "dns_name", "model", "base_mac", "status",
             "last_seen"], matches, csv_path)
    print(f"\n{len(matches)} device(s) matching {field} '{pattern}'")


def handle_show(ip):
    """Show full details for a single device including hardware inventory."""
    conn = _db()
    dev = conn.execute("SELECT * FROM devices WHERE ip = ?", (ip,)).fetchone()
    if not dev:
        print(f"Device {ip} not found in database.")
        conn.close()
        return

    # Basic info
    print(f"{'IP:':<18} {dev['ip']}")
    print(f"{'Hostname:':<18} {dev['hostname'] or '(unknown)'}")
    # dns_name column may be absent on very old DBs
    dns_name = dev["dns_name"] if "dns_name" in dev.keys() else None
    print(f"{'DNS name:':<18} {dns_name or '(unresolved)'}")
    print(f"{'Status:':<18} {dev['status']}")
    print(f"{'Model:':<18} {dev['model'] or '(unknown)'}")
    print(f"{'Serial:':<18} {dev['serial'] or '(unknown)'}")
    print(f"{'Firmware:':<18} {dev['firmware'] or '(unknown)'}")
    print(f"{'Base MAC:':<18} {dev['base_mac'] or '(unknown)'}")
    # chassis_type/member_count columns may be absent on very old DBs.
    chassis_type = dev["chassis_type"] if "chassis_type" in dev.keys() else None
    member_count = dev["member_count"] if "member_count" in dev.keys() else None
    if chassis_type or member_count:
        suffix = f" ({member_count} member{'s' if (member_count or 0) != 1 else ''})" if member_count else ""
        print(f"{'Chassis:':<18} {chassis_type or '(unknown)'}{suffix}")
    print(f"{'Protocol:':<18} {dev['proto'] or '(unknown)'}")
    print(f"{'Username:':<18} {dev['username'] or '(unknown)'}")
    print(f"{'First seen:':<18} {dev['first_seen']}")
    print(f"{'Last seen:':<18} {dev['last_seen']}")
    if dev["duplicate_of"]:
        print(f"{'Duplicate of:':<18} {dev['duplicate_of']}")
    if dev["fail_reason"]:
        print(f"{'Fail reason:':<18} {dev['fail_reason']}")

    # Backup history
    backups = conn.execute(
        "SELECT backed_up_at, changed, filename FROM backups WHERE ip = ? ORDER BY backed_up_at DESC LIMIT 5",
        (ip,)
    ).fetchall()
    if backups:
        print(f"\n--- Last {len(backups)} Backup(s) ---")
        for b in backups:
            status = "changed" if b["changed"] else "unchanged"
            print(f"  {b['backed_up_at']}  {status}  {b['filename']}")

    # Port flaps (physical up-from-down transitions since last 'clear flap')
    flap_rows = conn.execute("""
        SELECT interface, flap_count, first_seen, last_seen
        FROM port_flaps WHERE ip = ?
        ORDER BY flap_count DESC, interface
    """, (ip,)).fetchall()
    if flap_rows:
        total = sum(r["flap_count"] for r in flap_rows)
        print(f"\n--- Port Flaps ({len(flap_rows)} port(s), {total} flap(s)) ---")
        print(f"  {'FLAPS':>5}  {'INTERFACE':<16}  {'FIRST SEEN':<19}  {'LAST SEEN':<19}")
        for r in flap_rows:
            print(f"  {r['flap_count']:>5}  {r['interface']:<16}  "
                  f"{r['first_seen']:<19}  {r['last_seen']:<19}")

    # Stack / chassis members
    members = conn.execute("""
        SELECT member_id, role, serial, mac, model, status, updated_at
        FROM device_members WHERE ip = ? ORDER BY member_id
    """, (ip,)).fetchall()
    if members:
        print(f"\n--- Members ({len(members)}) ---")
        print(f"  {'#':>3}  {'ROLE':<12}  {'SERIAL':<15}  {'MAC':<18}  MODEL")
        for r in members:
            print(f"  {r['member_id']:>3}  "
                  f"{(r['role'] or '-'):<12}  "
                  f"{(r['serial'] or '-'):<15}  "
                  f"{(r['mac'] or '-'):<18}  "
                  f"{r['model'] or '-'}")

    # Hardware inventory
    hw = dev["hardware_info"]
    if hw:
        print(f"\n--- Hardware Inventory ---")
        print(hw)

    conn.close()


def handle_add(ip_arg):
    """Manually add device IP(s) to the database."""
    ips = [s.strip() for s in ip_arg.split(",") if s.strip()]
    added = 0
    for ip in ips:
        # Basic IP validation
        try:
            ipaddress.ip_address(ip)
        except ValueError:
            log.error("Invalid IP address: %s", ip)
            continue
        upsert_device(ip, status="active")
        log.info("Added %s to database", ip)
        added += 1
    if added:
        log.info("Added %d device(s) to database.", added)


def handle_remove(ip_arg):
    """Remove device IP(s) from the database."""
    # Special keyword: remove all failed
    if ip_arg.strip().lower() == "failed":
        count = remove_devices_by_status("failed")
        if count:
            log.info("Removed %d failed device(s) from database.", count)
        else:
            log.info("No failed devices to remove.")
        return

    # Special keyword: remove all duplicates
    if ip_arg.strip().lower() == "duplicates":
        count = remove_devices_by_status("duplicate")
        if count:
            log.info("Removed %d duplicate device(s) from database.", count)
        else:
            log.info("No duplicate devices to remove.")
        return

    ips = [s.strip() for s in ip_arg.split(",") if s.strip()]
    removed = 0
    for ip in ips:
        if remove_device(ip):
            log.info("Removed %s from database", ip)
            removed += 1
        else:
            log.warning("%s not found in database", ip)
    if removed:
        log.info("Removed %d device(s) from database.", removed)


def _detect_import_format(path):
    """Return 'csv' or 'list' based on extension and first non-comment line."""
    if path.lower().endswith(".csv"):
        return "csv"
    try:
        with open(path, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#"):
                    continue
                # CSV header almost always contains 'ip' (case-insensitive)
                # AND at least one comma.
                if "," in line and "ip" in line.lower():
                    return "csv"
                return "list"
    except OSError:
        pass
    return "list"


def _parse_csv_devices(path):
    """Parse a CSV file. Returns list of dicts with keys: ip, hostname, dns_name,
    model, status (any may be missing). 'ip' is required.

    Recognizes columns case-insensitively: ip, hostname, dns_name/dns,
    model, status.
    """
    import csv
    rows = []
    with open(path, encoding="utf-8", newline="") as f:
        reader = csv.DictReader(f)
        # Normalize fieldnames (lowercase, strip whitespace)
        if reader.fieldnames:
            reader.fieldnames = [(n or "").strip().lower() for n in reader.fieldnames]
        if not reader.fieldnames or "ip" not in reader.fieldnames:
            raise ValueError(
                f"CSV at {path} has no 'ip' column. Found: {reader.fieldnames}"
            )
        for raw in reader:
            row = {k: (v or "").strip() for k, v in raw.items() if k}
            ip = row.get("ip", "").strip()
            if not ip:
                continue
            # Support dns or dns_name
            dns_name = row.get("dns_name") or row.get("dns") or ""
            rows.append({
                "ip": ip,
                "hostname": row.get("hostname") or "",
                "dns_name": dns_name,
                "model": row.get("model") or "",
                "status": row.get("status") or "active",
            })
    return rows


def handle_import(cfg, path=None, fmt="auto"):
    """Import devices from a file into the database.

    path: explicit file path (overrides the configured device_file).
    fmt:  "auto" | "list" | "csv".
    """
    device_file = path if path else cfg["device_file"]
    if not os.path.exists(device_file):
        log.error("%s not found.", device_file)
        return

    if fmt == "auto":
        fmt = _detect_import_format(device_file)
    log.info("Importing from %s (format: %s)", device_file, fmt)

    if fmt == "csv":
        try:
            rows = _parse_csv_devices(device_file)
        except ValueError as e:
            log.error("%s", e)
            return
        if not rows:
            log.info("No devices found in %s.", device_file)
            return
        added = 0
        skipped = 0
        for row in rows:
            try:
                ipaddress.ip_address(row["ip"])
            except ValueError:
                log.warning("  skipping invalid IP: %s", row["ip"])
                skipped += 1
                continue
            upsert_device(
                row["ip"],
                hostname=row["hostname"] or None,
                dns_name=row["dns_name"] or None,
                model=row["model"] or None,
                status=row["status"] or "active",
                preserve_status=True,
            )
            added += 1
        log.info("Imported %d device(s) from %s%s.",
                 added, device_file,
                 f" ({skipped} skipped)" if skipped else "")
        return

    # --- plain list format ---
    ips = parse_devices(device_file)
    if not ips:
        log.info("No devices found in %s.", device_file)
        return
    added = 0
    skipped = 0
    for ip in ips:
        try:
            ipaddress.ip_address(ip)
        except ValueError:
            log.warning("  skipping invalid IP: %s", ip)
            skipped += 1
            continue
        upsert_device(ip, status="active", preserve_status=True)
        added += 1
    log.info("Imported %d device(s) from %s%s.",
             added, device_file,
             f" ({skipped} skipped)" if skipped else "")

    # Also import failed_devices.txt if it exists (only when importing the
    # default device_file — not when an explicit path is given).
    if not path:
        failed_file = os.path.join(os.path.dirname(device_file), "failed_devices.txt")
        if os.path.exists(failed_file):
            failed_ips = parse_failed_file(failed_file)
            for fip in failed_ips:
                try:
                    ipaddress.ip_address(fip)
                except ValueError:
                    continue
                upsert_device(fip, status="failed",
                              fail_reason="imported from failed_devices.txt",
                              preserve_status=True)
            log.info("Imported %d failed device(s) from %s.",
                     len(failed_ips), failed_file)


# ---------------------------------------------------------------------------
# Subcommand handlers (pure functions taking cfg / args as needed)
# ---------------------------------------------------------------------------

def handle_wipe_db():
    """Delete the active DB file after confirmation."""
    if not _acquire_advisory_lock("wipe"):
        return
    if not os.path.exists(DB_FILE):
        log.info("No database to wipe at %s", DB_FILE)
        return
    try:
        conn = sqlite3.connect(DB_FILE, timeout=30)
        dev_count = conn.execute("SELECT COUNT(*) FROM devices").fetchone()[0]
        bk_count = conn.execute("SELECT COUNT(*) FROM backups").fetchone()[0]
        conn.close()
        print(f"About to delete {DB_FILE}")
        print(f"  devices:         {dev_count}")
        print(f"  backup records:  {bk_count}")
    except sqlite3.DatabaseError:
        print(f"About to delete {DB_FILE} (not a valid netops DB)")
    resp = input("Delete this database? [y/N]: ").strip().lower()
    if resp != "y":
        print("Aborted.")
        return
    os.remove(DB_FILE)
    log.info("Deleted %s", DB_FILE)


def handle_cat_config(device, set_format=False):
    """Print the most recent saved config for a device to stdout.

    Resolves the device by IPv4 first, then hostname (case-insensitive).
    Exits 1 with a stderr message on miss — keeps stdout clean for
    'ssh netops@host netops cat-config X > local.cfg' usage.
    """
    import ipaddress
    conn = _db()
    conn.row_factory = sqlite3.Row
    try:
        ipaddress.IPv4Address(device)
        row = conn.execute(
            "SELECT ip, hostname FROM devices WHERE ip = ? LIMIT 1",
            (device,)).fetchone()
    except ValueError:
        row = conn.execute(
            "SELECT ip, hostname FROM devices "
            "WHERE LOWER(hostname) = LOWER(?) LIMIT 1",
            (device,)).fetchone()
    conn.close()
    if not row:
        print(f"cat-config: no device matches {device!r}", file=sys.stderr)
        sys.exit(1)
    ip = row["ip"]
    host = row["hostname"]
    # Try hostname_ip.cfg, fall back to ip.cfg
    candidates = []
    if host:
        if set_format:
            candidates.append(f"{host}_{ip}_set.cfg")
        candidates.append(f"{host}_{ip}.cfg")
    if set_format:
        candidates.append(f"{ip}_set.cfg")
    candidates.append(f"{ip}.cfg")
    for fname in candidates:
        path = os.path.join(CONFIGS_DIR, "current", fname)
        if os.path.isfile(path):
            with open(path, encoding="utf-8", errors="replace") as f:
                sys.stdout.write(f.read())
            return
    print(f"cat-config: no saved config found for {host or '?'} ({ip}) — "
          f"checked {', '.join(candidates)} in {CONFIGS_DIR}/current/",
          file=sys.stderr)
    sys.exit(1)


def _email_recipients():
    """Current [email] 'to' recipients from netops.conf, as a list."""
    import configparser
    cp = configparser.ConfigParser(interpolation=None)
    cp.optionxform = str
    try:
        cp.read(CONFIG_FILE, encoding="utf-8")
    except (configparser.Error, OSError):
        return []
    raw = cp.get("email", "to", fallback="") if cp.has_section("email") else ""
    return [a.strip() for a in raw.split(",") if a.strip()]


def _set_email_recipients(recipients):
    """Rewrite the [email] 'to' line in netops.conf, preserving comments.

    netops.conf is 0660 root:netops so the netops group can write it.
    Returns True on success.
    """
    new_to = "to = " + ", ".join(recipients)
    try:
        with open(CONFIG_FILE, encoding="utf-8") as f:
            lines = f.read().split("\n")
    except OSError as e:
        print(f"configure: cannot read {CONFIG_FILE}: {e}")
        return False
    out = []
    in_email = False
    wrote = False
    for line in lines:
        s = line.strip()
        if s.startswith("[") and s.endswith("]"):
            if in_email and not wrote:        # leaving [email], no 'to' seen
                out.append(new_to)
                wrote = True
            in_email = (s == "[email]")
            out.append(line)
            continue
        if (in_email and not wrote and "=" in s
                and s.split("=", 1)[0].strip() == "to"):
            out.append(new_to)
            wrote = True
            continue
        out.append(line)
    if in_email and not wrote:                # [email] was the final section
        out.append(new_to)
        wrote = True
    if not wrote:                             # no [email] section at all
        if out and out[-1].strip():
            out.append("")
        out += ["[email]", new_to]
    try:
        with open(CONFIG_FILE, "w", encoding="utf-8") as f:
            f.write("\n".join(out))
    except OSError as e:
        print(f"configure: cannot write {CONFIG_FILE}: {e}")
        print("  netops.conf must be group-writable (0660 root:netops).")
        return False
    return True


def handle_configure(tokens):
    """`configure` — change settings in netops.conf (currently email)."""
    if tokens[0] != "email":
        print(f"configure: unknown setting '{tokens[0]}' (expected 'email')")
        return
    rest = tokens[1:]
    if not rest:
        print("Usage: configure email {add <address>|remove <address>|list}")
        return
    action = rest[0]
    if action == "list":
        recips = _email_recipients()
        if recips:
            for r in recips:
                print(f"  {r}")
            print(f"\n{len(recips)} email recipient(s)")
        else:
            print("No email recipients configured ([email] 'to' is empty).")
        return
    if action not in ("add", "remove"):
        print(f"configure email: unknown action '{action}' (add|remove|list)")
        return
    if len(rest) != 2:
        print(f"Usage: configure email {action} <address>")
        return
    addr = rest[1]
    if "@" not in addr:
        print(f"configure email {action}: '{addr}' is not an email address")
        return
    recips = _email_recipients()
    if action == "add":
        if addr in recips:
            print(f"{addr} is already a recipient.")
            return
        recips.append(addr)
    else:
        if addr not in recips:
            print(f"{addr} is not a current recipient — "
                  "'configure email list' shows them.")
            return
        recips.remove(addr)
    if _set_email_recipients(recips):
        verb = "Added" if action == "add" else "Removed"
        print(f"{verb} {addr}.")
        print(f"Recipients now: {', '.join(recips) if recips else '(none)'}")


def handle_check_config(cfg):
    """Print effective config, system limits, and DB status."""
    caps = cfg["_thread_caps"]
    fd = caps["fd_limit"]
    mem = caps["total_mem_mb"]
    scan_cap = caps["scan_cap"]

    print(f"netops v{__version__}")
    print()
    print("--- System Limits ---")
    print(f"  OS:                  {sys.platform}")
    print(f"  ssh:                 {'found' if _HAS_SSH else 'NOT FOUND (required)'}")
    print(f"  telnet:              {'found' if _HAS_TELNET else 'not found (telnet fallback disabled)'}")
    print(f"  File descriptors:    {fd}  (raise with 'ulimit -n <value>')")
    print(f"  System memory:       {mem} MB  ({mem / 1024:.1f} GB)")
    print()
    print("--- Thread Limits ---")
    print(f"  scan_threads:")
    print(f"    configured:        {caps['raw_scan']}")
    print(f"    max (fd-based):    {scan_cap}  ({fd} fd - 100 reserved)")
    print(f"    effective:         {cfg['scan_threads']}")
    print(f"  backup_threads:")
    fd_bk = max((fd - 100) // 3, 10)
    mem_bk = max(int(mem * _SSH_MEM_FRACTION / _SSH_MEM_MB), 10)
    print(f"    configured:        {caps['raw_backup']}")
    print(f"    max (fd-based):    {fd_bk}  (({fd} fd - 100) / 3 fds per session)")
    print(f"    max (mem-based):   {mem_bk}  (25% of {mem} MB / {_SSH_MEM_MB} MB per session)")
    print(f"    effective:         {cfg['backup_threads']}  (limited by {'fd' if fd_bk <= mem_bk else 'memory'})")
    print()
    print("--- Database ---")
    if os.path.exists(DB_FILE):
        conn = _db()
        dev_count = conn.execute("SELECT COUNT(*) FROM devices").fetchone()[0]
        bk_count = conn.execute("SELECT COUNT(*) FROM backups").fetchone()[0]
        conn.close()
        print(f"  path:                {DB_FILE}")
        print(f"  devices:             {dev_count}")
        print(f"  backup records:      {bk_count}")
    else:
        print(f"  path:                {DB_FILE} (not created yet)")
    print()
    print("--- Config File ---")
    print(f"  path:                {CONFIG_FILE}")
    print(f"  storage_mode:        {cfg['storage_mode']}")
    if cfg["storage_mode"] == "git":
        print(f"  git_remote_url:      {cfg.get('git_remote_url') or '(none — local only)'}")
        print(f"  git_branch:          {cfg.get('git_branch') or 'main'}")
    print(f"  log_format:          {cfg['log_format']}")
    print(f"  device_file:         {cfg['device_file']}")
    print(f"  subnets:             {', '.join(cfg['subnets']) or '(none)'}")
    print(f"  ports:               {', '.join(str(p) for p in cfg['ports'])}")
    print(f"  scan timeout:        {cfg['timeout']}s")
    print(f"  usernames:           {', '.join(cfg['usernames'])}")
    pw_src = cfg.get("_pw_source", "none")
    pw_count = len(cfg.get("password_list", []))
    if pw_src == "env":
        print(f"  passwords:           {pw_count} (from NETOPS_PASSWORDS env var)")
    elif pw_src == "secrets.conf":
        print(f"  passwords:           {pw_count} (from {SECRETS_FILE})")
    else:
        print(f"  passwords:           (not set — add them to {os.path.basename(SECRETS_FILE)})")
    overrides = [k for k in cfg["passwords"] if k != "default"]
    if overrides:
        print(f"  password overrides:  {', '.join(overrides)}")
    user_pw_map = cfg.get("user_passwords") or {}
    if user_pw_map:
        print(f"  user_passwords:      {len(user_pw_map)} user(s) mapped")
        for user, pws in user_pw_map.items():
            print(f"    {user}: {len(pws)} password(s) — bulk list skipped for this user")


def run_probe(cfg, status_filter="all", csv_path=None, timeout=2.0, threads=100):
    """TCP reachability check for IPs already in the DB.

    Updates ssh_open/telnet_open columns. Does NOT change 'status' — a
    previously failed device that's now reachable is still failed until
    re-tested. Writes nothing to the DB if there are no matching rows.
    """
    conn = _db()
    if status_filter == "all":
        rows = conn.execute(
            "SELECT ip, hostname, dns_name, status FROM devices "
            "WHERE status != 'duplicate'"
        ).fetchall()
    else:
        rows = conn.execute(
            "SELECT ip, hostname, dns_name, status FROM devices WHERE status = ?",
            (status_filter,),
        ).fetchall()
    conn.close()

    rows = _sort_by_ip(rows)
    if not rows:
        log.info("No devices match filter '%s'.", status_filter)
        return

    ports = cfg.get("ports") or [22, 23]
    total = len(rows)
    log.info("Probing %d device(s) on ports %s (timeout %.1fs)...",
             total, ",".join(str(p) for p in ports), timeout)

    # Worker: return (ip, list_of_open_ports)
    def _probe_one(ip):
        return (ip, check_ports(ip, ports, timeout))

    results_by_ip = {}
    done = 0
    with ThreadPoolExecutor(max_workers=threads) as pool:
        futures = {pool.submit(_probe_one, r["ip"]): r["ip"] for r in rows}
        for future in as_completed(futures):
            ip, open_ports = future.result()
            results_by_ip[ip] = open_ports
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rProbing... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    # Update DB + build summary
    conn = _db()
    reachable = 0
    unreachable = 0
    ssh_count = 0
    telnet_count = 0
    both_count = 0
    display_rows = []
    for r in rows:
        ip = r["ip"]
        open_ports = results_by_ip.get(ip, [])
        has_ssh = 22 in open_ports
        has_telnet = 23 in open_ports
        conn.execute(
            "UPDATE devices SET ssh_open = ?, telnet_open = ? WHERE ip = ?",
            (1 if has_ssh else 0, 1 if has_telnet else 0, ip),
        )
        if open_ports:
            reachable += 1
            if has_ssh:
                ssh_count += 1
            if has_telnet:
                telnet_count += 1
            if has_ssh and has_telnet:
                both_count += 1
            reach_flag = "yes"
        else:
            unreachable += 1
            reach_flag = "no"
        display_rows.append({
            "ip": ip,
            "hostname": r["hostname"] or "",
            "dns_name": r["dns_name"] or "" if "dns_name" in r.keys() else "",
            "status": r["status"],
            "reachable": reach_flag,
            "ssh": "yes" if has_ssh else "no",
            "telnet": "yes" if has_telnet else "no",
        })
    conn.commit()
    conn.close()

    log.info("--- Probe Summary ---")
    log.info("Reachable:    %d  (SSH: %d, telnet: %d, both: %d)",
             reachable, ssh_count, telnet_count, both_count)
    log.info("Unreachable:  %d", unreachable)
    log.info("Total:        %d", total)

    if csv_path:
        _write_csv(csv_path,
                   ["ip", "hostname", "dns_name", "status",
                    "reachable", "ssh", "telnet"],
                   display_rows)


def run_test(cfg):
    """Test credentials against all devices (no backup)."""
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        sys.exit(1)

    device_file = cfg["device_file"]
    db_devices = get_devices(status="active") + get_devices(status="inactive")
    if db_devices:
        ips = [r["ip"] for r in db_devices]
    elif os.path.exists(device_file):
        ips = parse_devices(device_file)
        if ips:
            log.info("Database empty — reading from %s", device_file)
    else:
        ips = []

    if not ips:
        log.info("No devices to test. Run 'discover' first or 'import'.")
        sys.exit(0)

    # Resolve DNS for any devices without a name yet (first-test or
    # post-import without a preceding discover/scan).
    ips_needing_dns = []
    if db_devices:
        for r in db_devices:
            existing_dns = r["dns_name"] if "dns_name" in r.keys() else None
            if not existing_dns:
                ips_needing_dns.append(r["ip"])
    else:
        # No DB rows yet — resolve all
        ips_needing_dns = list(ips)

    if ips_needing_dns:
        log.info("Resolving reverse DNS for %d device(s)...", len(ips_needing_dns))
        dns_map = resolve_dns_batch(ips_needing_dns)
        conn = _db()
        for ip, name in dns_map.items():
            if name:
                conn.execute("UPDATE devices SET dns_name = ? WHERE ip = ?",
                             (name, ip))
        conn.commit()
        conn.close()

    def test_one(ip):
        conn = _db()
        row = conn.execute(
            "SELECT username, password_hash FROM devices WHERE ip = ? AND status = 'active'",
            (ip,)).fetchone()
        conn.close()
        known_user = row["username"] if row else None
        known_pw_hash = row["password_hash"] if row else None

        child, proto, username, pw_hash, ssh_open, telnet_open = connect_device(
            ip, cfg["usernames"], passwords["default"],
            password_list=cfg.get("password_list"),
            known_username=known_user, known_password_hash=known_pw_hash,
            user_passwords=cfg.get("user_passwords"),
        )
        if child is not None:
            # Defensive: any pexpect/identification failure here (e.g. a
            # Ruckus FastIron `enable` reauth confusing the session into
            # User Name: / Password: state that doesn't match PROMPT_RE)
            # leaves a damaged session — _detect_platform raises TIMEOUT.
            # Catch it: credentials WORKED (connect_device returned), so
            # this is still a PASS, just with empty identification.
            hostname = ""
            base_mac = ""
            serial = ""
            firmware = ""
            model = ""
            try:
                platform = _detect_platform(child)
                hostname = get_hostname(child)
                if platform == "junos":
                    ver = send_command(child, "show version | no-more", timeout=15)
                    hw = send_command(child, "show chassis hardware | no-more", timeout=15)
                    mac_out = send_command(child, "show chassis mac-addresses | no-more", timeout=15)
                    id_output = ver + "\n" + hw + "\n" + mac_out
                elif platform == "fastiron":
                    # Ruckus FastIron has no `show system`; `show version`
                    # alone contains HW (model), SW (firmware) and Serial #.
                    id_output = send_command(child, "show version", timeout=15)
                else:
                    sys_out = send_command(child, "show system", timeout=10)
                    ver_out = send_command(child, "show version", timeout=10)
                    id_output = sys_out + "\n" + ver_out
                dev_id = get_device_id(id_output)
                base_mac = dev_id.get("base_mac", "")
                serial = dev_id.get("serial", "")
                firmware = dev_id.get("firmware", "")
                model = dev_id.get("model", "")
                if not model and platform not in ("junos", "fastiron"):
                    for sys_line in id_output.splitlines():
                        sys_line = sys_line.strip()
                        if "system name" in sys_line.lower():
                            continue
                        if any(kw in sys_line.lower() for kw in
                               ["procurve", "aruba", "cisco", "juniper",
                                "palo", "switch", "router", "firewall"]):
                            model = sys_line.strip("; ")
                            break
            except Exception as e:
                log.debug("test_one: %s — identification failed: %s", ip, e)
            finally:
                try:
                    disconnect(child)
                except Exception:
                    pass
            label = hostname or ip
            if model:
                label = f"{label} ({model})"
            mac_info = f", mac={base_mac}" if base_mac else ""
            info = {"hostname": hostname, "base_mac": base_mac, "serial": serial,
                    "firmware": firmware, "model": model, "proto": proto,
                    "username": username, "password_hash": pw_hash,
                    "ssh_open": ssh_open, "telnet_open": telnet_open}
            return (ip, f"PASS proto={proto}, user={username}"
                        f"{mac_info}, host={label}", True, info)
        else:
            reason = classify_fail_reason(ssh_open, telnet_open)
            return (ip, f"FAIL — {reason}", False,
                    {"hostname": None, "base_mac": "", "serial": "",
                     "firmware": "", "model": "", "proto": None,
                     "username": None, "password_hash": None,
                     "ssh_open": ssh_open, "telnet_open": telnet_open,
                     "fail_reason": reason})

    results = []
    total = len(ips)
    done = 0
    with ThreadPoolExecutor(max_workers=cfg["backup_threads"]) as pool:
        futures = {pool.submit(test_one, ip): ip for ip in ips}
        for future in as_completed(futures):
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rTesting credentials... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()
            ip_for_future = futures[future]
            try:
                result = future.result()
            except Exception as e:
                log.error("test_one(%s) raised unhandled %s: %s",
                          ip_for_future, type(e).__name__, e)
                result = (ip_for_future,
                          f"FAIL — internal error: {type(e).__name__}",
                          False,
                          {"hostname": None, "base_mac": "", "serial": "",
                           "firmware": "", "model": "", "proto": None,
                           "username": None, "password_hash": None,
                           "ssh_open": None, "telnet_open": None,
                           "fail_reason": f"internal error: {type(e).__name__}"})
            results.append(result)
            # Incremental upsert so a crash later in the run doesn't lose
            # progress. The final batch below re-upserts the same data
            # (idempotent).
            try:
                _ip, _msg, ok, info = result
                if ok:
                    upsert_device(_ip, base_mac=info.get("base_mac"),
                                  hostname=info.get("hostname"),
                                  model=info.get("model"),
                                  serial=info.get("serial"),
                                  firmware=info.get("firmware"),
                                  proto=info.get("proto"),
                                  username=info.get("username"),
                                  password_hash=info.get("password_hash"),
                                  ssh_open=info.get("ssh_open"),
                                  telnet_open=info.get("telnet_open"),
                                  status="active")
                else:
                    upsert_device(_ip, status="inactive",
                                  fail_reason=info.get("fail_reason")
                                              or "all connection attempts failed")
            except Exception as e:
                log.debug("test incremental upsert failed: %s", e)

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    results.sort(key=lambda r: ipaddress.ip_address(r[0]))
    passed_count = 0
    failed_count = 0
    mac_groups = {}
    for ip, result, ok, info in results:
        print(f"  {ip}: {result}")
        if ok:
            passed_count += 1
            base_mac = info["base_mac"]
            if base_mac:
                mac_groups.setdefault(base_mac, []).append(ip)
            upsert_device(ip, base_mac=info["base_mac"],
                          hostname=info["hostname"], model=info.get("model"),
                          serial=info.get("serial"), firmware=info.get("firmware"),
                          hardware_info=info.get("hardware_info"),
                          proto=info["proto"], username=info["username"],
                          password_hash=info.get("password_hash"),
                          ssh_open=info.get("ssh_open"),
                          telnet_open=info.get("telnet_open"),
                          status="active")
        else:
            failed_count += 1
            upsert_device(ip, status="inactive",
                          fail_reason=info.get("fail_reason")
                                      or "all connection attempts failed")

    dupes = {mac: ips_list for mac, ips_list in mac_groups.items() if len(ips_list) > 1}
    if dupes:
        log.info("--- Duplicate IPs (same physical switch) ---")
        for mac, ips_list in dupes.items():
            log.info("  %s: %s", mac, ", ".join(ips_list))

    log.info("Passed: %d  Failed: %d", passed_count, failed_count)


def run_backup(cfg, sanitize=True):
    """Back up configs for all devices in the active DB (+ inactive fallback)."""
    # Backup fires daily at 01:30 and collided at :30 with the per-minute
    # monitor-stp tick holding the 'ssh' lock. A non-blocking acquire made
    # the nightly backup silently skip whenever it lost that race. Wait up
    # to 90 s (one stp tick is worst-case ~80 s) rather than miss the run.
    if not _acquire_advisory_lock("ssh", wait=90):
        return
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        sys.exit(1)

    device_file = cfg["device_file"]
    storage_mode = cfg["storage_mode"]

    # Backup only targets devices that successfully authenticated at some point.
    # Devices in 'inactive' (failed auth) are skipped — use 'retest' to
    # promote them back to 'active' once the underlying issue is fixed.
    db_devices = get_devices(status="active")
    if db_devices:
        ips = [r["ip"] for r in db_devices]
    elif os.path.exists(device_file):
        ips = parse_devices(device_file)
        if ips:
            log.info("Database empty — reading from %s", device_file)
    else:
        ips = []

    if not ips:
        log.info("No devices to back up. Run 'discover' first or 'import'.")
        sys.exit(0)

    os.makedirs(CONFIGS_DIR, exist_ok=True)

    successes = []
    failures = []

    def backup_one(ip):
        conn = _db()
        row = conn.execute(
            "SELECT username, password_hash FROM devices WHERE ip = ? AND status = 'active'",
            (ip,)).fetchone()
        conn.close()
        known_user = row["username"] if row else None
        known_pw_hash = row["password_hash"] if row else None

        _t0 = time.monotonic()
        _started_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        try:
            filename, config, info = backup_device(
                ip, passwords["default"], cfg["usernames"],
                sanitize=sanitize,
                password_list=cfg.get("password_list"),
                known_username=known_user, known_password_hash=known_pw_hash,
                user_passwords=cfg.get("user_passwords"),
            )
            _record_op("backup_device",
                       started_at=_started_at,
                       duration_ms=int((time.monotonic() - _t0) * 1000),
                       success=True, ip=ip, reason="backup")
            return (ip, filename, config, None, info)
        except Exception as e:
            _record_op("backup_device",
                       started_at=_started_at,
                       duration_ms=int((time.monotonic() - _t0) * 1000),
                       success=False, ip=ip, reason="backup",
                       error=str(e)[:200])
            return (ip, None, None, e,
                    {"hostname": None, "base_mac": "", "serial": "",
                     "firmware": "", "model": "", "hardware_info": "",
                     "proto": None, "username": None, "password_hash": None})

    results = []
    total = len(ips)
    done = 0
    with ThreadPoolExecutor(max_workers=cfg["backup_threads"]) as pool:
        futures = {pool.submit(backup_one, ip): ip for ip in ips}
        for future in as_completed(futures):
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rBacking up devices... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()
            results.append(future.result())

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    results.sort(key=lambda r: ipaddress.ip_address(r[0]))
    seen_macs = {}
    skipped = []
    for ip, filename, config, error, info in results:
        if error:
            log.error("  %s FAILED: %s", ip, error)
            failures.append(ip)
            upsert_device(ip, status="inactive", fail_reason=str(error))
            continue

        base_mac = info["base_mac"]

        if base_mac and base_mac in seen_macs:
            log.info("  %s SKIPPED (duplicate of %s, mac %s)",
                     ip, seen_macs[base_mac], base_mac)
            skipped.append(ip)
            upsert_device(ip, base_mac=base_mac, hostname=info["hostname"],
                          firmware=info.get("firmware"),
                          hardware_info=info.get("hardware_info"),
                          proto=info["proto"], username=info["username"],
                          password_hash=info.get("password_hash"),
                          ssh_open=info.get("ssh_open"),
                          telnet_open=info.get("telnet_open"),
                          status="duplicate", duplicate_of=seen_macs[base_mac])
            continue
        if base_mac:
            seen_macs[base_mac] = ip

        changed = False
        if storage_mode == "local":
            changed = save_config_local(filename, config)
            config_set = info.get("config_set")
            if config_set:
                set_filename = os.path.splitext(filename)[0] + "_set.cfg"
                save_config_local(set_filename, config_set)
            if changed:
                log.info("  %s OK (config updated)", ip)
            else:
                log.info("  %s OK (no changes)", ip)
        else:
            config_path = os.path.join(CONFIGS_DIR, filename)
            with open(config_path, "w", encoding="utf-8") as f:
                f.write(config)
            config_set = info.get("config_set")
            if config_set:
                set_path = os.path.splitext(config_path)[0] + "_set.cfg"
                with open(set_path, "w", encoding="utf-8") as f:
                    f.write(config_set)
            changed = True
            log.info("  %s OK", ip)

        successes.append(filename)

        config_hash = hashlib.sha256(config.encode()).hexdigest()
        record_backup(ip, base_mac=base_mac, hostname=info["hostname"],
                      filename=filename, config_hash=config_hash,
                      changed=changed)
        upsert_device(ip, base_mac=base_mac, hostname=info["hostname"],
                      serial=info.get("serial"), firmware=info.get("firmware"),
                      hardware_info=info.get("hardware_info"),
                      proto=info["proto"], username=info["username"],
                      password_hash=info.get("password_hash"),
                      ssh_open=info.get("ssh_open"),
                      telnet_open=info.get("telnet_open"),
                      status="active")

    if storage_mode == "git" and successes:
        try:
            git_commit_changes(
                successes,
                remote_url=cfg.get("git_remote_url", ""),
                branch=cfg.get("git_branch", "main"))
        except Exception as e:
            log.error("Git commit failed: %s", e)

    log.info("--- Summary ---")
    log.info("Success:    %d", len(successes))
    if skipped:
        log.info("Duplicates: %d (same switch, different IP)", len(skipped))
    log.info("Failed:     %d", len(failures))
    if failures:
        log.info("Failed devices: %s", ", ".join(failures))


def _format_outage_duration(alerted_at_str):
    """Render an alerted_at timestamp as '2h 35m' since now. '?' on parse failure."""
    try:
        t0 = datetime.strptime(alerted_at_str, "%Y-%m-%d %H:%M:%S")
        sec = max(0, int((datetime.now() - t0).total_seconds()))
        h, rem = divmod(sec, 3600)
        m, _ = divmod(rem, 60)
        return f"{h}h {m}m" if h else f"{m}m"
    except (TypeError, ValueError):
        return "?"


def _classify_reachability(ip, timeout=3.0):
    """Quick TCP probe to confirm/refute an SNMP-unreachable signal.

    Reuses check_ports (the same helper run_probe uses). Returns
    {'ssh': bool, 'telnet': bool}. Both False is the 'switch is dark'
    signal; either True means SNMP-specific issue (ACL, community,
    daemon) on an otherwise-reachable box.
    """
    open_ports = check_ports(ip, [22, 23], timeout)
    return {"ssh": 22 in open_ports, "telnet": 23 in open_ports}


def _send_unreachable_alert(cfg, downed):
    """Email a batched 'netops can't monitor' alert.

    Triggered after N consecutive failed polls. In auto poll mode this
    means BOTH the SNMP fast-path AND the SSH/CLI fallback failed to
    return STP data — not just a transient SNMP hiccup. The TCP probe
    (port 22 / 23 banner check) only confirms whether a daemon is
    listening; it does NOT mean SSH is usable for monitoring (creds,
    parser support, auth-rate limits can all still be broken).

    downed: list of dicts with keys ip, hostname, model, fails,
    last_ok, last_diag, probe (output of _classify_reachability).
    Partitions into 'no protocol responding' (TCP closed on both 22
    and 23) and 'daemon listens but unusable' (banner open, data
    collection still failing) so the operator can triage at a glance.
    """
    no_banner = [d for d in downed
                 if not d["probe"]["ssh"] and not d["probe"]["telnet"]]
    banner_up = [d for d in downed
                 if d["probe"]["ssh"] or d["probe"]["telnet"]]

    if no_banner and not banner_up:
        subject = (f"[netops] {len(no_banner)} switch(es) UNREACHABLE — "
                   f"no protocol responding")
    elif banner_up and not no_banner:
        subject = (f"[netops] netops can't monitor {len(banner_up)} switch(es) — "
                   f"SNMP+SSH/CLI both failed (daemon listens)")
    else:
        subject = (f"[netops] {len(no_banner)} unreachable + "
                   f"{len(banner_up)} unmonitorable (daemon listens)")

    lines = []
    if no_banner:
        lines.append(f"=== Unreachable — no protocol responding ({len(no_banner)}) ===")
        lines.append("SNMP poll failed AND TCP/22 + TCP/23 both closed. "
                     "Likely: switch off the network, mgmt VLAN/uplink down, or rebooting.")
        lines.append("")
        for d in no_banner:
            lines.append(f"  {d['hostname'] or d['ip']} ({d['ip']})  [{d['model'] or '?'}]")
            lines.append(f"    consecutive-fails: {d['fails']}  last-ok: {d['last_ok'] or '(never)'}")
            if d["last_diag"]:
                lines.append(f"    last-error: {d['last_diag']}")
        lines.append("")
    if banner_up:
        lines.append(f"=== Can't monitor — SNMP AND SSH/CLI both failed; "
                     f"TCP banner open ({len(banner_up)}) ===")
        lines.append("netops tried SNMP, then fell back to SSH/CLI; both failed. "
                     "TCP/22 (or /23) listening does NOT mean SSH is usable — "
                     "credentials may be wrong, the platform may be unsupported by "
                     "our parsers, or the device may be config-locked.")
        lines.append("Likely: missing/wrong creds, unsupported model, SNMP not "
                     "configured, or device-side config issue. Operator action "
                     "required — netops cannot see this switch's STP state.")
        lines.append("")
        for d in banner_up:
            banner = []
            if d["probe"]["ssh"]:    banner.append("SSH/22")
            if d["probe"]["telnet"]: banner.append("Telnet/23")
            lines.append(f"  {d['hostname'] or d['ip']} ({d['ip']})  [{d['model'] or '?'}]  "
                         f"TCP banner: {', '.join(banner)}")
            lines.append(f"    consecutive-fails: {d['fails']}  last-ok: {d['last_ok'] or '(never)'}")
            if d["last_diag"]:
                lines.append(f"    last-error: {d['last_diag']}")
        lines.append("")

    # Per-device upstream topology trace — Slice A. Walks the cached
    # topology_edges from each downed device toward the root bridge and
    # reports recent activity at each hop. Catches the common "the
    # uplink to this switch flapped" failure mode without the operator
    # needing to mentally walk the topology themselves.
    for d in downed:
        trace_lines = _format_topology_trace_lines(
            d["ip"], d["hostname"], max_hops=4, activity_window_min=5)
        if trace_lines:
            lines.append(f"=== Upstream trace from {d['hostname'] or d['ip']} ({d['ip']}) ===")
            lines.extend(trace_lines)
            lines.append("")

    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("Unreachable alert email %s (%d switch(es))", detail, len(downed))
    else:
        log.error("Unreachable alert email failed: %s", detail)


def _send_recovery_alert(cfg, recovered):
    """Email a batched 'switch recovered' alert.

    recovered: list of dicts with keys ip, hostname, model, alerted_at, downtime.
    """
    subject = f"[netops] {len(recovered)} switch(es) recovered"
    lines = ["Previously-unreachable switch(es) are responding again.", ""]
    for d in recovered:
        lines.append(f"  {d['hostname'] or d['ip']} ({d['ip']})  [{d['model'] or '?'}]")
        lines.append(f"    down since: {d['alerted_at']}  downtime: {d['downtime']}")
    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("Recovery alert email %s (%d switch(es))", detail, len(recovered))
    else:
        log.error("Recovery alert email failed: %s", detail)


# ---------------------------------------------------------------------------
# Topology mapping — fleet-wide LLDP scrape, populates topology_edges so the
# alert path can walk one or more hops upstream from an affected device.
# ---------------------------------------------------------------------------

def _parse_lldp_neighbors_bulk(text, platform):
    """Parse a fleet-style 'show lldp neighbors' output into a list of dicts.

    Each dict has: local_port, neighbor_chassis, neighbor_sysname,
    neighbor_port, neighbor_port_desc. Best-effort regex-based parsing —
    vendor table formats differ; we tolerate column-order changes by
    keying off the header row when we can detect one.

    Skips internal stack-member LLDP rows on Aruba-CX 6200/8325 stacks
    (interface format X/M/N where M is a member index — the stack
    members peer with each other internally and aren't useful for
    cross-switch topology).
    """
    rows = []
    if not text:
        return rows
    lines = text.splitlines()
    if platform == "junos":
        # Format:
        #   Local Interface    Parent Interface  Chassis Id          Port info        System Name
        #   ge-0/0/1           -                 ec:eb:b8:11:22:33   ge-1/0/1         AG-BDF-0
        # Junos returns logical iface names ('ge-0/0/1.0'); we canonicalize
        # to the physical name so the walk's stp_state lookup (which also
        # canonicalizes) finds the matching edge.
        for line in lines:
            line = line.rstrip()
            if not line or line.lstrip().startswith("Local Interface"):
                continue
            m = re.match(
                r"^\s*((?:ge|xe|et|fe|me|ae|lo|ms)-\S+|\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s*(.*)$",
                line)
            if not m:
                continue
            local, parent, chassis, port, sysname = m.groups()
            if local in ("ge-X/Y/Z",):
                continue
            local = local.split(".", 1)[0]
            rows.append({
                "local_port":         local,
                "neighbor_chassis":   chassis,
                "neighbor_sysname":   sysname.strip() or "",
                "neighbor_port":      port,
                "neighbor_port_desc": "",
            })
    elif platform == "aruba-cx":
        # Format:
        #   LOCAL PORT  | NEIGHBOR ID         | NEIGHBOR NAME | NEIGHBOR PORT
        #   1/1/27      | ec:eb:b8:11:22:33   | OTHER-SWITCH  | 1/1/24
        for line in lines:
            line = line.rstrip()
            if "|" not in line:
                continue
            parts = [p.strip() for p in line.split("|")]
            if len(parts) < 4:
                continue
            if parts[0].upper() in ("LOCAL PORT", "LOCAL"):
                continue
            local, chassis, sysname, neigh_port = parts[0], parts[1], parts[2], parts[3]
            if not local or not re.match(r"^\d+/\d+/\d+", local):
                continue
            rows.append({
                "local_port":         local,
                "neighbor_chassis":   chassis,
                "neighbor_sysname":   sysname,
                "neighbor_port":      neigh_port,
                "neighbor_port_desc": "",
            })
    else:  # procurve
        # Format (after a separator '----' line):
        #   LocalPort | ChassisId               PortId PortDescr SysName
        #   1         | ec eb b8 11 22 33       24     1/1/24    OTHER-SWITCH
        in_table = False
        for line in lines:
            line = line.rstrip()
            if re.match(r"^\s*[-+\s]+$", line):
                in_table = True
                continue
            if not in_table or not line.strip():
                continue
            parts = re.split(r"\s*\|\s*", line, maxsplit=1)
            if len(parts) != 2:
                continue
            local = parts[0].strip()
            rest = parts[1]
            # ChassisId is "aa bb cc dd ee ff" (6 hex pairs, space separated)
            m = re.match(
                r"^((?:[0-9a-fA-F]{2}\s+){5}[0-9a-fA-F]{2})\s+(\S+)\s+(\S+)\s+(.*)$",
                rest)
            if not m:
                continue
            chassis_spaced, port_id, port_desc, sysname = m.groups()
            chassis = re.sub(r"\s+", "", chassis_spaced)
            rows.append({
                "local_port":         local,
                "neighbor_chassis":   chassis,
                "neighbor_sysname":   sysname.strip(),
                "neighbor_port":      port_id,
                "neighbor_port_desc": port_desc.strip(),
            })
    return rows


def _resolve_edge_neighbor_ip(neighbor_chassis, neighbor_sysname, conn):
    """Return the IP of a topology neighbor if it's in our devices table.

    Match priority: chassis MAC -> base_mac, then sysname -> hostname.
    Same approach as Phase A's _resolve_neighbor_in_db but takes an
    already-open connection.
    """
    norm = _normalize_mac(neighbor_chassis or "")
    if norm:
        for r in conn.execute(
            "SELECT ip, base_mac FROM devices "
            "WHERE base_mac IS NOT NULL AND status IN ('active','inactive')"
        ).fetchall():
            if _normalize_mac(r["base_mac"]) == norm:
                return r["ip"]
    if neighbor_sysname:
        r = conn.execute(
            "SELECT ip FROM devices WHERE LOWER(hostname) = LOWER(?) "
            "AND status IN ('active','inactive') LIMIT 1",
            (neighbor_sysname.strip(),)).fetchone()
        if r:
            return r["ip"]
    return None


def _collect_lldp_snmp(ip, cred, snmp_cfg):
    """Walk LLDP-MIB::lldpRemTable via SNMP and return the same row dicts
    as _parse_lldp_neighbors_bulk. Works regardless of platform / login
    shell — only needs a working SNMP cred. Empty list on any failure.

    lldpRemTable is indexed by (lldpRemTimeMark, lldpRemLocalPortNum,
    lldpRemIndex). We map lldpRemLocalPortNum → human port name via
    ifName (preferred) or ifDescr (fallback) from IF-MIB. Most modern
    gear uses ifIndex as the local-port-num so this lookup just works.
    """
    timeout = int(snmp_cfg.get("timeout") or 5)
    retries = int(snmp_cfg.get("retries") or 1)
    try:
        # 1) Walk port-num → name (try ifName first, fall back to ifDescr)
        try:
            port_names = _snmp_walk_dict(cred, ip, "1.3.6.1.2.1.31.1.1.1.1",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)
        except SnmpCollectError:
            port_names = _snmp_walk_dict(cred, ip, "1.3.6.1.2.1.2.2.1.2",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)

        # 2) Walk the LLDP remote table — one walk per column we need
        chassis_ids   = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.5",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)
        chassis_subt  = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.4",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)
        port_ids      = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.7",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)
        port_id_subt  = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.6",
                                         timeout=timeout, retries=retries,
                                         op_timeout=20)
        try:
            port_descs = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.8",
                                          timeout=timeout, retries=retries,
                                          op_timeout=20)
        except SnmpCollectError:
            port_descs = {}
        try:
            sys_names  = _snmp_walk_dict(cred, ip, "1.0.8802.1.1.2.1.4.1.1.9",
                                          timeout=timeout, retries=retries,
                                          op_timeout=20)
        except SnmpCollectError:
            sys_names = {}
    except SnmpCollectError as e:
        log.debug("lldp snmp walk failed on %s: %s", ip, e)
        return []
    if not chassis_ids and not port_ids:
        return []

    # 3) Build rows. Index keys look like '0.514.1' — split to extract the
    # local port number (the middle component).
    rows = []
    for key in chassis_ids.keys():
        parts = str(key).split(".")
        if len(parts) != 3:
            continue
        try:
            local_port_num = int(parts[1])
        except ValueError:
            continue
        local_port = port_names.get(local_port_num) or f"port{local_port_num}"
        # Junos returns logical names ('ge-0/0/1.0'); canonicalize.
        local_port = local_port.split(".", 1)[0] if "-" in local_port else local_port
        chassis_raw = chassis_ids.get(key)
        chassis_str = _format_lldp_id(chassis_raw, chassis_subt.get(key))
        port_raw = port_ids.get(key)
        port_str = _format_lldp_id(port_raw, port_id_subt.get(key))
        rows.append({
            "local_port":         local_port,
            "neighbor_chassis":   chassis_str,
            "neighbor_sysname":   (sys_names.get(key) or "").strip()
                                    if isinstance(sys_names.get(key), str) else "",
            "neighbor_port":      port_str,
            "neighbor_port_desc": (port_descs.get(key) or "").strip()
                                    if isinstance(port_descs.get(key), str) else "",
        })
    return rows


def _format_lldp_id(raw, subtype):
    """Best-effort conversion of an LLDP chassis-id / port-id to a string.
    SubType 4 (macAddress) → 'aa:bb:cc:dd:ee:ff' format. Other subtypes,
    or any bytes we get, are decoded as ASCII (replacing invalid bytes)
    so the caller has a stable str to store and match on."""
    if raw is None:
        return ""
    if isinstance(raw, bytes):
        # MAC subtype is 4. Many devices also return bytes for ifName /
        # local IDs; in those cases we want a readable string.
        if subtype == 4 and len(raw) == 6:
            return ":".join(f"{b:02x}" for b in raw)
        try:
            return raw.decode("ascii", errors="replace").rstrip("\x00").strip()
        except Exception:
            return raw.hex()
    return str(raw).strip()


# Cap distinct MACs per port before treating it as a trunk/uplink and
# dropping it from port_macs. Access/edge ports (the only ones the flux
# signal and last-known-MAC fallback care about) sit far below this; a
# port above it is structurally a switch-link already covered by the LLDP
# topology graph. Keeps port_macs small on a 365-device fleet.
_FDB_PORT_MAC_CAP = 64


def _collect_fdb_snmp(ip, cred, snmp_cfg):
    """Walk the bridge forwarding DB via SNMP -> [(ifname, mac, vlan), ...]
    for access/edge ports only.

    Q-BRIDGE dot1qTpFdbTable first (VLAN-aware; index <vlan>.<6 MAC
    octets>), else classic BRIDGE-MIB dot1dTpFdbTable (index = 6 MAC
    octets, vlan=0). Table value is a dot1dBasePort, mapped
    base->ifIndex->ifName with the same chain the SNMP STP collector
    uses. Ports with > _FDB_PORT_MAC_CAP MACs are dropped as
    trunks/uplinks. Empty list on any failure — best-effort."""
    timeout = int(snmp_cfg.get("timeout") or 5)
    retries = int(snmp_cfg.get("retries") or 1)
    try:
        base_to_ifidx = _snmp_walk_dict(cred, ip, _OID_DOT1D_BASE_PORT_IFIDX,
                                        timeout=timeout, retries=retries,
                                        op_timeout=20)
        ifidx_to_name = _snmp_walk_dict(cred, ip, _OID_IF_NAME,
                                        timeout=timeout, retries=retries,
                                        op_timeout=20)
        vlan_aware = True
        try:
            fdb = _snmp_walk_dict(cred, ip, _OID_DOT1Q_TPFDB_PORT,
                                  timeout=timeout, retries=retries,
                                  op_timeout=30)
        except SnmpCollectError:
            fdb = {}
        if not fdb:
            vlan_aware = False
            fdb = _snmp_walk_dict(cred, ip, _OID_DOT1D_TPFDB_PORT,
                                  timeout=timeout, retries=retries,
                                  op_timeout=30)
    except SnmpCollectError as e:
        log.debug("fdb snmp walk failed on %s: %s", ip, e)
        return []
    if not fdb:
        return []

    by_port = {}   # ifname -> list[(mac, vlan)]
    for idx, base_port in fdb.items():
        octets = str(idx).split(".")
        if vlan_aware:
            if len(octets) < 7:
                continue
            try:
                vlan = int(octets[0])
            except ValueError:
                continue
        else:
            if len(octets) < 6:
                continue
            vlan = 0
        try:
            ovals = [int(o) for o in octets[-6:]]
        except ValueError:
            continue
        if any(o < 0 or o > 255 for o in ovals):
            continue
        if all(o == 0 for o in ovals):
            continue
        if ovals[0] & 1:               # group bit set -> multicast/broadcast
            continue
        try:
            bp = int(base_port)
        except (TypeError, ValueError):
            continue
        if bp <= 0:
            continue
        # _snmp_walk_dict stores all-digit trailing indexes as int keys
        # (base port, ifIndex), so look up with the int first; tolerate a
        # str key too in case that ever changes.
        ifidx = base_to_ifidx.get(bp, base_to_ifidx.get(str(bp)))
        if ifidx is None:
            continue
        name = ifidx_to_name.get(ifidx, ifidx_to_name.get(str(ifidx)))
        if not name:
            continue
        if "-" in name and "." in name:   # Junos 'ge-0/0/1.0' -> 'ge-0/0/1'
            name = name.split(".", 1)[0]
        mac = ":".join("%02x" % o for o in ovals)
        by_port.setdefault(name, []).append((mac, vlan))

    out = []
    for name, entries in by_port.items():
        if len(entries) > _FDB_PORT_MAC_CAP:
            continue                   # trunk/uplink — LLDP graph covers it
        for mac, vlan in entries:
            out.append((name, mac, vlan))
    return out


def _collect_lldp_one(ip, hostname, platform, cfg, password_cache):
    """Return {"lldp": [...], "fdb": [...]} for one device.

    LLDP tries SNMP first (works on more devices, no SSH session) then
    SSH fallback. FDB (port<->MAC history) is SNMP-only and harvested
    opportunistically on the same cached SNMP cred — SSH-only devices
    contribute no FDB. Empty lists on failure — best-effort, like
    run_backup."""
    snmp_cfg = cfg.get("snmp") or {}
    result = {"lldp": [], "fdb": []}
    conn = _db()
    conn.row_factory = sqlite3.Row
    row = conn.execute(
        "SELECT username, password_hash, snmp_enabled, snmp_proto, "
        "       snmp_community, snmp_v3_user "
        "FROM devices WHERE ip = ?", (ip,)
    ).fetchone()
    conn.close()

    # SNMP fast path — LLDP + FDB share the one cached SNMP cred.
    if row and row["snmp_enabled"] == 1:
        try:
            cred = _resolve_snmp_cred(row, snmp_cfg)
            result["lldp"] = _collect_lldp_snmp(ip, cred, snmp_cfg) or []
            try:
                result["fdb"] = _collect_fdb_snmp(ip, cred, snmp_cfg) or []
            except SnmpCollectError as e:
                log.debug("fdb snmp on %s: %s", ip, e)
            if result["lldp"]:
                return result
        except SnmpCollectError as e:
            log.debug("lldp snmp on %s: %s — falling back to SSH", ip, e)

    # SSH fallback — LLDP only (FDB keeps whatever SNMP produced, if any).
    if platform not in ("junos", "aruba-cx", "procurve"):
        return result
    if not row or not row["username"] or not row["password_hash"]:
        return result
    usernames = cfg.get("usernames") or []
    password_list = cfg.get("password_list") or []
    user_passwords = cfg.get("user_passwords") or {}
    child, proto, user, _, ssh_open, telnet_open = connect_device(
        ip, usernames, None, timeout=15,
        password_list=password_list, user_passwords=user_passwords,
        known_username=row["username"], known_password_hash=row["password_hash"],
        reason="monitor_topology_fallback")
    if child is None:
        return result
    try:
        if platform == "junos":
            cmd = "show lldp neighbors | no-more"
        elif platform == "aruba-cx":
            cmd = "show lldp neighbor-info"
        else:  # procurve
            cmd = "show lldp info remote-device"
        out = _strip_cli_noise(send_command(child, cmd, timeout=30))
    except Exception as e:
        log.debug("lldp ssh collect failed for %s: %s", ip, e)
        return result
    finally:
        disconnect(child)
    result["lldp"] = _parse_lldp_neighbors_bulk(out, platform)
    return result


def run_topology(cfg):
    """Fleet-wide LLDP scrape — populates topology_edges.

    Per-device path: SNMP first (works on any device with snmp_enabled=1,
    no SSH session required), SSH fallback for devices without SNMP or
    whose SNMP LLDP table is empty. Best-effort — a single device's
    failure doesn't abort the run.
    """
    # Topology fires every 30 min and historically COLLIDED at :00/:30
    # with the per-minute monitor-stp tick that also holds the 'ssh' lock.
    # On larger fleets stp's tick is 30-42 s so the non-blocking acquire
    # silently lost the race the majority of the time (PSC ~75 %, Drew
    # ~75 %, SLU ~58 % miss rate over a representative 12-hour window).
    # Wait up to 90 s for the lock (much shorter than a 30-min cycle) and
    # record a skip op_event if the wait expires so the miss isn't silent.
    _topo_started = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    if not _acquire_advisory_lock("ssh", wait=90):
        _record_op("monitor_topology_tick",
                   started_at=_topo_started,
                   duration_ms=0, success=False,
                   reason="ssh-lock-held",
                   extra="skipped=lock_held wait_s=90")
        return
    _topo_t0 = time.monotonic()
    _snmp_timing_start()
    db_devices = get_devices(status="active")
    # SNMP works on any device that's been probed (regardless of platform);
    # SSH only on the three platforms we have CLI parsers for. Including
    # snmp_enabled=1 devices broadens our coverage past the SSH-only set.
    targets = [r for r in db_devices
               if (r["platform"] in ("junos", "aruba-cx", "procurve"))
               or r["snmp_enabled"] == 1]
    if not targets:
        log.info("No supported devices to scrape LLDP from.")
        return

    threads = min(cfg.get("backup_threads", 50), max(1, len(targets)))
    log.info("Collecting LLDP from %d device(s) using %d threads...",
             len(targets), threads)

    results_by_ip = {}
    done = 0
    total = len(targets)
    with ThreadPoolExecutor(max_workers=threads) as pool:
        futures = {
            pool.submit(_collect_lldp_one, r["ip"], r["hostname"],
                        r["platform"], cfg, None): r
            for r in targets
        }
        for fut in as_completed(futures):
            r = futures[fut]
            try:
                rows = fut.result()
            except Exception as e:
                log.debug("lldp collect exception on %s: %s", r["ip"], e)
                rows = {"lldp": [], "fdb": []}
            results_by_ip[r["ip"]] = rows
            done += 1
            sys.stdout.write(f"\rLLDP scrape... ({done}/{total})  ")
            sys.stdout.flush()
    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    conn.row_factory = sqlite3.Row
    edges_written = 0
    macs_written = 0
    devices_with_neighbors = 0
    for src_ip, res in results_by_ip.items():
        lldp_rows = (res or {}).get("lldp") or []
        fdb_rows = (res or {}).get("fdb") or []
        if lldp_rows:
            devices_with_neighbors += 1
            for row in lldp_rows:
                neigh_ip = _resolve_edge_neighbor_ip(
                    row["neighbor_chassis"], row["neighbor_sysname"], conn)
                conn.execute("""
                    INSERT INTO topology_edges
                        (src_ip, src_port, neighbor_chassis, neighbor_sysname,
                         neighbor_port, neighbor_port_desc, neighbor_ip, last_seen)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(src_ip, src_port) DO UPDATE SET
                        neighbor_chassis = excluded.neighbor_chassis,
                        neighbor_sysname = excluded.neighbor_sysname,
                        neighbor_port = excluded.neighbor_port,
                        neighbor_port_desc = excluded.neighbor_port_desc,
                        neighbor_ip = excluded.neighbor_ip,
                        last_seen = excluded.last_seen
                """, (src_ip, row["local_port"],
                      row["neighbor_chassis"] or None,
                      row["neighbor_sysname"] or None,
                      row["neighbor_port"] or None,
                      row["neighbor_port_desc"] or None,
                      neigh_ip, now))
                edges_written += 1
        # FDB -> port_macs history (last-known MAC fallback + flux signal).
        for ifname, mac, vlan in fdb_rows:
            conn.execute("""
                INSERT INTO port_macs
                    (ip, interface, mac, vlan, oui_vendor,
                     first_seen, last_seen, times_seen)
                VALUES (?, ?, ?, ?, ?, ?, ?, 1)
                ON CONFLICT(ip, interface, mac, vlan) DO UPDATE SET
                    last_seen = excluded.last_seen,
                    times_seen = port_macs.times_seen + 1,
                    oui_vendor = excluded.oui_vendor
            """, (src_ip, ifname, mac, vlan, _oui_vendor_hint(mac), now, now))
            macs_written += 1
    conn.commit()
    # Edge-triggered non-trunk MAC-flux security notice. Evaluated here so
    # detection latency is <= one 30-min FDB harvest. Returns only the
    # ports that just crossed (suppressed until the episode clears); email
    # is sent after the DB handle is closed so SMTP doesn't hold it.
    try:
        new_flux = _evaluate_mac_flux_alerts(conn, cfg, now)
    except Exception as e:
        log.debug("mac-flux evaluation skipped: %s", e)
        new_flux = []
    conn.close()
    if new_flux:
        _send_flux_alert_email(cfg, new_flux)
    log.info("LLDP scrape complete: %d edge(s), %d port-MAC(s) recorded "
             "across %d device(s)", edges_written, macs_written,
             devices_with_neighbors)

    # Tick-level op_event with SNMP timing rollup
    _snmp_stats = _snmp_timing_stop()
    _topo_extra = {
        "edges_written": edges_written,
        "macs_written": macs_written,
        "devices_with_neighbors": devices_with_neighbors,
        "targets": len(targets),
    }
    if _snmp_stats:
        _topo_extra.update(_snmp_stats)
    _record_op("monitor_topology_tick",
               started_at=_topo_started,
               duration_ms=int((time.monotonic() - _topo_t0) * 1000),
               success=True,
               extra=_topo_extra)


def _walk_topology_upstream(start_ip, max_hops=4):
    """Walk topology_edges upstream from start_ip, following each device's
    current STP root port. Returns (hops, stop_reason) where hops is a
    list of dicts (one per *successfully walked* upstream device) and
    stop_reason is a string describing why we stopped.

    Each hop dict: {ip, hostname, model, via_port (on previous hop's side),
    neighbor_port (this hop's side), edge_last_seen, is_root}.

    "Upstream" = follow the port currently in STP Root role.
    """
    hops = []
    visited = {start_ip}
    cur_ip = start_ip
    stop_reason = None
    conn = _db()
    conn.row_factory = sqlite3.Row
    try:
        for _ in range(max_hops):
            # Find current device's root port (CIST instance preferred
            # if multiple). stp_state stores role per port; look for exact
            # 'Root' (Junos) or 'ROOT' (Aruba-CX / ProCurve).
            root_port_row = conn.execute(
                "SELECT interface FROM stp_state "
                "WHERE ip = ? AND role IN ('Root', 'ROOT') "
                "ORDER BY interface LIMIT 1", (cur_ip,)).fetchone()
            if root_port_row is None:
                is_root_row = conn.execute(
                    "SELECT is_root FROM stp_root_state "
                    "WHERE ip = ? AND instance = 'CIST'", (cur_ip,)).fetchone()
                stop_reason = (
                    f"{cur_ip} appears to be the root bridge (no Root-port)"
                    if is_root_row and is_root_row["is_root"]
                    else f"no STP Root-port recorded for {cur_ip}"
                )
                break
            via_port = root_port_row["interface"]
            # Junos STP reports logical iface ('ge-0/0/8.0'); topology_edges
            # is keyed on the physical name ('ge-0/0/8').
            via_port_phys = via_port.split(".", 1)[0]
            edge = conn.execute(
                "SELECT neighbor_ip, neighbor_sysname, neighbor_port, "
                "       neighbor_chassis, last_seen "
                "FROM topology_edges WHERE src_ip = ? AND src_port = ?",
                (cur_ip, via_port_phys)).fetchone()
            if edge is None:
                stop_reason = (f"no topology edge cached for {cur_ip} "
                               f"{via_port_phys} — run 'monitor topology'")
                break
            if edge["neighbor_ip"] is None:
                stop_reason = (
                    f"{cur_ip} {via_port_phys}'s LLDP neighbor "
                    f"(chassis={edge['neighbor_chassis']!r}, "
                    f"sysname={edge['neighbor_sysname']!r}) is not in "
                    f"netops devices — endpoint or undiscovered switch")
                break
            if edge["neighbor_ip"] in visited:
                stop_reason = (f"cycle detected — neighbor {edge['neighbor_ip']} "
                               f"already on the path")
                break
            neigh = conn.execute(
                "SELECT hostname, model FROM devices WHERE ip = ?",
                (edge["neighbor_ip"],)).fetchone()
            hops.append({
                "ip": edge["neighbor_ip"],
                "hostname": neigh["hostname"] if neigh else None,
                "model":    neigh["model"]    if neigh else None,
                "via_port": via_port_phys,
                "neighbor_port": edge["neighbor_port"],
                "edge_last_seen": edge["last_seen"],
                "is_root": False,
            })
            # Did we just land on the root bridge? If so, mark and stop.
            is_root_row = conn.execute(
                "SELECT is_root FROM stp_root_state "
                "WHERE ip = ? AND instance = 'CIST'",
                (edge["neighbor_ip"],)).fetchone()
            if is_root_row and is_root_row["is_root"]:
                hops[-1]["is_root"] = True
                stop_reason = "reached root bridge"
                break
            visited.add(edge["neighbor_ip"])
            cur_ip = edge["neighbor_ip"]
        else:
            stop_reason = f"reached max_hops={max_hops}"
    finally:
        conn.close()
    return hops, stop_reason


def _format_topology_trace_lines(start_ip, start_hostname, max_hops=4,
                                  activity_window_min=5):
    """Render the multi-hop upstream trace block for an alert email.

    For each hop, attach a one-line summary of recent (in the last N
    minutes) port flaps + STP changes + root-bridge changes on that
    device. Operator can see which hop had nearby activity at the time
    of the alert — that's usually the proximate cause.

    Returns an empty list if there's no useful trace (e.g. no
    topology_edges data populated yet).
    """
    hops, stop_reason = _walk_topology_upstream(start_ip, max_hops=max_hops)
    lines = [f"--- Upstream topology trace (toward root, up to {max_hops} hops) ---"]
    lines.append(f"  hop 0  this device  {start_hostname or '?'} ({start_ip})")
    conn = _db()
    conn.row_factory = sqlite3.Row
    try:
        for i, h in enumerate(hops, start=1):
            label = h["hostname"] or "?"
            via = h["via_port"] or "?"
            n_port = h["neighbor_port"] or "?"
            head = (f"  hop {i}  via {via} ↔ {n_port}  "
                    f"{label} ({h['ip']})")
            if h.get("model"):
                head += f"  [{h['model']}]"
            if h.get("is_root"):
                head += "  ★ ROOT BRIDGE"
            lines.append(head)
            # Recent activity on THIS hop's device in the activity window.
            window_clause = f"-{int(activity_window_min)} minutes"
            flaps = conn.execute("""
                SELECT interface, flap_count, last_seen FROM port_flaps
                WHERE ip = ? AND last_seen >= datetime('now','localtime',?)
                ORDER BY last_seen DESC LIMIT 5
            """, (h["ip"], window_clause)).fetchall()
            stp_chg = conn.execute("""
                SELECT interface, old_role, old_state, new_role, new_state, changed_at
                FROM stp_changes
                WHERE ip = ? AND changed_at >= datetime('now','localtime',?)
                ORDER BY changed_at DESC LIMIT 5
            """, (h["ip"], window_clause)).fetchall()
            root_chg = conn.execute("""
                SELECT instance, changed_at FROM stp_root_changes
                WHERE ip = ? AND changed_at >= datetime('now','localtime',?)
                ORDER BY changed_at DESC LIMIT 3
            """, (h["ip"], window_clause)).fetchall()
            if not flaps and not stp_chg and not root_chg:
                lines.append(f"         no port-flap / STP / root-change activity in last {activity_window_min}m")
            for r in flaps:
                lines.append(f"         port_flap  {r['interface']:14s} "
                             f"count={r['flap_count']:>4d} last={r['last_seen']}")
            for r in stp_chg:
                lines.append(f"         stp_change {r['interface']:14s} "
                             f"{r['old_role']}/{r['old_state']} -> "
                             f"{r['new_role']}/{r['new_state']} @ {r['changed_at']}")
            for r in root_chg:
                lines.append(f"         root_change inst={r['instance']} @ {r['changed_at']}")
    finally:
        conn.close()
    if stop_reason:
        lines.append(f"  trace stopped: {stop_reason}")
    return lines


def run_monitor(cfg, mode, detail=False):
    """Poll device state across active devices.

    mode="stp":  poll STP + interface link state, confirm STP changes across
                 two polls, email alerts, and count port flaps.
    mode="flap": poll interface link state only, count port flaps. No STP
                 commands are sent; existing STP state is left untouched.

    The per-device platform (junos|aruba-cx|procurve) is cached on the
    `devices` row; a cache miss triggers a `show version` classification
    that is then stored for subsequent polls.
    """
    if mode not in ("stp", "flap"):
        raise ValueError(f"run_monitor: invalid mode {mode!r}")
    if not _acquire_advisory_lock("ssh"):
        return
    label = "STP" if mode == "stp" else "port"
    passwords = cfg["passwords"]
    if not passwords["default"] and not cfg.get("password_list") and not cfg.get("user_passwords"):
        log.error("No credentials configured. Set 'passwords' in [backup] or mappings in [user_passwords] (netops.conf or secrets.conf).")
        sys.exit(1)

    # Per-tick instrumentation — accumulates SNMP query timing across all
    # worker threads, rolled up to one op_events row at tick end.
    _tick_t0 = time.monotonic()
    _tick_started_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    _snmp_timing_start()

    db_devices = get_devices(status="active")
    if not db_devices:
        log.info("No active devices. Run 'discover' first.")
        sys.exit(0)

    ips = [r["ip"] for r in db_devices]
    hostnames = {r["ip"]: r["hostname"] or r["ip"] for r in db_devices}
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    # Pre-resolve passwords for known-good connections.
    # Monitoring should ONLY use stored credentials — never sweep all combos,
    # because Juniper (and others) impose progressive login delays after
    # failed attempts, causing timeouts that cascade.
    all_pws = list(cfg.get("password_list") or [])
    for u, pws in (cfg.get("user_passwords") or {}).items():
        all_pws.extend(pws)
    pw_by_hash = {hashlib.sha256(pw.encode()).hexdigest(): pw for pw in all_pws}

    snmp_cfg = cfg.get("snmp") or {}
    poll_mode_cfg = cfg.get("stp_poll_mode", "auto")  # auto|snmp|ssh

    def poll_one(ip):
        """Poll a single device. Transport is chosen per device:
          - SNMP when cfg.stp_mode allows it and the device has snmp_enabled=1
            (cached cred from discover). Single-roundtrip per OID column,
            no RADIUS auth, sub-second per device.
          - SSH otherwise — legacy path. Same return shape.

        Returns (ip, ports, ifaces, stp_mode, root_entries, last_flapped) where:
          ports        = list of STP port dicts (empty in flap mode or when STP disabled)
          ifaces       = {interface: link_up_bool}
          stp_mode     = lowercased mode string ('mstp'/'rstp'/'stp'); None when
                         not determinable from this transport (existing DB value
                         is preserved by the caller).
          root_entries = list of root-state dicts (one per STP instance; currently
                         just CIST). Empty in flap mode or when STP is disabled.
          ports is None when the poll failed (credentials/connection/parse).
        """
        conn = _db()
        row = conn.execute(
            "SELECT username, password_hash, platform, snmp_enabled, snmp_proto,"
            " snmp_community, snmp_v3_user "
            "FROM devices WHERE ip = ? AND status = 'active'",
            (ip,)).fetchone()
        conn.close()
        known_user = row["username"] if row else None
        known_pw_hash = row["password_hash"] if row else None
        cached_platform = row["platform"] if row and "platform" in row.keys() else None

        # ----- SNMP fast path -----
        # Per-device decision: SNMP when allowed, the device is probed-good,
        # and we have a cached platform (snmp_collect_stp needs platform up
        # front; we can't run 'show version' without an SSH session).
        snmp_enabled = (row["snmp_enabled"] if row and "snmp_enabled" in row.keys()
                        else None)
        snmp_allowed = poll_mode_cfg in ("auto", "snmp") and snmp_enabled == 1
        if snmp_allowed and cached_platform in ("junos", "aruba-cx", "procurve"):
            try:
                cred = _resolve_snmp_cred(row, snmp_cfg)
                ports, ifaces, stp_mode_v, root_entries, last_flapped = \
                    snmp_collect_stp(ip, cred, cached_platform)
                log.debug(
                    "monitor-%s: %s — snmp poll: %d STP port(s), %d iface(s), "
                    "mode=%s, root_instances=%d, %d flap timestamps",
                    mode, ip, len(ports), len(ifaces), stp_mode_v,
                    len(root_entries),
                    sum(1 for v in last_flapped.values() if v is not None))
                return (ip, ports, ifaces, stp_mode_v, root_entries, last_flapped)
            except SnmpCollectError as e:
                # Hard failure: stp_mode='snmp' treats this as failure for the
                # poll (no SSH fallback). 'auto' falls through to SSH below so
                # transient SNMP issues don't drop the device from monitoring.
                log.debug("monitor-%s: %s — snmp failed: %s", mode, ip, e)
                if poll_mode_cfg == "snmp":
                    return (ip, None, {}, None, [], {})
                # else fall through to SSH

        if poll_mode_cfg == "snmp":
            # snmp-only mode but device isn't snmp-capable yet (or platform
            # missing). Skip rather than reverting to SSH — the operator chose
            # this mode explicitly.
            log.debug("monitor-%s: %s — snmp-only mode but device not "
                      "snmp-ready (snmp_enabled=%s platform=%s); skipping",
                      mode, ip, snmp_enabled, cached_platform)
            return (ip, None, {}, None, [], {})

        if not known_user or not known_pw_hash:
            log.debug("monitor-%s: %s — no stored credentials, skipping", mode, ip)
            return (ip, None, {}, None, [], {})

        known_pw = pw_by_hash.get(known_pw_hash)
        if not known_pw:
            log.debug("monitor-%s: %s — stored password hash not in current config, skipping", mode, ip)
            return (ip, None, {}, None, [], {})

        # ssh_connect handles modern-first negotiation and falls back to
        # legacy algos only when the device's negotiation error advertises
        # them. Algorithm negotiation failures don't count as failed
        # logins on Juniper (the auth phase hasn't started), so the smart
        # retry loop is safe even on Junos.
        _ssh_t0 = time.monotonic()
        _ssh_started = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        child = ssh_connect(ip, known_user, known_pw, timeout=30)
        _record_op("ssh_connect",
                   started_at=_ssh_started,
                   duration_ms=int((time.monotonic() - _ssh_t0) * 1000),
                   success=(child is not None),
                   ip=ip, reason="monitor_stp_fallback")
        if child is None:
            log.debug("monitor-%s: %s — SSH failed as %s", mode, ip, known_user)
            return (ip, None, {}, None, [], {})

        ports = []
        ifaces = {}
        stp_mode = None
        root_entries = []
        try:
            platform = cached_platform
            if platform not in ("junos", "aruba-cx", "procurve"):
                platform = _classify_platform(child)
                c = _db()
                c.execute("UPDATE devices SET platform = ? WHERE ip = ?", (platform, ip))
                c.commit()
                c.close()
                log.debug("monitor-%s: %s — classified platform=%s", mode, ip, platform)

            if mode == "stp":
                if platform == "junos":
                    out = send_command(child, "show spanning-tree interface | no-more", timeout=15)
                    ports = parse_stp_junos(out)
                    # 'show spanning-tree bridge' gives us mode + CIST root-bridge
                    # info in one shot.
                    bridge_out = send_command(child, "show spanning-tree bridge | no-more", timeout=10)
                    stp_mode = parse_stp_mode_junos(bridge_out)
                    root_entries = parse_stp_root_junos(bridge_out)
                else:
                    out = send_command(child, "show spanning-tree", timeout=15)
                    if "Spanning-tree is disabled" in out:
                        log.debug("monitor-%s: %s — STP disabled", mode, ip)
                        disconnect(child)
                        return (ip, [], {}, None, [], {})
                    if platform == "aruba-cx":
                        ports = parse_stp_aruba_cx(out)
                        stp_mode = parse_stp_mode_aruba_cx(out)
                        root_entries = parse_stp_root_aruba_cx(out)
                    else:
                        ports = parse_stp_procurve(out)
                        stp_mode = parse_stp_mode_procurve(out)
                        root_entries = parse_stp_root_procurve(out)

            # Pull link state AND per-port last-flapped in one command where
            # vendor supports it. Junos filtered form keeps output compact
            # (2 lines per port); ArubaOS-CX's full 'show interface' is
            # larger (~500KB for a VSF stack) but parseable. ProCurve lacks
            # a native last-flapped field so last_flapped stays empty.
            last_flapped = {}
            if platform == "junos":
                iface_out = send_command(
                    child,
                    'show interfaces | match "^Physical|^  Last flapped" | no-more',
                    timeout=25)
                ifaces = parse_ifaces_junos(iface_out)
                last_flapped = parse_junos_iface_flaps(iface_out)
            elif platform == "aruba-cx":
                iface_out = send_command(child, "show interface", timeout=40)
                ifaces = parse_ifaces_aruba_cx(iface_out)
                last_flapped = parse_aruba_cx_iface_flaps(iface_out)
            else:
                iface_out = send_command(child, "show interfaces brief", timeout=20)
                ifaces = parse_ifaces_procurve(iface_out)
        except Exception as e:
            log.debug("monitor-%s: %s — poll failed: %s", mode, ip, e)
            disconnect(child)
            return (ip, None, {}, None, [], {})
        disconnect(child)
        log.debug("monitor-%s: %s — %d STP port(s), %d interface(s) polled, "
                  "mode=%s, root_instances=%d, %d flap timestamps",
                  mode, ip, len(ports), len(ifaces), stp_mode,
                  len(root_entries), sum(1 for v in last_flapped.values() if v is not None))
        return (ip, ports, ifaces, stp_mode, root_entries, last_flapped)

    # --- Poll all devices ---
    results = []
    total = len(ips)
    done = 0
    threads = cfg["backup_threads"]
    log.info("Using %d concurrent threads for %s polling", threads, label)
    with ThreadPoolExecutor(max_workers=threads) as pool:
        futures = {pool.submit(poll_one, ip): ip for ip in ips}
        for future in as_completed(futures):
            done += 1
            pct = done * 100 // total
            sys.stdout.write(f"\rPolling {label} state... ({done}/{total}) {pct}%  ")
            sys.stdout.flush()
            results.append(future.result())

    sys.stdout.write("\r" + " " * 120 + "\r")
    sys.stdout.flush()

    # --- Compare against baseline and store ---
    expected_mode = (cfg.get("stp_expected_mode") or "mstp").lower()
    conn = _db()
    changes = []              # confirmed STP changes (emit alerts for these)
    new_pending = 0           # counter: new STP changes buffered this poll
    dropped_pending = 0       # counter: pending changes that reverted (transient)
    flap_suppressed = 0       # counter: confirmed STP changes dropped as flap artifacts
    classifier_suppressed = 0 # counter: confirmed STP changes tagged as snmp_transient
    suspected_fp = []         # changes the classifier tagged as suspected false-positive
    root_changes = []         # confirmed root-bridge changes (per device/instance)
    new_root_pending = 0      # counter: new root changes buffered this poll
    dropped_root_pending = 0  # counter: root-change pending that reverted
    flap_events = 0           # counter: observed up-from-down transitions
    polled = 0
    failed = 0
    stp_disabled = 0
    stp_mismatched = 0
    # Reachability tracking: collect devices that cross the unreachable
    # threshold this tick (first email) and devices that recover (clearing
    # email). Both are batched into single emails at end of monitor so a
    # site-wide outage doesn't spam N separate messages.
    unreachable_threshold = int(cfg.get("snmp_unreachable_polls", 3))
    newly_unreachable = []   # [{ip, hostname, model, fails, last_ok, last_diag}]
    newly_recovered = []     # [{ip, hostname, model, alerted_at, downtime}]

    def _on_poll_success(ip_):
        """Reset fail counter; if device was previously alerted as down,
        clear the alert flag and queue a recovery email."""
        r = conn.execute(
            "SELECT hostname, model, snmp_consecutive_fails, "
            "unreachable_alerted_at FROM devices WHERE ip = ?", (ip_,)
        ).fetchone()
        if not r:
            return
        was_alerted = r["unreachable_alerted_at"]
        if (r["snmp_consecutive_fails"] or 0) > 0 or was_alerted:
            conn.execute(
                "UPDATE devices SET snmp_consecutive_fails = 0, "
                "unreachable_alerted_at = NULL WHERE ip = ?", (ip_,))
        if was_alerted:
            newly_recovered.append({
                "ip": ip_,
                "hostname": r["hostname"],
                "model": r["model"],
                "alerted_at": was_alerted,
                "downtime": _format_outage_duration(was_alerted),
            })

    def _on_poll_fail(ip_):
        """Increment fail counter; if it just crossed the threshold and we
        haven't already alerted for this outage, queue the device for the
        unreachable email (TCP probe runs after the loop)."""
        r = conn.execute(
            "SELECT hostname, model, snmp_consecutive_fails, "
            "unreachable_alerted_at, snmp_last_ok, snmp_diag "
            "FROM devices WHERE ip = ?", (ip_,)
        ).fetchone()
        if not r:
            return
        new_count = (r["snmp_consecutive_fails"] or 0) + 1
        conn.execute(
            "UPDATE devices SET snmp_consecutive_fails = ? WHERE ip = ?",
            (new_count, ip_))
        if new_count >= unreachable_threshold and not r["unreachable_alerted_at"]:
            newly_unreachable.append({
                "ip": ip_,
                "hostname": r["hostname"],
                "model": r["model"],
                "fails": new_count,
                "last_ok": r["snmp_last_ok"],
                "last_diag": r["snmp_diag"],
            })

    for ip, ports, ifaces, stp_mode, root_entries, last_flapped in results:
        if ports is None:
            failed += 1
            _on_poll_fail(ip)
            # Don't touch stp_enabled — keep last known value
            continue
        _on_poll_success(ip)
        hostname = hostnames.get(ip, ip)
        if mode == "stp":
            if not ports:
                stp_disabled += 1
                # stp_disabled_since: only set if not already set. Preserves
                # the original transition timestamp across successive
                # consecutive-disabled polls so the digest can require a
                # sustained-disabled duration before listing (avoids
                # false-positives from transient empty-SNMP-table polls).
                conn.execute(
                    "UPDATE devices SET stp_enabled = 0, stp_mode = NULL, "
                    "stp_last_check = ?, "
                    "stp_disabled_since = COALESCE(stp_disabled_since, ?) "
                    "WHERE ip = ?",
                    (now, now, ip),
                )
                # An STP-disabled device has no meaningful root info either;
                # clear any stale entries so 'show spanning-tree root' isn't misleading.
                conn.execute("DELETE FROM stp_root_state WHERE ip = ?", (ip,))
                continue
            polled += 1
            # --- Persist root-bridge state (one row per STP instance) ---
            # root_priority/root_mac are two-phase confirmed (same shape as
            # stp_pending_changes): a new reading goes into stp_root_pending and
            # only overwrites stp_root_state after the NEXT poll still sees it.
            # bridge_*, tcn_count, last_tcn_seconds, updated_at update in place
            # every poll (they're not alert drivers and stale values would lie
            # to the 'show spanning-tree root' query and to the cross-switch consensus).
            for re_ in root_entries:
                inst = re_["instance"]
                new_t = (re_.get("root_priority"), re_.get("root_mac"))
                bridge_prio = re_.get("bridge_priority")
                bridge_mac = re_.get("bridge_mac")
                tcn = re_.get("tcn_count")
                last_tcn = re_.get("last_tcn_seconds")

                prev_row = conn.execute(
                    "SELECT root_priority, root_mac FROM stp_root_state "
                    "WHERE ip = ? AND instance = ?", (ip, inst)).fetchone()
                prev_t = (prev_row["root_priority"], prev_row["root_mac"]) if prev_row else None

                pend_row = conn.execute(
                    "SELECT old_priority, old_mac, new_priority, new_mac "
                    "FROM stp_root_pending WHERE ip = ? AND instance = ?",
                    (ip, inst)).fetchone()

                # Decide what root_priority/root_mac to store in stp_root_state
                # this poll. Only "confirmed" values are kept there; the cross-
                # switch consensus query reads from stp_root_state, so it too
                # sees only confirmed roots.
                confirmed_t = prev_t
                if pend_row:
                    pend_old = (pend_row["old_priority"], pend_row["old_mac"])
                    pend_new = (pend_row["new_priority"], pend_row["new_mac"])
                    if new_t == pend_new:
                        # Confirmed — alert and move state forward
                        root_changes.append({
                            "ip": ip, "hostname": hostname, "instance": inst,
                            "old_priority": pend_old[0], "old_mac": pend_old[1],
                            "new_priority": new_t[0], "new_mac": new_t[1],
                        })
                        conn.execute(
                            "DELETE FROM stp_root_pending WHERE ip = ? AND instance = ?",
                            (ip, inst))
                        confirmed_t = new_t
                    elif new_t == pend_old:
                        # Reverted — silent, drop pending
                        conn.execute(
                            "DELETE FROM stp_root_pending WHERE ip = ? AND instance = ?",
                            (ip, inst))
                        dropped_root_pending += 1
                        confirmed_t = pend_old
                    else:
                        # Rebased — keep original old, update new to latest reading
                        conn.execute(
                            "UPDATE stp_root_pending SET new_priority = ?, new_mac = ? "
                            "WHERE ip = ? AND instance = ?",
                            (new_t[0], new_t[1], ip, inst))
                        confirmed_t = pend_old
                else:
                    if prev_t is None:
                        # First sighting of this (ip, instance) — record as
                        # baseline without alerting.
                        confirmed_t = new_t
                    elif new_t != prev_t:
                        # New change — enter pending for next-poll confirmation.
                        conn.execute("""
                            INSERT INTO stp_root_pending
                                (ip, instance, old_priority, old_mac,
                                 new_priority, new_mac, first_seen)
                            VALUES (?, ?, ?, ?, ?, ?, ?)
                        """, (ip, inst, prev_t[0], prev_t[1],
                              new_t[0], new_t[1], now))
                        new_root_pending += 1
                        confirmed_t = prev_t

                conf_prio, conf_mac = confirmed_t if confirmed_t else (None, None)
                is_root = 1 if (conf_mac and bridge_mac and conf_mac == bridge_mac) else 0
                conn.execute("""
                    INSERT INTO stp_root_state
                        (ip, instance, root_priority, root_mac,
                         bridge_priority, bridge_mac, is_root,
                         tcn_count, last_tcn_seconds, updated_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(ip, instance) DO UPDATE SET
                        root_priority = excluded.root_priority,
                        root_mac = excluded.root_mac,
                        bridge_priority = excluded.bridge_priority,
                        bridge_mac = excluded.bridge_mac,
                        is_root = excluded.is_root,
                        tcn_count = excluded.tcn_count,
                        last_tcn_seconds = excluded.last_tcn_seconds,
                        updated_at = excluded.updated_at
                """, (ip, inst, conf_prio, conf_mac,
                      bridge_prio, bridge_mac, is_root,
                      tcn, last_tcn, now))
            # Only overwrite stp_mode when we successfully parsed a value; a
            # failed parse leaves the previous value intact (conservative, so
            # a transient output glitch doesn't flip a device in/out of the
            # mismatch list on a single poll).
            up_count = sum(1 for v in ifaces.values() if v)
            if stp_mode is not None:
                # Clear stp_disabled_since on the 0→1 transition (or first
                # poll). Idempotent: NULL stays NULL on enabled→enabled.
                conn.execute(
                    "UPDATE devices SET stp_enabled = 1, stp_mode = ?, "
                    "stp_last_check = ?, link_up_count = ?, "
                    "stp_disabled_since = NULL WHERE ip = ?",
                    (stp_mode, now, up_count, ip),
                )
                if stp_mode != expected_mode:
                    stp_mismatched += 1
            else:
                conn.execute(
                    "UPDATE devices SET stp_enabled = 1, stp_last_check = ?, "
                    "link_up_count = ?, stp_disabled_since = NULL "
                    "WHERE ip = ?",
                    (now, up_count, ip),
                )
        else:
            polled += 1

        # --- Port flap tracking (physical link-up/down) ---
        prev_port_rows = conn.execute(
            "SELECT interface, link_up, last_seen FROM port_state WHERE ip = ?", (ip,)
        ).fetchall()
        prev_link = {r["interface"]: bool(r["link_up"]) for r in prev_port_rows}
        # Per-iface previous poll timestamp; used to compute the gap between
        # polls so we can compare it against the vendor's 'last flapped X ago'
        # value and detect sub-poll flaps our up-from-down observation misses.
        prev_last_seen = {r["interface"]: r["last_seen"] for r in prev_port_rows}
        # Set of interfaces that completed a flap cycle during THIS poll —
        # used below to suppress STP state-change alerts whose transition is
        # a byproduct of the flap rather than a real topology event.
        flapped_this_poll = set()
        now_dt = datetime.strptime(now, "%Y-%m-%d %H:%M:%S")
        for iface, link_up in ifaces.items():
            was_up = prev_link.get(iface)
            observed_flap = (was_up is False and link_up)
            # Inferred flap: vendor says the port's last flap was AFTER our
            # previous poll on this iface. Means the port flapped during the
            # gap — we missed it because the up-from-down transition didn't
            # straddle two polls. Only applies to iface that already had a
            # prev_last_seen (i.e., we polled it before).
            inferred_flap = False
            flap_sec = last_flapped.get(iface)
            if (flap_sec is not None and flap_sec > 0
                    and not observed_flap
                    and iface in prev_last_seen):
                try:
                    prev_dt = datetime.strptime(prev_last_seen[iface],
                                                "%Y-%m-%d %H:%M:%S")
                    gap_sec = int((now_dt - prev_dt).total_seconds())
                    if 0 < flap_sec < gap_sec:
                        inferred_flap = True
                except (TypeError, ValueError):
                    pass
            if observed_flap or inferred_flap:
                # one completed flap cycle
                conn.execute("""
                    INSERT INTO port_flaps (ip, hostname, interface, flap_count, first_seen, last_seen)
                    VALUES (?, ?, ?, 1, ?, ?)
                    ON CONFLICT(ip, interface) DO UPDATE SET
                        hostname = excluded.hostname,
                        flap_count = flap_count + 1,
                        last_seen = excluded.last_seen
                """, (ip, hostname, iface, now, now))
                flap_events += 1
                flapped_this_poll.add(iface)
            conn.execute("""
                INSERT INTO port_state (ip, interface, link_up, last_seen)
                VALUES (?, ?, ?, ?)
                ON CONFLICT(ip, interface) DO UPDATE SET
                    link_up = excluded.link_up,
                    last_seen = excluded.last_seen
            """, (ip, iface, 1 if link_up else 0, now))

        # STP analysis only runs in stp mode; flap mode skips it entirely.
        if mode != "stp":
            continue

        # --- STP change detection with 1-poll confirmation ---
        prev_rows = conn.execute(
            "SELECT interface, role, state, edge_expiry FROM stp_state WHERE ip = ?",
            (ip,)
        ).fetchall()
        prev = {r["interface"]: (r["role"], r["state"]) for r in prev_rows}

        pending_rows = conn.execute("""
            SELECT interface, old_role, old_state, new_role, new_state, first_seen,
                   edge_expiry_at_first_seen, tcn_at_first_seen
            FROM stp_pending_changes WHERE ip = ?
        """, (ip,)).fetchall()
        pending = {r["interface"]: (
            (r["old_role"], r["old_state"]),
            (r["new_role"], r["new_state"]),
            r["first_seen"],
            r["edge_expiry_at_first_seen"],
            r["tcn_at_first_seen"],
        ) for r in pending_rows}

        # --- Phantom-evidence inputs (Junos SNMP path only) ---
        # Per-port edge_expiry from this poll's SNMP walk; bridge-wide
        # CIST TC count from root_entries. Both are None for non-Junos
        # devices and for any platform/path where the SNMP walk didn't
        # populate them — the downstream comparison treats None as
        # 'unknown' and skips logging rather than mis-classifying.
        current_expiry = {p["interface"]: p.get("edge_expiry") for p in ports}
        current_tcn = None
        for re_ in root_entries:
            if re_["instance"] == "CIST" and re_.get("tcn_count") is not None:
                current_tcn = re_["tcn_count"]
                break

        # Snapshot flap last-seen times for this device so the confirmation
        # branch can check whether a recorded flap falls inside the pending
        # window. Recovery alerts ('BLK -> FWD' after the link came back up)
        # don't coincide with an up-from-down transition in the confirmation
        # poll itself, so in-poll checks alone miss them.
        flap_last_seen = {r["interface"]: r["last_seen"] for r in conn.execute(
            "SELECT interface, last_seen FROM port_flaps WHERE ip = ?", (ip,)
        ).fetchall()}

        current = {p["interface"]: (p["role"], p["state"]) for p in ports}
        GONE = ("GONE", "GONE")

        # Union of all interfaces we've ever seen for this device that are
        # still in play (current poll, last confirmed state, or pending).
        all_ifaces = set(current) | set(prev) | set(pending)

        for iface in all_ifaces:
            new_t = current.get(iface, GONE)
            if iface in pending:
                (old_t, pend_new_t, pend_first_seen,
                 pend_edge_expiry, pend_tcn) = pending[iface]
                if new_t == pend_new_t:
                    # Confirmed — same change persisted to this poll. Before
                    # alerting, check whether this transition is a link-flap
                    # artifact and should be suppressed. Three signals:
                    #   1. Port is currently down.
                    #   2. Port completed an up-from-down cycle in this poll.
                    #   3. port_flaps.last_seen >= pending.first_seen — a flap
                    #      was observed at or after we first noticed the STP
                    #      change. Catches the recovery side ('BLK -> FWD'
                    #      after the link came back up a poll ago), which
                    #      signals 1 and 2 miss by the time the port is
                    #      stable-up again.
                    # STP reports Junos logical names ('ge-1/0/0.0') but our
                    # ifaces/flap tracking keys off the physical port name
                    # ('ge-1/0/0'), so strip the '.N' subunit before lookup.
                    iface_phys = iface.split(".", 1)[0]
                    link_down = ifaces.get(iface_phys) is False
                    in_poll_flap = iface_phys in flapped_this_poll
                    last_flap = flap_last_seen.get(iface_phys)
                    in_window_flap = (last_flap is not None and pend_first_seen
                                      and last_flap >= pend_first_seen)
                    if link_down or in_poll_flap or in_window_flap:
                        conn.execute("DELETE FROM stp_pending_changes WHERE ip = ? AND interface = ?",
                                     (ip, iface))
                        flap_suppressed += 1
                        # Leave stp_state at old_t so a later recovery to old_t
                        # is silent and a later true transition still compares
                        # against the last stable reference.
                        continue
                    # Phantom-evidence log: when a state change confirms,
                    # compare the per-port state-machine activity counter
                    # (edge_expiry) and bridge-wide TC count between
                    # pending-creation and now. delta=0 on both means the
                    # state machine never ran and no TC propagated, yet
                    # the MIB reported a state transition — direct
                    # evidence of a Junos MIB-cache vs state-machine race.
                    cur_expiry = current_expiry.get(iface)
                    if (pend_edge_expiry is not None
                            and cur_expiry is not None):
                        edge_delta = cur_expiry - pend_edge_expiry
                    else:
                        edge_delta = None
                    if pend_tcn is not None and current_tcn is not None:
                        tcn_delta = current_tcn - pend_tcn
                    else:
                        tcn_delta = None
                    log.info(
                        "phantom-evidence: %s (%s) %s  %s/%s -> %s/%s  "
                        "edge_delta=%s tcn_delta=%s",
                        ip, hostname, iface,
                        old_t[0], old_t[1], new_t[0], new_t[1],
                        edge_delta if edge_delta is not None else "?",
                        tcn_delta if tcn_delta is not None else "?",
                    )
                    changes.append({
                        "ip": ip, "hostname": hostname, "interface": iface,
                        "old_role": old_t[0], "old_state": old_t[1],
                        "new_role": new_t[0], "new_state": new_t[1],
                    })
                    conn.execute("DELETE FROM stp_pending_changes WHERE ip = ? AND interface = ?",
                                 (ip, iface))
                    if new_t == GONE:
                        conn.execute("DELETE FROM stp_state WHERE ip = ? AND interface = ?",
                                     (ip, iface))
                    else:
                        conn.execute("""
                            INSERT INTO stp_state (ip, interface, role, state, last_seen, edge_expiry)
                            VALUES (?, ?, ?, ?, ?, ?)
                            ON CONFLICT(ip, interface) DO UPDATE SET
                                role = excluded.role, state = excluded.state,
                                last_seen = excluded.last_seen,
                                edge_expiry = excluded.edge_expiry
                        """, (ip, iface, new_t[0], new_t[1], now,
                              current_expiry.get(iface)))
                elif new_t == old_t:
                    # Reverted — transient, silently drop
                    conn.execute("DELETE FROM stp_pending_changes WHERE ip = ? AND interface = ?",
                                 (ip, iface))
                    dropped_pending += 1
                else:
                    # State changed to yet another value — re-check the rebased
                    # (old_t, new_t) pair. If it's no longer alertable (e.g. the
                    # original pending caught a transient step that now rolled into
                    # a link-down), drop the pending just like a revert; otherwise
                    # rebase new_t and keep old_t so a later revert still detects.
                    if should_alert_stp_change(old_t, new_t):
                        conn.execute("""
                            UPDATE stp_pending_changes
                            SET new_role = ?, new_state = ?
                            WHERE ip = ? AND interface = ?
                        """, (new_t[0], new_t[1], ip, iface))
                    else:
                        conn.execute("DELETE FROM stp_pending_changes WHERE ip = ? AND interface = ?",
                                     (ip, iface))
                        dropped_pending += 1
            else:
                old_t = prev.get(iface)
                if old_t is None:
                    # First time we've seen this port — just record it, no alert.
                    # Skip transient states (LRN/LIS) so we don't latch a mid-
                    # convergence snapshot as the reference point.
                    if new_t != GONE and not _is_stp_transient_state(*new_t):
                        conn.execute("""
                            INSERT INTO stp_state (ip, interface, role, state, last_seen, edge_expiry)
                            VALUES (?, ?, ?, ?, ?, ?)
                        """, (ip, iface, new_t[0], new_t[1], now,
                              current_expiry.get(iface)))
                elif new_t != old_t:
                    if should_alert_stp_change(old_t, new_t):
                        conn.execute("""
                            INSERT INTO stp_pending_changes
                                (ip, hostname, interface, old_role, old_state,
                                 new_role, new_state, first_seen,
                                 edge_expiry_at_first_seen, tcn_at_first_seen)
                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                        """, (ip, hostname, iface,
                              old_t[0], old_t[1], new_t[0], new_t[1], now,
                              current_expiry.get(iface), current_tcn))
                        new_pending += 1
                    # else: filtered (link-flap artifact) — no pending, no alert,
                    # and stp_state stays at old_t so recovery is also silent.

        # Refresh edge_expiry for every port we polled this round, not just
        # the ones whose state changed. The phantom-evidence delta needs a
        # *fresh* prior value (the one captured into stp_pending_changes
        # at first-seen time gets compared to *this* poll's reading); if
        # we only wrote edge_expiry on state changes, stable ports would
        # carry NULL forever and the delta would be uncomputable. Cheap:
        # a single UPDATE per port, indexed lookup, no I/O on the hot path.
        for iface, expiry in current_expiry.items():
            if expiry is not None:
                conn.execute(
                    "UPDATE stp_state SET edge_expiry = ? "
                    "WHERE ip = ? AND interface = ?",
                    (expiry, ip, iface),
                )

    # Record confirmed changes
    for ch in changes:
        conn.execute("""
            INSERT INTO stp_changes (ip, hostname, interface, old_role, old_state,
                                     new_role, new_state, changed_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (ch["ip"], ch["hostname"], ch["interface"],
              ch["old_role"], ch["old_state"],
              ch["new_role"], ch["new_state"], now))

    for ch in root_changes:
        conn.execute("""
            INSERT INTO stp_root_changes
                (ip, hostname, instance, old_priority, old_mac,
                 new_priority, new_mac, changed_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (ch["ip"], ch["hostname"], ch["instance"],
              ch["old_priority"], ch["old_mac"],
              ch["new_priority"], ch["new_mac"], now))

    conn.commit()
    conn.close()

    # --- Investigation + post-investigation flap filter ---
    # Runs before the summary report so the logged counts reflect the
    # final, post-filter numbers. Investigation happens once per affected
    # switch (one SSH session per IP, multiple ports per session).
    investigations = {}
    if mode == "stp" and changes:
        by_switch = {}
        for ch in changes:
            by_switch.setdefault(ch["ip"], []).append(ch["interface"])
        for inv_ip, inv_ports in by_switch.items():
            for port, info in investigate_stp_ports(inv_ip, inv_ports, cfg).items():
                investigations[(inv_ip, port)] = info

        # If the vendor's own 'last flapped' field says the port flapped
        # within FLAP_SUPPRESS_SEC, the STP transition is a flap artifact.
        # Authoritative source of truth — catches sub-poll flaps our
        # port_flaps table can't see.
        FLAP_SUPPRESS_SEC = 300
        kept = []
        for ch in changes:
            info = investigations.get((ch["ip"], ch["interface"]))
            flap_sec = None
            if info and info.get("ok"):
                out = info.get("interface_stats", "") or ""
                flap_sec = (parse_last_flapped_junos(out)
                            if "Last flapped" in out
                            else parse_last_flapped_aruba_cx(out))
            if flap_sec is not None and flap_sec < FLAP_SUPPRESS_SEC:
                log.info("flap-suppressed (investigation): %s (%s) %s — "
                         "last flapped %ds ago",
                         ch["ip"], ch["hostname"], ch["interface"], flap_sec)
                flap_suppressed += 1
                continue
            kept.append(ch)
        changes = kept

        # --- Classifier-based suspected-false-positive partition ---
        # The heuristic classifier (see _classify_stp_event) inspects the
        # investigation output and tags each surviving change. The
        # 'snmp_transient' label is the strongest false-positive signal we
        # have — it fires only when the port has *no* port_flaps row, *no*
        # BPDU input, link is up, and no recent topology change. On Junos
        # this is the documented MIB-cache-vs-state-machine race for
        # edge-boundary ports. We *partition* rather than drop so the
        # operator still sees the events; the alert email renders them in
        # a separate "Suspected false positives" section and subject line
        # reflects only the actionable count.
        suspected_fp = []
        kept = []
        for ch in changes:
            info = investigations.get((ch["ip"], ch["interface"]))
            label = LIKELY_UNCLASSIFIED
            if info and info.get("ok") and info.get("parsed"):
                pf, tcn_s = _stp_event_context(ch["ip"], ch["interface"])
                label, _conf, _ev = _classify_stp_event(
                    info["parsed"], pf, tcn_s,
                    old_role=ch.get("old_role"), new_role=ch.get("new_role"))
            ch["_classifier_label"] = label
            if label == LIKELY_SNMP_TRANSIENT:
                suspected_fp.append(ch)
            else:
                kept.append(ch)
        changes = kept
        classifier_suppressed = len(suspected_fp)

    # --- Report ---
    if mode == "stp":
        conn = _db()
        blocked_count = conn.execute("""
            SELECT COUNT(*) FROM stp_state
            WHERE (role = 'ALT' AND state = 'BLK')
               OR (role = 'Alternate' AND state = 'Blocking')
        """).fetchone()[0]
        root_mismatches, not_in_service, unassigned = _detect_root_mismatches(
            conn, now, cfg.get("stp_domains") or {})
        conn.close()

        log.info("--- STP Monitor Summary ---")
        log.info("Polled:       %d device(s)", polled)
        log.info("STP disabled: %d device(s)", stp_disabled)
        log.info("Mismatched:   %d device(s) not running %s", stp_mismatched, expected_mode)
        log.info("Failed:       %d device(s)", failed)
        log.info("Confirmed:    %d STP change(s)", len(changes))
        log.info("Pending:      %d new change(s) awaiting next poll", new_pending)
        log.info("Transient:    %d pending change(s) dropped (reverted)", dropped_pending)
        log.info("Flap-suppress: %d confirmed change(s) dropped as flap artifacts", flap_suppressed)
        log.info("Classifier-fp: %d confirmed change(s) tagged as snmp_transient (suspected false positive)",
                 classifier_suppressed)
        log.info("Flaps:        %d port flap(s) recorded", flap_events)
        log.info("Blocked:      %d port(s) currently in blocking (run 'show spanning-tree blocked' for details)",
                 blocked_count)
        log.info("Root changes: %d confirmed, %d pending, %d reverted",
                 len(root_changes), new_root_pending, dropped_root_pending)
        log.info("Root disagree: %d (domain, instance) pair(s) with root-bridge disagreement",
                 len(root_mismatches))
        log.info("Not in service: %d device(s) excluded from consensus (no data-plane links)",
                 not_in_service)
        if unassigned:
            log.info("Unassigned: %d device(s) — IPs not matched by any [stp_domains] prefix",
                     unassigned)

        if changes or suspected_fp:
            # Sort by hostname; stable sort preserves per-device interface order
            # from the switch's own output (natural numbering like ge-0/0/2
            # before ge-0/0/10, which a string sort would flip).
            changes.sort(key=lambda c: c["hostname"].lower())
            suspected_fp.sort(key=lambda c: c["hostname"].lower())
            if changes:
                log.info("--- Confirmed STP Changes ---")
                for ch in changes:
                    blocking = " *** BLOCKING ***" if is_stp_blocking(ch) else ""
                    log.info("  %s (%s) %s: %s/%s -> %s/%s%s",
                             ch["ip"], ch["hostname"], ch["interface"],
                             ch["old_role"], ch["old_state"],
                             ch["new_role"], ch["new_state"],
                             blocking)
            if suspected_fp:
                log.info("--- Suspected False Positives (snmp_transient) ---")
                for ch in suspected_fp:
                    log.info("  %s (%s) %s: %s/%s -> %s/%s",
                             ch["ip"], ch["hostname"], ch["interface"],
                             ch["old_role"], ch["old_state"],
                             ch["new_role"], ch["new_state"])
            # Email only when there's at least one actionable change. All-FP
            # ticks stay in the log + the daily digest summary.
            if changes:
                _send_stp_alert(cfg, changes, investigations,
                                suspected_fp=suspected_fp)

        if root_changes:
            root_changes.sort(key=lambda c: (c["instance"], c["hostname"].lower()))
            log.info("--- Confirmed STP Root-Bridge Changes ---")
            for ch in root_changes:
                log.info("  %s (%s) %s: %s/%s -> %s/%s",
                         ch["ip"], ch["hostname"], ch["instance"],
                         ch["old_priority"], ch["old_mac"],
                         ch["new_priority"], ch["new_mac"])
            _send_root_change_alert(cfg, root_changes)

        if root_mismatches:
            # Log only — the daily 'digest stp' email is the alert channel for
            # root disagreement (cadence matches STP-disabled / mode-mismatch
            # config issues, which also surface only through the digest).
            log.info("--- STP Root-Bridge Disagreement (see 'digest stp' for alert) ---")
            for mm in root_mismatches:
                log.info("  domain=%s instance=%s — %d distinct roots:",
                         mm["domain"], mm["instance"], len(mm["groups"]))
                # Groups are ordered smallest-first; the LAST group is the
                # majority/agreeing one. Collapse its device list to a count
                # unless --detail was passed.
                majority_idx = len(mm["groups"]) - 1
                for i, grp in enumerate(mm["groups"]):
                    is_majority = (i == majority_idx)
                    suffix = " agree (pass --detail to list)" if is_majority and not detail else ""
                    log.info("    priority=%s mac=%s — %d switch(es)%s",
                             grp["root_priority"], grp["root_mac"],
                             len(grp["devices"]), suffix)
                    if is_majority and not detail:
                        continue
                    for ip, hostname in grp["devices"]:
                        log.info("      %s (%s)", hostname, ip)
    else:
        log.info("--- Flap Monitor Summary ---")
        log.info("Polled:       %d device(s)", polled)
        log.info("Failed:       %d device(s)", failed)
        log.info("Flaps:        %d port flap(s) recorded", flap_events)

    # --- Reachability alerts ---
    # Per-device TCP probe runs only for devices that just crossed the
    # threshold this tick, so an idle minute does zero extra network I/O.
    # Recovery emails fire on the first successful poll after an alert.
    if newly_unreachable or newly_recovered:
        if newly_unreachable:
            log.info("--- Unreachable: %d device(s) crossed threshold ---",
                     len(newly_unreachable))
            probe_threads = min(20, max(1, len(newly_unreachable)))
            with ThreadPoolExecutor(max_workers=probe_threads) as pool:
                futures = {pool.submit(_classify_reachability, d["ip"]): d
                           for d in newly_unreachable}
                for fut in as_completed(futures):
                    futures[fut]["probe"] = fut.result()
            # Mark alerted_at first so a same-tick email failure can't cause
            # re-alerting next minute (the operator sees one log line per
            # failed-email attempt; the alarm itself fires once per outage).
            now_ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            conn2 = _db()
            for d in newly_unreachable:
                conn2.execute(
                    "UPDATE devices SET unreachable_alerted_at = ? WHERE ip = ?",
                    (now_ts, d["ip"]))
            conn2.commit()
            conn2.close()
            _send_unreachable_alert(cfg, newly_unreachable)
        if newly_recovered:
            log.info("--- Recovered: %d device(s) ---", len(newly_recovered))
            _send_recovery_alert(cfg, newly_recovered)

    # --- Tick instrumentation — write one op_event row for this tick ---
    _tick_duration_ms = int((time.monotonic() - _tick_t0) * 1000)
    _snmp_stats = _snmp_timing_stop()
    _tick_extra = {
        "polled": polled,
        "failed": failed,
        "flap_events": flap_events,
        "confirmed": len(changes),
    }
    if mode == "stp":
        _tick_extra["stp_disabled"] = stp_disabled
    if _snmp_stats:
        _tick_extra.update(_snmp_stats)
    _record_op(f"monitor_{mode}_tick",
               started_at=_tick_started_at,
               duration_ms=_tick_duration_ms,
               success=True,
               extra=_tick_extra)


def _send_email(cfg, subject, body):
    """Send an email via the [email] config. Returns (ok, error_message)."""
    import smtplib
    from email.mime.text import MIMEText

    smtp_server = cfg.get("smtp_server")
    if not smtp_server:
        return False, "no [email] section configured (smtp_server missing)"

    smtp_port = int(cfg.get("smtp_port", 587))
    from_addr = cfg.get("email_from", "netops@localhost")
    to_addrs = [a.strip() for a in cfg.get("email_to", "").split(",") if a.strip()]
    if not to_addrs:
        return False, "no recipients configured in [email] to="

    msg = MIMEText(body)
    msg["Subject"] = subject
    msg["From"] = from_addr
    msg["To"] = ", ".join(to_addrs)

    try:
        smtp_user = cfg.get("smtp_username")
        smtp_pass = cfg.get("smtp_password")
        with smtplib.SMTP(smtp_server, smtp_port, timeout=30) as server:
            server.ehlo()
            if smtp_port != 25:
                server.starttls()
            if smtp_user and smtp_pass:
                server.login(smtp_user, smtp_pass)
            server.sendmail(from_addr, to_addrs, msg.as_string())
        return True, f"sent to {', '.join(to_addrs)}"
    except Exception as e:
        return False, f"SMTP error: {e}"


_JUNOS_PROMPT_RE = re.compile(r"^\w+@\S*[>#]?$")
_JUNOS_MODE_RE = re.compile(r"^\{(master|backup)(?::\d+)?\}$")
# Junos displays a line-wrapped command as 'user@host> ...<rest-of-command>'.
# Not substantive output — matches only when preceded by a prompt.
_JUNOS_ECHO_CONT_RE = re.compile(r"^\w+@\S+>\s*\.\.\.")


def _strip_cli_noise(output):
    """Strip leading command-echo noise and trailing prompt/mode-tag lines
    left behind by Junos's multi-line command display. Conservative — only
    matches prompt-shaped lines, never substantive output.
    """
    if not output:
        return output
    lines = output.splitlines()
    while lines:
        s = lines[0].strip()
        if (not s
                or _JUNOS_PROMPT_RE.match(s)
                or _JUNOS_ECHO_CONT_RE.match(s)
                or s.startswith(("show ", "set ", "request ", "restart "))):
            lines.pop(0)
            continue
        break
    while lines:
        s = lines[-1].strip()
        if (not s
                or _JUNOS_PROMPT_RE.match(s)
                or _JUNOS_MODE_RE.match(s)):
            lines.pop()
            continue
        break
    return "\n".join(lines)


def investigate_stp_ports(ip, ports, cfg):
    """Re-connect to a switch and gather context for one or more alerted
    STP ports in a single SSH session. Returns {port: info_dict} where
    info_dict has keys: ok, error, stp_detail, lldp_neighbor, interface_stats.

    Output is captured raw (not parsed) — the reader is a human looking at
    an email or terminal; structured extraction would add work for no value.
    """
    results = {p: {"ok": False, "error": None,
                   "stp_detail": "", "lldp_neighbor": "", "interface_stats": "",
                   "stp_stats": "", "mac_table": "", "parsed": {}}
               for p in ports}

    conn = _db()
    row = conn.execute(
        "SELECT hostname, username, password_hash, platform FROM devices WHERE ip = ?",
        (ip,)).fetchone()
    conn.close()
    if not row:
        err = f"device {ip} not in DB"
        for p in ports:
            results[p]["error"] = err
        return results

    platform = row["platform"]
    if platform not in ("junos", "aruba-cx", "procurve"):
        err = f"unsupported platform: {platform!r}"
        for p in ports:
            results[p]["error"] = err
        return results

    usernames = cfg.get("usernames") or []
    password_list = cfg.get("password_list") or []
    user_passwords = cfg.get("user_passwords") or {}
    known_user = row["username"]
    known_pw_hash = row["password_hash"]

    child, proto, user, _, ssh_open, telnet_open = connect_device(
        ip, usernames, None, timeout=15,
        password_list=password_list, user_passwords=user_passwords,
        known_username=known_user, known_password_hash=known_pw_hash,
        reason="investigate_stp")
    if child is None:
        err = f"SSH connect failed (ssh_open={ssh_open}, telnet_open={telnet_open})"
        for p in ports:
            results[p]["error"] = err
        return results

    try:
        for port in ports:
            # Junos STP output reports logical port names ('ge-1/0/11.0'); the
            # operational CLI commands take the physical form ('ge-1/0/11').
            port_cli = port.split(".")[0] if platform == "junos" else port
            stp_stats_cmd = None
            if platform == "junos":
                stp_cmd = f"show spanning-tree interface {port_cli} detail | no-more"
                # Junos's per-interface form is already verbose (it shows
                # Chassis-ID, Port-ID, System name when the neighbor sends
                # it). The 'detail' keyword isn't valid on this command —
                # parser falls back to chassis-MAC matching for endpoints
                # that don't advertise System name.
                lldp_cmd = f"show lldp neighbors interface {port_cli} | no-more"
                # Plain 'show interfaces' (not 'extensive') — has physical link,
                # last flapped, carrier transitions, and error counters without
                # the per-queue/CoS noise that makes 'extensive' ~90 lines.
                iface_cmd = f"show interfaces {port_cli} | no-more"
                # Junos doesn't include BPDU counters in 'spanning-tree
                # interface detail' — the classifier needs them, so pull
                # them with a separate one-line statistics command.
                stp_stats_cmd = f"show spanning-tree statistics interface {port_cli} | no-more"
                mac_cmd = f"show ethernet-switching table interface {port_cli} | no-more"
            elif platform == "procurve":
                stp_cmd = f"show spanning-tree {port_cli} detail"
                lldp_cmd = f"show lldp info remote-device {port_cli}"
                iface_cmd = f"show interfaces {port_cli}"
                mac_cmd = f"show mac-address {port_cli}"
            else:  # aruba-cx
                stp_cmd = f"show spanning-tree interface {port_cli}"
                lldp_cmd = f"show lldp neighbor-info {port_cli}"
                iface_cmd = f"show interface {port_cli}"
                mac_cmd = f"show mac-address-table interface {port_cli}"

            info = results[port]
            try:
                info["stp_detail"] = _strip_cli_noise(send_command(child, stp_cmd, timeout=20))
            except Exception as e:
                info["stp_detail"] = f"(error: {e})"
            try:
                info["lldp_neighbor"] = _strip_cli_noise(send_command(child, lldp_cmd, timeout=20))
            except Exception as e:
                info["lldp_neighbor"] = f"(error: {e})"
            try:
                info["interface_stats"] = _strip_cli_noise(send_command(child, iface_cmd, timeout=25))
            except Exception as e:
                info["interface_stats"] = f"(error: {e})"
            if stp_stats_cmd:
                try:
                    info["stp_stats"] = _strip_cli_noise(send_command(child, stp_stats_cmd, timeout=15))
                except Exception as e:
                    info["stp_stats"] = f"(error: {e})"
            try:
                info["mac_table"] = _strip_cli_noise(send_command(child, mac_cmd, timeout=20))
            except Exception as e:
                info["mac_table"] = f"(error: {e})"

            # Parse the raw CLI for the heuristic classifier. Parser
            # failure is non-fatal — caller still gets the raw text.
            try:
                if platform == "junos":
                    info["parsed"] = _parse_junos_investigation(
                        info["stp_detail"], info["interface_stats"], info["stp_stats"])
                elif platform == "aruba-cx":
                    info["parsed"] = _parse_arubacx_investigation(
                        info["stp_detail"], info["interface_stats"])
                else:  # procurve
                    info["parsed"] = _parse_procurve_investigation(
                        info["stp_detail"], info["interface_stats"])
            except Exception as e:
                info["parsed"] = {}
                log.debug("classifier parse failed for %s %s: %s", ip, port, e)

            info["ok"] = True
    finally:
        disconnect(child)
    return results


def _format_root_bridge_lines(ip):
    """Read this switch's current view of the root bridge from stp_root_state
    (populated by the same monitor-stp poll). Returns a list of formatted
    lines for inclusion in the investigation block. Empty list if we have
    no root data for this device (e.g. STP disabled or first poll)."""
    conn = _db()
    rows = conn.execute("""
        SELECT instance, root_priority, root_mac,
               bridge_priority, bridge_mac, is_root,
               tcn_count, last_tcn_seconds
        FROM stp_root_state WHERE ip = ? ORDER BY instance
    """, (ip,)).fetchall()
    conn.close()
    if not rows:
        return []
    out = []
    for r in rows:
        rp = r["root_priority"]; rm = r["root_mac"]
        bp = r["bridge_priority"]; bm = r["bridge_mac"]
        is_root = "yes" if r["is_root"] else "no"
        line = (f"{r['instance']}: root priority={rp} mac={rm}  "
                f"this bridge priority={bp} mac={bm}  is_root={is_root}")
        if r["tcn_count"] is not None:
            tcn_bit = f"tcn_count={r['tcn_count']}"
            if r["last_tcn_seconds"] is not None:
                tcn_bit += f", last tcn {r['last_tcn_seconds']}s ago"
            line += f"  ({tcn_bit})"
        out.append(line)
    return out


# ============================================================
# STP-event classifier — heuristic, runs at investigate time
# ============================================================
# Operates on data we already collect (port_flaps, stp_root_state) plus
# CLI fields parsed from the live investigate_stp_ports SSH session.
# Output is one extra block at the top of the investigation: likely cause
# + suggested action + the evidence that fired the rule.

# Stable identifiers; copy lives in _STP_CLASSIFIER_COPY for easy edit.
LIKELY_PHYSICAL_INSTABILITY = "physical_link_instability"
LIKELY_ONE_WAY_LINK         = "one_way_link"
LIKELY_SNMP_TRANSIENT       = "snmp_transient_or_collector"
LIKELY_TC_FLUSH             = "topology_change_flush"
LIKELY_REAL_STP_EVENT       = "real_stp_topology_change"
LIKELY_UNCLASSIFIED         = "unclassified"

_STP_CLASSIFIER_COPY = {
    LIKELY_PHYSICAL_INSTABILITY: (
        "Physical link instability",
        "Check the cable, SFP/optic, and port hardware on this interface.",
    ),
    LIKELY_ONE_WAY_LINK: (
        "Suspected one-way link",
        "TX is active but RX is silent. Inspect the fiber strand pair, "
        "swap optics on either end, and verify the device on the other side.",
    ),
    LIKELY_SNMP_TRANSIENT: (
        "Likely SNMP/MIB transient — no real STP event",
        "No physical flap, no BPDU input, port has been up for an extended "
        "period. Operator action probably not required; flag for tool review "
        "if the pattern persists.",
    ),
    LIKELY_TC_FLUSH: (
        "Topology-change flush",
        "A TCN propagated through this bridge recently. The port itself is "
        "stable; brief learning state is expected during MAC-table refresh.",
    ),
    LIKELY_REAL_STP_EVENT: (
        "Genuine STP topology change",
        "Investigate which device became the new designated bridge upstream "
        "and why.",
    ),
    LIKELY_UNCLASSIFIED: (
        "Unable to classify automatically",
        "Manual review recommended; the available signals don't fit a known "
        "pattern.",
    ),
}

# Compact tags for the daily digest's per-port line; the full label is too
# long for a fixed-width inline display. Keep widths consistent — we render
# them in a left-padded column.
_STP_CLASSIFIER_SHORT_TAG = {
    LIKELY_PHYSICAL_INSTABILITY: "physical",
    LIKELY_ONE_WAY_LINK:         "one_way",
    LIKELY_SNMP_TRANSIENT:       "snmp_transient",
    LIKELY_TC_FLUSH:             "tc_flush",
    LIKELY_REAL_STP_EVENT:       "real_event",
    LIKELY_UNCLASSIFIED:         "unknown",
}

# Tunables. Module-level so they're easy to find and adjust.
_HIGH_FLAP_THRESHOLD   = 5
_TC_FLUSH_WINDOW_S     = 300        # 5 minutes
_HIGH_EDGE_EXPIRY_RATE = 100        # absolute count, used as a noise floor


def _parse_junos_investigation(stp_detail, interface_stats, stp_stats):
    """Junos: extract BPDU/edge/link signals from the three CLI dumps.
    Missing fields → None; callers must tolerate."""
    p = {}
    m = re.search(r"Edge delay while expiry count\s*:\s*(\d+)", stp_detail or "")
    p["edge_delay_expiry"] = int(m.group(1)) if m else None
    m = re.search(r"Rcvd info while expiry count\s*:\s*(\d+)", stp_detail or "")
    p["rcvd_info_expiry"] = int(m.group(1)) if m else None
    m = re.search(r"Boundary port\s*:\s*(\w+)", stp_detail or "")
    p["boundary_port"] = (m.group(1).lower() == "yes") if m else None
    m = re.search(r"Link type\s*:\s*(\S+)", stp_detail or "")
    p["link_type"] = m.group(1) if m else None
    p["is_edge"] = bool(p["link_type"] and "EDGE" in p["link_type"].upper())

    m = re.search(r"Physical link is\s+(Up|Down)", interface_stats or "")
    p["link_up"] = (m.group(1) == "Up") if m else None
    m = re.search(r"Last flapped\s*:\s*.+?\(([^)]+) ago\)", interface_stats or "")
    p["last_flapped_text"] = m.group(1) if m else None
    m = re.search(r"Input rate\s*:\s*(\d+)\s*bps", interface_stats or "")
    p["input_rate_bps"] = int(m.group(1)) if m else None
    m = re.search(r"Output rate\s*:\s*(\d+)\s*bps", interface_stats or "")
    p["output_rate_bps"] = int(m.group(1)) if m else None

    # show spanning-tree statistics interface — single data row after header.
    m = re.search(r"^\s*\S+\s+(\d+)\s+(\d+)", stp_stats or "", re.M)
    if m:
        p["bpdus_sent"]     = int(m.group(1))
        p["bpdus_received"] = int(m.group(2))
    else:
        p["bpdus_sent"]     = None
        p["bpdus_received"] = None
    return p


def _parse_arubacx_investigation(stp_detail, interface_stats):
    """Aruba-CX: 'show spanning-tree interface' already exposes BPDU
    counters, so no separate stats command is needed."""
    p = {}
    m = re.search(r"BPDU Tx Count\s*:\s*(\d+)", stp_detail or "")
    p["bpdus_sent"] = int(m.group(1)) if m else None
    m = re.search(r"BPDU Rx Count\s*:\s*(\d+)", stp_detail or "")
    p["bpdus_received"] = int(m.group(1)) if m else None
    m = re.search(r"Admin Edge Port\s*:\s*(\S+)", stp_detail or "")
    aep = m.group(1).lower() if m else None
    p["is_edge"] = (aep in ("admin-edge", "auto-edge")) if aep is not None else None
    m = re.search(r"Link Type\s*:\s*(\S+)", stp_detail or "")
    p["link_type"] = m.group(1) if m else None
    # AOS-CX doesn't expose the equivalent edge-delay timer counter.
    p["edge_delay_expiry"] = None
    p["boundary_port"]     = None

    # 'Interface 1/1/1 is up' / 'is down'
    m = re.search(r"Interface\s+\S+\s+is\s+(up|down)", interface_stats or "", re.I)
    p["link_up"] = (m.group(1).lower() == "up") if m else None
    # 'Link state: up for X' / 'Link state: down for X'
    m = re.search(r"Link state\s*:\s*(?:up|down)\s+for\s+(.+?)(?:\s+\(|\n|$)",
                  interface_stats or "", re.I)
    p["last_flapped_text"] = m.group(1).strip() if m else None
    # Three numeric columns (RX, TX, Total) on the Mbits/sec row.
    m = re.search(r"Mbits / sec\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)",
                  interface_stats or "")
    if m:
        p["input_rate_bps"]  = int(float(m.group(1)) * 1_000_000)
        p["output_rate_bps"] = int(float(m.group(2)) * 1_000_000)
    else:
        p["input_rate_bps"]  = None
        p["output_rate_bps"] = None
    return p


def _parse_procurve_investigation(stp_detail, interface_stats):
    """ProCurve: BPDU counts are in 'show spanning-tree <port> detail'
    (six-column MST/CFG/TCN Tx/Rx table). Sums MST_Tx+CFG_Tx for sent and
    MST_Rx+CFG_Rx for received — TCN is excluded since it counts notifications,
    not regular BPDU traffic, and would inflate the figure for ports that
    don't normally carry STP traffic.
    """
    p = {}
    m = re.search(
        r"-{5,}\s+-{5,}\s+-{5,}\s+-{5,}\s+-{5,}\s+-{5,}\s+"
        r"(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)",
        stp_detail or ""
    )
    if m:
        mst_tx, mst_rx, cfg_tx, cfg_rx, _tcn_tx, _tcn_rx = (int(x) for x in m.groups())
        p["bpdus_sent"]     = mst_tx + cfg_tx
        p["bpdus_received"] = mst_rx + cfg_rx
    else:
        p["bpdus_sent"]     = None
        p["bpdus_received"] = None

    m = re.search(r"OperEdgePort\s*:\s*(\w+)", stp_detail or "")
    p["is_edge"] = (m.group(1).lower() == "yes") if m else None
    m = re.search(r"MST Region Boundary\s*:\s*(\w+)", stp_detail or "")
    p["boundary_port"] = (m.group(1).lower() == "yes") if m else None
    p["link_type"]         = None
    p["edge_delay_expiry"] = None  # not exposed in ProCurve CLI

    m = re.search(r"Link Status\s*:\s*(Up|Down)", interface_stats or "", re.I)
    p["link_up"] = (m.group(1).lower() == "up") if m else None
    p["last_flapped_text"] = None  # not in 'show interfaces' output

    # 'Total Rx(Kbps) : N    Total Tx(Kbps) : N' → bps
    m = re.search(r"Total Rx\(Kbps\)\s*:\s*(\d+)", interface_stats or "")
    p["input_rate_bps"] = int(m.group(1)) * 1000 if m else None
    m = re.search(r"Total Tx\(Kbps\)\s*:\s*(\d+)", interface_stats or "")
    p["output_rate_bps"] = int(m.group(1)) * 1000 if m else None
    return p


def _stp_event_context(ip, interface):
    """Pull port_flaps row + most-recent TCN at this bridge from the DB.
    last_tcn_seconds = MIN across instances (smallest seconds-ago = most
    recent topology change)."""
    conn = _db()
    pf = conn.execute(
        "SELECT * FROM port_flaps WHERE ip = ? AND interface = ?",
        (ip, interface)
    ).fetchone()
    tcn_row = conn.execute(
        "SELECT MIN(last_tcn_seconds) AS tcn_s FROM stp_root_state "
        "WHERE ip = ? AND last_tcn_seconds IS NOT NULL",
        (ip,)
    ).fetchone()
    conn.close()
    return pf, (tcn_row["tcn_s"] if tcn_row and tcn_row["tcn_s"] is not None else None)


def _classify_stp_event(parsed, port_flap_row, last_tcn_seconds,
                        old_role=None, new_role=None):
    """Pure-function classifier. Returns (label, confidence, evidence_lines).
    Confidence ('high'|'medium'|'low') is rendered for emphasis only — it
    never gates control flow.

    old_role/new_role: when both are provided and differ, the snmp_transient
    rule is gated off — a role change (e.g. DESG -> BACKUP) is a strong
    signal of a real STP topology event regardless of BPDU history. The
    'snmp_transient' rule is *only* meant to catch state-only flickers
    where the JUNIPER MIB cache reports a stale state for a port whose
    actual role and link are unchanged.
    """
    ev = []
    flap_count   = port_flap_row["flap_count"] if port_flap_row else 0
    last_flap_at = port_flap_row["last_seen"]  if port_flap_row else None
    bpdus_rx     = parsed.get("bpdus_received")
    role_changed = (old_role is not None and new_role is not None
                    and old_role != new_role)

    # 1) Real STP topology change via role flip — DESG <-> BACKUP, DESG
    # <-> ALT, etc. The role machine actually moved, regardless of BPDU
    # history. Highest priority because a role flip is unambiguous: the
    # other rules (one_way, physical, snmp_transient) are heuristics that
    # can fire spuriously on a port that just got promoted/demoted (no
    # BPDUs received on a brand-new DESG, edge_delay_expiry accumulating
    # over weeks, etc.). When the device tells us the role flipped, we
    # trust it.
    if role_changed:
        ev.append(f"role changed: {old_role} -> {new_role} "
                  "(real STP role-machine event)")
        if (bpdus_rx or 0) > 0:
            ev.append(f"BPDUs received={bpdus_rx} (active state machine)")
        return (LIKELY_REAL_STP_EVENT, "medium", ev)

    # 2) Physical link instability — observed flapping at the port.
    # Outranks the one_way rule because flap_count is observed history
    # (real-world transitions recorded by the cron). An actively-flapping
    # port can momentarily satisfy one_way's conditions (link 'up' between
    # flaps, BPDUs not received during a down moment), and we don't want
    # to misdiagnose a flapping cable as a one-way link.
    if flap_count >= _HIGH_FLAP_THRESHOLD:
        ev.append(f"port_flaps.flap_count={flap_count} "
                  f"(≥{_HIGH_FLAP_THRESHOLD} threshold)")
        if last_flap_at:
            ev.append(f"last flap @ {last_flap_at}")
        return (LIKELY_PHYSICAL_INSTABILITY, "high", ev)

    # 3) One-way link — link up, TX flowing, no RX, no BPDUs in. Gated on
    # *evidence of recent state-machine flux* so a one-shot input_rate_bps
    # snapshot of 0 bps on a steady port doesn't trip the rule. The gate
    # is: flap_count > 0 OR edge_delay_expiry above noise floor. Either
    # one means the port has been actively transitioning recently — both
    # are zero on a port that's just sitting there with no real traffic.
    edge_expiry = parsed.get("edge_delay_expiry") or 0
    state_churn = (flap_count > 0) or (edge_expiry > _HIGH_EDGE_EXPIRY_RATE)
    if (parsed.get("link_up") is True
            and (parsed.get("output_rate_bps") or 0) > 0
            and parsed.get("input_rate_bps") == 0
            and bpdus_rx == 0
            and state_churn):
        ev.append(f"link up; output={parsed['output_rate_bps']} bps; input=0 bps")
        ev.append("BPDUs received=0")
        if flap_count > 0:
            ev.append(f"port_flaps.flap_count={flap_count}")
        if edge_expiry > _HIGH_EDGE_EXPIRY_RATE:
            ev.append(f"edge_delay_expiry={edge_expiry} (high)")
        return (LIKELY_ONE_WAY_LINK, "high", ev)

    # 4) TC flush — recent topology change drove a brief learning state.
    if last_tcn_seconds is not None and last_tcn_seconds < _TC_FLUSH_WINDOW_S:
        ev.append(f"TC at this bridge {last_tcn_seconds}s ago "
                  f"(< {_TC_FLUSH_WINDOW_S}s window)")
        return (LIKELY_TC_FLUSH, "medium", ev)

    # 5) SNMP/MIB phantom — no flap row, no BPDU input, port stable, no TC.
    # Role-changed events are caught by rule 3 above and never land here.
    if (port_flap_row is None
            and bpdus_rx == 0
            and parsed.get("link_up") is True
            and (last_tcn_seconds is None
                 or last_tcn_seconds >= _TC_FLUSH_WINDOW_S)):
        ev.append("no port_flaps row (no observed link transitions)")
        ev.append("BPDUs received=0")
        if parsed.get("last_flapped_text"):
            ev.append(f"port up for {parsed['last_flapped_text']}")
        if parsed.get("is_edge") or parsed.get("boundary_port"):
            ev.append("edge/boundary port")
        return (LIKELY_SNMP_TRANSIENT, "medium", ev)

    # 6) Active state machine — BPDUs flowing without a role change.
    # Lower confidence than rule 3 because there's no specific event
    # signal beyond "BPDUs are arriving" — operator should still look but
    # the diagnosis is more "investigate" than "definite topology shift".
    if (bpdus_rx or 0) > 0 and parsed.get("link_up"):
        ev.append(f"BPDUs received={bpdus_rx} (active state machine)")
        return (LIKELY_REAL_STP_EVENT, "low", ev)

    return (LIKELY_UNCLASSIFIED, "low", ev)


def _format_classification_block(label, confidence, evidence_lines):
    """Render the classifier output as a fixed-width block matching the
    other investigation sections (--- Title --- header style)."""
    title, action = _STP_CLASSIFIER_COPY[label]
    out = [f"--- Likely cause ({confidence} confidence): {title} ---"]
    if evidence_lines:
        for line in evidence_lines:
            out.append(f"  • {line}")
    else:
        out.append("  (no signals matched any rule)")
    out.append(f"Suggested action: {action}")
    return out


# Vendor OUI hints for the per-port MAC table snapshot. Not exhaustive —
# covers the common gear an on-call admin needs to recognize at a glance
# (phones / APs / network kit / endpoint OSes). For an unmapped OUI we
# print "(unknown vendor)" rather than failing; the operator can still
# look it up manually in the IEEE OUI registry.
_OUI_VENDOR_HINTS = {
    # Cisco (selection — Cisco owns hundreds of OUIs)
    "0000": "Xerox/older", "0001": "Cisco?", "00000c": "Cisco",
    "001011": "Cisco", "00142a": "Cisco", "001b67": "Cisco",
    "001c0e": "Cisco", "001e4a": "Cisco", "001e7a": "Cisco",
    "0024c4": "Cisco", "0026cb": "Cisco", "00405d": "Cisco",
    "446d57": "Cisco", "503de5": "Cisco", "70b3d5": "Cisco",
    # Juniper
    "001a82": "Juniper", "002083": "Juniper", "002586": "Juniper",
    "0026ce": "Juniper", "5c5e89": "Juniper", "78fe3d": "Juniper",
    "9051b6": "Juniper", "a8d0e5": "Juniper",
    # HPE/Aruba Networking (acquired Aruba 2015; OUIs span both eras)
    "001635": "HP",        "001871": "HP",        "001e0b": "HP",
    "0024a8": "HP",        "002655": "HP",        "00306e": "HP",
    "00184e": "HP",        "00246c": "Aruba/HPE", "002a4b": "HPE Networking",
    "6cf37f": "HPE Networking", "84d47e": "HPE Networking",
    "94f128": "HPE Networking", "c0917e": "HPE Networking",
    "ecebb8": "HPE Networking",
    # Aruba IAP / APs (older OUI block before merger)
    "001a1e": "Aruba",     "0024fd": "Aruba",     "ac1610": "Aruba AP",
    # IP phones — Polycom, Mitel, Yealink
    "0004f2": "Polycom", "64167f": "Polycom",
    "00085d": "Mitel",   "001565": "Mitel/Yealink",  "805ec0": "Yealink",
    # Apple
    "000393": "Apple", "000a27": "Apple", "001ec2": "Apple",
    "002608": "Apple", "002500": "Apple", "0026bb": "Apple",
    "0c74c2": "Apple", "14c213": "Apple", "78fd94": "Apple",
    # Microsoft (Hyper-V virtual NICs, Surface, etc.)
    "00125a": "Microsoft", "00155d": "Microsoft (Hyper-V)",
    "001dd8": "Microsoft", "0050f2": "Microsoft", "7c1e52": "Microsoft",
    # VMware
    "000569": "VMware", "000c29": "VMware",
    "001456": "VMware", "005056": "VMware",
    # Intel (NICs)
    "0002b3": "Intel", "001b21": "Intel", "001e64": "Intel",
    "0019d1": "Intel", "1c697a": "Intel", "8c1645": "Intel",
    # Dell
    "001422": "Dell", "00188b": "Dell", "001ec9": "Dell", "002219": "Dell",
    "78ac44": "Dell", "b083fe": "Dell", "b4e9b0": "Dell", "f48e38": "Dell",
    # Lenovo
    "001cb3": "Lenovo", "0021cc": "Lenovo", "002435": "Lenovo",
    "447766": "Lenovo", "705a0f": "Lenovo",
    # Printers
    "001ba9": "Brother", "30055c": "Brother",
    "0007e9": "Canon",   "001e8f": "Canon",
    "00194b": "Lexmark", "002564": "Lexmark",
    # Ubiquiti
    "002722": "Ubiquiti", "00154d": "Ubiquiti",
    "78a351": "Ubiquiti", "fc92bc": "Ubiquiti", "78cb6e": "Ubiquiti",
    # Misc
    "00114c": "Aerohive",
    "0090fb": "Portwell",
    "001a4b": "HP/Compaq",
}


def _normalize_mac(mac):
    """Return a lowercase hex-only MAC (no separators), or '' if not a MAC."""
    if not mac:
        return ""
    s = re.sub(r"[^0-9a-fA-F]", "", mac).lower()
    return s if len(s) == 12 else ""


# Full IEEE OUI assignments — loaded lazily from the first available source
# (Debian ieee-data is the freshest; the .deb-bundled CSV is the floor so
# this always works even without the package; Wireshark's manuf as a third
# alternative). Parsed once per process and cached. Empty dict means we
# couldn't find any source.
_OUI_DB = None  # tuple (db_36, db_28, db_24): {hex_prefix_lower: vendor}


def _load_oui_db():
    """Return cached (db_36, db_28, db_24) — IEEE OUI assignments by tier.

    IEEE publishes three OUI registries (MA-S 36-bit, MA-M 28-bit, MA-L
    24-bit). Smaller-block vendors and many recent allocations live in
    MA-M/MA-S, so a 24-bit-only lookup misses them. We bundle all three
    (always fresh at release time) and consult them longest-prefix-first
    in _oui_vendor_hint.

    Priority: bundled (always fresh) > apt-installed ieee-data (24-bit
    only, may be old) > Wireshark manuf (24-bit only). The bundled CSVs
    are the floor — netops works correctly even without any other OUI
    package installed. Cached after first parse; {} dicts on any failure.
    """
    global _OUI_DB
    if _OUI_DB is not None:
        return _OUI_DB
    db_36, db_28, db_24 = {}, {}, {}

    def _load_csv(path, dst, hexlen):
        try:
            import csv
            with open(path, encoding="utf-8", errors="replace") as f:
                r = csv.reader(f)
                next(r, None)  # header
                for row in r:
                    if len(row) >= 3 and len(row[1]) == hexlen:
                        dst[row[1].lower()] = row[2].strip()
            return True
        except Exception as e:
            log.debug("OUI CSV load %s: %s", path, e)
            return False

    # Bundled first — always fresh at .deb release time, covers all three tiers.
    bundled_ok = False
    for path, dst, hexlen in (("/usr/share/netops/oui36.csv", db_36, 9),
                              ("/usr/share/netops/mam.csv",   db_28, 7),
                              ("/usr/share/netops/oui.csv",   db_24, 6)):
        if os.path.exists(path) and _load_csv(path, dst, hexlen):
            bundled_ok = True

    # If bundled missing (dev checkout, future-stripped package, etc.) fall
    # back to system-installed 24-bit lists. These are 24-bit-only — no
    # MA-M/MA-S coverage in this path. Better than nothing.
    if not bundled_ok:
        hex_line = re.compile(
            r'^([0-9A-Fa-f]{2})[-:]([0-9A-Fa-f]{2})[-:]([0-9A-Fa-f]{2})'
            r'\s+(?:\(hex\)\s+)?([^#\n]+?)\s*(?:#.*)?$')
        for path in ("/usr/share/ieee-data/oui.txt",
                     "/usr/share/wireshark/manuf"):
            if not os.path.exists(path):
                continue
            try:
                with open(path, encoding="utf-8", errors="replace") as f:
                    for line in f:
                        m = hex_line.match(line)
                        if m:
                            db_24[(m.group(1) + m.group(2)
                                   + m.group(3)).lower()] = m.group(4).strip()
                if db_24:
                    break
            except Exception as e:
                log.debug("OUI fallback %s: %s", path, e)

    _OUI_DB = (db_36, db_28, db_24)
    return _OUI_DB


def _oui_vendor_hint(mac):
    """Look up the operator-facing vendor name for a MAC. The curated
    _OUI_VENDOR_HINTS dict wins first (its hand-picked short names are
    nicer in alerts than the verbose IEEE org names), then we fall back to
    the full IEEE OUI assignments DB by longest-prefix match: MA-S
    (36-bit, 9 hex) -> MA-M (28-bit, 7 hex) -> MA-L (24-bit, 6 hex).
    Also tries 4-hex against the curated dict for legacy short-OUI
    entries we registered.

    Returns None when no vendor can be found. Callers that write the
    result into port_macs.oui_vendor get a NULL — that way a future
    registry update reaches the cached row via the live-lookup fallback
    instead of being shadowed by a stale literal '(unknown vendor)' that
    has to be backfilled out by hand."""
    norm = _normalize_mac(mac)
    if not norm:
        return None
    for plen in (6, 4):
        v = _OUI_VENDOR_HINTS.get(norm[:plen])
        if v:
            return v
    db_36, db_28, db_24 = _load_oui_db()
    v = (db_36.get(norm[:9]) if len(norm) >= 9 else None) \
        or (db_28.get(norm[:7]) if len(norm) >= 7 else None) \
        or db_24.get(norm[:6])
    return v  # may be None — display callers add the "(unknown vendor)" label


# Match the three common MAC string formats: aa:bb:cc:dd:ee:ff,
# aabb-ccdd-eeff (HP/ProCurve), aabb.ccdd.eeff (Cisco dot).
_MAC_RE = re.compile(
    r"(?:[0-9a-fA-F]{2}[:-]){5}[0-9a-fA-F]{2}"
    r"|[0-9a-fA-F]{6}-[0-9a-fA-F]{6}"
    r"|[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}"
)


def _format_mac_table_section(mac_table_text, limit=10):
    """Extract MACs from CLI output and render an operator-friendly list.

    Strips obvious self-MAC noise (e.g. the switch's own base MAC) is left
    to the caller — vendor CLIs don't typically include the switch MAC in
    a per-port table query, so we don't try to guess.

    limit caps the number of MACs rendered (trunk ports can have hundreds).
    """
    if not mac_table_text:
        return ["(no MAC table output)"]
    macs = []
    seen = set()
    for raw in _MAC_RE.findall(mac_table_text):
        norm = _normalize_mac(raw)
        if not norm or norm in seen:
            continue
        seen.add(norm)
        macs.append(raw)
    if not macs:
        return ["(no MACs learned on this port)"]
    lines = []
    for i, mac in enumerate(macs[:limit]):
        vendor = _oui_vendor_hint(mac) or "(unknown vendor)"
        lines.append(f"  {mac}  [{vendor}]")
    if len(macs) > limit:
        lines.append(f"  ... and {len(macs) - limit} more MAC(s)")
    return lines


def _format_port_macs_history(ip, interface, limit=10):
    """Last-known MAC(s) on a port from the port_macs FDB history, newest
    first, with OUI vendor + age. The STP-alert / flap-digest fallback for
    when the live FDB is empty because the port is down/flapping — the
    exact case where the operator most needs to know what *was* there.
    Returns [] on miss (no history, or pre-3.8.0 DB without the table)."""
    try:
        conn = _db()
        conn.row_factory = sqlite3.Row
        rows = conn.execute(
            "SELECT mac, vlan, oui_vendor, last_seen "
            "FROM port_macs WHERE ip = ? AND interface = ? "
            "ORDER BY last_seen DESC LIMIT ?",
            (ip, interface, limit)
        ).fetchall()
        conn.close()
    except Exception:
        return []
    if not rows:
        return []
    now = datetime.now()
    out = []
    for r in rows:
        try:
            seen = datetime.strptime(r["last_seen"], "%Y-%m-%d %H:%M:%S")
            secs = max(0, int((now - seen).total_seconds()))
            if secs < 3600:
                age = f"{secs // 60}m ago"
            elif secs < 86400:
                age = f"{secs // 3600}h ago"
            else:
                age = f"{secs // 86400}d ago"
        except (ValueError, TypeError):
            age = "age unknown"
        vendor = r["oui_vendor"] or _oui_vendor_hint(r["mac"]) or "(unknown vendor)"
        vlan = f", vlan {r['vlan']}" if r["vlan"] else ""
        out.append(f"{r['mac']}  [{vendor}]  (last seen {age}{vlan})")
    return out


def _port_macs_digest_hint(ip, interface):
    """One-line last-known-MAC hint for the flap digest (newest entry +
    'and N more'), or '' when there's no port_macs history. Lets an admin
    identify the flapping device without logging into the switch."""
    hist = _format_port_macs_history(ip, interface, limit=4)
    if not hist:
        return ""
    extra = f"   (+{len(hist) - 1} more recent MAC(s))" if len(hist) > 1 else ""
    return f"         last-known MAC: {hist[0]}{extra}"


# LAG / trunk-named interfaces across our vendors: ProCurve Trk4,
# Junos ae0, Aruba-CX lag1, Cisco-ish Po1 / Port-Channel1.
_LAG_IFACE_RE = re.compile(
    r'(?:trk|ae|lag|po|port-?channel)\s*\d+$', re.IGNORECASE)


def _compute_mac_flux(conn, threshold, ceiling, max_vlans,
                       max_times_seen=0, max_span_sec=0):
    """Access/edge ports whose ROTATING-MAC count over the trailing 24h is
    in [threshold, ceiling]. Hybrid non-trunk classifier — a port is
    excluded when ANY of:
      * the interface is LAG/trunk-named (Trk/ae/lag/po/port-channel),
      * it has ANY LLDP neighbor (present in topology_edges at all — a
        switch/AP link, not a bare access host),
      * its distinct-MAC count exceeds the uplink ceiling (structurally an
        aggregation/uplink, not a single rogue device),
      * its MACs span more than max_vlans VLANs — a true access port is
        one data VLAN (+ maybe a voice VLAN); a port carrying many VLANs
        is a trunk or a virtualization host (e.g. a VMware vSwitch uplink,
        which legitimately shows many VM MACs and would otherwise false-
        alarm on every vMotion). VLAN-unaware (dot1d, vlan=0) ports count
        as a single VLAN and are unaffected.

    Rotation filter (3.8.7+) — only MACs that look like brief passers-
    through count toward the threshold; stable residents do not. A MAC
    qualifies as "rotating" when BOTH of these hold:
      * its `times_seen` (= number of 30-min FDB harvests it appeared in)
        is <= max_times_seen, AND
      * its (last_seen - first_seen) span is < max_span_sec.
    Either parameter set to 0 (or non-positive) disables that part of
    the filter and falls back to the 3.8.6 "count all 24h distincts"
    semantics — preserved so operators with a legacy config still see
    the original behavior until they opt in. The classifier name
    matches its semantic intent: a port where the MAC neighborhood is
    ROTATING (rogue switch with rotating clients, hot-desk wall jack)
    rather than where many devices are STABLY present (conference-room
    AV stack, multi-device desk).

    Returns [{ip, hostname, interface, distinct, vlans}], busiest first.
    threshold <= 0 disables. conn needs a sqlite3.Row row_factory."""
    if threshold <= 0:
        return []
    since = (datetime.now() - timedelta(hours=24)).strftime("%Y-%m-%d %H:%M:%S")
    where = ["last_seen >= ?"]
    params = [since]
    if max_times_seen and max_times_seen > 0:
        where.append("times_seen <= ?")
        params.append(max_times_seen)
    if max_span_sec and max_span_sec > 0:
        # span in seconds — julianday() returns days, *86400 -> seconds.
        where.append(
            "(julianday(last_seen) - julianday(first_seen)) * 86400 < ?")
        params.append(max_span_sec)
    params.append(threshold)
    try:
        rows = conn.execute(f"""
            SELECT ip, interface,
                   COUNT(DISTINCT mac)  AS distinct_macs,
                   COUNT(DISTINCT vlan) AS vlan_count
            FROM port_macs
            WHERE {' AND '.join(where)}
            GROUP BY ip, interface
            HAVING distinct_macs >= ?
        """, params).fetchall()
    except sqlite3.OperationalError:
        return []                       # pre-3.8.0 DB without port_macs
    if not rows:
        return []
    # ANY LLDP neighbor => switch/AP link, not a bare access host.
    linked = {(e["src_ip"], e["src_port"]) for e in conn.execute(
        "SELECT src_ip, src_port FROM topology_edges").fetchall()}
    out = []
    for r in rows:
        if ceiling and ceiling > 0 and r["distinct_macs"] > ceiling:
            continue                    # structurally an uplink/aggregation
        if max_vlans and max_vlans > 0 and r["vlan_count"] > max_vlans:
            continue                    # multi-VLAN trunk / hypervisor uplink
        if (r["ip"], r["interface"]) in linked:
            continue                    # has an LLDP neighbor (switch/AP)
        if _LAG_IFACE_RE.match((r["interface"] or "").strip()):
            continue                    # LAG / trunk-named interface
        hn = conn.execute("SELECT hostname FROM devices WHERE ip = ?",
                           (r["ip"],)).fetchone()
        out.append({
            "ip": r["ip"],
            "hostname": (hn["hostname"] if hn and hn["hostname"] else r["ip"]),
            "interface": r["interface"],
            "distinct": r["distinct_macs"],
            "vlans": r["vlan_count"],
        })
    out.sort(key=lambda d: (-d["distinct"], d["hostname"], d["interface"]))
    return out


def _get_flap_silenced_ports(conn):
    """Currently-active port-flap silences as a set of (ip, interface)
    tuples. Excludes expired entries (`expires_at IS NOT NULL AND
    expires_at < now`). Callers that filter the digest also want
    expired rows garbage-collected periodically — see
    _gc_expired_flap_silences()."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    try:
        rows = conn.execute(
            "SELECT ip, interface FROM port_flap_whitelist "
            "WHERE expires_at IS NULL OR expires_at > ?",
            (now,)).fetchall()
    except sqlite3.OperationalError:
        return set()                    # pre-3.8.10 DB without the table
    return {(r["ip"], r["interface"]) for r in rows}


def _gc_expired_flap_silences(conn):
    """Delete port_flap_whitelist rows whose expires_at has passed.
    Called from digest_flap so the table stays bounded without a
    dedicated cleanup timer. Safe to call when the table doesn't
    exist (pre-3.8.10 DB) — swallows the OperationalError."""
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    try:
        cur = conn.execute(
            "DELETE FROM port_flap_whitelist "
            "WHERE expires_at IS NOT NULL AND expires_at < ?",
            (now,))
        conn.commit()
        if cur.rowcount:
            log.info("Garbage-collected %d expired flap silence(s).",
                     cur.rowcount)
    except sqlite3.OperationalError:
        pass


def _get_excluded_ports(conn, cfg):
    """Union of the cfg-managed and DB-managed flux-whitelist sets.

    Returns a set of (ip, interface) tuples. The cfg key
    [monitor] mac_flux_excluded_ports (3.8.7) is the version-controlled
    surface; the mac_flux_whitelist DB table (3.8.8) is the operator-
    managed surface edited via `netops whitelist {add|remove}`. Either
    or both may be populated; their union is the exclusion list."""
    excluded = set(cfg.get("mac_flux_excluded_ports", set()))
    try:
        for r in conn.execute(
                "SELECT ip, interface FROM mac_flux_whitelist").fetchall():
            excluded.add((r["ip"], r["interface"]))
    except sqlite3.OperationalError:
        pass                       # pre-3.8.8 DB without the table
    return excluded


def handle_whitelist(args):
    """`netops whitelist {add|remove|list} [--flap|--flux|--all] [-r REASON] [--until DATE]`

    Manages two operator-managed exclusion tables:
      mac_flux_whitelist  -- approved ports that should never page the
                             MAC-flux security signal (3.8.8+)
      port_flap_whitelist -- approved/known-noisy ports that should not
                             surface in `digest flap` emails (3.8.10+)

    Backward-compat: if --flap / --flux / --all are all unset, defaults
    to --flux so the 3.8.8 CLI keeps working unchanged.

    --until YYYY-MM-DD applies only to --flap entries (the flap silence
    is the time-bounded variant). For --flux it's accepted but warned
    against; the cfg surface treats flux entries as permanent."""
    action = args.action
    # Category selector — backward-compat: bare CLI = flux.
    want_flap = bool(getattr(args, "flap", False))
    want_flux = bool(getattr(args, "flux", False)) or \
                (not want_flap and not getattr(args, "all", False))
    want_all  = bool(getattr(args, "all", False))
    until     = getattr(args, "until", None)
    if until and not want_flap:
        print("whitelist: --until is only meaningful for --flap silences; "
              "mac-flux entries are permanent. Ignoring --until.")
        until = None

    # ------------------------------------------------------------------
    if action == "list":
        conn = _db(); conn.row_factory = sqlite3.Row
        printed_any = False
        if want_flux or want_all:
            try:
                rows = conn.execute(
                    "SELECT ip, interface, reason, added_at, added_by "
                    "FROM mac_flux_whitelist ORDER BY ip, interface"
                ).fetchall()
                print("=== MAC-flux whitelist (mac_flux_whitelist) ===")
                if rows:
                    _print_table(
                        ["ip", "interface", "added_at", "added_by", "reason"],
                        [(r["ip"], r["interface"], r["added_at"],
                          r["added_by"] or "", r["reason"] or "")
                         for r in rows])
                    printed_any = True
                else:
                    print("(empty)")
                cfg_excl = sorted(getattr(args, "_cfg_excluded", set()))
                if cfg_excl:
                    print("\nAlso excluded via [monitor] mac_flux_excluded_ports:")
                    for ip, iface in cfg_excl:
                        print(f"  {ip}  {iface}")
            except sqlite3.OperationalError:
                print("(no mac_flux_whitelist table — initialize DB first)")
        if want_flap or want_all:
            if want_all:
                print()
            try:
                rows = conn.execute(
                    "SELECT ip, interface, reason, added_at, added_by, expires_at "
                    "FROM port_flap_whitelist ORDER BY ip, interface"
                ).fetchall()
                print("=== Port-flap silences (port_flap_whitelist) ===")
                if rows:
                    _print_table(
                        ["ip", "interface", "added_at", "added_by",
                         "expires_at", "reason"],
                        [(r["ip"], r["interface"], r["added_at"],
                          r["added_by"] or "",
                          r["expires_at"] or "(permanent)",
                          r["reason"] or "") for r in rows])
                    printed_any = True
                else:
                    print("(empty)")
            except sqlite3.OperationalError:
                print("(no port_flap_whitelist table — initialize DB first)")
        return

    # ------------------------------------------------------------------
    # add / remove: require ip + interface
    if not args.ip or not args.interface:
        print(f"whitelist {action}: ip and interface are required")
        sys.exit(2)
    ip, iface = args.ip.strip(), args.interface.strip()
    added_by = (os.environ.get("SUDO_USER")
                or os.environ.get("USER") or "unknown")
    added_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    # Normalize --until: accept YYYY-MM-DD or YYYY-MM-DD HH:MM:SS.
    expires_at = None
    if until:
        u = until.strip()
        if len(u) == 10:                # bare date -> end of day
            u = f"{u} 23:59:59"
        try:
            datetime.strptime(u, "%Y-%m-%d %H:%M:%S")
            expires_at = u
        except ValueError:
            print(f"whitelist add: --until {until!r} must be YYYY-MM-DD "
                  f"or 'YYYY-MM-DD HH:MM:SS'")
            sys.exit(2)

    table  = "port_flap_whitelist" if want_flap else "mac_flux_whitelist"
    label  = "flap silence"          if want_flap else "MAC-flux whitelist"
    alerts = "(no live-alert clearing for flap silences)" if want_flap \
             else None
    conn = _db()

    if action == "add":
        if want_flap:
            conn.execute(
                "INSERT OR REPLACE INTO port_flap_whitelist "
                "(ip, interface, reason, added_at, added_by, expires_at) "
                "VALUES (?, ?, ?, ?, ?, ?)",
                (ip, iface, args.reason, added_at, added_by, expires_at))
        else:
            conn.execute(
                "INSERT OR REPLACE INTO mac_flux_whitelist "
                "(ip, interface, reason, added_at, added_by) "
                "VALUES (?, ?, ?, ?, ?)",
                (ip, iface, args.reason, added_at, added_by))
            # If the port currently has an active alert, mark it cleared so
            # operator intent is reflected immediately rather than waiting
            # up to 30 min for the next topology tick to reconcile.
            conn.execute(
                "UPDATE mac_flux_alerts SET cleared_at = ? "
                "WHERE ip = ? AND interface = ? AND cleared_at IS NULL",
                (added_at, ip, iface))
        conn.commit()
        msg = f"added {ip} {iface} to {label}"
        if args.reason:
            msg += f" — {args.reason}"
        if expires_at:
            msg += f" (expires {expires_at})"
        msg += f"  [{added_by}]"
        print(msg)
        return

    if action == "remove":
        cur = conn.execute(
            f"DELETE FROM {table} WHERE ip = ? AND interface = ?",
            (ip, iface))
        conn.commit()
        if cur.rowcount:
            print(f"removed {ip} {iface} from {label}")
        else:
            cfg_hint = ("if it's listed in [monitor] mac_flux_excluded_ports "
                        "in netops.conf, edit that file instead") \
                       if not want_flap else \
                       "no entry to remove"
            print(f"(no entry for {ip} {iface} in {table}; {cfg_hint})")
        return


def _evaluate_mac_flux_alerts(conn, cfg, now):
    """Reconcile current non-trunk MAC-flux against the mac_flux_alerts
    dedup table. Edge-triggered: returns the ports that JUST crossed the
    threshold this episode (so the caller emails once), suppresses while
    an episode is ongoing, and marks cleared_at when a port drops back
    under threshold so a later re-crossing re-alerts. Mirrors the
    stp_pending / *_alerted_at preservation pattern. conn needs a
    sqlite3.Row row_factory."""
    threshold = cfg.get("mac_flux_threshold_24h", 5)
    flux = _compute_mac_flux(conn, threshold,
                               cfg.get("mac_flux_uplink_ceiling", 24),
                               cfg.get("mac_flux_max_vlans", 2),
                               cfg.get("mac_flux_rotation_max_times_seen", 3),
                               cfg.get("mac_flux_rotation_max_span_sec", 7200))
    excluded = _get_excluded_ports(conn, cfg)
    if excluded:
        flux = [p for p in flux if (p["ip"], p["interface"]) not in excluded]
    try:
        existing = {(r["ip"], r["interface"]): r for r in conn.execute(
            "SELECT ip, interface, cleared_at FROM mac_flux_alerts"
        ).fetchall()}
    except sqlite3.OperationalError:
        return []                       # pre-3.8.0 DB without the table
    cur_keys = set()
    new_alerts = []
    for c in flux:
        key = (c["ip"], c["interface"])
        cur_keys.add(key)
        ex = existing.get(key)
        if ex is None or ex["cleared_at"] is not None:
            conn.execute(
                "INSERT OR REPLACE INTO mac_flux_alerts "
                "(ip, interface, distinct_macs, notified_at, cleared_at) "
                "VALUES (?, ?, ?, ?, NULL)",
                (c["ip"], c["interface"], c["distinct"], now))
            new_alerts.append(c)
        else:
            conn.execute(
                "UPDATE mac_flux_alerts SET distinct_macs = ? "
                "WHERE ip = ? AND interface = ?",
                (c["distinct"], c["ip"], c["interface"]))
    for key, ex in existing.items():
        if key not in cur_keys and ex["cleared_at"] is None:
            conn.execute(
                "UPDATE mac_flux_alerts SET cleared_at = ? "
                "WHERE ip = ? AND interface = ?", (now, key[0], key[1]))
    conn.commit()
    return new_alerts


def _send_flux_alert_email(cfg, new_alerts):
    """One-shot security notice for non-trunk ports that just crossed the
    24h MAC-flux threshold. Best-effort; logs the outcome."""
    thr = cfg.get("mac_flux_threshold_24h", 5)
    subject = (f"[netops] SECURITY: MAC flux on {len(new_alerts)} "
               "non-trunk port(s)")
    lines = [
        f"{len(new_alerts)} non-trunk port(s) crossed the MAC-flux "
        f"threshold (>= {thr} distinct MACs in the trailing 24h).",
        "Possible unauthorized mini-switch/hub, an AP, MAC spoofing, or "
        "rapid device swapping. One notice per episode — no re-page until "
        "the port settles and the flux recurs.",
        "",
    ]
    for c in new_alerts:
        lines.append(f"=== {c['hostname']} ({c['ip']}) {c['interface']} "
                     f"— {c['distinct']} distinct MAC(s)/24h"
                     f", {c.get('vlans', 1)} vlan(s) ===")
        for h in _format_port_macs_history(c["ip"], c["interface"], limit=12):
            lines.append(f"  {h}")
        lines.append("")
    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("MAC-flux security email %s (%d port(s))",
                 detail, len(new_alerts))
    else:
        log.error("MAC-flux security email failed: %s", detail)


def _lookup_port_description(ip, hostname, interface, platform):
    """Find the operator's description/name for this port from the most
    recent saved config. Returns '' on miss (no backup, missing port, or
    no description configured). Best-effort — fast and silent on failure.
    """
    candidates = []
    if hostname:
        candidates.append(f"{hostname}_{ip}.cfg")
    candidates.append(f"{ip}.cfg")
    config_path = None
    for fname in candidates:
        path = os.path.join(CONFIGS_DIR, "current", fname)
        if os.path.isfile(path):
            config_path = path
            break
    if not config_path:
        return ""
    iface_phys = interface.split(".", 1)[0]
    try:
        # For Junos prefer the _set.cfg sibling (line-per-statement, simple
        # regex). For hierarchical Junos / Aruba-CX / ProCurve we walk the
        # interface block.
        if platform == "junos":
            stem = os.path.splitext(os.path.basename(config_path))[0]
            set_path = os.path.join(os.path.dirname(config_path),
                                    f"{stem}_set.cfg")
            if os.path.isfile(set_path):
                with open(set_path, encoding="utf-8", errors="replace") as f:
                    for line in f:
                        m = re.match(
                            rf"^set interfaces {re.escape(iface_phys)} "
                            rf'description "?([^"\n]+?)"?\s*$', line)
                        if m:
                            return m.group(1).strip()
                return ""
            # Hierarchical fallback
            with open(config_path, encoding="utf-8", errors="replace") as f:
                text = f.read()
            m = re.search(
                rf"\b{re.escape(iface_phys)}\b\s*\{{[^}}]*?"
                rf'description\s+"?([^";\n]+?)"?\s*;', text, re.DOTALL)
            return m.group(1).strip() if m else ""
        elif platform == "aruba-cx":
            with open(config_path, encoding="utf-8", errors="replace") as f:
                lines = f.readlines()
            in_iface = False
            for line in lines:
                stripped = line.strip()
                if stripped.startswith("interface "):
                    in_iface = (stripped == f"interface {iface_phys}")
                    continue
                if in_iface and stripped.startswith("description "):
                    return stripped[len("description "):].strip().strip('"')
                if in_iface and stripped and not stripped.startswith(" ") \
                        and not line.startswith(" ") and not line.startswith("\t"):
                    # New top-level stanza — exit interface block
                    if not stripped.startswith("!"):
                        in_iface = False
            return ""
        else:  # procurve
            with open(config_path, encoding="utf-8", errors="replace") as f:
                lines = f.readlines()
            in_iface = False
            for line in lines:
                stripped = line.strip()
                if stripped.startswith("interface "):
                    rest = stripped[len("interface "):].strip()
                    in_iface = (rest == iface_phys)
                    continue
                if in_iface and stripped.startswith("name "):
                    return stripped[len("name "):].strip().strip('"')
                if in_iface and stripped == "exit":
                    in_iface = False
            return ""
    except OSError:
        return ""


def _parse_lldp_neighbor(lldp_text, platform):
    """Best-effort extraction of LLDP-neighbor fields from per-port detail.

    Returns dict with keys (any can be empty string):
      system_name, system_description, port_id, port_description,
      chassis_id, chassis_type, capabilities.

    Vendor output formats differ; we use the same loose key-value scan for
    all three and let missing keys come back empty. A missing system_name
    is normal for endpoint devices that don't run LLDP-MED with the host
    name TLV — the DB cross-reference falls back to chassis MAC in that
    case.
    """
    out = {"system_name": "", "system_description": "", "port_id": "",
           "port_description": "", "chassis_id": "", "chassis_type": "",
           "capabilities": ""}
    if not lldp_text:
        return out
    # Junos / Aruba-CX / ProCurve all use a flavor of "Key  : Value".
    # ProCurve uses single-space separation in some lines; we match both
    # ": " and just ":" with surrounding whitespace.
    # Each pattern allows the separator between word parts to be space,
    # hyphen, underscore, or nothing (covers "System Name", "SysName",
    # "Chassis-ID", "ChassisId", etc.). Case-insensitive via re.IGNORECASE
    # at match time.
    kv_patterns = {
        "system_name":        r"(?:System[-_ ]?Name|SysName|Chassis[-_ ]?Name)\s*:?\s*(\S.*?)\s*$",
        "system_description": r"(?:System[-_ ]?Descr(?:iption)?|Chassis[-_ ]?Description|SystemDescr)\s*:?\s*(\S.*?)\s*$",
        "port_id":            r"^\s*Port[-_ ]?Id\s*:?\s*(\S.*?)\s*$",
        "port_description":   r"^\s*Port[-_ ]?Descr(?:iption|iption)?\s*:?\s*(\S.*?)\s*$",
        "chassis_id":         r"^\s*Chassis[-_ ]?Id\s*:?\s*(\S.*?)\s*$",
        "chassis_type":       r"^\s*Chassis[-_ ]?Type\s*:?\s*(\S.*?)\s*$",
    }
    cap_re = re.compile(r"(?:System\s*)?[Cc]apabilities", re.MULTILINE)
    for line in lldp_text.splitlines():
        for field, pat in kv_patterns.items():
            if out[field]:
                continue
            m = re.search(pat, line, re.IGNORECASE)
            if m:
                out[field] = m.group(1).strip()
                break
    # ProCurve writes ChassisId with internal spaces: "ec eb b8 11 22 33".
    # Collapse whitespace in chassis_id so downstream MAC matching works.
    if out["chassis_id"]:
        out["chassis_id"] = re.sub(r"\s+", "", out["chassis_id"]).strip()
    # System capabilities arrive as a multi-line block on Junos/Aruba-CX;
    # we just record whether 'Bridge' appears, which is the bit we use to
    # tell "endpoint" from "another switch" when the DB lookup misses.
    if cap_re.search(lldp_text):
        out["capabilities"] = ("bridge" if re.search(
            r"[Bb]ridge", lldp_text) else "non-bridge")
    return out


def _resolve_neighbor_in_db(parsed):
    """Return the matched devices row (or None) for an LLDP neighbor.

    Match priority:
      1) chassis MAC (12 hex chars) → devices.base_mac
      2) system_name → devices.hostname (case-insensitive)
    A miss usually means the neighbor is an endpoint (workstation / phone /
    AP) not tracked by netops, or a switch we haven't discovered yet.
    """
    chassis_norm = _normalize_mac(parsed.get("chassis_id"))
    sys_name = (parsed.get("system_name") or "").strip()
    if not chassis_norm and not sys_name:
        return None
    conn = _db()
    conn.row_factory = sqlite3.Row
    row = None
    if chassis_norm:
        # base_mac is stored lowercase, hyphenated as "aaaa-aaaa-aaaa"
        # (ProCurve style) or plain hex. Normalize both sides.
        norm_target = chassis_norm
        for r in conn.execute(
            "SELECT * FROM devices WHERE base_mac IS NOT NULL "
            "AND status IN ('active','inactive')"
        ).fetchall():
            stored = _normalize_mac(r["base_mac"])
            if stored and stored == norm_target:
                row = r
                break
    if row is None and sys_name:
        row = conn.execute(
            "SELECT * FROM devices "
            "WHERE LOWER(hostname) = LOWER(?) "
            "AND status IN ('active','inactive') LIMIT 1",
            (sys_name,)).fetchone()
    conn.close()
    return row


def _format_upstream_context_lines(ch, info):
    """Build the 'Upstream context (one hop)' lines for the email.

    Phase A: one-hop trace. Parse the LLDP neighbor on the alerted port,
    cross-reference against our devices DB, and report:
      - what's on the other side (endpoint vs tracked switch),
      - if a tracked switch, that switch's recent STP / root-change activity
        in the last 60 minutes.

    Empty list if there's no LLDP output at all (the caller already shows
    '(no neighbor detected on this port)' in the raw section).
    """
    raw = (info.get("lldp_neighbor") or "").strip()
    if not raw or raw.startswith("(error:"):
        return []
    conn = _db()
    conn.row_factory = sqlite3.Row
    plat_row = conn.execute(
        "SELECT platform FROM devices WHERE ip = ?", (ch["ip"],)
    ).fetchone()
    conn.close()
    platform = plat_row["platform"] if plat_row and plat_row["platform"] else ""
    parsed = _parse_lldp_neighbor(raw, platform)
    lines = ["--- Upstream context (one hop via LLDP) ---"]
    if not parsed["chassis_id"] and not parsed["system_name"]:
        lines.append("  (LLDP output didn't include a chassis-id or system-name "
                     "we could parse — see raw section below)")
        return lines

    # Identifier line — what we extracted from the neighbor's LLDP TLVs.
    ident_bits = []
    if parsed["system_name"]:
        ident_bits.append(f"system={parsed['system_name']}")
    if parsed["chassis_id"]:
        # Render as the same string format we usually display MACs in if
        # it parses as a MAC; otherwise show as-is (some endpoints set
        # chassis to a serial number).
        mac_norm = _normalize_mac(parsed["chassis_id"])
        if mac_norm:
            mac_disp = ":".join(mac_norm[i:i+2] for i in range(0, 12, 2))
            vendor = _oui_vendor_hint(mac_norm)
            ident_bits.append(f"chassis={mac_disp} [{vendor}]")
        else:
            ident_bits.append(f"chassis={parsed['chassis_id']}")
    if parsed["port_id"]:
        ident_bits.append(f"neighbor-port={parsed['port_id']}")
    lines.append("  " + " · ".join(ident_bits))
    if parsed["port_description"]:
        lines.append(f"  neighbor-port-desc: {parsed['port_description']}")

    matched = _resolve_neighbor_in_db(parsed)
    if matched is None:
        # No DB match — almost always an endpoint (printer / phone / AP /
        # workstation). The capabilities TLV usually tells us, but the
        # MAC's OUI vendor hint is the most useful one-line summary.
        cap = parsed.get("capabilities", "")
        if cap == "bridge":
            lines.append("  not tracked in netops devices — capabilities include "
                         "Bridge, so this is likely a switch we haven't discovered yet")
        else:
            lines.append("  not tracked in netops devices — likely an endpoint "
                         "(workstation / phone / AP / printer)")
        return lines

    # Matched a known device — report its current STP-related activity.
    lines.append(f"  matched netops device: {matched['hostname'] or '?'} "
                 f"({matched['ip']})  platform={matched['platform'] or '?'}  "
                 f"model={matched['model'] or '?'}")

    # Recent neighbor STP changes (last 60 min) — limited to its side of the link
    # if we can identify it, otherwise the full set.
    conn = _db()
    rows = conn.execute("""
        SELECT interface, old_role, old_state, new_role, new_state, changed_at
        FROM stp_changes
        WHERE ip = ?
          AND changed_at >= datetime('now', 'localtime', '-60 minutes')
        ORDER BY changed_at DESC
        LIMIT 8
    """, (matched["ip"],)).fetchall()
    if rows:
        lines.append(f"  STP changes on this neighbor in last 60 min ({len(rows)}):")
        for r in rows:
            lines.append(f"    {r['changed_at']}  {r['interface']:14s} "
                         f"{r['old_role']}/{r['old_state']} -> "
                         f"{r['new_role']}/{r['new_state']}")
    else:
        lines.append("  STP changes on this neighbor in last 60 min: none")
    root_rows = conn.execute("""
        SELECT instance, old_priority, old_mac, new_priority, new_mac, changed_at
        FROM stp_root_changes
        WHERE ip = ?
          AND changed_at >= datetime('now', 'localtime', '-60 minutes')
        ORDER BY changed_at DESC
        LIMIT 4
    """, (matched["ip"],)).fetchall()
    if root_rows:
        lines.append(f"  Root-bridge changes on this neighbor in last 60 min ({len(root_rows)}):")
        for r in root_rows:
            lines.append(f"    {r['changed_at']}  inst={r['instance']}  "
                         f"prio {r['old_priority']}->{r['new_priority']}  "
                         f"mac {r['old_mac']}->{r['new_mac']}")
    else:
        lines.append("  Root-bridge changes on this neighbor in last 60 min: none")
    conn.close()
    return lines


def _format_investigation_block(ch, info):
    """Format one investigation block for the alert email / stdout.

    Ordering is top-down — big picture first: root bridge -> port STP
    detail -> LLDP neighbor -> physical interface. The reader gets the
    'where is root?' context before drilling into port-level detail.
    """
    lines = []
    header = f"=== {ch['hostname']} ({ch['ip']}) {ch['interface']} — investigation ==="
    lines.append("")
    lines.append(header)
    if not info.get("ok"):
        lines.append(f"  (investigation failed: {info.get('error') or 'unknown error'})")
        return lines

    # Port description from the most recent saved config — surfaces the
    # operator's own label ("Room 204 phone", "AP-Lobby-East") right under
    # the header so the reader knows what's on this port before reading
    # any other section. Best-effort lookup; empty on miss.
    conn = _db()
    plat_row = conn.execute(
        "SELECT platform FROM devices WHERE ip = ?", (ch["ip"],)
    ).fetchone()
    conn.close()
    if plat_row and plat_row["platform"]:
        desc = _lookup_port_description(
            ch["ip"], ch["hostname"], ch["interface"], plat_row["platform"])
        if desc:
            lines.append(f"  port description (from saved config): {desc}")

    # Heuristic classifier verdict — operator-friendly summary at the top,
    # before the raw CLI dumps. Falls back gracefully when 'parsed' is
    # missing or empty (e.g. older code paths that don't populate it).
    if info.get("parsed"):
        pf, tcn_s = _stp_event_context(ch["ip"], ch["interface"])
        label, conf, evidence = _classify_stp_event(
            info["parsed"], pf, tcn_s,
            old_role=ch.get("old_role"), new_role=ch.get("new_role"))
        lines.extend(_format_classification_block(label, conf, evidence))
        lines.append("")

    # One-hop upstream context — parse LLDP, cross-reference the neighbor
    # against our devices table, and surface that neighbor's recent STP /
    # root-change activity. For endpoints (workstation/phone/AP/printer)
    # we just note that nothing else needs investigating one hop out.
    upstream_lines = _format_upstream_context_lines(ch, info)
    if upstream_lines:
        lines.extend(upstream_lines)
        lines.append("")

    root_lines = _format_root_bridge_lines(ch["ip"])
    if root_lines:
        lines.append("--- Root bridge (this switch's view) ---")
        for rl in root_lines:
            lines.append(f"  {rl}")
        lines.append("")
    lines.append("--- STP interface detail ---")
    lines.append((info["stp_detail"] or "").strip() or "(no output)")
    lines.append("")
    lines.append("--- LLDP neighbor ---")
    lldp = (info["lldp_neighbor"] or "").strip()
    lines.append(lldp or "(no neighbor detected on this port)")
    lines.append("")
    lines.append("--- MAC table (this port, with OUI vendor hints) ---")
    mac_lines = _format_mac_table_section(info.get("mac_table") or "")
    for ml in mac_lines:
        lines.append(ml)
    # Live FDB is empty exactly when the port is down/flapping — fall back
    # to the port_macs history so the operator still sees what was there.
    if mac_lines in (["(no MACs learned on this port)"],
                     ["(no MAC table output)"]):
        hist = _format_port_macs_history(ch["ip"], ch["interface"])
        if hist:
            lines.append("  last-known on this port (FDB history):")
            for hl in hist:
                lines.append(f"    {hl}")
    lines.append("")
    lines.append("--- Interface status ---")
    lines.append((info["interface_stats"] or "").strip() or "(no output)")
    return lines


def _send_stp_alert(cfg, changes, investigations=None, suspected_fp=None):
    """Send email alert for STP state changes if [email] is configured.

    investigations: optional {(ip, interface): info_dict} produced by
    investigate_stp_ports; when provided, the email body appends a
    per-port investigation block below the summary.

    suspected_fp: optional list of changes the classifier tagged as
    snmp_transient — rendered in a separate "Suspected false positives"
    section so the operator can verify the classification without acting
    on them. Subject line counts only `changes`.
    """
    if not cfg.get("smtp_server"):
        log.debug("No [email] config — skipping email alert")
        return

    suspected_fp = suspected_fp or []
    subject = f"[netops] STP changes confirmed — {len(changes)} port(s)"
    if suspected_fp:
        subject += f" (+ {len(suspected_fp)} suspected false-positive)"
    body_lines = [f"STP state changes (confirmed across two consecutive polls) at "
                  f"{datetime.now():%Y-%m-%d %H:%M}:\n"]
    for ch in changes:
        blocking = " *** BLOCKING ***" if is_stp_blocking(ch) else ""
        body_lines.append(
            f"  {ch['hostname']} ({ch['ip']}) {ch['interface']}: "
            f"{ch['old_role']}/{ch['old_state']} -> {ch['new_role']}/{ch['new_state']}{blocking}"
        )

    if investigations:
        for ch in changes:
            info = investigations.get((ch["ip"], ch["interface"]))
            if info is not None:
                body_lines.extend(_format_investigation_block(ch, info))

    if suspected_fp:
        body_lines.append("")
        body_lines.append(
            f"=== Suspected false positives ({len(suspected_fp)}) ===")
        body_lines.append(
            "These changes confirmed across two polls but the classifier "
            "flagged them as snmp_transient (no port_flaps row, no BPDU "
            "input, link stable, no recent TC). Probably MIB-cache vs "
            "state-machine race on Junos edge/boundary ports. Verify the "
            "investigation block agrees before acting.")
        for ch in suspected_fp:
            body_lines.append(
                f"  {ch['hostname']} ({ch['ip']}) {ch['interface']}: "
                f"{ch['old_role']}/{ch['old_state']} -> "
                f"{ch['new_role']}/{ch['new_state']}"
            )
        if investigations:
            for ch in suspected_fp:
                info = investigations.get((ch["ip"], ch["interface"]))
                if info is not None:
                    body_lines.extend(_format_investigation_block(ch, info))

    ok, detail = _send_email(cfg, subject, "\n".join(body_lines))
    if ok:
        log.info("Email alert %s", detail)
    else:
        log.error("Email alert failed: %s", detail)


def _send_root_change_alert(cfg, changes):
    """Email alert when a switch's view of the STP root bridge changes
    (confirmed across two consecutive polls)."""
    if not cfg.get("smtp_server"):
        log.debug("No [email] config — skipping root-change email")
        return
    subject = (f"[netops] STP root-bridge change confirmed — "
               f"{len(changes)} device(s)")
    lines = [f"STP root-bridge changes (confirmed across two consecutive polls) at "
             f"{datetime.now():%Y-%m-%d %H:%M}:\n"]
    for ch in changes:
        lines.append(
            f"  {ch['hostname']} ({ch['ip']}) {ch['instance']}: "
            f"{ch['old_priority']}/{ch['old_mac']} -> "
            f"{ch['new_priority']}/{ch['new_mac']}"
        )
    ok, detail = _send_email(cfg, subject, "\n".join(lines))
    if ok:
        log.info("Root-change email %s", detail)
    else:
        log.error("Root-change email failed: %s", detail)


_DEFAULT_STP_DOMAIN = "(default)"


def _ip_to_stp_domain(ip_str, subnet_map):
    """Return the STP domain name for ip_str by longest-prefix match.

    Empty subnet_map -> '(default)' for every IP (backward-compatible
    single-domain behavior). Configured subnet_map -> the matching
    domain name, or None if no prefix matches (caller treats None as
    'unassigned' and excludes from consensus checks).
    """
    if not subnet_map:
        return _DEFAULT_STP_DOMAIN
    try:
        addr = ipaddress.ip_address(ip_str)
    except ValueError:
        return None
    best = None
    for net, name in subnet_map.items():
        if addr in net and (best is None or net.prefixlen > best[0].prefixlen):
            best = (net, name)
    return best[1] if best else None


def _detect_root_mismatches(conn, poll_timestamp, subnet_map):
    """Find STP instances where this poll's responsive, in-service switches
    disagree on the root bridge — within each STP domain.

    Only considers rows with updated_at == poll_timestamp so stale entries
    from a previous poll can't falsify consensus. Excludes:
      - devices with link_up_count == 0 (reachable via management only).
      - devices whose IP doesn't match any [stp_domains] prefix when domains
        are configured (unassigned).

    Returns (mismatches, not_in_service_count, unassigned_count). Mismatches:
        [{'domain': 'main-campus', 'instance': 'CIST',
          'groups': [{'root_priority': 0, 'root_mac': '..',
                      'devices': [(ip, hostname), ...]}, ...]}, ...]
    Only (domain, instance) pairs with >1 distinct (priority, mac) group
    are returned.
    """
    rows = conn.execute("""
        SELECT s.instance, s.root_priority, s.root_mac, s.ip,
               COALESCE(d.hostname, s.ip) AS hostname,
               d.link_up_count
        FROM stp_root_state s
        LEFT JOIN devices d ON d.ip = s.ip
        WHERE s.updated_at = ? AND s.root_mac IS NOT NULL
        ORDER BY s.instance, s.root_priority, s.root_mac, hostname
    """, (poll_timestamp,)).fetchall()

    not_in_service = 0
    unassigned = 0
    by_domain_instance = {}
    for r in rows:
        if r["link_up_count"] is not None and r["link_up_count"] <= 0:
            not_in_service += 1
            continue
        domain = _ip_to_stp_domain(r["ip"], subnet_map)
        if domain is None:
            unassigned += 1
            continue
        key = (r["root_priority"], r["root_mac"])
        by_domain_instance.setdefault(
            (domain, r["instance"]), {}
        ).setdefault(key, []).append((r["ip"], r["hostname"]))

    mismatches = []
    for (domain, instance), groups in by_domain_instance.items():
        if len(groups) < 2:
            continue
        # Smallest group first — the minority/outlier roots are the interesting
        # signal; the majority/agreeing group sorts to the end so it can be
        # hidden or collapsed by the caller.
        ordered = sorted(groups.items(), key=lambda kv: (len(kv[1]), kv[0]))
        mismatches.append({
            "domain": domain,
            "instance": instance,
            "groups": [
                {"root_priority": k[0], "root_mac": k[1], "devices": v}
                for k, v in ordered
            ],
        })
    # Stable order: domain name, then instance.
    mismatches.sort(key=lambda m: (m["domain"], m["instance"]))
    return mismatches, not_in_service, unassigned


def handle_flap_digest(cfg, min_count=1, reset=False):
    """Email a digest of port flap counters (optionally clearing them after).

    min_count filters the report to ports with >= N flaps. reset=True clears
    all of port_flaps after a successful send; otherwise counters are kept
    so retention can be managed independently via `clear flap`.
    """
    conn = _db()
    conn.row_factory = sqlite3.Row
    # 3.8.10: garbage-collect expired flap silences first so the
    # silenced-set we apply below doesn't include stale entries.
    _gc_expired_flap_silences(conn)
    silenced = _get_flap_silenced_ports(conn)
    rows = conn.execute("""
        SELECT ip, hostname, interface, flap_count, first_seen, last_seen
        FROM port_flaps
        WHERE flap_count >= ?
        ORDER BY flap_count DESC, hostname, interface
    """, (min_count,)).fetchall()
    conn.close()
    silenced_skipped = sum(1 for r in rows
                           if (r["ip"], r["interface"]) in silenced)
    if silenced:
        rows = [r for r in rows
                if (r["ip"], r["interface"]) not in silenced]

    flux_conn = _db()
    flux_conn.row_factory = sqlite3.Row
    flux = _compute_mac_flux(flux_conn,
                               cfg.get("mac_flux_threshold_24h", 5),
                               cfg.get("mac_flux_uplink_ceiling", 24),
                               cfg.get("mac_flux_max_vlans", 2),
                               cfg.get("mac_flux_rotation_max_times_seen", 3),
                               cfg.get("mac_flux_rotation_max_span_sec", 7200))
    excluded = _get_excluded_ports(flux_conn, cfg)
    if excluded:
        flux = [p for p in flux if (p["ip"], p["interface"]) not in excluded]
    flux_conn.close()

    if not rows and not flux:
        log.info("No port flaps >= %d and no MAC flux to report.", min_count)
        return

    total_flaps = sum(r["flap_count"] for r in rows)
    subj_bits = []
    if rows:
        subj_bits.append(f"{len(rows)} flapping port(s), {total_flaps} flap(s)")
    if flux:
        subj_bits.append(f"{len(flux)} MAC-flux port(s)")
    if not subj_bits and silenced_skipped:
        # Nothing to email — but we still want the log to record that
        # silenced rows were skipped.
        log.info("No reportable flaps (silenced=%d, flux=0).", silenced_skipped)
        return
    subject = "[netops] Port flap digest — " + "; ".join(subj_bits)
    body_lines = [f"Port flap digest at {datetime.now():%Y-%m-%d %H:%M}"]
    if rows:
        sil_note = (f"  (+{silenced_skipped} silenced port(s) filtered out — "
                    f"see `netops whitelist list --flap`)") if silenced_skipped \
                   else ""
        body_lines += [
            f"Ports with >= {min_count} flap(s): {len(rows)} (total {total_flaps} flaps){sil_note}",
            "",
            "  FLAPS  HOSTNAME / IP                         INTERFACE         FIRST SEEN           LAST SEEN",
            "  -----  ------------------------------------  ----------------  -------------------  -------------------",
        ]
        for r in rows:
            label = f"{r['hostname'] or r['ip']} ({r['ip']})"
            body_lines.append(
                f"  {r['flap_count']:>5}  {label:<36}  {r['interface']:<16}  "
                f"{r['first_seen']:<19}  {r['last_seen']:<19}"
            )
            hint = _port_macs_digest_hint(r["ip"], r["interface"])
            if hint:
                body_lines.append(hint)
    if flux:
        thr = cfg.get("mac_flux_threshold_24h", 5)
        if rows:
            body_lines.append("")
        body_lines += [
            f"--- MAC flux (non-trunk; >= {thr} distinct MAC(s) in 24h — "
            "possible unauthorized device) ---",
            "  MACS  VLANS  HOSTNAME / IP                         INTERFACE",
            "  ----  -----  ------------------------------------  ----------------",
        ]
        for c in flux:
            label = f"{c['hostname']} ({c['ip']})"
            body_lines.append(
                f"  {c['distinct']:>4}  {c.get('vlans', 1):>5}  "
                f"{label:<36}  {c['interface']:<16}")
            hint = _port_macs_digest_hint(c["ip"], c["interface"])
            if hint:
                body_lines.append(hint)
    body = "\n".join(body_lines)

    log.info("--- Port Flap Digest ---")
    for line in body_lines[1:]:
        log.info(line)

    ok, detail = _send_email(cfg, subject, body)
    if ok:
        log.info("Flap digest email %s", detail)
        if reset:
            conn = _db()
            conn.execute("DELETE FROM port_flaps")
            conn.commit()
            conn.close()
            log.info("Flap counters reset.")
    else:
        log.error("Flap digest email failed: %s — counters NOT reset", detail)


def handle_stp_digest(cfg, detail=False):
    """Email a single digest covering STP-disabled, STP-mode-mismatched
    devices, and the current root-bridge view per instance.

    Queries current state from the DB (populated by 'monitor stp'). Sends
    nothing when all three sections are empty. Does not modify any state —
    run as often as you want; designed for once-a-day cron.

    detail=True lists agreeing switches in full (the majority group per
    disagreeing instance, and instances where all switches agree). Default
    keeps the email focused on the outliers.
    """
    expected = (cfg.get("stp_expected_mode") or "mstp").lower()
    conn = _db()
    # Require sustained-disabled state before listing. stp_disabled_since
    # is set to the timestamp when stp_enabled first transitioned to 0
    # (preserved across consecutive-disabled polls) and cleared back to
    # NULL on 0→1. Filter keeps:
    #   - Devices that have been disabled for ≥ SUSTAINED_MIN (real)
    #   - Devices with NULL since (legacy rows from before tracking, OR
    #     devices that were already disabled when the column was added —
    #     conservative: show them, they're presumed real)
    # Excludes: transient false-positives that just became disabled
    # within the past SUSTAINED_MIN minutes (e.g. NC-BDF on 2026-05-11
    # where a port-flap storm produced an empty SNMP table for one poll).
    STP_DISABLED_SUSTAINED_MIN = 15
    disabled_rows = conn.execute("""
        SELECT ip, hostname, dns_name, model, stp_last_check
        FROM devices
        WHERE stp_enabled = 0
          AND status IN ('active', 'inactive')
          AND (stp_disabled_since IS NULL
               OR stp_disabled_since < datetime('now','localtime',?))
        ORDER BY hostname, ip
    """, (f"-{STP_DISABLED_SUSTAINED_MIN} minutes",)).fetchall()
    mismatch_rows = conn.execute("""
        SELECT ip, hostname, dns_name, model, stp_mode, stp_last_check
        FROM devices
        WHERE stp_mode IS NOT NULL
          AND lower(stp_mode) != ?
          AND status IN ('active', 'inactive')
        ORDER BY hostname, ip
    """, (expected,)).fetchall()
    # Exclude devices with zero data-plane links from the consensus check —
    # they're reachable via management only ('not in service') and can't
    # meaningfully be in or out of STP consensus.
    root_rows = conn.execute("""
        SELECT s.instance, s.root_priority, s.root_mac,
               s.ip, COALESCE(d.hostname, s.ip) AS hostname
        FROM stp_root_state s
        LEFT JOIN devices d ON d.ip = s.ip
        WHERE s.root_mac IS NOT NULL
          AND (d.link_up_count IS NULL OR d.link_up_count > 0)
        ORDER BY s.instance, s.root_priority, s.root_mac, hostname
    """).fetchall()
    # Recent STP port changes (last 24 h) — input for the per-port classifier
    # section. Hostname pulled here so the render loop doesn't reopen the
    # connection.
    since_24h = (datetime.now() - timedelta(hours=24)).strftime("%Y-%m-%d %H:%M:%S")
    recent_rows = conn.execute("""
        SELECT c.ip, c.interface,
               COUNT(*) AS change_count,
               MAX(c.changed_at) AS latest_change,
               COALESCE(d.hostname, c.ip) AS hostname
        FROM stp_changes c
        LEFT JOIN devices d ON d.ip = c.ip
        WHERE c.changed_at > ?
        GROUP BY c.ip, c.interface
        ORDER BY change_count DESC, latest_change DESC
    """, (since_24h,)).fetchall()
    conn.close()

    # Group root view by (domain, instance) for the digest section + detect
    # disagreements. Multi-site networks legitimately have different roots
    # in different L2 domains; the [stp_domains] config tells us how to
    # split things. Devices unmatched by any prefix are excluded with a
    # count for the operator to act on.
    subnet_map = cfg.get("stp_domains") or {}
    n_unassigned = 0
    by_domain_instance = {}
    for r in root_rows:
        domain = _ip_to_stp_domain(r["ip"], subnet_map)
        if domain is None:
            n_unassigned += 1
            continue
        by_domain_instance.setdefault(
            (domain, r["instance"]), {}
        ).setdefault((r["root_priority"], r["root_mac"]), []).append(
            (r["ip"], r["hostname"]))
    disagree_keys = [k for k, g in by_domain_instance.items() if len(g) > 1]

    n_dis = len(disabled_rows)
    n_mis = len(mismatch_rows)
    n_dq = len(disagree_keys)
    n_recent = len(recent_rows)
    if n_dis == 0 and n_mis == 0 and n_dq == 0 and n_recent == 0:
        log.info("No STP configuration issues or recent port changes to report.")
        return

    subject_bits = []
    if n_dis:    subject_bits.append(f"{n_dis} disabled")
    if n_mis:    subject_bits.append(f"{n_mis} mode-mismatch")
    if n_dq:     subject_bits.append(f"{n_dq} root-disagreement")
    if n_recent: subject_bits.append(f"{n_recent} recent port change(s)")
    subject = (f"[netops] STP digest — {', '.join(subject_bits)}"
               if subject_bits else "[netops] STP digest")

    lines = [
        f"STP configuration digest at {datetime.now():%Y-%m-%d %H:%M}",
        f"Expected mode: {expected}",
        "",
    ]
    if n_dis:
        lines.append(f"--- STP disabled ({n_dis}) ---")
        for r in disabled_rows:
            label = r["hostname"] or r["dns_name"] or r["ip"]
            last = r["stp_last_check"] or "(never polled)"
            lines.append(f"  {label} ({r['ip']})  [{r['model'] or '?'}]  last-checked {last}")
        lines.append("")
    if n_mis:
        lines.append(f"--- STP mode mismatch (expected: {expected}) ({n_mis}) ---")
        for r in mismatch_rows:
            label = r["hostname"] or r["dns_name"] or r["ip"]
            last = r["stp_last_check"] or "(never polled)"
            lines.append(f"  {label} ({r['ip']})  running: {r['stp_mode']}  [{r['model'] or '?'}]  last-checked {last}")
        lines.append("")
    # Root-bridge section:
    #   default (no --detail): show only (domain, instance) pairs with
    #     disagreement; within those, list minority groups in full and
    #     collapse the majority group to a count-only line. Pairs where
    #     everyone agrees are omitted since the email only fires when
    #     SOMETHING is wrong.
    #   --detail: show every (domain, instance) pair and every device.
    renderable = [(k, g) for k, g in by_domain_instance.items()
                  if detail or len(g) > 1]
    if renderable:
        section_label = ("Root bridge per STP domain + instance"
                         if subnet_map else "Root bridge per STP instance")
        lines.append(f"--- {section_label} ---")
        # Disagreeing pairs first; agreeing ones only appear in --detail
        # mode and sort last.
        ordered_pairs = sorted(
            renderable, key=lambda kv: (len(kv[1]) == 1, kv[0]))
        for (domain, inst), groups in ordered_pairs:
            label = f"{domain} / {inst}" if subnet_map else inst
            if len(groups) == 1:
                (prio, mac), devs = next(iter(groups.items()))
                lines.append(f"  {label}: all {len(devs)} device(s) agree — "
                             f"priority={prio} mac={mac}")
                if detail:
                    for ip, hostname in devs:
                        lines.append(f"      {hostname} ({ip})")
            else:
                lines.append(f"  {label}: *** DISAGREEMENT — "
                             f"{len(groups)} distinct roots ***")
                ordered = sorted(groups.items(), key=lambda kv: (len(kv[1]), kv[0]))
                majority_idx = len(ordered) - 1
                for i, ((prio, mac), devs) in enumerate(ordered):
                    is_majority = (i == majority_idx)
                    suffix = " agree (pass --detail to list)" if is_majority and not detail else ""
                    lines.append(f"    priority={prio} mac={mac}  "
                                 f"({len(devs)} switch(es)){suffix}")
                    if is_majority and not detail:
                        continue
                    for ip, hostname in devs:
                        lines.append(f"      {hostname} ({ip})")
        lines.append("")

    # Recent port changes — runs investigate_stp_ports per device to get
    # BPDU/edge/link signals, then renders one classifier-tagged line per
    # port. Bounded by the number of unique devices with confirmed STP
    # changes in the last 24 h. Investigation failure → 'unknown' tag,
    # not a hard error.
    if n_recent:
        recent_classifications = {}
        ports_by_device = {}
        for r in recent_rows:
            ports_by_device.setdefault(r["ip"], []).append(r["interface"])
        # Look up the most-recent (old_role, new_role) per port so the
        # classifier can apply the role-change guard for snmp_transient.
        latest_roles_conn = _db()
        latest_roles = {}
        for r in recent_rows:
            lr = latest_roles_conn.execute(
                "SELECT old_role, new_role FROM stp_changes "
                "WHERE ip = ? AND interface = ? "
                "ORDER BY changed_at DESC LIMIT 1",
                (r["ip"], r["interface"])
            ).fetchone()
            if lr:
                latest_roles[(r["ip"], r["interface"])] = (
                    lr["old_role"], lr["new_role"])
        latest_roles_conn.close()

        for ip, ifaces in ports_by_device.items():
            try:
                inv = investigate_stp_ports(ip, ifaces, cfg)
            except Exception as e:
                log.debug("digest investigation failed for %s: %s", ip, e)
                inv = {p: {"ok": False, "parsed": {}} for p in ifaces}
            for iface, info in inv.items():
                pf, tcn_s = _stp_event_context(ip, iface)
                old_r, new_r = latest_roles.get((ip, iface), (None, None))
                label, _conf, _ev = _classify_stp_event(
                    info.get("parsed") or {}, pf, tcn_s,
                    old_role=old_r, new_role=new_r)
                recent_classifications[(ip, iface)] = label

        lines.append(f"--- Recent STP port changes (last 24h) ({n_recent}) ---")
        for r in recent_rows:
            label = recent_classifications.get(
                (r["ip"], r["interface"]), LIKELY_UNCLASSIFIED)
            tag = _STP_CLASSIFIER_SHORT_TAG.get(label, "unknown")
            lines.append(
                f"  [{tag:<14}] {r['hostname']} ({r['ip']}) {r['interface']}  "
                f"{r['change_count']} change(s) — latest @ {r['latest_change']}"
            )
        lines.append("")

    if n_unassigned:
        lines.append(f"--- Unassigned ({n_unassigned} device(s)) ---")
        lines.append("These IPs aren't matched by any [stp_domains] prefix and "
                     "were excluded from the consensus check.")
        lines.append("Add their subnets to [stp_domains] in netops.conf to include them.")
        lines.append("")

    # Reference legend — always rendered so an on-call recipient who isn't
    # familiar with the report can interpret it without out-of-band docs.
    lines.append("--- Reference ---")
    lines.append("")
    lines.append("Section meanings:")
    lines.append("  STP disabled               Switches with stp_enabled=0 for at "
                 "least 15 minutes (sustained-state filter)")
    lines.append("                             avoids transient false positives "
                 "from one-off empty SNMP polls.")
    lines.append("  STP mode mismatch          Running mode (rstp/stp/etc.) "
                 "differs from the expected mode for")
    lines.append("                             this network (configurable, "
                 "default: mstp).")
    lines.append("  STP root-bridge consensus  Per (domain, instance): root-bridge "
                 "MAC agreement vs. disagreement")
    lines.append("                             across switches that should share "
                 "a domain.")
    lines.append("  Recent STP port changes    Confirmed role/state transitions in "
                 "the last 24h, tagged by the")
    lines.append("                             heuristic classifier (see below).")
    lines.append("")
    lines.append("STP port-change classifier tags (in [brackets] on each port-change line):")
    lines.append("  [physical      ]  Physical link instability — check the cable, "
                 "SFP/optic, and port hardware.")
    lines.append("  [one_way       ]  Suspected one-way link — TX active, RX silent. "
                 "Swap optics, verify fiber pair.")
    lines.append("  [snmp_transient]  Likely SNMP/MIB-cache transient — no real STP "
                 "event observed. Operator action")
    lines.append("                    probably not required; flag for tool review "
                 "if the pattern persists.")
    lines.append("  [tc_flush      ]  Topology-change flush — a TCN propagated "
                 "through this bridge. The port itself")
    lines.append("                    is stable; brief learning state is expected "
                 "during MAC-table refresh.")
    lines.append("  [real_event    ]  Genuine STP topology change — investigate "
                 "which device became the new")
    lines.append("                    designated bridge upstream and why.")
    lines.append("  [unknown       ]  Available signals don't fit a known pattern; "
                 "manual review recommended.")

    body = "\n".join(lines)
    log.info("--- STP Config Digest ---")
    log.info("Disabled:      %d device(s)", n_dis)
    log.info("Mismatched:    %d device(s) not running %s", n_mis, expected)
    log.info("Root disagree: %d instance(s)", n_dq)
    log.info("Recent ports:  %d (last 24h)", n_recent)
    ok, detail = _send_email(cfg, subject, body)
    if ok:
        log.info("STP digest email %s", detail)
    else:
        log.error("STP digest email failed: %s", detail)


def handle_backup_digest(cfg, detail=False):
    """Email a weekly backup-status report: never-backed-up devices, stale
    backups, recent config changes, and (with detail=True) a full inventory.

    Always emits the email when there's at least one active device — this
    is a status report, not an issue alert, so recipients expect it on a
    fixed weekly cadence. Stale threshold is [monitor] backup_stale_days
    (default 7).
    """
    stale_days = int(cfg.get("backup_stale_days", 7))
    now_dt = datetime.now()
    now = now_dt.strftime("%Y-%m-%d %H:%M:%S")
    stale_cutoff = (now_dt - timedelta(days=stale_days)).strftime("%Y-%m-%d %H:%M:%S")
    week_cutoff  = (now_dt - timedelta(days=7)).strftime("%Y-%m-%d %H:%M:%S")

    conn = _db()
    rows = conn.execute("""
        SELECT d.ip, d.hostname, d.dns_name, d.model, d.status, d.last_seen,
               (SELECT MAX(backed_up_at) FROM backups WHERE ip = d.ip) AS last_backup,
               (SELECT MAX(backed_up_at) FROM backups
                  WHERE ip = d.ip AND changed = 1)                      AS last_change
        FROM devices d
        WHERE d.status IN ('active', 'inactive')
        ORDER BY d.hostname, d.ip
    """).fetchall()
    conn.close()
    if not rows:
        log.info("No active/inactive devices — skipping backup digest.")
        return

    active   = [r for r in rows if r["status"] == "active"]
    inactive = [r for r in rows if r["status"] == "inactive"]
    never    = [r for r in rows if not r["last_backup"]]
    # Stale = has a backup but it's older than the cutoff. Inactive devices
    # with no recent backup attempt surface here too.
    stale    = [r for r in rows if r["last_backup"] and r["last_backup"] < stale_cutoff]
    recent_changed = [r for r in rows if r["last_change"] and r["last_change"] >= week_cutoff]

    def _label(r):
        return r["hostname"] or r["dns_name"] or r["ip"]

    def _fmt(ts, when_null="(never)"):
        return ts or when_null

    subject_bits = []
    if never:          subject_bits.append(f"{len(never)} never backed up")
    if stale:          subject_bits.append(f"{len(stale)} stale")
    if recent_changed: subject_bits.append(f"{len(recent_changed)} changed")
    subject = (f"[netops] Weekly backup digest — {', '.join(subject_bits)}"
               if subject_bits else "[netops] Weekly backup digest — all healthy")

    lines = [
        f"Weekly backup digest at {now_dt:%Y-%m-%d %H:%M}",
        f"Fleet: {len(active)} active, {len(inactive)} inactive. "
        f"Stale threshold: {stale_days} day(s).",
        "",
    ]
    if never:
        lines.append(f"--- Never backed up ({len(never)}) ---")
        for r in never:
            lines.append(f"  {_label(r)} ({r['ip']})  [{r['model'] or '?'}]  "
                         f"status={r['status']}  last-polled={_fmt(r['last_seen'])}")
        lines.append("")
    if stale:
        lines.append(f"--- Stale backup (> {stale_days} days) ({len(stale)}) ---")
        for r in stale:
            lines.append(
                f"  {_label(r)} ({r['ip']})  status={r['status']}  "
                f"last-backup={_fmt(r['last_backup'])}  "
                f"last-polled={_fmt(r['last_seen'])}")
        lines.append("")
    if recent_changed:
        lines.append(f"--- Config changes in the past 7 days ({len(recent_changed)}) ---")
        for r in recent_changed:
            lines.append(f"  {_label(r)} ({r['ip']})  last-change={r['last_change']}")
        lines.append("")

    healthy = [r for r in rows if r not in never and r not in stale]
    if detail:
        lines.append(f"--- Full inventory ({len(rows)}) ---")
        for r in rows:
            lines.append(
                f"  {_label(r)} ({r['ip']})  status={r['status']}  "
                f"backup={_fmt(r['last_backup'])}  "
                f"change={_fmt(r['last_change'], '(never)')}  "
                f"polled={_fmt(r['last_seen'])}")
    else:
        lines.append(f"--- Healthy ({len(healthy)} device(s) with backup within "
                     f"{stale_days} day(s); pass --detail to list) ---")

    body = "\n".join(lines)
    log.info("--- Backup Digest ---")
    log.info("Never backed up: %d", len(never))
    log.info("Stale:           %d (> %d days)", len(stale), stale_days)
    log.info("Changed (7d):    %d", len(recent_changed))
    log.info("Healthy:         %d", len(healthy))
    ok, send_detail = _send_email(cfg, subject, body)
    if ok:
        log.info("Backup digest email %s", send_detail)
    else:
        log.error("Backup digest email failed: %s", send_detail)


def _parse_op_extra(extra_str):
    """Parse 'k=v k=v' extra column into a dict. Numeric values are
    coerced to int; everything else stays string."""
    if not extra_str:
        return {}
    out = {}
    for tok in extra_str.split():
        if "=" not in tok:
            continue
        k, _, v = tok.partition("=")
        try:
            out[k] = int(v)
        except ValueError:
            out[k] = v
    return out


def _grep_log_lines(pattern, since_dt, max_matches=2000):
    """Count + sample log lines matching a regex pattern that fall on or
    after since_dt. Returns (count, sample_first_line_or_None)."""
    log_path = LOG_FILE
    if not os.path.exists(log_path):
        return 0, None
    rx = re.compile(pattern)
    # Date prefix: '2026-05-17 17:25:16' — easy lexical compare
    since_str = since_dt.strftime("%Y-%m-%d %H:%M:%S")
    count = 0
    first = None
    try:
        # Read tail-ish (last 50 MB) to avoid pathological reads
        sz = os.path.getsize(log_path)
        with open(log_path, "rb") as f:
            if sz > 50 * 1024 * 1024:
                f.seek(-50 * 1024 * 1024, os.SEEK_END)
                f.readline()  # discard partial line
            for raw in f:
                try:
                    line = raw.decode("utf-8", errors="replace")
                except Exception:
                    continue
                if line[:19] < since_str:
                    continue
                if rx.search(line):
                    count += 1
                    if first is None:
                        first = line.strip()
                    if count >= max_matches:
                        break
    except OSError:
        pass
    return count, first


def handle_health_digest(cfg, retain_days=35):
    """Email a weekly enterprise health digest.

    Aggregates op_events from the past 7 days, fleet inventory, backup
    health, STP/flap activity, reachability alerts, system-level metrics,
    and operator action items. Prunes op_events rows older than
    retain_days at the end of the run.
    """
    now_dt = datetime.now()
    week_ago_dt = now_dt - timedelta(days=7)
    now_str = now_dt.strftime("%Y-%m-%d %H:%M:%S")
    week_ago_str = week_ago_dt.strftime("%Y-%m-%d %H:%M:%S")
    expected_ticks = 7 * 24 * 60   # one per minute

    conn = _db()
    conn.row_factory = sqlite3.Row

    # --- Section 1: Fleet inventory --------------------------------------
    fleet = {r["status"]: r["n"] for r in conn.execute(
        "SELECT status, COUNT(*) AS n FROM devices GROUP BY status")}
    fleet_total = sum(fleet.values())
    new_this_week = conn.execute(
        "SELECT COUNT(*) FROM devices WHERE first_seen >= ?",
        (week_ago_str,)).fetchone()[0]
    currently_flagged = list(conn.execute(
        "SELECT ip, hostname, snmp_consecutive_fails, unreachable_alerted_at "
        "FROM devices WHERE unreachable_alerted_at IS NOT NULL "
        "ORDER BY unreachable_alerted_at"))

    # --- Section 2: Backups ---------------------------------------------
    backup_rows = list(conn.execute(
        "SELECT success, duration_ms FROM op_events "
        "WHERE op_type='backup_device' AND started_at >= ?", (week_ago_str,)))
    backup_total = len(backup_rows)
    backup_ok = sum(1 for r in backup_rows if r["success"])
    backup_fail = backup_total - backup_ok
    bk_durs = [r["duration_ms"] for r in backup_rows if r["success"]]
    bk_avg = int(sum(bk_durs) / len(bk_durs)) if bk_durs else 0
    bk_min = min(bk_durs) if bk_durs else 0
    bk_max = max(bk_durs) if bk_durs else 0
    config_changes_week = conn.execute(
        "SELECT COUNT(*) FROM backups WHERE backed_up_at >= ? AND changed = 1",
        (week_ago_str,)).fetchone()[0]
    stale_backups = list(conn.execute("""
        SELECT d.ip, d.hostname,
               (SELECT MAX(backed_up_at) FROM backups b WHERE b.ip = d.ip) AS last
        FROM devices d
        WHERE d.status = 'active'
        ORDER BY d.hostname
    """))
    stale_backups = [r for r in stale_backups
                     if r["last"] is None or r["last"] < week_ago_str]

    # --- Section 3: STP / flap activity ---------------------------------
    stp_changes_week = conn.execute(
        "SELECT COUNT(*) FROM stp_changes WHERE changed_at >= ?",
        (week_ago_str,)).fetchone()[0]
    root_changes_week = conn.execute(
        "SELECT COUNT(*) FROM stp_root_changes WHERE changed_at >= ?",
        (week_ago_str,)).fetchone()[0]
    top_changers = list(conn.execute("""
        SELECT hostname, interface, COUNT(*) AS n
        FROM stp_changes WHERE changed_at >= ?
        GROUP BY hostname, interface ORDER BY n DESC LIMIT 5
    """, (week_ago_str,)))
    # Flap events: sum extra.flap_events from monitor_stp_tick rows
    flap_total_week = 0
    tick_rows = list(conn.execute(
        "SELECT duration_ms, extra FROM op_events "
        "WHERE op_type='monitor_stp_tick' AND started_at >= ?",
        (week_ago_str,)))
    for r in tick_rows:
        fe = _parse_op_extra(r["extra"]).get("flap_events", 0)
        if isinstance(fe, int):
            flap_total_week += fe
    top_flappers = list(conn.execute("""
        SELECT hostname, interface, flap_count, last_seen FROM port_flaps
        ORDER BY flap_count DESC LIMIT 5
    """))

    # --- Section 4: Reachability events (parse log) ---------------------
    unreach_count, unreach_sample = _grep_log_lines(
        r"Unreachable: \d+ device\(s\) crossed threshold", week_ago_dt)
    recovery_count, recovery_sample = _grep_log_lines(
        r"Recovered: \d+ device\(s\)", week_ago_dt)

    # --- Section 5: System performance ----------------------------------
    ticks_total = len(tick_rows)
    tick_durs = [r["duration_ms"] for r in tick_rows]
    tick_avg = int(sum(tick_durs) / len(tick_durs)) if tick_durs else 0
    tick_min = min(tick_durs) if tick_durs else 0
    tick_max = max(tick_durs) if tick_durs else 0
    # SNMP rollup: sum snmp_count, weighted mean of snmp_avg_ms, min/max envelopes
    snmp_count_total = 0
    snmp_failed_total = 0
    snmp_min_ms = None
    snmp_max_ms = 0
    snmp_weighted_sum_ms = 0
    for r in tick_rows:
        ex = _parse_op_extra(r["extra"])
        sc = ex.get("snmp_count", 0)
        if isinstance(sc, int) and sc > 0:
            snmp_count_total += sc
            sf = ex.get("snmp_failed", 0)
            if isinstance(sf, int):
                snmp_failed_total += sf
            mn = ex.get("snmp_min_ms")
            mx = ex.get("snmp_max_ms")
            av = ex.get("snmp_avg_ms")
            if isinstance(mn, int):
                snmp_min_ms = mn if snmp_min_ms is None else min(snmp_min_ms, mn)
            if isinstance(mx, int):
                snmp_max_ms = max(snmp_max_ms, mx)
            if isinstance(av, int):
                snmp_weighted_sum_ms += av * sc
    snmp_avg_ms = int(snmp_weighted_sum_ms / snmp_count_total) if snmp_count_total else 0
    # SSH connect events by reason
    ssh_by_reason = {}
    for r in conn.execute(
        "SELECT reason, success, duration_ms FROM op_events "
        "WHERE op_type IN ('ssh_connect', 'telnet_connect') "
        "  AND started_at >= ?", (week_ago_str,)
    ):
        rs = r["reason"] or "(unknown)"
        bucket = ssh_by_reason.setdefault(rs,
                    {"total": 0, "ok": 0, "fail": 0, "durs": []})
        bucket["total"] += 1
        if r["success"]:
            bucket["ok"] += 1
            bucket["durs"].append(r["duration_ms"])
        else:
            bucket["fail"] += 1
    # Cron-skip count
    skip_count, _ = _grep_log_lines(
        r"holds the .ssh. lock", week_ago_dt)
    # Email path: count send-success/-failure log lines
    email_ok, _ = _grep_log_lines(r"email sent", week_ago_dt)
    email_fail, _ = _grep_log_lines(
        r"(?:digest|alert) email failed", week_ago_dt)
    # Tracebacks
    traceback_count, traceback_sample = _grep_log_lines(
        r"^Traceback \(most recent call last\)", week_ago_dt)

    # --- Section 6: netops-box health -----------------------------------
    db_size = os.path.getsize(DB_FILE) if os.path.exists(DB_FILE) else 0
    log_path = LOG_FILE
    log_size = os.path.getsize(log_path) if os.path.exists(log_path) else 0
    try:
        du = shutil.disk_usage(STATE_DIR)
        disk_free_pct = (du.free * 100.0) / du.total
        disk_free_gb = du.free / (1024 ** 3)
    except Exception:
        disk_free_pct, disk_free_gb = 0.0, 0.0

    # --- Section 7: Operator action items -------------------------------
    no_creds = list(conn.execute(
        "SELECT ip, hostname FROM devices "
        "WHERE status='active' AND (username IS NULL OR password_hash IS NULL) "
        "ORDER BY hostname LIMIT 10"))
    no_lldp_count = conn.execute("""
        SELECT COUNT(*) FROM devices d
        WHERE d.status='active'
          AND NOT EXISTS (SELECT 1 FROM topology_edges t WHERE t.src_ip = d.ip)
    """).fetchone()[0]

    conn.close()

    # ---- Render -----------------------------------------------------------
    def _fmt_size(n):
        for unit in ("B", "KB", "MB", "GB"):
            if n < 1024 or unit == "GB":
                return f"{n:.1f} {unit}" if unit != "B" else f"{int(n)} B"
            n /= 1024

    subject_bits = []
    if currently_flagged: subject_bits.append(f"{len(currently_flagged)} unreachable")
    if stale_backups:     subject_bits.append(f"{len(stale_backups)} stale backups")
    if traceback_count:   subject_bits.append(f"{traceback_count} tracebacks")
    subject = (f"[netops] Weekly health digest — {', '.join(subject_bits)}"
               if subject_bits else "[netops] Weekly health digest — all green")

    lines = [
        f"Weekly health digest — {week_ago_dt:%Y-%m-%d} to {now_dt:%Y-%m-%d %H:%M}",
        "",
        "=== Fleet ===",
        f"  total devices:           {fleet_total} "
        f"({fleet.get('active', 0)} active, {fleet.get('inactive', 0)} inactive, "
        f"{fleet.get('failed', 0)} failed)",
        f"  new this week:           {new_this_week}",
        f"  currently flagged down:  {len(currently_flagged)}",
    ]
    for r in currently_flagged[:8]:
        lines.append(f"    - {r['hostname'] or '?'} ({r['ip']})  "
                     f"alerted_at={r['unreachable_alerted_at']}  "
                     f"fails={r['snmp_consecutive_fails']}")
    if len(currently_flagged) > 8:
        lines.append(f"    ... and {len(currently_flagged) - 8} more")

    lines += [
        "",
        "=== Backups ===",
        f"  attempts this week:      {backup_total} "
        f"({backup_ok} ok, {backup_fail} failed)",
        f"  per-device duration:     avg {bk_avg/1000:.1f}s "
        f"(min {bk_min/1000:.1f}s, max {bk_max/1000:.1f}s)",
        f"  config changes detected: {config_changes_week}",
        f"  stale (no backup ≥ 7d):  {len(stale_backups)}",
    ]
    for r in stale_backups[:8]:
        last = r["last"] or "(never)"
        lines.append(f"    - {r['hostname'] or '?'} ({r['ip']})  last={last}")
    if len(stale_backups) > 8:
        lines.append(f"    ... and {len(stale_backups) - 8} more")

    lines += [
        "",
        "=== STP / flap activity ===",
        f"  confirmed STP changes:   {stp_changes_week}",
        f"  root-bridge changes:     {root_changes_week}",
        f"  port flap events:        {flap_total_week} "
        f"(cumulative across all monitor-stp ticks)",
    ]
    if top_changers:
        lines.append("  top STP changers (week):")
        for r in top_changers:
            lines.append(f"    {r['hostname']:32}  {r['interface']:14}  "
                         f"{r['n']} change(s)")
    if top_flappers:
        lines.append("  top port flappers (cumulative count since last reset):")
        for r in top_flappers:
            host = (r['hostname'] or '?')[:32]
            lines.append(f"    {host:32}  {r['interface']:14}  "
                         f"flaps={r['flap_count']:>4d}  last={r['last_seen']}")

    lines += [
        "",
        "=== Reachability events ===",
        f"  unreachable-alert fires: {unreach_count}",
        f"  recovery emails:         {recovery_count}",
    ]
    if unreach_sample:
        lines.append(f"  most recent unreachable: {unreach_sample[:160]}")

    lines += [
        "",
        "=== System performance ===",
        f"  monitor-stp ticks:       {ticks_total} of expected {expected_ticks} "
        f"({100.0 * ticks_total / expected_ticks if expected_ticks else 0:.1f}%)",
        f"  tick duration:           avg {tick_avg/1000:.1f}s "
        f"(min {tick_min/1000:.1f}s, max {tick_max/1000:.1f}s)",
        f"  SNMP queries:            {snmp_count_total:,} "
        f"({snmp_failed_total} failed)",
    ]
    if snmp_count_total:
        lines.append(f"  SNMP latency:            avg {snmp_avg_ms} ms "
                     f"(min {snmp_min_ms} ms, max {snmp_max_ms} ms)")
    if ssh_by_reason:
        lines.append("  SSH/telnet attempts by reason:")
        for rs, b in sorted(ssh_by_reason.items(),
                            key=lambda kv: -kv[1]["total"]):
            avg = (sum(b["durs"]) / len(b["durs"]) / 1000) if b["durs"] else 0
            lines.append(f"    {rs:30}  total={b['total']:>4d} "
                         f"ok={b['ok']:>4d} fail={b['fail']:>3d}  "
                         f"avg-on-success {avg:.1f}s")
    lines += [
        f"  cron-skips (flock):      {skip_count}",
        f"  email sends (ok / fail): {email_ok} / {email_fail}",
        f"  log tracebacks:          {traceback_count}",
    ]
    if traceback_sample and traceback_count:
        lines.append(f"    most recent: {traceback_sample[:160]}")

    lines += [
        "",
        "=== netops-box health ===",
        f"  netops.db size:          {_fmt_size(db_size)}",
        f"  netops.log size:         {_fmt_size(log_size)}",
        f"  filesystem free:         {disk_free_gb:.1f} GB "
        f"({disk_free_pct:.1f}%)",
    ]

    lines += [
        "",
        "=== Operator action items ===",
    ]
    if no_creds:
        lines.append(f"  devices without stored creds ({len(no_creds)}):")
        for r in no_creds[:5]:
            lines.append(f"    - {r['hostname'] or '?'} ({r['ip']})")
        if len(no_creds) > 5:
            lines.append(f"    ... and {len(no_creds) - 5} more")
    if no_lldp_count:
        lines.append(f"  active devices with no LLDP edges in topology graph: "
                     f"{no_lldp_count}")
    if currently_flagged:
        lines.append(f"  unreachable_alerted_at stuck on "
                     f"{len(currently_flagged)} device(s) — "
                     f"investigate or clear manually if resolved")
    if not (no_creds or no_lldp_count or currently_flagged):
        lines.append("  nothing requires attention 🎉")

    body = "\n".join(lines)
    log.info("--- Weekly health digest ---")
    log.info("Fleet:        %d total, %d active", fleet_total, fleet.get("active", 0))
    log.info("Backups:      %d/%d ok", backup_ok, backup_total)
    log.info("STP changes:  %d (confirmed, last 7d)", stp_changes_week)
    log.info("Tracebacks:   %d", traceback_count)
    ok, detail = _send_email(cfg, subject, body)
    if ok:
        log.info("Weekly health digest email %s", detail)
    else:
        log.error("Weekly health digest email failed: %s", detail)

    # --- Retention: prune old op_events rows -----------------------------
    cutoff_dt = now_dt - timedelta(days=retain_days)
    cutoff_str = cutoff_dt.strftime("%Y-%m-%d %H:%M:%S")
    conn = _db()
    deleted = conn.execute(
        "DELETE FROM op_events WHERE started_at < ?", (cutoff_str,))
    n_deleted = deleted.rowcount
    # port_macs FDB history shares the op_events retention window. Also
    # drop long-cleared flux-alert episodes so that table stays bounded
    # (active / not-yet-cleared rows are always kept).
    try:
        pm = conn.execute(
            "DELETE FROM port_macs WHERE last_seen < ?", (cutoff_str,))
        n_pm = pm.rowcount
        conn.execute(
            "DELETE FROM mac_flux_alerts "
            "WHERE cleared_at IS NOT NULL AND cleared_at < ?", (cutoff_str,))
    except sqlite3.OperationalError:
        n_pm = 0                        # pre-3.8.0 DB without the tables
    conn.commit()
    conn.close()
    if n_deleted:
        log.info("Pruned %d op_events row(s) older than %d days",
                 n_deleted, retain_days)
    if n_pm:
        log.info("Pruned %d port_macs row(s) older than %d days",
                 n_pm, retain_days)


def handle_investigate_stp(cfg, ip, port):
    """Run the STP investigation for a single (ip, port) and print to stdout.

    Retrospective counterpart to the auto-attached investigation that
    monitor stp emails on confirmed changes — use this to drill down on a
    port whose alert already came and went, or to check baseline state.
    """
    results = investigate_stp_ports(ip, [port], cfg)
    info = results[port]
    # Reuse the same block formatter as the email so stdout and email look
    # identical — the operator learns one layout.
    conn = _db()
    row = conn.execute("SELECT hostname FROM devices WHERE ip = ?", (ip,)).fetchone()
    conn.close()
    ch = {"ip": ip, "interface": port,
          "hostname": (row["hostname"] if row else ip) or ip}
    for line in _format_investigation_block(ch, info):
        print(line)
    return 0 if info.get("ok") else 1


def handle_flap_reset(min_count=0, dry_run=False):
    """Clear port flap counters. With min_count>0, only clears rows at or
    above that threshold. dry_run prints what would be cleared without
    deleting.
    """
    conn = _db()
    rows = conn.execute("""
        SELECT ip, hostname, interface, flap_count
        FROM port_flaps
        WHERE flap_count >= ?
        ORDER BY flap_count DESC, hostname, interface
    """, (min_count,)).fetchall()

    if not rows:
        log.info("No port flap rows >= %d to clear.", min_count)
        conn.close()
        return

    log.info("--- Port Flap Reset%s ---", " (dry-run)" if dry_run else "")
    log.info("  FLAPS  HOSTNAME / IP                         INTERFACE")
    for r in rows:
        label = f"{r['hostname'] or r['ip']} ({r['ip']})"
        log.info(f"  {r['flap_count']:>5}  {label:<36}  {r['interface']}")

    if dry_run:
        log.info("Dry run: %d row(s) would be cleared.", len(rows))
        conn.close()
        return

    conn.execute("DELETE FROM port_flaps WHERE flap_count >= ?", (min_count,))
    conn.commit()
    conn.close()
    log.info("Cleared %d port flap row(s) with flap_count >= %d.",
             len(rows), min_count)


def handle_test_email(cfg, target):
    """Send a canned test email. target is 'all' or one configured address."""
    recipients = [a.strip() for a in cfg.get("email_to", "").split(",") if a.strip()]
    if not recipients:
        print("No recipients configured in [email] to= — nothing to test.")
        return
    if target == "all":
        send_to = recipients
    elif target in recipients:
        send_to = [target]
    else:
        print(f"test-email: '{target}' is not a configured recipient.")
        print(f"  Configured: {', '.join(recipients)}")
        print("  Use 'test-email all', or one of the addresses above.")
        return

    smtp_server = cfg.get("smtp_server") or "(unset)"
    smtp_port = cfg.get("smtp_port", 587)
    from_addr = cfg.get("email_from", "netops@localhost")
    auth = "yes" if (cfg.get("smtp_username") and cfg.get("smtp_password")) else "no"

    print("Email configuration:")
    print(f"  smtp_server: {smtp_server}")
    print(f"  smtp_port:   {smtp_port}")
    print(f"  from:        {from_addr}")
    print(f"  to:          {', '.join(send_to)}")
    print(f"  auth:        {auth}")
    print()

    cfg = dict(cfg)
    cfg["email_to"] = ", ".join(send_to)

    subject = "[netops] test email"
    body = (
        f"This is a test message from netops v{__version__}.\n"
        f"Sent at {datetime.now():%Y-%m-%d %H:%M:%S}.\n\n"
        "If you received this, your [email] configuration is working."
    )
    print("Sending test message...")
    ok, detail = _send_email(cfg, subject, body)
    if ok:
        print(f"OK — {detail}")
    else:
        print(f"FAILED — {detail}")


def _apply_cli_overrides(cfg, args):
    """Apply per-invocation subnet/username/password overrides from CLI args."""
    if getattr(args, "subnet", None):
        cfg["subnets"] = [s.strip() for s in args.subnet.split(",") if s.strip()]
    if getattr(args, "username", None):
        cfg["usernames"] = [u.strip() for u in args.username.split(",") if u.strip()]
    if getattr(args, "password", None):
        cfg["passwords"]["default"] = args.password


def _require_subnets(cfg):
    if not cfg["subnets"]:
        log.error("No subnets configured. Use -s or set subnets in netops.conf [scan].")
        sys.exit(1)


def _warn_thread_caps(cfg):
    caps = cfg["_thread_caps"]
    if caps["scan_capped"]:
        log.warning(
            "scan_threads reduced from %d to %d — %s",
            caps["raw_scan"], cfg["scan_threads"], caps["scan_limit_reason"],
        )
    if caps["backup_capped"]:
        log.warning(
            "backup_threads reduced from %d to %d — %s",
            caps["raw_backup"], cfg["backup_threads"],
            caps["backup_limit_reason"],
        )
    log.debug(
        "Threads: scan=%d (cap %d), backup=%d (cap %d), "
        "fd_limit=%d, memory=%d MB",
        cfg["scan_threads"], caps["scan_cap"],
        cfg["backup_threads"], caps["backup_cap"],
        caps["fd_limit"], caps["total_mem_mb"],
    )


def handle_refresh_dns():
    """Reverse-DNS every device in the DB and update the dns_name column."""
    conn = _db()
    ips = [r["ip"] for r in conn.execute(
        "SELECT ip FROM devices WHERE status IN ('active', 'inactive', 'failed', 'duplicate')"
    ).fetchall()]
    conn.close()
    if not ips:
        log.info("No devices in DB.")
        return
    log.info("Resolving reverse DNS for %d device(s)...", len(ips))
    dns_map = resolve_dns_batch(ips)

    conn = _db()
    updated = 0
    for ip, name in dns_map.items():
        conn.execute("UPDATE devices SET dns_name = ? WHERE ip = ?", (name, ip))
        if name:
            updated += 1
    conn.commit()
    conn.close()

    resolved = sum(1 for v in dns_map.values() if v)
    log.info("DNS refresh complete: %d/%d resolved to a name", resolved, len(ips))


def run_console(cfg):
    """Interactive REPL for netops. Re-parses each line as a subcommand.

    When the environment variable NETOPS_ADMIN is set (populated by the
    authorized_keys 'command=' wrapper for SSH console access), the
    prompt reflects that identity and every command is logged with it
    for audit. Access is gated entirely by SSH key auth at the OS layer —
    no in-REPL password. A local shell run (NETOPS_ADMIN unset) renders
    a plain 'netops>' prompt.
    """
    import cmd
    import shlex
    try:
        import readline
    except ImportError:
        readline = None

    admin = os.environ.get("NETOPS_ADMIN", "").strip() or None

    # Per-user history file so each admin's ~/.netops_history_<admin> is
    # isolated. Falls back to the classic path when identity isn't set
    # (local shell invocation).
    hist_suffix = f"_{admin}" if admin else ""
    history_file = os.path.expanduser(f"~/.netops_history{hist_suffix}")
    if readline:
        try:
            readline.read_history_file(history_file)
        except (FileNotFoundError, OSError):
            pass
        readline.set_history_length(1000)

    # Cache device IPs/hostnames for tab completion
    def _device_names():
        try:
            conn = _db()
            rows = conn.execute(
                "SELECT ip, hostname FROM devices WHERE status IN ('active', 'inactive')"
            ).fetchall()
            conn.close()
            names = []
            for r in rows:
                names.append(r["ip"])
                if r["hostname"]:
                    names.append(r["hostname"])
            return names
        except Exception:
            return []

    class NetopsShell(cmd.Cmd):
        _admin = admin

        intro = (
            f"netops v{__version__} — interactive console. "
            "Type '?' for commands, '<command> ?' for help on a command, "
            "'exit' to leave.\nDB: {db}"
            + (f"\nUser: {admin}" if admin else "")
        ).format(db=DB_FILE)

        # Console adds help/exit/quit on top of the shared command summaries.
        _command_summaries = COMMAND_SUMMARIES + [
            ("help", "Show help ('help <cmd>' or '<cmd> ?')"),
            ("exit", "Leave the console"),
            ("quit", "Leave the console"),
        ]
        _commands = [c for c, _ in _command_summaries]
        _monitor_targets = MONITOR_TARGETS
        _single_flap_targets = SINGLE_FLAP_TARGETS

        # Junos-style prompt. user@netops> when NETOPS_ADMIN is set
        # (SSH console), plain netops> otherwise (local shell).
        prompt = (f"{admin}@netops> " if admin else "netops> ")

        # Console-only extras (help/exit/quit) on top of the shared HELP_DETAILS.
        _help_details = {
            **HELP_DETAILS,
            "help": [
                "Usage: help [<command>]  or  <command> ?",
                "  Without args: list all commands.",
                "  With a command: show detailed usage for that command.",
            ],
            "exit": ["Usage: exit   — leave the console."],
            "quit": ["Usage: quit   — leave the console."],
        }

        def _show_root_help(self):
            print("Commands:")
            for c, summary in self._command_summaries:
                print(f"  {c:<14} {summary}")
            print("\nType '<command> ?' for details on a specific command.")

        def _show_command_help(self, command):
            """Print context-sensitive help for a command (Cisco-style `command ?`)."""
            details = self._help_details.get(command)
            if not details:
                print(f"No help available for '{command}'")
                return
            for line in details:
                print(line)
            if command == "show":
                print("  Options:")
                print("    --csv FILE      write to CSV file instead of a table")
                print("  port-flaps filters:")
                print("    --ip IP         show only this device")
                print("    --interface IF  show only this interface (exact match)")
                print("    --min-count N   only rows with >= N flaps (default 1)")

        def onecmd(self, line):
            """Intercept trailing '?' for Cisco-style context help."""
            stripped = line.strip()
            # Match '?' alone, 'list?', 'list ?', 'show ?', etc.
            if stripped.endswith("?"):
                head = stripped[:-1].strip()
                if not head:
                    self._show_root_help()
                    return False
                command = head.split()[0]
                if command in self._commands:
                    self._show_command_help(command)
                    return False
                print(f"Unknown command '{command}'. Type '?' for command list.")
                return False
            return super().onecmd(line)

        # Commands that require a positional argument. Bare invocation in the
        # REPL routes to '<cmd> ?' help instead of argparse's usage dump. When
        # the second token is present but not in the choice set, we show the
        # same help with a short error line. None = no finite choice list
        # (validation happens downstream in _dispatch or the handler).
        _required_arg = {
            "monitor": {"stp", "flap"},
            "clear":   {"flap"},
            "digest":  {"flap", "stp", "backup", "health"},
            "show":    None,
            "configure": None,
            "test-email": None,
            "add":     None,
            "remove":  None,
        }
        _digest_targets = [
            ("flap",   "Port flap counters"),
            ("stp",    "STP disabled + mode-mismatched devices"),
            ("backup", "Weekly backup status + stale backups + recent changes"),
        ]

        # Legacy hyphenated command forms that map to the current device-style
        # grouped verbs. Lets users paste or remember older syntax without
        # hitting argparse dumps.
        _legacy_map = {
            "monitor-stp":  ["monitor", "stp"],
            "monitor-flap": ["monitor", "flap"],
            "digest-stp":   ["digest", "stp"],
            "digest-flap":  ["digest", "flap"],
            "clear-flap":   ["clear", "flap"],
        }

        def default(self, line):
            """Route non-builtin lines through the same argparse as CLI."""
            try:
                argv = shlex.split(line)
            except ValueError as e:
                print(f"parse error: {e}")
                return
            if not argv:
                return
            # Accept 'netops <cmd>' (shell-style paste) by stripping the
            # redundant binary name — we're already inside the netops REPL.
            if argv[0] == "netops":
                argv = argv[1:]
            if not argv:
                self._show_root_help()
                return
            # Translate legacy hyphenated forms to device-style groups.
            if argv[0] in self._legacy_map:
                argv = self._legacy_map[argv[0]] + argv[1:]
            cmd = argv[0]
            # Audit log every command with the admin identity so netops.log
            # shows who ran what over the SSH console.
            log.info("repl: user=%s cmd=%s",
                     self._admin or "(local)", " ".join(argv))
            # Unknown first token — show friendly help instead of argparse's
            # usage dump + 'invalid choice' message.
            if cmd not in self._commands:
                print(f"% Unknown command '{cmd}'. Type '?' for command list.")
                return
            if cmd in self._required_arg:
                if len(argv) == 1:
                    self._show_command_help(cmd)
                    return
                choices = self._required_arg[cmd]
                if choices is not None and argv[1] not in choices:
                    print(f"% Invalid target '{argv[1]}'.")
                    self._show_command_help(cmd)
                    return
            # Rebuild args using the main parser (so all subcommands work)
            old_argv = sys.argv
            try:
                sys.argv = ["netops"] + argv
                args = parse_args()
                _dispatch(args, cfg)
            except SystemExit:
                # argparse calls sys.exit on error or --help; swallow it
                pass
            except KeyboardInterrupt:
                print("\n^C")
            except Exception as e:
                print(f"error: {e}")
            finally:
                sys.argv = old_argv

        def completenames(self, text, *ignored):
            return [c for c in self._commands if c.startswith(text)]

        def completedefault(self, text, line, begidx, endidx):
            # Tab-complete: show → categories + device names; remove → device names;
            # monitor/digest/clear → their target list.
            parts = line[:begidx].split()
            if parts and parts[0] == "show":
                cands = _complete_show(parts, text)
                # completing the pattern of `show devices ip|name <TAB>`
                if (len(parts) == 3 and parts[1] == "devices"
                        and parts[2] in ("ip", "name")):
                    cands += [n for n in _device_names() if n.startswith(text)]
                return cands
            if parts and parts[0] == "configure":
                if len(parts) == 1:
                    return [o for o in ("email",) if o.startswith(text)]
                if len(parts) == 2 and parts[1] == "email":
                    return [o for o in ("add", "remove", "list")
                            if o.startswith(text)]
                if (len(parts) == 3 and parts[1] == "email"
                        and parts[2] == "remove"):
                    return [r for r in _email_recipients()
                            if r.startswith(text)]
                return []
            if parts and parts[0] == "test-email" and len(parts) == 1:
                opts = ["all"] + _email_recipients()
                return [o for o in opts if o.startswith(text)]
            if parts and parts[0] == "remove":
                return [n for n in _device_names() if n.startswith(text)]
            if parts and parts[0] == "monitor":
                return [t for t, _ in self._monitor_targets if t.startswith(text)]
            if parts and parts[0] == "digest":
                return [t for t, _ in self._digest_targets if t.startswith(text)]
            if parts and parts[0] == "clear":
                return [t for t, _ in self._single_flap_targets if t.startswith(text)]
            return []

        # Built-in commands
        def do_exit(self, arg):
            """Exit the console."""
            return True

        def do_quit(self, arg):
            """Exit the console."""
            return True

        def do_EOF(self, arg):
            """Ctrl-D exits."""
            print()
            return True

        def emptyline(self):
            pass  # don't repeat last command on empty enter

        def do_help(self, arg):
            """help [<command>]   — list commands, or details for one."""
            arg = (arg or "").strip()
            if arg:
                if arg in self._commands:
                    self._show_command_help(arg)
                else:
                    print(f"Unknown command '{arg}'. Type '?' for command list.")
                return
            self._show_root_help()

    try:
        NetopsShell().cmdloop()
    except KeyboardInterrupt:
        print("\n^C (use 'exit' to leave)")
    finally:
        if readline:
            try:
                readline.write_history_file(history_file)
            except OSError:
                pass


def _dispatch(args, cfg):
    """Shared dispatch used by both main() and the console REPL."""
    cmd = args.command
    if cmd == "show":
        # Junos-style hierarchical grammar (resolved by _resolve_show):
        #   show spanning-tree [state] / show devices switch [status] /
        #   show devices ip|name <pattern> [detail] / show <flat-category>.
        kind, val = _resolve_show(args.arg)
        if kind == "error":
            print(val)
            return
        if kind == "filter":
            field, pattern, detail = val
            return handle_devices_filter(field, pattern, detail,
                                         csv_path=args.csv)
        if kind == "stp-summary":
            if args.csv:
                print("show spanning-tree: --csv applies to a single state "
                      "(e.g. show spanning-tree blocked --csv FILE); ignoring.")
            for state in _SHOW_STP_STATES:
                print(f"\n----- spanning-tree {state} -----")
                handle_list(_SHOW_STP_CATEGORY[state],
                            ip=args.ip, interface=args.interface,
                            min_count=args.min_count, show_all=False)
            return
        if kind == "category-all":
            return handle_list(val, csv_path=args.csv,
                               ip=args.ip, interface=args.interface,
                               min_count=args.min_count, show_all=True)
        return handle_list(val, csv_path=args.csv,
                           ip=args.ip, interface=args.interface,
                           min_count=args.min_count, show_all=False)
    if cmd == "add":
        return handle_add(args.ip)
    if cmd == "remove":
        return handle_remove(args.ip)
    if cmd == "whitelist":
        return handle_whitelist(args)
    if cmd == "import":
        return handle_import(cfg, path=args.path, fmt=args.format)
    if cmd == "export":
        return handle_export(cfg)
    if cmd == "cat-config":
        return handle_cat_config(args.device, set_format=getattr(args, "set", False))
    if cmd == "configure":
        return handle_configure(args.tokens)
    if cmd == "test-email":
        return handle_test_email(cfg, args.target)
    if cmd == "check-config":
        return handle_check_config(cfg)
    if cmd == "wipe-db":
        return handle_wipe_db()
    # Connection-using commands
    _warn_thread_caps(cfg)
    _apply_cli_overrides(cfg, args)
    if cmd == "scan":
        _require_subnets(cfg)
        return run_scan(cfg, dry_run=args.dry_run)
    if cmd in ("discover", "discovery"):
        _require_subnets(cfg)
        return run_discover(cfg, dry_run=args.dry_run,
                             email=getattr(args, "email", False))
    if cmd == "retest":
        return run_retest(cfg)
    if cmd == "probe":
        return run_probe(cfg, status_filter=args.status,
                         csv_path=args.csv, timeout=args.timeout,
                         threads=args.threads)
    if cmd == "test":
        return run_test(cfg)
    if cmd == "backup":
        return run_backup(cfg, sanitize=not args.no_sanitize)
    if cmd == "monitor" and args.target == "topology":
        return run_topology(cfg)
    if cmd == "monitor":
        return run_monitor(cfg, mode=args.target, detail=getattr(args, "detail", False))
    if cmd == "digest" and args.target == "flap":
        return handle_flap_digest(cfg, min_count=args.min_count, reset=args.reset)
    if cmd == "digest" and args.target == "stp":
        return handle_stp_digest(cfg, detail=getattr(args, "detail", False))
    if cmd == "digest" and args.target == "backup":
        return handle_backup_digest(cfg, detail=getattr(args, "detail", False))
    if cmd == "digest" and args.target == "health":
        return handle_health_digest(cfg)
    if cmd == "clear" and args.target == "flap":
        return handle_flap_reset(min_count=args.min_count, dry_run=args.dry_run)
    if cmd == "investigate" and args.target == "stp":
        return handle_investigate_stp(cfg, args.ip, args.port)
    if cmd == "console":
        return run_console(cfg)
    if cmd == "refresh-dns":
        return handle_refresh_dns()


def main():
    args = parse_args()
    cfg = load_config()
    # Shared flags use argparse.SUPPRESS default — attributes may be missing.
    dbg = getattr(args, "debug", False)
    dbgfile = getattr(args, "debug_file", None)
    logmode = getattr(args, "log_mode", "append")
    db_override = getattr(args, "db", None)

    debug_file = (os.path.abspath(os.path.expanduser(dbgfile))
                  if dbgfile else None)
    setup_logging(debug=dbg, debug_file=debug_file,
                  log_format=cfg["log_format"], log_mode=logmode)

    # --- Resolve DB file (before init_db / any DB access) ---
    if db_override:
        global DB_FILE
        DB_FILE = os.path.abspath(os.path.expanduser(db_override))

    cmd = args.command

    # wipe-db runs before init_db so we don't recreate the file we're about to delete.
    if cmd == "wipe-db":
        handle_wipe_db()
        return

    init_db()
    check_tools()
    _dispatch(args, cfg)


if __name__ == "__main__":
    main()
