From 7b44f689bc26b92f735af963947538666ecb62b6 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Thu, 8 Oct 2026 16:07:55 +0200 Subject: [PATCH] feat: report a dpkg left halfway and when the host's own automatic updates last succeeded An unattended upgrade cut off while it built an initrd left a kernel package half-configured on an Ubuntu host. For three months apt refused every upgrade, while apt.systemd.daily exited quietly and the timers reported success (NetOrk/netork#738). HOST_STATUS_COMMAND now also reads, without root: - dpkg's state the way apt checks it: numbered files left in /var/lib/dpkg/updates, or a package half-installed, unpacked, half-configured, waiting for triggers or flagged for reinstallation. While dpkg, apt, aptitude or unattended-upgrade runs, packages are halfway on purpose, so the answer is then None. - /var/lib/apt/periodic/upgrade-stamp, which apt.systemd.daily touches only after unattended-upgrade succeeded, and the interval of APT::Periodic::Unattended-Upgrade in apt's units. update-success-stamp is not read: every apt-get update touches it, whoever runs it. HostStatusDict gains package_manager_interrupted, interrupted_packages, auto_updates_last_success and auto_updates_interval, all NotRequired. --- README.md | 6 +- napalm_device_types/host_status.py | 96 +++++++++++++++++++-- napalm_device_types/models.py | 19 ++++ pyproject.toml | 2 +- tests/test_host_status.py | 134 ++++++++++++++++++++++++++++- 5 files changed, 247 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 59b1280..e8cd863 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,10 @@ too, and `hasattr(driver, "get_kernel_facts")` has to stay truthful. `HostStatusMixin` (`get_host_status`) is mixed in the same way: whether a Linux host needs a reboot to finish an update (`/var/run/reboot-required`, `needs-restarting -r`, or a newer -kernel of the running flavour installed) and whether it patches itself (unattended-upgrades, -dnf-automatic). `package_updates` holds the shared apt and dnf parsers: apt's suites become +kernel of the running flavour installed), whether it patches itself (unattended-upgrades, +dnf-automatic) and when that last succeeded (apt's `upgrade-stamp`), and whether dpkg was +left halfway (a journal in `/var/lib/dpkg/updates` or a half-configured package; unknown +while a package manager is running). `package_updates` holds the shared apt and dnf parsers: apt's suites become an update's `origin`, a `-security` suite makes it a security update, and dnf's security advisories do the same. diff --git a/napalm_device_types/host_status.py b/napalm_device_types/host_status.py index 943db00..31fb7d5 100644 --- a/napalm_device_types/host_status.py +++ b/napalm_device_types/host_status.py @@ -24,7 +24,22 @@ without a ``/lib/modules`` of its own. **Auto updates** is apt's ``APT::Periodic::Unattended-Upgrade`` (set, not "0", and ``apt-daily-upgrade.timer`` not disabled) or an enabled dnf-automatic timer. -It is ``None`` on a host with neither apt nor dnf-automatic. +It is ``None`` on a host with neither apt nor dnf-automatic. When they last ran +successfully is ``/var/lib/apt/periodic/upgrade-stamp``, which +``apt.systemd.daily`` touches only after unattended-upgrade succeeded. +``update-success-stamp`` would say nothing: every ``apt-get update`` touches it, +whoever runs it. How often they are meant to run is the same setting, in apt's +units (a bare number is days). + +**An interrupted package manager** is dpkg left halfway, the way apt checks it: +numbered files left in ``/var/lib/dpkg/updates``, or a package in a state only +a run cut off leaves behind (half-installed, unpacked, half-configured, waiting +for triggers, or flagged for reinstallation). apt then refuses to install +anything until ``dpkg --configure -a`` has run, while ``apt.systemd.daily`` +exits quietly every day (NetOrk/netork#738). While dpkg, apt, aptitude or +unattended-upgrade is running, packages are halfway on purpose -- also between +two of apt's dpkg calls -- so the answer is then ``None``, as on a host without +dpkg. """ from __future__ import annotations @@ -32,7 +47,7 @@ from __future__ import annotations import re from typing import Dict, List, Optional, Tuple, TYPE_CHECKING -from napalm_device_types.models import HostStatusDict +from napalm_device_types.models import HostStatusDict, InterruptedPackageDict from napalm_device_types.terminal import strip_terminal_codes _BEGIN = "HSTAT_BEGIN" @@ -40,6 +55,11 @@ _END = "HSTAT_END" _REBOOT_FILE = "/var/run/reboot-required" _APT_TIMER = "apt-daily-upgrade.timer" _DNF_TIMERS = ("dnf-automatic.timer", "dnf-automatic-install.timer") +_UPGRADE_STAMP = "/var/lib/apt/periodic/upgrade-stamp" +_DPKG_JOURNAL = "/var/lib/dpkg/updates" +#: The processes that leave packages halfway while they work (``comm``, at most +#: 15 characters: unattended-upgrade shows as ``unattended-upgr``). +_PACKAGE_MANAGERS = "dpkg|apt|apt-get|aptitude|unattended-upgr" #: One line, POSIX ``sh``, read-only, no privileges. The frame markers are #: printed in two halves so that an echoing transport does not show them early. @@ -56,7 +76,13 @@ HOST_STATUS_COMMAND = ( "echo '[freebsd-kernel]'; freebsd-version -k; " "echo '[freebsd-running]'; freebsd-version -r; fi; " "if command -v apt-config >/dev/null 2>&1; then echo '[apt-config]'; " - "apt-config dump 2>/dev/null | grep '^APT::Periodic::Unattended-Upgrade '; fi; " + "apt-config dump 2>/dev/null | grep '^APT::Periodic::Unattended-Upgrade '; " + f"echo '[upgrade-stamp]'; stat -c %Y {_UPGRADE_STAMP} 2>/dev/null; fi; " + "if command -v dpkg-query >/dev/null 2>&1; then " + f"ps -e -o comm= 2>/dev/null | grep -qxE '{_PACKAGE_MANAGERS}' && echo '[dpkg-busy]'; " + f"echo '[dpkg-journal]'; ls -1 {_DPKG_JOURNAL} 2>/dev/null | head -n 20; " + "echo '[dpkg-audit]'; dpkg-query -W -f='${db:Status-Abbrev} ${Package}\\n' 2>/dev/null " + "| awk 'substr($0, 2, 1) ~ /[HUFWt]/ || substr($0, 3, 1) == \"R\"' | head -n 50; fi; " f"echo '[timers]'; for u in {_APT_TIMER} {' '.join(_DNF_TIMERS)}; do " 'printf \'%s %s\\n\' "$u" "$(systemctl is-enabled "$u" 2>/dev/null)"; done; ' "printf '%s%s\\n' HSTAT_ END; } 2>/dev/null | cat" @@ -64,6 +90,17 @@ HOST_STATUS_COMMAND = ( _PERIODIC = re.compile(r'^APT::Periodic::Unattended-Upgrade\s+"([^"]*)"') _OFF_STATES = frozenset({"disabled", "masked"}) +_INTERVAL = re.compile(r"(\d+)([smhd]?)") +_UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "": 86400} +_JOURNAL_ENTRY = re.compile(r"\d+") +#: dpkg's second status letter -> its word, for the states a cut-off run leaves. +_HALFWAY = { + "H": "half-installed", + "U": "unpacked", + "F": "half-configured", + "W": "triggers-awaited", + "t": "triggers-pending", +} def _sections(output: str) -> Dict[str, List[str]]: @@ -143,17 +180,54 @@ def _timer_states(sections: Dict[str, List[str]]) -> Dict[str, str]: return states +def _periodic(sections: Dict[str, List[str]]) -> Optional[str]: + """The value of ``APT::Periodic::Unattended-Upgrade``, or None when it is not set.""" + lines = sections.get("apt-config") or [] + match = next(filter(None, (_PERIODIC.match(line) for line in lines)), None) + return match.group(1) if match else None + + def _auto_updates(sections: Dict[str, List[str]]) -> Optional[bool]: timers = _timer_states(sections) if any(timers.get(t) == "enabled" for t in _DNF_TIMERS): return True if "apt-config" not in sections: return None - match = next(filter(None, (_PERIODIC.match(line) for line in sections["apt-config"])), None) - switched_on = match is not None and match.group(1) not in ("", "0") + switched_on = _periodic(sections) not in (None, "", "0") return switched_on and timers.get(_APT_TIMER) not in _OFF_STATES +def _interval(sections: Dict[str, List[str]]) -> Optional[int]: + """apt's interval in seconds, the way ``apt.systemd.daily`` reads it; 0 for "always".""" + value = _periodic(sections) + if value == "always": + return 0 + match = _INTERVAL.fullmatch(value or "") + if not match or int(match.group(1)) == 0: + return None + return int(match.group(1)) * _UNIT_SECONDS[match.group(2)] + + +def _last_success(sections: Dict[str, List[str]]) -> Optional[int]: + stamp = (sections.get("upgrade-stamp") or [""])[0] + return int(stamp) if stamp.isdigit() else None + + +def _halfway(line: str) -> Optional[InterruptedPackageDict]: + status, _, name = line.partition(" ") + name = name.strip() + state = "reinstall-required" if status[2:3] == "R" else _HALFWAY.get(status[1:2]) + return {"name": name, "state": state} if name and state else None + + +def _dpkg(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], List[InterruptedPackageDict]]: + if "dpkg-audit" not in sections or "dpkg-busy" in sections: + return None, [] + packages = [p for p in map(_halfway, sections["dpkg-audit"]) if p is not None] + journal = any(_JOURNAL_ENTRY.fullmatch(f) for f in sections.get("dpkg-journal") or []) + return bool(packages) or journal, packages + + def parse_host_status(output: str) -> HostStatusDict: """Parse what :data:`HOST_STATUS_COMMAND` printed. @@ -161,10 +235,15 @@ def parse_host_status(output: str) -> HostStatusDict: """ sections = _sections(output) required, reason = _reboot(sections) + interrupted, packages = _dpkg(sections) return { "reboot_required": required, "reboot_reason": reason, "auto_updates": _auto_updates(sections), + "auto_updates_last_success": _last_success(sections), + "auto_updates_interval": _interval(sections), + "package_manager_interrupted": interrupted, + "interrupted_packages": packages, } @@ -185,11 +264,16 @@ class HostStatusMixin: def get_host_status(self) -> HostStatusDict: """ - Returns whether the host needs a reboot and whether it patches itself. + Returns whether the host needs a reboot, whether it patches itself, and + whether its package manager was left halfway. * reboot_required (bool or None) * reboot_reason (string or None) * auto_updates (bool or None) + * auto_updates_last_success (Unix time or None) + * auto_updates_interval (seconds or None) + * package_manager_interrupted (bool or None) + * interrupted_packages (list of {name, state}) :raises ValueError: if the host's output carried no intact report. """ diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index e758484..d17dcdb 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -75,6 +75,15 @@ class UpdateDict(TypedDict): security: NotRequired[Optional[bool]] +class InterruptedPackageDict(TypedDict): + """A package an interrupted package manager run left halfway.""" + + name: str + #: dpkg's word for it: "half-installed", "unpacked", "half-configured", + #: "triggers-awaited", "triggers-pending", or "reinstall-required". + state: str + + class HostStatusDict(TypedDict): """What a host says about its own patch state (``HostStatusMixin.get_host_status``).""" @@ -84,6 +93,16 @@ class HostStatusDict(TypedDict): reboot_reason: Optional[str] #: True when the host installs updates on its own (unattended-upgrades, dnf-automatic). auto_updates: Optional[bool] + #: True when a package manager run was cut off and has to be finished + #: (``dpkg --configure -a``) before anything else installs; None when it + #: cannot tell, or a package manager is running right now. + package_manager_interrupted: NotRequired[Optional[bool]] + #: The packages that run left halfway; empty when only its journal says so. + interrupted_packages: NotRequired[List[InterruptedPackageDict]] + #: When the host's own automatic updates last ran successfully (Unix time). + auto_updates_last_success: NotRequired[Optional[int]] + #: How often they are meant to run, in seconds; 0 for every time the timer fires. + auto_updates_interval: NotRequired[Optional[int]] # --------------------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 30a15e4..09410b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.2.0" +version = "4.3.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_host_status.py b/tests/test_host_status.py index 81f1208..a8fa9f9 100644 --- a/tests/test_host_status.py +++ b/tests/test_host_status.py @@ -29,8 +29,12 @@ def _wire( modules=("6.8.0-139-generic", "6.8.0-142-generic"), needs_restarting=None, periodic=None, + stamp=None, timer="enabled", dnf_timers=("not-found", "not-found"), + dpkg=None, + journal=(), + busy=False, ): lines = ["HSTAT_BEGIN"] if reboot_file: @@ -39,7 +43,11 @@ def _wire( lines += ["[needs-restarting]", str(needs_restarting)] lines += ["[kernel]", running, "[modules]", *modules] if periodic is not None: - lines += ["[apt-config]", *periodic] + lines += ["[apt-config]", *periodic, "[upgrade-stamp]", *([str(stamp)] if stamp else [])] + if dpkg is not None: + if busy: + lines.append("[dpkg-busy]") + lines += ["[dpkg-journal]", *journal, "[dpkg-audit]", *dpkg] lines += [ "[timers]", f"apt-daily-upgrade.timer {timer}", @@ -243,3 +251,127 @@ class TestFreeBSD: assert "command -v freebsd-version" in HOST_STATUS_COMMAND assert "freebsd-version -k" in HOST_STATUS_COMMAND assert "freebsd-version -r" in HOST_STATUS_COMMAND + + +class TestInterruptedDpkg: + """dpkg left halfway: apt refuses to install anything until + ``dpkg --configure -a`` has run, and the daily apt timers report success + all the same (NetOrk/netork#738). + + The fixture is aris (Ubuntu 24.04): unattended-upgrades was cut off on + 2026-07-03 while update-initramfs built the initrd for 6.8.0-134, and the + kernel package stayed half-configured for three months. + """ + + ARIS = ("iF linux-image-6.8.0-134-generic",) + + def test_a_half_configured_kernel_is_an_interrupted_dpkg(self): + status = parse_host_status(_wire(dpkg=self.ARIS)) + + assert status["package_manager_interrupted"] is True + assert status["interrupted_packages"] == [ + {"name": "linux-image-6.8.0-134-generic", "state": "half-configured"} + ] + + def test_a_clean_dpkg_is_not_interrupted(self): + status = parse_host_status(_wire(dpkg=())) + + assert status["package_manager_interrupted"] is False + assert status["interrupted_packages"] == [] + + @pytest.mark.parametrize( + ("line", "state"), + [ + ("iH base-files", "half-installed"), + ("iU base-files", "unpacked"), + ("iF base-files", "half-configured"), + ("iW base-files", "triggers-awaited"), + ("it base-files", "triggers-pending"), + ("iUR base-files", "reinstall-required"), + ("hF base-files", "half-configured"), + ], + ) + def test_every_state_an_interrupted_run_leaves(self, line, state): + status = parse_host_status(_wire(dpkg=(line,))) + + assert status["interrupted_packages"] == [{"name": "base-files", "state": state}] + + def test_a_journal_left_behind_is_interrupted_on_its_own(self): + """What apt itself checks: numbered files in /var/lib/dpkg/updates.""" + status = parse_host_status(_wire(dpkg=(), journal=("0000", "0001"))) + + assert status["package_manager_interrupted"] is True + assert status["interrupted_packages"] == [] + + def test_only_numbered_journal_files_count(self): + """apt ignores anything else there, such as dpkg's tmp.i.""" + status = parse_host_status(_wire(dpkg=(), journal=("tmp.i",))) + + assert status["package_manager_interrupted"] is False + + def test_a_running_package_manager_makes_it_unknown(self): + """Mid-upgrade, packages are unpacked and not yet configured, also + between two of apt's dpkg calls. That is work in progress, not an + interruption.""" + status = parse_host_status(_wire(dpkg=("iU base-files",), busy=True)) + + assert status["package_manager_interrupted"] is None + assert status["interrupted_packages"] == [] + + def test_without_dpkg_it_is_unknown(self): + status = parse_host_status(_wire()) + + assert status["package_manager_interrupted"] is None + assert status["interrupted_packages"] == [] + + def test_the_command_asks_dpkg_only_where_it_exists(self): + assert "command -v dpkg-query" in HOST_STATUS_COMMAND + assert "/var/lib/dpkg/updates" in HOST_STATUS_COMMAND + + def test_it_never_repairs(self): + assert "--configure" not in HOST_STATUS_COMMAND + + +class TestAutoUpdatesLastSuccess: + """apt.systemd.daily touches upgrade-stamp only after unattended-upgrade + succeeded. update-success-stamp says nothing: netOrk's own daily + ``apt-get update`` touches it as well (NetOrk/netork#738).""" + + def test_the_upgrade_stamp_is_the_last_successful_run(self): + status = parse_host_status(_wire(periodic=UNATTENDED, stamp=1751525820)) + + assert status["auto_updates_last_success"] == 1751525820 + assert status["auto_updates_interval"] == 86400 + + def test_no_stamp_is_unknown(self): + status = parse_host_status(_wire(periodic=UNATTENDED)) + + assert status["auto_updates_last_success"] is None + + def test_without_apt_both_are_unknown(self): + status = parse_host_status(_wire()) + + assert status["auto_updates_last_success"] is None + assert status["auto_updates_interval"] is None + + @pytest.mark.parametrize( + ("value", "seconds"), + [ + ('"1"', 86400), + ('"7d"', 7 * 86400), + ('"12h"', 43200), + ('"30m"', 1800), + ('"90s"', 90), + ('"always"', 0), + ('"0"', None), + ('"weekly"', None), + ], + ) + def test_the_interval_in_apts_units(self, value, seconds): + periodic = [f"APT::Periodic::Unattended-Upgrade {value};"] + + assert parse_host_status(_wire(periodic=periodic))["auto_updates_interval"] == seconds + + def test_the_command_reads_the_upgrade_stamp_only(self): + assert "/var/lib/apt/periodic/upgrade-stamp" in HOST_STATUS_COMMAND + assert "update-success-stamp" not in HOST_STATUS_COMMAND -- 2.54.0