Merge pull request 'feat: report a dpkg left halfway and when the host's own automatic updates last succeeded' (#25) from feat/dpkg-interrupted into main

README.md

@@ -81,8 +81,10 @@ too, and `hasattr(driver, "get_kernel_facts")` has to stay truthful.
 
 `HostStatusMixin` (`get_host_status`) is mixed in the same way: whether a Linux host needs
 a reboot to finish an update (`/var/run/reboot-required`, `needs-restarting -r`, or a newer
-kernel of the running flavour installed) and whether it patches itself (unattended-upgrades,
-dnf-automatic). `package_updates` holds the shared apt and dnf parsers: apt's suites become
+kernel of the running flavour installed), whether it patches itself (unattended-upgrades,
+dnf-automatic) and when that last succeeded (apt's `upgrade-stamp`), and whether dpkg was
+left halfway (a journal in `/var/lib/dpkg/updates` or a half-configured package; unknown
+while a package manager is running). `package_updates` holds the shared apt and dnf parsers: apt's suites become
 an update's `origin`, a `-security` suite makes it a security update, and dnf's security
 advisories do the same.
 

napalm_device_types/host_status.py

@@ -24,7 +24,22 @@ without a ``/lib/modules`` of its own.
 
 **Auto updates** is apt's ``APT::Periodic::Unattended-Upgrade`` (set, not "0",
 and ``apt-daily-upgrade.timer`` not disabled) or an enabled dnf-automatic timer.
-It is ``None`` on a host with neither apt nor dnf-automatic.
+It is ``None`` on a host with neither apt nor dnf-automatic. When they last ran
+successfully is ``/var/lib/apt/periodic/upgrade-stamp``, which
+``apt.systemd.daily`` touches only after unattended-upgrade succeeded.
+``update-success-stamp`` would say nothing: every ``apt-get update`` touches it,
+whoever runs it. How often they are meant to run is the same setting, in apt's
+units (a bare number is days).
+
+**An interrupted package manager** is dpkg left halfway, the way apt checks it:
+numbered files left in ``/var/lib/dpkg/updates``, or a package in a state only
+a run cut off leaves behind (half-installed, unpacked, half-configured, waiting
+for triggers, or flagged for reinstallation). apt then refuses to install
+anything until ``dpkg --configure -a`` has run, while ``apt.systemd.daily``
+exits quietly every day (NetOrk/netork#738). While dpkg, apt, aptitude or
+unattended-upgrade is running, packages are halfway on purpose -- also between
+two of apt's dpkg calls -- so the answer is then ``None``, as on a host without
+dpkg.
 """
 
 from __future__ import annotations
@@ -32,7 +47,7 @@ from __future__ import annotations
 import re
 from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
 
-from napalm_device_types.models import HostStatusDict
+from napalm_device_types.models import HostStatusDict, InterruptedPackageDict
 from napalm_device_types.terminal import strip_terminal_codes
 
 _BEGIN = "HSTAT_BEGIN"
@@ -40,6 +55,11 @@ _END = "HSTAT_END"
 _REBOOT_FILE = "/var/run/reboot-required"
 _APT_TIMER = "apt-daily-upgrade.timer"
 _DNF_TIMERS = ("dnf-automatic.timer", "dnf-automatic-install.timer")
+_UPGRADE_STAMP = "/var/lib/apt/periodic/upgrade-stamp"
+_DPKG_JOURNAL = "/var/lib/dpkg/updates"
+#: The processes that leave packages halfway while they work (``comm``, at most
+#: 15 characters: unattended-upgrade shows as ``unattended-upgr``).
+_PACKAGE_MANAGERS = "dpkg|apt|apt-get|aptitude|unattended-upgr"
 
 #: One line, POSIX ``sh``, read-only, no privileges. The frame markers are
 #: printed in two halves so that an echoing transport does not show them early.
@@ -56,7 +76,13 @@ HOST_STATUS_COMMAND = (
     "echo '[freebsd-kernel]'; freebsd-version -k; "
     "echo '[freebsd-running]'; freebsd-version -r; fi; "
     "if command -v apt-config >/dev/null 2>&1; then echo '[apt-config]'; "
-    "apt-config dump 2>/dev/null | grep '^APT::Periodic::Unattended-Upgrade '; fi; "
+    "apt-config dump 2>/dev/null | grep '^APT::Periodic::Unattended-Upgrade '; "
+    f"echo '[upgrade-stamp]'; stat -c %Y {_UPGRADE_STAMP} 2>/dev/null; fi; "
+    "if command -v dpkg-query >/dev/null 2>&1; then "
+    f"ps -e -o comm= 2>/dev/null | grep -qxE '{_PACKAGE_MANAGERS}' && echo '[dpkg-busy]'; "
+    f"echo '[dpkg-journal]'; ls -1 {_DPKG_JOURNAL} 2>/dev/null | head -n 20; "
+    "echo '[dpkg-audit]'; dpkg-query -W -f='${db:Status-Abbrev} ${Package}\\n' 2>/dev/null "
+    "| awk 'substr($0, 2, 1) ~ /[HUFWt]/ || substr($0, 3, 1) == \"R\"' | head -n 50; fi; "
     f"echo '[timers]'; for u in {_APT_TIMER} {' '.join(_DNF_TIMERS)}; do "
     'printf \'%s %s\\n\' "$u" "$(systemctl is-enabled "$u" 2>/dev/null)"; done; '
     "printf '%s%s\\n' HSTAT_ END; } 2>/dev/null | cat"
@@ -64,6 +90,17 @@ HOST_STATUS_COMMAND = (
 
 _PERIODIC = re.compile(r'^APT::Periodic::Unattended-Upgrade\s+"([^"]*)"')
 _OFF_STATES = frozenset({"disabled", "masked"})
+_INTERVAL = re.compile(r"(\d+)([smhd]?)")
+_UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "": 86400}
+_JOURNAL_ENTRY = re.compile(r"\d+")
+#: dpkg's second status letter -> its word, for the states a cut-off run leaves.
+_HALFWAY = {
+    "H": "half-installed",
+    "U": "unpacked",
+    "F": "half-configured",
+    "W": "triggers-awaited",
+    "t": "triggers-pending",
+}
 
 
 def _sections(output: str) -> Dict[str, List[str]]:
@@ -143,17 +180,54 @@ def _timer_states(sections: Dict[str, List[str]]) -> Dict[str, str]:
     return states
 
 
+def _periodic(sections: Dict[str, List[str]]) -> Optional[str]:
+    """The value of ``APT::Periodic::Unattended-Upgrade``, or None when it is not set."""
+    lines = sections.get("apt-config") or []
+    match = next(filter(None, (_PERIODIC.match(line) for line in lines)), None)
+    return match.group(1) if match else None
+
+
 def _auto_updates(sections: Dict[str, List[str]]) -> Optional[bool]:
     timers = _timer_states(sections)
     if any(timers.get(t) == "enabled" for t in _DNF_TIMERS):
         return True
     if "apt-config" not in sections:
         return None
-    match = next(filter(None, (_PERIODIC.match(line) for line in sections["apt-config"])), None)
-    switched_on = match is not None and match.group(1) not in ("", "0")
+    switched_on = _periodic(sections) not in (None, "", "0")
     return switched_on and timers.get(_APT_TIMER) not in _OFF_STATES
 
 
+def _interval(sections: Dict[str, List[str]]) -> Optional[int]:
+    """apt's interval in seconds, the way ``apt.systemd.daily`` reads it; 0 for "always"."""
+    value = _periodic(sections)
+    if value == "always":
+        return 0
+    match = _INTERVAL.fullmatch(value or "")
+    if not match or int(match.group(1)) == 0:
+        return None
+    return int(match.group(1)) * _UNIT_SECONDS[match.group(2)]
+
+
+def _last_success(sections: Dict[str, List[str]]) -> Optional[int]:
+    stamp = (sections.get("upgrade-stamp") or [""])[0]
+    return int(stamp) if stamp.isdigit() else None
+
+
+def _halfway(line: str) -> Optional[InterruptedPackageDict]:
+    status, _, name = line.partition(" ")
+    name = name.strip()
+    state = "reinstall-required" if status[2:3] == "R" else _HALFWAY.get(status[1:2])
+    return {"name": name, "state": state} if name and state else None
+
+
+def _dpkg(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], List[InterruptedPackageDict]]:
+    if "dpkg-audit" not in sections or "dpkg-busy" in sections:
+        return None, []
+    packages = [p for p in map(_halfway, sections["dpkg-audit"]) if p is not None]
+    journal = any(_JOURNAL_ENTRY.fullmatch(f) for f in sections.get("dpkg-journal") or [])
+    return bool(packages) or journal, packages
+
+
 def parse_host_status(output: str) -> HostStatusDict:
     """Parse what :data:`HOST_STATUS_COMMAND` printed.
 
@@ -161,10 +235,15 @@ def parse_host_status(output: str) -> HostStatusDict:
     """
     sections = _sections(output)
     required, reason = _reboot(sections)
+    interrupted, packages = _dpkg(sections)
     return {
         "reboot_required": required,
         "reboot_reason": reason,
         "auto_updates": _auto_updates(sections),
+        "auto_updates_last_success": _last_success(sections),
+        "auto_updates_interval": _interval(sections),
+        "package_manager_interrupted": interrupted,
+        "interrupted_packages": packages,
     }
 
 
@@ -185,11 +264,16 @@ class HostStatusMixin:
 
     def get_host_status(self) -> HostStatusDict:
         """
-        Returns whether the host needs a reboot and whether it patches itself.
+        Returns whether the host needs a reboot, whether it patches itself, and
+        whether its package manager was left halfway.
 
         * reboot_required (bool or None)
         * reboot_reason (string or None)
         * auto_updates (bool or None)
+        * auto_updates_last_success (Unix time or None)
+        * auto_updates_interval (seconds or None)
+        * package_manager_interrupted (bool or None)
+        * interrupted_packages (list of {name, state})
 
         :raises ValueError: if the host's output carried no intact report.
         """

napalm_device_types/models.py

@@ -75,6 +75,15 @@ class UpdateDict(TypedDict):
     security: NotRequired[Optional[bool]]
 
 
+class InterruptedPackageDict(TypedDict):
+    """A package an interrupted package manager run left halfway."""
+
+    name: str
+    #: dpkg's word for it: "half-installed", "unpacked", "half-configured",
+    #: "triggers-awaited", "triggers-pending", or "reinstall-required".
+    state: str
+
+
 class HostStatusDict(TypedDict):
     """What a host says about its own patch state (``HostStatusMixin.get_host_status``)."""
 
@@ -84,6 +93,16 @@ class HostStatusDict(TypedDict):
     reboot_reason: Optional[str]
     #: True when the host installs updates on its own (unattended-upgrades, dnf-automatic).
     auto_updates: Optional[bool]
+    #: True when a package manager run was cut off and has to be finished
+    #: (``dpkg --configure -a``) before anything else installs; None when it
+    #: cannot tell, or a package manager is running right now.
+    package_manager_interrupted: NotRequired[Optional[bool]]
+    #: The packages that run left halfway; empty when only its journal says so.
+    interrupted_packages: NotRequired[List[InterruptedPackageDict]]
+    #: When the host's own automatic updates last ran successfully (Unix time).
+    auto_updates_last_success: NotRequired[Optional[int]]
+    #: How often they are meant to run, in seconds; 0 for every time the timer fires.
+    auto_updates_interval: NotRequired[Optional[int]]
 
 
 # ---------------------------------------------------------------------------

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "napalm-device-types"
-version = "4.2.0"
+version = "4.3.0"
 description = "Abstract device-type base classes for NAPALM drivers"
 readme = "README.md"
 requires-python = ">=3.10"

tests/test_host_status.py

@@ -29,8 +29,12 @@ def _wire(
     modules=("6.8.0-139-generic", "6.8.0-142-generic"),
     needs_restarting=None,
     periodic=None,
+    stamp=None,
     timer="enabled",
     dnf_timers=("not-found", "not-found"),
+    dpkg=None,
+    journal=(),
+    busy=False,
 ):
     lines = ["HSTAT_BEGIN"]
     if reboot_file:
@@ -39,7 +43,11 @@ def _wire(
         lines += ["[needs-restarting]", str(needs_restarting)]
     lines += ["[kernel]", running, "[modules]", *modules]
     if periodic is not None:
-        lines += ["[apt-config]", *periodic]
+        lines += ["[apt-config]", *periodic, "[upgrade-stamp]", *([str(stamp)] if stamp else [])]
+    if dpkg is not None:
+        if busy:
+            lines.append("[dpkg-busy]")
+        lines += ["[dpkg-journal]", *journal, "[dpkg-audit]", *dpkg]
     lines += [
         "[timers]",
         f"apt-daily-upgrade.timer {timer}",
@@ -243,3 +251,127 @@ class TestFreeBSD:
         assert "command -v freebsd-version" in HOST_STATUS_COMMAND
         assert "freebsd-version -k" in HOST_STATUS_COMMAND
         assert "freebsd-version -r" in HOST_STATUS_COMMAND
+
+
+class TestInterruptedDpkg:
+    """dpkg left halfway: apt refuses to install anything until
+    ``dpkg --configure -a`` has run, and the daily apt timers report success
+    all the same (NetOrk/netork#738).
+
+    The fixture is aris (Ubuntu 24.04): unattended-upgrades was cut off on
+    2026-07-03 while update-initramfs built the initrd for 6.8.0-134, and the
+    kernel package stayed half-configured for three months.
+    """
+
+    ARIS = ("iF  linux-image-6.8.0-134-generic",)
+
+    def test_a_half_configured_kernel_is_an_interrupted_dpkg(self):
+        status = parse_host_status(_wire(dpkg=self.ARIS))
+
+        assert status["package_manager_interrupted"] is True
+        assert status["interrupted_packages"] == [
+            {"name": "linux-image-6.8.0-134-generic", "state": "half-configured"}
+        ]
+
+    def test_a_clean_dpkg_is_not_interrupted(self):
+        status = parse_host_status(_wire(dpkg=()))
+
+        assert status["package_manager_interrupted"] is False
+        assert status["interrupted_packages"] == []
+
+    @pytest.mark.parametrize(
+        ("line", "state"),
+        [
+            ("iH  base-files", "half-installed"),
+            ("iU  base-files", "unpacked"),
+            ("iF  base-files", "half-configured"),
+            ("iW  base-files", "triggers-awaited"),
+            ("it  base-files", "triggers-pending"),
+            ("iUR base-files", "reinstall-required"),
+            ("hF  base-files", "half-configured"),
+        ],
+    )
+    def test_every_state_an_interrupted_run_leaves(self, line, state):
+        status = parse_host_status(_wire(dpkg=(line,)))
+
+        assert status["interrupted_packages"] == [{"name": "base-files", "state": state}]
+
+    def test_a_journal_left_behind_is_interrupted_on_its_own(self):
+        """What apt itself checks: numbered files in /var/lib/dpkg/updates."""
+        status = parse_host_status(_wire(dpkg=(), journal=("0000", "0001")))
+
+        assert status["package_manager_interrupted"] is True
+        assert status["interrupted_packages"] == []
+
+    def test_only_numbered_journal_files_count(self):
+        """apt ignores anything else there, such as dpkg's tmp.i."""
+        status = parse_host_status(_wire(dpkg=(), journal=("tmp.i",)))
+
+        assert status["package_manager_interrupted"] is False
+
+    def test_a_running_package_manager_makes_it_unknown(self):
+        """Mid-upgrade, packages are unpacked and not yet configured, also
+        between two of apt's dpkg calls. That is work in progress, not an
+        interruption."""
+        status = parse_host_status(_wire(dpkg=("iU  base-files",), busy=True))
+
+        assert status["package_manager_interrupted"] is None
+        assert status["interrupted_packages"] == []
+
+    def test_without_dpkg_it_is_unknown(self):
+        status = parse_host_status(_wire())
+
+        assert status["package_manager_interrupted"] is None
+        assert status["interrupted_packages"] == []
+
+    def test_the_command_asks_dpkg_only_where_it_exists(self):
+        assert "command -v dpkg-query" in HOST_STATUS_COMMAND
+        assert "/var/lib/dpkg/updates" in HOST_STATUS_COMMAND
+
+    def test_it_never_repairs(self):
+        assert "--configure" not in HOST_STATUS_COMMAND
+
+
+class TestAutoUpdatesLastSuccess:
+    """apt.systemd.daily touches upgrade-stamp only after unattended-upgrade
+    succeeded. update-success-stamp says nothing: netOrk's own daily
+    ``apt-get update`` touches it as well (NetOrk/netork#738)."""
+
+    def test_the_upgrade_stamp_is_the_last_successful_run(self):
+        status = parse_host_status(_wire(periodic=UNATTENDED, stamp=1751525820))
+
+        assert status["auto_updates_last_success"] == 1751525820
+        assert status["auto_updates_interval"] == 86400
+
+    def test_no_stamp_is_unknown(self):
+        status = parse_host_status(_wire(periodic=UNATTENDED))
+
+        assert status["auto_updates_last_success"] is None
+
+    def test_without_apt_both_are_unknown(self):
+        status = parse_host_status(_wire())
+
+        assert status["auto_updates_last_success"] is None
+        assert status["auto_updates_interval"] is None
+
+    @pytest.mark.parametrize(
+        ("value", "seconds"),
+        [
+            ('"1"', 86400),
+            ('"7d"', 7 * 86400),
+            ('"12h"', 43200),
+            ('"30m"', 1800),
+            ('"90s"', 90),
+            ('"always"', 0),
+            ('"0"', None),
+            ('"weekly"', None),
+        ],
+    )
+    def test_the_interval_in_apts_units(self, value, seconds):
+        periodic = [f"APT::Periodic::Unattended-Upgrade {value};"]
+
+        assert parse_host_status(_wire(periodic=periodic))["auto_updates_interval"] == seconds
+
+    def test_the_command_reads_the_upgrade_stamp_only(self):
+        assert "/var/lib/apt/periodic/upgrade-stamp" in HOST_STATUS_COMMAND
+        assert "update-success-stamp" not in HOST_STATUS_COMMAND