From b8b89acee1ab2c493007a4d87026e64333e1a09e Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Wed, 7 Oct 2026 07:20:27 +0200 Subject: [PATCH] feat: read the sockets of a host without ss from netstat, and procd's services OpenWrt has no ss: the listening-socket command fell through to [no-ss] and the reading raised ListeningSocketsUnavailable. Without ss it now runs netstat -lntup (busybox and net-tools both), and the cgroups are read for its PIDs alike. - netstat names a socket's process as PID/Program; a UDP line has no state column, "-" is a socket without a process, and net-tools prints tcp6/udp6 and program names with a space ("sshd: /usr/sbin"). - On OpenWrt the cgroup is /services//, so the unit is the procd service -- a jailed one too, whose PID is not the one procd reports (dnsmasq under ujail). - A host with neither tool still raises ListeningSocketsUnavailable. Checked against a real OpenWrt 25.12.2 access point, and that a Linux and a Proxmox host still read the same sockets with the longer command. 2.5.0. For netOrk#673. --- README.md | 4 +- napalm_device_types/listening.py | 108 ++++++++++++++++++++------ pyproject.toml | 2 +- tests/test_listening_sockets.py | 126 ++++++++++++++++++++++++++++++- 4 files changed, 211 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index f150a80..5ac73ee 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,9 @@ from `/proc//cgroup`, in one round trip. A driver supplies `_run_listening_sockets_command(command, privileged=)`; the command arrives as one `sh -c` argument, so a `sudo -n` prefix covers all of it. Without root `ss` names only the login user's processes, and the reading says so (`attributed: false`) instead of failing. A host -without `ss` raises `ListeningSocketsUnavailable`. +without `ss` is read with `netstat -lntup` (OpenWrt's busybox, old net-tools); on OpenWrt the +cgroup names the procd service (`/services//`). A host with neither raises +`ListeningSocketsUnavailable`. **Update readers raise when they cannot read.** `get_available_updates` returns an empty list only when nothing is pending; netOrk keeps "pending since" per package, and an empty diff --git a/napalm_device_types/listening.py b/napalm_device_types/listening.py index e69661a..68c5240 100644 --- a/napalm_device_types/listening.py +++ b/napalm_device_types/listening.py @@ -21,8 +21,13 @@ the reading says it is not ``attributed``. **No ``-H``.** iproute2 before 4.10 has no option to leave out the header and fails on it, which would read as nothing listening. The parse skips the header -instead. A host without ``ss`` at all (busybox, QNAP) raises -:class:`ListeningSocketsUnavailable`. +instead. + +**Without ``ss``, ``netstat``.** OpenWrt's busybox and old net-tools hosts have +no ``ss``; ``netstat -lntup`` lists the same sockets with ``PID/Program``, and +the cgroups are read for its PIDs alike. On OpenWrt the cgroup names the procd +service (``/services//``), a jailed one too, whose PID is not the +one procd reports. A host with neither raises :class:`ListeningSocketsUnavailable`. """ from __future__ import annotations @@ -42,23 +47,37 @@ _RC_RE = re.compile(r"^__SS_RC=(\d+)$") #: One line, POSIX ``sh``, read-only. The frame markers are printed in two #: halves so that a transport which echoes the command does not show them early. #: ``ss`` is in ``/usr/sbin`` on some systems, outside a login user's ``PATH``. +#: ``netstat`` names a socket's process as ``PID/Program``, ``ss`` as ``pid=PID``. LISTENING_SOCKETS_COMMAND = ( "PATH=$PATH:/usr/sbin:/sbin; " "printf '%s%s\\n' SOCK_ BEGIN; " "if command -v ss >/dev/null 2>&1; then " - "s=$(ss -lntup 2>&1); r=$?; echo '[ss]'; printf '%s\\n' \"$s\"; echo \"__SS_RC=$r\"; " + "t=ss; s=$(ss -lntup 2>&1); r=$?; " + "pids=$(printf '%s\\n' \"$s\" | grep -o 'pid=[0-9]*' | cut -d= -f2); " + "elif command -v netstat >/dev/null 2>&1; then " + "t=netstat; s=$(netstat -lntup 2>&1); r=$?; " + "pids=$(printf '%s\\n' \"$s\" | grep -o ' [0-9][0-9]*/' | tr -d ' /'); " + "else t=; fi; " + "if [ -n \"$t\" ]; then " + "echo \"[$t]\"; printf '%s\\n' \"$s\"; echo \"__SS_RC=$r\"; " "echo '[cgroups]'; " - "for p in $(printf '%s\\n' \"$s\" | grep -o 'pid=[0-9]*' | cut -d= -f2 | sort -u); do " + "for p in $(printf '%s\\n' \"$pids\" | sort -u); do " "sed \"s|^|$p |\" /proc/$p/cgroup 2>/dev/null; done; " "else echo '[no-ss]'; fi; " "printf '%s%s\\n' SOCK_ END" ) _PROTOCOLS = frozenset({"tcp", "udp"}) +#: netstat names the IPv6 sockets of net-tools ``tcp6``/``udp6``; busybox does not. +_NETSTAT_PROTOCOLS = {"tcp": "tcp", "tcp6": "tcp", "udp": "udp", "udp6": "udp"} +#: ``1604/dropbear``; net-tools prints ``700/sshd: /usr/sbin``. +_PROGRAM_RE = re.compile(r"^(\d+)/(\S*)") #: ``users:(("nginx",pid=901,fd=6),("nginx",pid=900,fd=6))`` _USER_RE = re.compile(r'\("((?:[^"\\]|\\.)*)",pid=(\d+),fd=\d+\)') #: The service a cgroup path runs in: its deepest ``*.service`` component. _SERVICE_RE = re.compile(r"/([^/]+)\.service(?=/|$)") +#: OpenWrt's procd: ``/services//``. +_PROCD_RE = re.compile(r"^/services/([^/]+)(?:/|$)") #: A container's cgroup: ``docker-.scope`` (systemd driver), ``/docker/`` (cgroupfs). _CONTAINER_RE = re.compile( r"(?:docker|libpod)-([0-9a-f]{64})\.scope|/(?:docker|libpod)/([0-9a-f]{64})(?=/|$)" @@ -66,7 +85,7 @@ _CONTAINER_RE = re.compile( class ListeningSocketsUnavailable(NotImplementedError): - """The host has no ``ss``; there is nothing to read and nothing to retry.""" + """The host has neither ``ss`` nor ``netstat``; nothing to read, nothing to retry.""" def _frame(output: str) -> List[str]: @@ -117,7 +136,10 @@ def _cgroup_paths(lines: List[str]) -> Dict[int, str]: def _unit(path: Optional[str]) -> Optional[str]: services = _SERVICE_RE.findall(path or "") - return services[-1] if services else None + if services: + return str(services[-1]) + procd = _PROCD_RE.match(path or "") + return str(procd.group(1)) if procd else None def _container(path: Optional[str]) -> Optional[str]: @@ -125,19 +147,17 @@ def _container(path: Optional[str]) -> Optional[str]: return (match.group(1) or match.group(2)) if match else None -def _socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict]: - parts = line.split() - if len(parts) < 5 or parts[0] not in _PROTOCOLS: - return None - local = _split_local(parts[4]) - if local is None: - return None +def _entry( + proto: str, + local: Tuple[str, Optional[str], int], + process: Optional[str], + pid: Optional[int], + paths: Dict[int, str], +) -> ListeningSocketDict: address, interface, port = local - users = _USER_RE.findall(line) - process, pid = (users[0][0], int(users[0][1])) if users else (None, None) path = paths.get(pid) if pid is not None else None return { - "proto": parts[0], + "proto": proto, "address": address, "port": port, "interface": interface, @@ -148,29 +168,67 @@ def _socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict]: } +def _ss_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict]: + parts = line.split() + if len(parts) < 5 or parts[0] not in _PROTOCOLS: + return None + local = _split_local(parts[4]) + if local is None: + return None + users = _USER_RE.findall(line) + process, pid = (users[0][0], int(users[0][1])) if users else (None, None) + return _entry(parts[0], local, process, pid, paths) + + +def _netstat_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict]: + """``Proto Recv-Q Send-Q Local Foreign [State] PID/Program`` -- a UDP line + has no state, and ``-`` is a socket without a process.""" + parts = line.split() + proto = _NETSTAT_PROTOCOLS.get(parts[0]) if parts else None + if proto is None or len(parts) < 6: + return None + local = _split_local(parts[3]) + if local is None: + return None + process, pid = None, None + for token in parts[5:7]: + program = _PROGRAM_RE.match(token) + if program: + pid, process = int(program.group(1)), program.group(2).rstrip(":") or None + break + return _entry(proto, local, process, pid, paths) + + +_PARSERS = {"ss": _ss_socket, "netstat": _netstat_socket} + + def parse_listening_sockets(output: str) -> List[ListeningSocketDict]: """Parse what :data:`LISTENING_SOCKETS_COMMAND` printed, sorted by protocol, port and address. - :raises ListeningSocketsUnavailable: when the host has no ``ss``. - :raises ValueError: when the output carries no intact report, or ``ss`` failed. + :raises ListeningSocketsUnavailable: when the host has neither ``ss`` nor ``netstat``. + :raises ValueError: when the output carries no intact report, or the tool failed. """ sections = _sections(_frame(output)) if _NO_SS in sections: - raise ListeningSocketsUnavailable("the host has no ss") - ss_lines = sections.get("ss", []) - statuses = [m.group(1) for m in map(_RC_RE.match, ss_lines) if m] + raise ListeningSocketsUnavailable("the host has neither ss nor netstat") + tool = "netstat" if "netstat" in sections else "ss" + lines = sections.get(tool, []) + statuses = [m.group(1) for m in map(_RC_RE.match, lines) if m] if not statuses or statuses[-1] != "0": - detail = " ".join(line for line in ss_lines if not _RC_RE.match(line))[:200] - raise ValueError(f"ss did not list the sockets: {detail or 'no exit status'}") + detail = " ".join(line for line in lines if not _RC_RE.match(line))[:200] + raise ValueError(f"{tool} did not list the sockets: {detail or 'no exit status'}") paths = _cgroup_paths(sections.get("cgroups", [])) - sockets = [s for s in (_socket(line, paths) for line in ss_lines) if s is not None] + parse = _PARSERS[tool] + sockets = [s for s in (parse(line, paths) for line in lines) if s is not None] return sorted(sockets, key=lambda s: (s["proto"], s["port"], s["address"], s["interface"] or "")) class ListeningSocketsMixin: """Adds :meth:`get_listening_sockets` to a driver that can run a command on a Linux host. + With ``ss``, or ``netstat`` where there is none (OpenWrt's busybox). + The template form (README, "Function classes"): the command and its parse are the same everywhere, so they are concrete here, and a driver supplies only :meth:`_run_listening_sockets_command` -- how a command reaches its @@ -205,7 +263,7 @@ class ListeningSocketsMixin: ], } - :raises ListeningSocketsUnavailable: if the host has no ``ss``. + :raises ListeningSocketsUnavailable: if the host has neither ``ss`` nor ``netstat``. :raises ValueError: if neither reading carried an intact report. """ command = f"sh -c {quote(LISTENING_SOCKETS_COMMAND)}" diff --git a/pyproject.toml b/pyproject.toml index 119c1cf..a320c85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "2.4.0" +version = "2.5.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_listening_sockets.py b/tests/test_listening_sockets.py index bc7eb3a..86b228f 100644 --- a/tests/test_listening_sockets.py +++ b/tests/test_listening_sockets.py @@ -5,10 +5,15 @@ listens on -- ``0.0.0.0:5432`` is, ``127.0.0.1:5432`` is not. Reading that is the same on every Linux host: ``ss`` for the sockets, ``/proc//cgroup`` for the systemd unit or container a process belongs to. So both live here once, and a driver only carries the command across (#658 in netOrk). + +A host without ``ss`` -- OpenWrt's busybox, an old net-tools box -- is read +with ``netstat -lntup`` instead, and on OpenWrt the cgroup names the procd +service (#673 in netOrk). """ from __future__ import annotations +import os import shutil import subprocess @@ -145,6 +150,14 @@ class TestCgroups: assert self._unit_and_container(lines) == ("wg-quick@wg0", None) + def test_a_procd_service_on_openwrt(self): + """procd puts every instance into /services//, a jailed + one too -- its PID is not the one procd reports, its cgroup is.""" + assert self._unit_and_container("5 0::/services/dnsmasq/cfg01411c\n") == ( + "dnsmasq", + None, + ) + def test_a_login_session_is_no_unit(self): assert self._unit_and_container("5 0::/user.slice/user-1000.slice/session-3.scope\n") == ( None, @@ -176,7 +189,7 @@ class TestFailures: with pytest.raises(ValueError): parse_listening_sockets(_wire()[:-len("SOCK_END\n")]) - def test_a_host_without_ss_is_unavailable(self): + def test_a_host_without_ss_or_netstat_is_unavailable(self): with pytest.raises(ListeningSocketsUnavailable): parse_listening_sockets("SOCK_BEGIN\n[no-ss]\nSOCK_END\n") @@ -190,6 +203,115 @@ class TestFailures: assert not issubclass(ListeningSocketsUnavailable, ValueError) +# busybox netstat on OpenWrt 25.12, addresses replaced by documentation ones. +BUSYBOX = """Active Internet connections (only servers) +Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name +tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1604/dropbear +tcp 0 0 0.0.0.0:443 0.0.0.0:* LISTEN 1969/uhttpd +tcp 0 0 192.0.2.15:53 0.0.0.0:* LISTEN 1495/dnsmasq +tcp 0 0 127.0.0.1:53 0.0.0.0:* LISTEN 1495/dnsmasq +tcp 0 0 :::22 :::* LISTEN 1604/dropbear +tcp 0 0 fe80::1:53 :::* LISTEN 1495/dnsmasq +tcp 0 0 ::1:53 :::* LISTEN 1495/dnsmasq +udp 0 0 192.0.2.15:53 0.0.0.0:* 1495/dnsmasq +udp 0 0 0.0.0.0:161 0.0.0.0:* 3173/snmpd +udp 0 0 0.0.0.0:5353 0.0.0.0:* - +""" +PROCD = """1495 0::/services/dnsmasq/cfg01411c +1604 0::/services/dropbear/instance1 +1969 0::/services/uhttpd/instance1 +3173 0::/services/snmpd/instance1 +""" +# net-tools netstat on an old Debian: tcp6/udp6, and a program name with a space. +NET_TOOLS = """Active Internet connections (only servers) +Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name +tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 700/sshd: /usr/sbin +tcp6 0 0 :::22 :::* LISTEN 700/sshd: /usr/sbin +udp6 0 0 :::5353 :::* - +""" + + +def _netstat(out: str = BUSYBOX, cgroups: str = PROCD, *, rc: int = 0) -> str: + return f"SOCK_BEGIN\n[netstat]\n{out}__SS_RC={rc}\n[cgroups]\n{cgroups}SOCK_END\n" + + +class TestNetstat: + """A host without ss (#673 in netOrk): OpenWrt's busybox, old net-tools.""" + + def test_a_socket_comes_with_its_process_and_procd_service(self): + sockets = _by_port(parse_listening_sockets(_netstat())) + + assert sockets[("tcp", 22, "0.0.0.0")] == { + "proto": "tcp", + "address": "0.0.0.0", + "port": 22, + "interface": None, + "process": "dropbear", + "pid": 1604, + "unit": "dropbear", + "container_id": None, + } + + @pytest.mark.parametrize( + "proto, port, address", + [("tcp", 22, "::"), ("tcp", 53, "fe80::1"), ("tcp", 53, "::1"), ("tcp", 53, "192.0.2.15")], + ) + def test_every_address_form(self, proto, port, address): + assert (proto, port, address) in _by_port(parse_listening_sockets(_netstat())) + + def test_a_udp_socket_has_no_state_column(self): + snmpd = _by_port(parse_listening_sockets(_netstat()))[("udp", 161, "0.0.0.0")] + + assert (snmpd["process"], snmpd["pid"], snmpd["unit"]) == ("snmpd", 3173, "snmpd") + + def test_a_socket_without_a_process_is_kept(self): + mdns = _by_port(parse_listening_sockets(_netstat()))[("udp", 5353, "0.0.0.0")] + + assert (mdns["process"], mdns["pid"], mdns["unit"]) == (None, None, None) + + def test_the_headers_are_no_sockets(self): + assert len(parse_listening_sockets(_netstat())) == 10 + + def test_net_tools_names_ipv6_and_programs_its_own_way(self): + sockets = _by_port(parse_listening_sockets(_netstat(NET_TOOLS, ""))) + + assert sockets[("tcp", 22, "::")]["process"] == "sshd" + assert sockets[("tcp", 22, "::")]["pid"] == 700 + assert ("udp", 5353, "::") in sockets + + def test_netstat_failing_raises(self): + with pytest.raises(ValueError): + parse_listening_sockets(_netstat("netstat: invalid option -- 'p'\n", "", rc=1)) + + +class TestTheNetstatFallback: + """The command itself, on a host where ss is missing and netstat is not.""" + + @pytest.mark.skipif( + any(os.path.exists(f"{d}/ss") for d in ("/usr/sbin", "/sbin")), + reason="ss sits on the PATH the command always adds", + ) + def test_it_reads_netstat_when_there_is_no_ss(self, tmp_path): + for tool in ("grep", "cut", "sort", "sed", "tr", "cat"): + (tmp_path / tool).symlink_to(shutil.which(tool)) + stub = tmp_path / "netstat" + stub.write_text(f"#!/bin/sh\ncat <<'EOF'\n{BUSYBOX}EOF\n") + stub.chmod(0o755) + + out = subprocess.run( + ["/bin/sh", "-c", LISTENING_SOCKETS_COMMAND], + capture_output=True, + text=True, + env={"PATH": str(tmp_path)}, + timeout=30, + ).stdout + + assert "[netstat]" in out + sockets = _by_port(parse_listening_sockets(out)) + assert sockets[("tcp", 443, "0.0.0.0")]["process"] == "uhttpd" + assert len(sockets) == 10 + + class TestTheCommand: def test_the_frame_is_not_in_the_command_itself(self): """An echoing transport prints the command back; the markers must only @@ -260,7 +382,7 @@ class TestTheTemplate: assert len(reading["sockets"]) == 9 assert [p for _c, p in driver.calls] == [True, False] - def test_a_host_without_ss_is_not_asked_twice(self): + def test_a_host_without_either_is_not_asked_twice(self): driver = _Driver("SOCK_BEGIN\n[no-ss]\nSOCK_END\n") with pytest.raises(ListeningSocketsUnavailable):