feat: a public command channel and container engine access

netOrk reached a host's shell through napalm-linux's private `_send`: an
interactive PTY with stdout and stderr merged and no exit code. For Docker
it went further and opened its own paramiko connections around the driver.

This adds the access layer only, no container model (NetOrk/netork#765):

- `channel.py`: `CommandChannelMixin` declares `run_command()` (stdout,
  stderr, exit code) and `open_stream()` (a `ByteStream` to a running
  command). Declared under TYPE_CHECKING, so hasattr stays truthful.
  `run_on_transport()` and `open_stream_on_transport()` implement both on
  a paramiko exec channel for any SSH driver; `ParamikoExecStream` drains
  stderr on every read so the shared window never stalls.
- `container_engine.py`: `ContainerEngineMixin` with `container_engines()`
  and `open_container_engine()`. The returned `ContainerEngineConnection`
  opens the engine API over `<binary> system dial-stdio` and runs the CLI
  with caller-chosen arguments. The driver decides the binary through
  `_container_engine_binary()`; callers never see the path.
- README: why the container model lives in netOrk and not here.

Additive; the Docker*Dict declarations stay until netOrk no longer reads
them. Version 2.6.0.

Refs #17

napalm_device_types/__init__.py

@@ -34,7 +34,9 @@ Role bases -- what a device *is*:
 Function classes -- what a device *can do*. Shared behaviour lives here once
 instead of being restated on every role that happens to need it:
 
+* :class:`~napalm_device_types.channel.CommandChannelMixin`
 * :class:`~napalm_device_types.config_lifecycle.ConfigLifecycleMixin`
+* :class:`~napalm_device_types.container_engine.ContainerEngineMixin`
 * :class:`~napalm_device_types.dhcp.DhcpServerMixin`
 * :class:`~napalm_device_types.firewall_rules.FirewallRuleMixin`
 * :class:`~napalm_device_types.health_metrics.HealthMetricsMixin`
@@ -58,7 +60,20 @@ Introspection -- :func:`~napalm_device_types.roles.roles_of`,
 
 from napalm_device_types.base import DeviceTypeDriver, FingerprintRule, PortSpec
 from napalm_device_types.access_point import AccessPointDriver
+from napalm_device_types.channel import (
+    ByteStream,
+    CommandChannelMixin,
+    CommandResult,
+    ParamikoExecStream,
+    open_stream_on_transport,
+    run_on_transport,
+)
 from napalm_device_types.config_lifecycle import ConfigLifecycleMixin
+from napalm_device_types.container_engine import (
+    ContainerEngineConnection,
+    ContainerEngineMixin,
+    ContainerEngineUnavailable,
+)
 from napalm_device_types.dhcp import DhcpServerMixin, normalize_cidr, normalize_mac
 from napalm_device_types.firewall import FirewallDriver
 from napalm_device_types.hypervisor import HypervisorDriver
@@ -104,6 +119,15 @@ from napalm_device_types.switch import SwitchDriver
 
 __all__ = [
     "AccessPointDriver",
+    "ByteStream",
+    "CommandChannelMixin",
+    "CommandResult",
+    "ContainerEngineConnection",
+    "ContainerEngineMixin",
+    "ContainerEngineUnavailable",
+    "ParamikoExecStream",
+    "open_stream_on_transport",
+    "run_on_transport",
     "ConfigLifecycleMixin",
     "DeviceTypeDriver",
     "DhcpServerMixin",

napalm_device_types/base.py

@@ -12,6 +12,7 @@ from typing import NamedTuple
 
 from napalm.base import NetworkDriver
 
+from napalm_device_types.channel import CommandChannelMixin
 from napalm_device_types.host_reboot import HostRebootMixin
 from napalm_device_types.ping_sweep import PingSweepMixin
 
@@ -48,7 +49,7 @@ class PortSpec(NamedTuple):
     mandatory: bool = False
 
 
-class DeviceTypeDriver(PingSweepMixin, HostRebootMixin, NetworkDriver):
+class DeviceTypeDriver(PingSweepMixin, HostRebootMixin, CommandChannelMixin, NetworkDriver):
     """Common base for all netOrk device-type drivers.
 
     Sits between napalm.base.NetworkDriver and the type-specific abstract

napalm_device_types/channel.py

@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+"""The command channel: a public way to run a command on a host, or open a stream.
+
+A driver already reaches its host, over SSH, a REST API or WinRM. What it did
+not offer was a *public* way for the layer above to use that reach: netOrk sent
+shell strings through napalm-linux's private ``_send``, an interactive PTY that
+merges stdout and stderr and has no exit code.
+
+:class:`CommandChannelMixin` declares two methods, and a driver that can
+implements them:
+
+* :meth:`run_command` runs a command and returns stdout, stderr and the exit code.
+* :meth:`open_stream` starts a command and returns a :class:`ByteStream` to its
+  stdin and stdout. That is how netOrk speaks the Docker Engine API, over
+  ``docker system dial-stdio`` (see :mod:`napalm_device_types.container_engine`).
+
+Like the role bases, the mixin only declares (under ``TYPE_CHECKING``), so
+``hasattr(driver, "run_command")`` stays a truthful answer.
+
+:func:`run_on_transport` and :func:`open_stream_on_transport` are the SSH
+implementation over a paramiko ``Transport``, for any SSH driver: an exec
+channel, so no PTY, separate stderr and a real exit status. How a command gains
+root (``sudo -S`` with the password on stdin, ``sudo -n``, or nothing as root)
+stays with the driver.
+"""
+
+from __future__ import annotations
+
+import socket
+import time
+from typing import Any, NamedTuple, Optional, Protocol, TYPE_CHECKING
+
+#: How much of a stream's stderr is kept. Enough to tell "permission denied"
+#: from "no such command"; a stream that writes megabytes there keeps its tail.
+STDERR_TAIL = 64 * 1024
+
+_READ = 32 * 1024
+_POLL = 0.01
+
+
+class CommandResult(NamedTuple):
+    """What a command printed, and how it ended."""
+
+    stdout: str
+    stderr: str
+    exit_code: int
+
+
+class ByteStream(Protocol):
+    """A running command's stdin and stdout, as bytes.
+
+    ``read`` returns ``b""`` at end of stream and raises :class:`TimeoutError`
+    when nothing arrives in time. ``write`` never closes anything:
+    ``dial-stdio`` hands the daemon a half-close as "client gone", and the
+    daemon then answers an unfinished request with HTTP 499 (netork#771). Only
+    ``close_write`` closes the writing side.
+    """
+
+    def read(self, max_bytes: int, timeout: Optional[float] = None) -> bytes: ...
+
+    def write(self, data: bytes) -> None: ...
+
+    def close_write(self) -> None: ...
+
+    def close(self) -> None: ...
+
+    @property
+    def exit_status(self) -> Optional[int]: ...
+
+    @property
+    def stderr(self) -> str: ...
+
+
+class CommandChannelMixin:
+    """Declares the channel; a driver that can reach a shell on its host implements it."""
+
+    if TYPE_CHECKING:  # pragma: no cover - declared for type checkers only
+
+        def run_command(
+            self,
+            command: str,
+            *,
+            privileged: bool = False,
+            timeout: float = 60,
+            stdin: Optional[bytes] = None,
+        ) -> CommandResult:
+            """Run *command* with the host's shell; *stdin*, if given, is written first."""
+            ...
+
+        def open_stream(self, command: str, *, privileged: bool = False) -> ByteStream:
+            """Start *command* and return a stream to its stdin and stdout."""
+            ...
+
+
+class ParamikoExecStream:
+    """A :class:`ByteStream` over a paramiko exec channel.
+
+    stderr is drained on every read: stdout and stderr share one window, and an
+    unread stderr would stall the stream.
+    """
+
+    def __init__(self, channel: Any, *, stderr_limit: int = STDERR_TAIL) -> None:
+        self._channel = channel
+        self._stderr = b""
+        self._limit = stderr_limit
+
+    def _drain_stderr(self) -> None:
+        while self._channel.recv_stderr_ready():
+            self._stderr = (self._stderr + self._channel.recv_stderr(_READ))[-self._limit :]
+
+    def read(self, max_bytes: int, timeout: Optional[float] = None) -> bytes:
+        self._channel.settimeout(timeout)
+        try:
+            return self._channel.recv(max_bytes)
+        except socket.timeout:
+            raise TimeoutError(f"no data within {timeout}s") from None
+        finally:
+            self._drain_stderr()
+
+    def write(self, data: bytes) -> None:
+        self._channel.sendall(data)
+
+    def close_write(self) -> None:
+        self._channel.shutdown_write()
+
+    def close(self) -> None:
+        self._channel.close()
+
+    @property
+    def exit_status(self) -> Optional[int]:
+        if not self._channel.exit_status_ready():
+            return None
+        return self._channel.recv_exit_status()
+
+    @property
+    def stderr(self) -> str:
+        self._drain_stderr()
+        return self._stderr.decode("utf-8", "replace")
+
+
+def _collect(channel: Any, command: str, timeout: float) -> tuple:
+    out, err = b"", b""
+    deadline = time.monotonic() + timeout
+    while True:
+        if channel.recv_ready():
+            out += channel.recv(_READ)
+        elif channel.recv_stderr_ready():
+            err += channel.recv_stderr(_READ)
+        elif channel.exit_status_ready():
+            return out, err
+        elif time.monotonic() > deadline:
+            raise TimeoutError(f"{command!r} did not finish within {timeout}s")
+        else:
+            time.sleep(_POLL)
+
+
+def run_on_transport(
+    transport: Any, command: str, *, stdin: Optional[bytes] = None, timeout: float = 60
+) -> CommandResult:
+    """Run *command* on an exec channel of the paramiko *transport*."""
+    channel = transport.open_session()
+    try:
+        channel.settimeout(timeout)
+        channel.exec_command(command)
+        if stdin is not None:
+            channel.sendall(stdin)
+        channel.shutdown_write()
+        out, err = _collect(channel, command, timeout)
+        return CommandResult(
+            out.decode("utf-8", "replace"), err.decode("utf-8", "replace"), channel.recv_exit_status()
+        )
+    finally:
+        channel.close()
+
+
+def open_stream_on_transport(
+    transport: Any, command: str, *, stdin_prefix: Optional[bytes] = None
+) -> ParamikoExecStream:
+    """Start *command* on an exec channel and return its stream.
+
+    *stdin_prefix* is written before anything else, which is how a sudo password
+    reaches ``sudo -S`` ahead of the stream's own bytes.
+    """
+    channel = transport.open_session()
+    channel.exec_command(command)
+    if stdin_prefix:
+        channel.sendall(stdin_prefix)
+    return ParamikoExecStream(channel)

napalm_device_types/container_engine.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+"""Access to a host's container engine, and nothing more.
+
+This package abstracts the connection, the driver builds it, and netOrk does
+the talking (NetOrk/netork#765). So there is no container model here and no
+parsing:
+
+* :meth:`ContainerEngineMixin.container_engines` says which engines the host
+  offers, and which API each speaks.
+* :meth:`ContainerEngineMixin.open_container_engine` returns a
+  :class:`ContainerEngineConnection`. It opens a stream to the engine's API
+  (``docker system dial-stdio``), and it runs the engine's CLI with arguments
+  the caller chooses. The CLI is there for what the API cannot do: compose,
+  pulls that need the host user's registry login, and private registry digests.
+
+The driver decides *how* the engine is reached. The hook is
+:meth:`ContainerEngineMixin._container_engine_binary`. QNAP returns its
+Container Station path, so callers never see where the binary lives. The
+command runs through the driver's own :class:`~napalm_device_types.channel.CommandChannelMixin`,
+whose privilege handling applies unchanged; Docker normally needs none, because
+the login user is in the ``docker`` group.
+
+Mixed in by a driver, not by a role base: a Windows host is an
+:class:`~napalm_device_types.os.OSDriver` too, and without a shell channel it
+has no ``dial-stdio`` to offer. ``hasattr(driver, "open_container_engine")``
+stays truthful.
+"""
+
+from __future__ import annotations
+
+import shlex
+from typing import Any, Dict, List, Optional, Sequence, TYPE_CHECKING
+
+from napalm_device_types.channel import ByteStream, CommandResult
+from napalm_device_types.models import ContainerEngineDict
+
+#: Engines this package knows how to reach, and the API each speaks.
+ENGINES: Dict[str, str] = {"docker": "docker-engine"}
+
+_PROBE_TIMEOUT = 15
+_CLI_TIMEOUT = 120
+
+
+class ContainerEngineUnavailable(NotImplementedError):
+    """The host offers no such engine, or this package does not know how to reach it."""
+
+
+class ContainerEngineConnection:
+    """The way to one container engine on one host.
+
+    Built by :meth:`ContainerEngineMixin.open_container_engine`. Callers pass
+    arguments; binary, quoting and the channel are the driver's.
+    """
+
+    def __init__(self, driver: Any, engine: str, binary: str) -> None:
+        self._driver = driver
+        self._binary = binary
+        self.engine = engine
+
+    def _command(self, args: Sequence[str]) -> str:
+        return shlex.join([self._binary, *args])
+
+    def open_api(self) -> ByteStream:
+        """A stream to the engine's API: HTTP/1.1 over ``system dial-stdio``."""
+        return self._driver.open_stream(self._command(["system", "dial-stdio"]))
+
+    def run_cli(
+        self, args: Sequence[str], *, stdin: Optional[bytes] = None, timeout: float = _CLI_TIMEOUT
+    ) -> CommandResult:
+        """Run the engine's CLI with *args* and wait for it."""
+        return self._driver.run_command(self._command(args), timeout=timeout, stdin=stdin)
+
+    def stream_cli(self, args: Sequence[str], *, merge_stderr: bool = False) -> ByteStream:
+        """Start the engine's CLI with *args* and stream its output.
+
+        *merge_stderr* folds stderr into the stream, for tools that report
+        progress there (``compose up``).
+        """
+        command = self._command(args)
+        return self._driver.open_stream(f"{command} 2>&1" if merge_stderr else command)
+
+
+class ContainerEngineMixin:
+    """Adds container engine access to a driver that implements the channel."""
+
+    if TYPE_CHECKING:  # pragma: no cover - declared for type checkers only
+
+        def run_command(
+            self,
+            command: str,
+            *,
+            privileged: bool = False,
+            timeout: float = 60,
+            stdin: Optional[bytes] = None,
+        ) -> CommandResult: ...
+
+        def open_stream(self, command: str, *, privileged: bool = False) -> ByteStream: ...
+
+    def _container_engine_binary(self, engine: str) -> str:
+        """The host command that is *engine*'s CLI. Override where it is not on PATH."""
+        return engine
+
+    def container_engines(self) -> List[ContainerEngineDict]:
+        """
+        Returns the container engines the host offers:
+
+        * engine (string) - e.g. "docker"
+        * api (string) - what its API is, e.g. "docker-engine"
+
+        An engine is listed when its CLI exists on the host. Whether the login
+        user may use it is decided by the first API call, which says
+        "permission denied" in the stream's stderr.
+        """
+        found: List[ContainerEngineDict] = []
+        for engine, api in ENGINES.items():
+            probe = shlex.join(["command", "-v", self._container_engine_binary(engine)])
+            result = self.run_command(probe, timeout=_PROBE_TIMEOUT)
+            if result.exit_code == 0 and result.stdout.strip():
+                found.append({"engine": engine, "api": api})
+        return found
+
+    def open_container_engine(self, engine: str) -> ContainerEngineConnection:
+        """The connection to *engine*; :class:`ContainerEngineUnavailable` if unknown."""
+        if engine not in ENGINES:
+            raise ContainerEngineUnavailable(f"no way to reach container engine {engine!r}")
+        return ContainerEngineConnection(self, engine, self._container_engine_binary(engine))

napalm_device_types/models.py

@@ -871,6 +871,17 @@ class DockerInfoDict(TypedDict):
     outdated_images: NotRequired[List[str]]      # image names with a newer remote digest
 
 
+class ContainerEngineDict(TypedDict):
+    """One container engine a host offers (``container_engines()``).
+
+    Only how to reach it: which engine, and which API it speaks. What runs on
+    it is read and modelled above the driver, in netOrk (netork#765).
+    """
+
+    engine: str     # "docker"
+    api: str        # "docker-engine": the Docker Engine API over ``system dial-stdio``
+
+
 class DeviceActionResultDict(TypedDict):
     """Return value of ``run_device_action()``."""