132 lines
5.4 KiB
Python
132 lines
5.4 KiB
Python
"""A load-bearing deploy step must not report success when it failed.
|
|
|
|
`deploy.sh` runs its real work over ssh. The exit status ssh hands back
|
|
is the status of the *remote* command — and when that command ends in a pipe,
|
|
the status belongs to the last stage of the pipe, not to the work:
|
|
|
|
ssh "$SERVER" "docker compose pull … 2>&1 | grep -E 'Pulled|Already|Error'"
|
|
|
|
A failed pull prints a line containing `Error`, `grep` matches it and exits 0, and
|
|
the deploy walks on to swap the containers. `set -o pipefail` cannot see this: the
|
|
remote pipeline ran inside the remote shell and from here the ssh succeeded.
|
|
|
|
A sibling service hit the same shape on 2026-09-15 with
|
|
`alembic upgrade head 2>&1 | tail -5`. The migration raised a TypeError, the
|
|
script printed the traceback and swapped the containers anyway, and the service
|
|
answered 500 on every request touching the changed table for eleven minutes.
|
|
|
|
So: tail and grep locally, on captured output, where `pipefail` and `set -e` can
|
|
still act. The rule below is deliberately narrow — it covers only the steps whose
|
|
failure must stop a deploy. `docker image prune | tail -1` is cosmetic and stays
|
|
exempt, because a rule that forbids every pipe would be worked around rather than
|
|
followed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
|
|
|
|
#: The steps a deploy must not walk past. Anything else may pipe as it likes.
|
|
LOAD_BEARING = re.compile(
|
|
r"docker pull|compose[^\n]*\bpull\b|compose[^\n]*\bup -d\b|alembic upgrade|/app/deploy/"
|
|
)
|
|
|
|
#: Filters that replace the exit status of everything before them.
|
|
SWALLOWS_STATUS = re.compile(r"\|\s*(tail|head|grep)\b")
|
|
|
|
#: Every ssh line ends `| sed "s/^/[${SERVER}] /"`, which runs *here*. Local
|
|
#: pipes are covered by `set -o pipefail`; only what runs on the far side is
|
|
#: invisible, so the local suffix is cut off before the check.
|
|
LOCAL_SUFFIX = '| sed "s/^/'
|
|
|
|
|
|
def _logical_lines() -> list[str]:
|
|
"""The script with backslash-continuations joined, so one command is one line.
|
|
|
|
Comments are dropped. The first version of this check flagged the comment
|
|
inside `run_remote` that quotes the offending line in order to explain it —
|
|
prose about a statement is not the statement, and a checker that cannot tell
|
|
them apart makes the explanation unwritable.
|
|
"""
|
|
joined = DEPLOY.read_text().replace("\\\n", " ")
|
|
return [line for line in joined.splitlines() if not line.lstrip().startswith("#")]
|
|
|
|
|
|
def _remote_part(line: str) -> str:
|
|
head, _, _ = line.partition(LOCAL_SUFFIX)
|
|
return head
|
|
|
|
|
|
def _offenders() -> list[str]:
|
|
return [
|
|
" ".join(line.split())[:120]
|
|
for line in _logical_lines()
|
|
if "ssh " in line
|
|
and LOAD_BEARING.search(line)
|
|
and SWALLOWS_STATUS.search(_remote_part(line))
|
|
]
|
|
|
|
|
|
def test_the_detector_sees_the_shape_it_exists_for() -> None:
|
|
"""Guards the guard, against the exact line that caused the outage."""
|
|
broken = 'ssh -n "$SERVER" "cd ~/x && docker compose pull api 2>&1 | grep -E \'Pulled\'"'
|
|
|
|
assert LOAD_BEARING.search(broken)
|
|
assert SWALLOWS_STATUS.search(_remote_part(broken))
|
|
|
|
|
|
def test_a_local_tail_is_not_an_offence() -> None:
|
|
"""`pipefail` already covers this side of the ssh; the rule is about the far
|
|
side. A check that flagged both would push the fix in the wrong direction."""
|
|
fine = 'ssh -n "$SERVER" "docker compose pull api" | sed "s/^/[x] /" | tail -3'
|
|
|
|
assert not SWALLOWS_STATUS.search(_remote_part(fine))
|
|
|
|
|
|
def test_the_script_is_there_and_has_ssh_steps() -> None:
|
|
"""An unreadable or renamed script would make the test below vacuously green."""
|
|
assert [line for line in _logical_lines() if "ssh " in line]
|
|
|
|
|
|
def test_no_load_bearing_step_hides_its_exit_status() -> None:
|
|
offenders = _offenders()
|
|
|
|
assert not offenders, "remote pipes swallow the status of:\n " + "\n ".join(offenders)
|
|
|
|
|
|
class TestTheHostRecordsOnlyWhatItRuns:
|
|
"""`~/netork/.env` on the server is a record, not an intention.
|
|
|
|
It used to be written before the pull. A deploy that then failed — a tag the
|
|
registry does not have, say — left the host recording a version it had never
|
|
run, and the next `docker compose up` a human typed there reached for an
|
|
image that does not exist. Observed on a test host while verifying the
|
|
failure path above: the deploy stopped exactly where it should, and still
|
|
left `NETORK_VERSION=main-0000000` behind.
|
|
|
|
So it is written after the swap and after the image-id check agreed, and this
|
|
holds that order. Positional rather than behavioural, which is the honest
|
|
limit of a static check — but the order is the whole property.
|
|
"""
|
|
|
|
@staticmethod
|
|
def _position(needle: str) -> int:
|
|
text = DEPLOY.read_text()
|
|
assert text.count(needle) == 1, f"{needle!r} appears {text.count(needle)} times"
|
|
return text.index(needle)
|
|
|
|
def test_env_is_recorded_after_the_image_check(self) -> None:
|
|
verified = self._position("Verified: containers run")
|
|
recorded = self._position("Recording registry settings")
|
|
|
|
assert recorded > verified, (
|
|
"the host records NETORK_VERSION before the deploy is verified; "
|
|
"a failed deploy then leaves it pointing at a version never run"
|
|
)
|
|
|
|
def test_env_is_recorded_after_the_pull(self) -> None:
|
|
assert self._position("Recording registry settings") > self._position("Pulling images...")
|