#!/usr/bin/env python3 """ Raise an alert from your own code, and let one place decide what happens to it. Your program knows something is wrong. It should not also know who is on call, whether it is 3am, or whether this is the fortieth time this minute. That is this file's job — it is the only thing you call, and the only thing you edit when the policy changes. # from anything, on this machine $ python3 alert.py --sev1 --service md-gateway "Market data gap on CME" "No ticks for 47s" $ echo '{"severity":"sev2","service":"oms","title":"Acks slow"}' | python3 alert.py # from Python import alert alert.raise_("sev1", "md-gateway", "Market data gap on CME", "No ticks for 47s") Register it so the last form works, and so the phone can raise one too: "actions": { "alert": {"run": "python3 ~/bin/alert.py", "description": "raise an alert"} } What it does with one, in order: 1. Writes it to the incident store — the same file `incidents.py` serves to the Incidents mini-app, so the phone sees it whether or not anyone was reached. 2. Drops it if the same thing was raised within the dedupe window. Forty identical errors a minute is one incident. 3. Follows the policy below: who to reach, by what, at this severity, at this hour. Nothing else in your code needs to know any of it. Policy lives in ~/.config/palmtop/alerts.json, and the defaults are sane: { "dedupe_seconds": 300, "quiet_hours": [22, 7], "routes": { "sev1": ["phone", "notify", "sms", "call"], "sev2": ["phone", "notify", "sms"], "sev3": ["phone", "notify"] }, "who": "+447700900123", "quiet_hours_exempt": ["sev1"] } `routes` names actions registered on this computer (notify.py, twilio_call.py); an action that is missing is skipped and said so in the result, because an alerting path that fails silently is worse than none. One route is built in: `phone` seals the alert to a phone's board through the relay (to_phone.py, beside this file), which is the route that works from a production server — it needs no hub, no pairing and nothing running at the desk. This file runs perfectly well on that server: it reads its own policy, keeps its own dedupe state, and calls its own Twilio, so nothing depends on a laptop being awake. """ import argparse import json import os import shutil import subprocess import sys import time HOME = os.path.expanduser("~/.config/palmtop") POLICY = os.path.join(HOME, "alerts.json") STORE = os.path.expanduser(os.environ.get("INCIDENTS_FILE", "~/.local/share/palmtop-incidents.json")) SEEN = os.path.expanduser("~/.local/share/palmtop-alert-seen.json") DEFAULTS = { "dedupe_seconds": 300, "quiet_hours": [22, 7], "quiet_hours_exempt": ["sev1"], "routes": {"sev1": ["phone", "notify", "sms", "call"], "sev2": ["phone", "notify", "sms"], "sev3": ["phone", "notify"]}, "who": "", } def load(path, dflt): try: with open(path, encoding="utf-8") as f: return json.load(f) except (OSError, ValueError): return dflt def save(path, value): os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as f: json.dump(value, f) def policy(): return {**DEFAULTS, **load(POLICY, {})} def quiet_now(p, now=None): start, end = (p.get("quiet_hours") or [None, None])[:2] if start is None: return False h = (now or time.localtime()).tm_hour return start <= h or h < end if start > end else start <= h < end def action(name, args): """Run a registered action the way the hub would — by name, from this computer's config.""" cfg = load(os.path.join(HOME, "config.json"), {}) a = (cfg.get("actions") or {}).get(name) if not isinstance(a, dict) or not a.get("run"): return {"ok": False, "error": f"no action called '{name}' on this computer"} if not a.get("enabled", True): return {"ok": False, "error": f"'{name}' is switched off"} try: r = subprocess.run(a["run"], shell=True, input=json.dumps(args), text=True, capture_output=True, timeout=int(a.get("timeout", 60))) except subprocess.TimeoutExpired: return {"ok": False, "error": f"'{name}' timed out"} if r.returncode != 0: return {"ok": False, "error": ((r.stderr or "").strip() or f"exit {r.returncode}")[:200]} return {"ok": True, "output": (r.stdout or "").strip()[:200]} def to_phone(sev, service, title, detail, key): """The built-in 'phone' route: Palmtop Link puts it on the phone's board. The one route that needs nothing at the desk to be awake — and nothing here either, beyond Palmtop Link being installed and enrolled on this machine (priorstates.com/palmtop/link/guide.html).""" link = os.environ.get("PALMTOP_LINK_BIN") or shutil.which("palmtop-link") if not link: return {"ok": False, "error": "palmtop-link is not installed on this machine " "(curl -fsSL https://priorstates.com/palmtop/link/install.sh | sh)"} payload = json.dumps({"sev": sev, "service": service, "title": title, "detail": detail, "key": key}) try: r = subprocess.run([link, "send", "--stdin", "--json"], input=payload, text=True, capture_output=True, timeout=30) except (subprocess.TimeoutExpired, OSError) as e: return {"ok": False, "error": f"palmtop-link: {e}"} try: return json.loads(r.stdout or "{}") except ValueError: return {"ok": r.returncode == 0, "error": (r.stderr or "").strip()[:200]} def raise_(severity="sev2", service="", title="", detail="", dedupe_key=None): p = policy() sev = severity if severity in ("sev1", "sev2", "sev3") else "sev2" now = int(time.time() * 1000) key = dedupe_key or f"{service}|{title}" seen = load(SEEN, {}) last = seen.get(key, 0) if now - last < int(p["dedupe_seconds"]) * 1000: return {"ok": True, "deduped": True, "since_seconds": round((now - last) / 1000)} seen[key] = now save(SEEN, {k: v for k, v in seen.items() if now - v < 86400_000}) rows = load(STORE, []) rows.append({"id": f"a{now}", "at": now, "severity": sev, "service": service, "title": title or "Something is wrong", "detail": detail, "state": "open", "acked_by": ""}) save(STORE, rows[-500:]) routes = list((p.get("routes") or {}).get(sev) or []) quiet = quiet_now(p) and sev not in (p.get("quiet_hours_exempt") or []) if quiet: routes = [r for r in routes if r in ("notify", "phone")] # the desk can shout, and a board is quiet; a call at 3am is not said = f"{sev.upper()} {service}: {title}".strip() sent = {} for name in routes: if name == "phone": sent[name] = to_phone(sev, service, title, detail, key) continue args = {"title": f"{sev.upper()} {service}".strip(), "body": title, "kind": "urgent" if sev == "sev1" else "quiet"} \ if name == "notify" else {"to": p.get("who", ""), "message": said + (f". {detail}" if detail else ""), "mode": "sms" if name == "sms" else "call"} sent[name] = action(name, args) return {"ok": True, "severity": sev, "recorded": True, "quiet_hours": quiet, "sent": {k: (v.get("ok") and "sent" or v.get("error")) for k, v in sent.items()}} def args_in(): """The arguments, as JSON on stdin. Never blocks: a scheduler or a CI runner can leave stdin open and empty, and an action that waits there holds a slot on the hub until it is killed. isatty() alone does not catch that case.""" if sys.stdin.isatty(): return {} try: import select if not select.select([sys.stdin], [], [], 0.5)[0]: return {} except Exception: # noqa: BLE001 — no select for pipes on this platform; read anyway pass try: return json.load(sys.stdin) or {} except ValueError: return {} def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[1]) ap.add_argument("title", nargs="?", default="") ap.add_argument("detail", nargs="?", default="") ap.add_argument("--service", default="") ap.add_argument("--sev1", action="store_const", const="sev1", dest="severity") ap.add_argument("--sev2", action="store_const", const="sev2", dest="severity") ap.add_argument("--sev3", action="store_const", const="sev3", dest="severity") ap.add_argument("--key", default=None, help="what counts as 'the same alert' for deduping") args = ap.parse_args() # Read stdin only when the caller gave nothing else — that is how the hub invokes it. # With a title on the command line, never touch stdin: a scheduler may leave it open # and empty, and waiting on it would hang the alert saying something is wrong. body = {} if (args.title or args.severity or args.service) else args_in() out = raise_( severity=body.get("severity") or args.severity or "sev2", service=body.get("service") or args.service, title=body.get("title") or args.title, detail=body.get("detail") or args.detail, dedupe_key=body.get("key") or args.key, ) json.dump(out, sys.stdout) if __name__ == "__main__": main()