Merge dev/fleet: fleet v1

2026-08-07 01:02:06 +02:00
parent f8bfa71f5f 184378ae57
commit ea06442d44
5 changed files with 450 additions and 0 deletions
--- a/fleet/README.md
+++ b/fleet/README.md
@@ -0,0 +1,60 @@
+# fleet — one-shot homelab health snapshot
+
+Replaces the hand-rolled `ssh pi5-claude sudo claude-docker ps …`
+loops from every runbook with a single command that checks **all
+hosts**: container states, disk fill and failed systemd units.
+Read-only by construction — it only runs `ps`/`df`/`systemctl` through
+the read-only claude-docker wrapper, so it needs no new permissions.
+From [`doc/tool-parity.md`](../doc/tool-parity.md) §4.2.
+
+## Usage
+
+```
+fleet status [--host NAME]     # all hosts, or one
+```
+
+```
+== pi5 ==
+  containers: 54/58 healthy
+  disk /                 81% used, 82G free
+  disk /srv/storage1     58% used, 1.2T free
+  failed units: 0 system, 0 user
+...
+19 finding(s):
+  pi5: container dozzle: exited (Exited (0) 5 months ago)
+  brasse-linux01: failed unit: cpupower-gui.service
+```
+
+| Exit | Meaning |
+|------|---------|
+| 0 | everything healthy |
+| 2 | findings printed (unhealthy/exited containers, disk ≥ warn %, failed units) |
+| 1 | error (unknown host, config problem) |
+
+Output is a stable text table — an agent can diff it between runs.
+Unreachable hosts become findings, not crashes. Gitea Actions'
+transient job containers are skipped.
+
+## Config
+
+`~/.config/fleet/config.json`, created on first run (`FLEET_CONFIG`
+overrides):
+
+```json
+{
+  "disk_warn_pct": 85,
+  "hosts": [
+    {"name": "pi5", "ssh": "pi5-claude", "docker_cmd": "sudo claude-docker", "df_targets": "/ /srv/storage1", "user_units": false},
+    {"name": "brasse-linux01", "ssh": "", "docker_cmd": "docker", "df_targets": "/ /home", "user_units": true}
+  ]
+}
+```
+
+`ssh: ""` means run locally. Add hosts by appending entries.
+
+## Build & test
+
+```bash
+go test ./...
+go build -o build/fleet .
+```
--- a/fleet/check/check.go
+++ b/fleet/check/check.go
@@ -0,0 +1,118 @@
+// Package check parses the health data fleet collects (docker ps,
+// df, systemctl --failed) into stable rows and findings an agent can
+// diff between runs.
+package check
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// Container is one row of docker ps output.
+type Container struct {
+	Name   string
+	State  string // running, exited, restarting, ...
+	Status string // "Up 2 hours (healthy)", "Exited (1) 3 days ago"
+	Image  string
+}
+
+// Healthy reports whether the container looks fine: running, and not
+// flagged unhealthy.
+func (c Container) Healthy() bool {
+	return c.State == "running" && !strings.Contains(c.Status, "unhealthy")
+}
+
+// ParseDockerPS parses `docker ps -a --format
+// "{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Image}}"`. CI job
+// containers (Gitea Actions workers) are transient and skipped.
+func ParseDockerPS(out string) []Container {
+	var rows []Container
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Split(line, "\t")
+		if len(parts) < 4 {
+			continue
+		}
+		if strings.HasPrefix(parts[0], "GITEA-ACTIONS-TASK-") {
+			continue
+		}
+		rows = append(rows, Container{Name: parts[0], State: parts[1], Status: parts[2], Image: parts[3]})
+	}
+	return rows
+}
+
+// Mount is one row of df output.
+type Mount struct {
+	Target  string
+	UsedPct int
+	Avail   string
+}
+
+// ParseDF parses `df -h --output=target,pcent,avail`.
+func ParseDF(out string) []Mount {
+	var rows []Mount
+	for i, line := range strings.Split(out, "\n") {
+		fields := strings.Fields(line)
+		if i == 0 || len(fields) < 3 { // header
+			continue
+		}
+		pct, err := strconv.Atoi(strings.TrimSuffix(fields[1], "%"))
+		if err != nil {
+			continue
+		}
+		rows = append(rows, Mount{Target: fields[0], UsedPct: pct, Avail: fields[2]})
+	}
+	return rows
+}
+
+// ParseFailedUnits parses `systemctl --failed --no-legend --plain`
+// into unit names.
+func ParseFailedUnits(out string) []string {
+	var units []string
+	for _, line := range strings.Split(out, "\n") {
+		fields := strings.Fields(strings.TrimPrefix(strings.TrimSpace(line), "● "))
+		if len(fields) == 0 {
+			continue
+		}
+		u := fields[0]
+		if strings.Contains(u, ".") {
+			units = append(units, u)
+		}
+	}
+	return units
+}
+
+// Finding is one thing worth attention.
+type Finding struct {
+	Host string
+	Text string
+}
+
+func (f Finding) String() string { return f.Host + ": " + f.Text }
+
+// Evaluate turns parsed data into findings. diskWarnPct is the fill
+// grade that counts as a problem.
+func Evaluate(host string, containers []Container, mounts []Mount, failedSys, failedUser []string, diskWarnPct int) []Finding {
+	var out []Finding
+	for _, c := range containers {
+		if !c.Healthy() {
+			out = append(out, Finding{host, fmt.Sprintf("container %s: %s (%s)", c.Name, c.State, c.Status)})
+		}
+	}
+	for _, m := range mounts {
+		if m.UsedPct >= diskWarnPct {
+			out = append(out, Finding{host, fmt.Sprintf("disk %s at %d%% (%s left)", m.Target, m.UsedPct, m.Avail)})
+		}
+	}
+	for _, u := range failedSys {
+		out = append(out, Finding{host, "failed unit: " + u})
+	}
+	for _, u := range failedUser {
+		out = append(out, Finding{host, "failed user unit: " + u})
+	}
+	return out
+}
--- a/fleet/check/check_test.go
+++ b/fleet/check/check_test.go
@@ -0,0 +1,78 @@
+package check
+
+import (
+	"strings"
+	"testing"
+)
+
+const psOut = `GITEA-ACTIONS-TASK-120-WORKFLOW-x	running	Up 2 minutes	catthehacker/ubuntu:act-latest
+helm-hub	running	Up 58 minutes	localhost:5000/helmd:latest
+gitea-d	running	Up 27 hours (healthy)	gitea/gitea:1.24
+deluge	exited	Exited (1) 3 days ago	linuxserver/deluge
+kuma	running	Up 4 days (unhealthy)	louislam/uptime-kuma:1`
+
+func TestParseDockerPS(t *testing.T) {
+	rows := ParseDockerPS(psOut)
+	if len(rows) != 4 {
+		t.Fatalf("got %d rows, want 4 (CI worker skipped)", len(rows))
+	}
+	if rows[0].Name != "helm-hub" || !rows[0].Healthy() {
+		t.Errorf("helm-hub should be healthy: %+v", rows[0])
+	}
+	for _, r := range rows {
+		switch r.Name {
+		case "deluge":
+			if r.Healthy() {
+				t.Error("exited container marked healthy")
+			}
+		case "kuma":
+			if r.Healthy() {
+				t.Error("unhealthy container marked healthy")
+			}
+		}
+	}
+}
+
+const dfOut = `Mounted on    Use% Avail
+/              81%   82G
+/srv/storage1  58%  1.2T`
+
+func TestParseDF(t *testing.T) {
+	rows := ParseDF(dfOut)
+	if len(rows) != 2 || rows[0].Target != "/" || rows[0].UsedPct != 81 || rows[1].Avail != "1.2T" {
+		t.Errorf("df parse wrong: %+v", rows)
+	}
+}
+
+func TestParseFailedUnits(t *testing.T) {
+	units := ParseFailedUnits("● backup.service loaded failed failed Nightly backup\nfoo.timer loaded failed failed X\n\n")
+	if len(units) != 2 || units[0] != "backup.service" || units[1] != "foo.timer" {
+		t.Errorf("failed units: %v", units)
+	}
+	if len(ParseFailedUnits("")) != 0 {
+		t.Error("empty output should give no units")
+	}
+}
+
+func TestEvaluate(t *testing.T) {
+	findings := Evaluate("pi5",
+		ParseDockerPS(psOut),
+		ParseDF(dfOut),
+		[]string{"backup.service"}, nil, 80)
+	var text []string
+	for _, f := range findings {
+		text = append(text, f.String())
+	}
+	joined := strings.Join(text, "\n")
+	for _, want := range []string{"deluge", "kuma", "disk / at 81%", "backup.service"} {
+		if !strings.Contains(joined, want) {
+			t.Errorf("findings missing %q:\n%s", want, joined)
+		}
+	}
+	if strings.Contains(joined, "storage1") {
+		t.Error("58% disk should not be a finding at threshold 80")
+	}
+	if len(findings) != 4 {
+		t.Errorf("got %d findings, want 4", len(findings))
+	}
+}
--- a/fleet/go.mod
+++ b/fleet/go.mod
@@ -0,0 +1,3 @@
+module gitea.brasse-pc.eu/brasse/agent-tools/fleet
+
+go 1.24
--- a/fleet/main.go
+++ b/fleet/main.go
@@ -0,0 +1,191 @@
+// fleet takes a one-shot health snapshot of the homelab: container
+// states, disk fill and failed systemd units per host — the loop every
+// runbook used to hand-roll, done once and properly. Read-only by
+// construction. See doc/tool-parity.md §4.2.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"gitea.brasse-pc.eu/brasse/agent-tools/fleet/check"
+)
+
+var version = "dev"
+
+const usage = `fleet - one-shot homelab health snapshot
+
+Usage:
+  fleet status [--host NAME] [--all]      containers, disks, failed units
+  fleet version
+
+Exit codes: 0 = everything healthy, 2 = findings, 1 = error.
+
+Hosts come from ~/.config/fleet/config.json (created on first run;
+FLEET_CONFIG overrides). Default: the Pi5 through the read-only
+claude-docker wrapper, and this workstation locally. All commands are
+read-only (ps/df/systemctl status) — mutations stay in the supervised
+tmux flow.
+`
+
+type host struct {
+	Name      string `json:"name"`
+	SSH       string `json:"ssh"`        // ssh host alias, "" = run locally
+	DockerCmd string `json:"docker_cmd"` // e.g. "sudo claude-docker" or "docker"
+	DFTargets string `json:"df_targets"` // space-separated mount points
+	UserUnits bool   `json:"user_units"` // also check systemctl --user --failed
+}
+
+type config struct {
+	DiskWarnPct int    `json:"disk_warn_pct"`
+	Hosts       []host `json:"hosts"`
+}
+
+func defaultConfig() *config {
+	return &config{
+		DiskWarnPct: 85,
+		Hosts: []host{
+			{Name: "pi5", SSH: "pi5-claude", DockerCmd: "sudo claude-docker", DFTargets: "/ /srv/storage1", UserUnits: false},
+			{Name: "brasse-linux01", SSH: "", DockerCmd: "docker", DFTargets: "/ /home", UserUnits: true},
+		},
+	}
+}
+
+func configPath() string {
+	if p := os.Getenv("FLEET_CONFIG"); p != "" {
+		return p
+	}
+	dir, err := os.UserConfigDir()
+	if err != nil {
+		dir = "."
+	}
+	return filepath.Join(dir, "fleet", "config.json")
+}
+
+func loadConfig() *config {
+	cfg := defaultConfig()
+	path := configPath()
+	data, err := os.ReadFile(path)
+	if os.IsNotExist(err) {
+		if err := os.MkdirAll(filepath.Dir(path), 0o755); err == nil {
+			out, _ := json.MarshalIndent(cfg, "", "  ")
+			os.WriteFile(path, append(out, '\n'), 0o600)
+			fmt.Fprintf(os.Stderr, "fleet: created %s\n", path)
+		}
+		return cfg
+	}
+	if err == nil {
+		json.Unmarshal(data, cfg)
+	}
+	return cfg
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Print(usage)
+		os.Exit(2)
+	}
+	switch os.Args[1] {
+	case "status":
+		cmdStatus(os.Args[2:])
+	case "version", "--version", "-v":
+		fmt.Println("fleet", version)
+	case "help", "--help", "-h":
+		fmt.Print(usage)
+	default:
+		die("unknown command %q — run 'fleet help'", os.Args[1])
+	}
+}
+
+// run executes a read-only command locally or over ssh, with a
+// timeout so one dead host cannot hang the snapshot.
+func run(h host, cmdline string) (string, error) {
+	var c *exec.Cmd
+	if h.SSH != "" {
+		c = exec.Command("ssh", "-o", "ConnectTimeout=10", h.SSH, cmdline)
+	} else {
+		c = exec.Command("sh", "-c", cmdline)
+	}
+	done := make(chan struct{})
+	var out []byte
+	var err error
+	go func() { out, err = c.CombinedOutput(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(45 * time.Second):
+		c.Process.Kill()
+		return "", fmt.Errorf("timeout")
+	}
+	return string(out), err
+}
+
+func cmdStatus(args []string) {
+	fs := flag.NewFlagSet("status", flag.ExitOnError)
+	hostFilter := fs.String("host", "", "only this host (default: all)")
+	fs.Parse(args)
+	cfg := loadConfig()
+
+	var allFindings []check.Finding
+	checked := 0
+	for _, h := range cfg.Hosts {
+		if *hostFilter != "" && h.Name != *hostFilter {
+			continue
+		}
+		checked++
+		fmt.Printf("== %s ==\n", h.Name)
+
+		psOut, err := run(h, h.DockerCmd+` ps -a --format '{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Image}}'`)
+		if err != nil {
+			fmt.Printf("  docker: UNREACHABLE (%v)\n", err)
+			allFindings = append(allFindings, check.Finding{Host: h.Name, Text: "docker unreachable: " + strings.TrimSpace(psOut)})
+		}
+		containers := check.ParseDockerPS(psOut)
+		up := 0
+		for _, c := range containers {
+			if c.Healthy() {
+				up++
+			}
+		}
+		fmt.Printf("  containers: %d/%d healthy\n", up, len(containers))
+
+		dfOut, _ := run(h, "df -h --output=target,pcent,avail "+h.DFTargets)
+		mounts := check.ParseDF(dfOut)
+		for _, m := range mounts {
+			fmt.Printf("  disk %-16s %3d%% used, %s free\n", m.Target, m.UsedPct, m.Avail)
+		}
+
+		sysOut, _ := run(h, "systemctl --failed --no-legend --plain")
+		failedSys := check.ParseFailedUnits(sysOut)
+		var failedUser []string
+		if h.UserUnits {
+			userOut, _ := run(h, "systemctl --user --failed --no-legend --plain")
+			failedUser = check.ParseFailedUnits(userOut)
+		}
+		fmt.Printf("  failed units: %d system, %d user\n", len(failedSys), len(failedUser))
+
+		allFindings = append(allFindings, check.Evaluate(h.Name, containers, mounts, failedSys, failedUser, cfg.DiskWarnPct)...)
+	}
+	if checked == 0 {
+		die("no host named %q in the config", *hostFilter)
+	}
+	if len(allFindings) == 0 {
+		fmt.Println("\nall healthy")
+		return
+	}
+	fmt.Printf("\n%d finding(s):\n", len(allFindings))
+	for _, f := range allFindings {
+		fmt.Println("  " + f.String())
+	}
+	os.Exit(2)
+}
+
+func die(format string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, "fleet: "+format+"\n", args...)
+	os.Exit(1)
+}