diff --git a/fleet/README.md b/fleet/README.md new file mode 100644 index 0000000..103162f --- /dev/null +++ b/fleet/README.md @@ -0,0 +1,60 @@ +# fleet — one-shot homelab health snapshot + +Replaces the hand-rolled `ssh pi5-claude sudo claude-docker ps …` +loops from every runbook with a single command that checks **all +hosts**: container states, disk fill and failed systemd units. +Read-only by construction — it only runs `ps`/`df`/`systemctl` through +the read-only claude-docker wrapper, so it needs no new permissions. +From [`doc/tool-parity.md`](../doc/tool-parity.md) §4.2. + +## Usage + +``` +fleet status [--host NAME] # all hosts, or one +``` + +``` +== pi5 == + containers: 54/58 healthy + disk / 81% used, 82G free + disk /srv/storage1 58% used, 1.2T free + failed units: 0 system, 0 user +... +19 finding(s): + pi5: container dozzle: exited (Exited (0) 5 months ago) + brasse-linux01: failed unit: cpupower-gui.service +``` + +| Exit | Meaning | +|------|---------| +| 0 | everything healthy | +| 2 | findings printed (unhealthy/exited containers, disk ≥ warn %, failed units) | +| 1 | error (unknown host, config problem) | + +Output is a stable text table — an agent can diff it between runs. +Unreachable hosts become findings, not crashes. Gitea Actions' +transient job containers are skipped. + +## Config + +`~/.config/fleet/config.json`, created on first run (`FLEET_CONFIG` +overrides): + +```json +{ + "disk_warn_pct": 85, + "hosts": [ + {"name": "pi5", "ssh": "pi5-claude", "docker_cmd": "sudo claude-docker", "df_targets": "/ /srv/storage1", "user_units": false}, + {"name": "brasse-linux01", "ssh": "", "docker_cmd": "docker", "df_targets": "/ /home", "user_units": true} + ] +} +``` + +`ssh: ""` means run locally. Add hosts by appending entries. + +## Build & test + +```bash +go test ./... +go build -o build/fleet . +``` diff --git a/fleet/check/check.go b/fleet/check/check.go new file mode 100644 index 0000000..5221ad0 --- /dev/null +++ b/fleet/check/check.go @@ -0,0 +1,118 @@ +// Package check parses the health data fleet collects (docker ps, +// df, systemctl --failed) into stable rows and findings an agent can +// diff between runs. +package check + +import ( + "fmt" + "strconv" + "strings" +) + +// Container is one row of docker ps output. +type Container struct { + Name string + State string // running, exited, restarting, ... + Status string // "Up 2 hours (healthy)", "Exited (1) 3 days ago" + Image string +} + +// Healthy reports whether the container looks fine: running, and not +// flagged unhealthy. +func (c Container) Healthy() bool { + return c.State == "running" && !strings.Contains(c.Status, "unhealthy") +} + +// ParseDockerPS parses `docker ps -a --format +// "{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Image}}"`. CI job +// containers (Gitea Actions workers) are transient and skipped. +func ParseDockerPS(out string) []Container { + var rows []Container + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Split(line, "\t") + if len(parts) < 4 { + continue + } + if strings.HasPrefix(parts[0], "GITEA-ACTIONS-TASK-") { + continue + } + rows = append(rows, Container{Name: parts[0], State: parts[1], Status: parts[2], Image: parts[3]}) + } + return rows +} + +// Mount is one row of df output. +type Mount struct { + Target string + UsedPct int + Avail string +} + +// ParseDF parses `df -h --output=target,pcent,avail`. +func ParseDF(out string) []Mount { + var rows []Mount + for i, line := range strings.Split(out, "\n") { + fields := strings.Fields(line) + if i == 0 || len(fields) < 3 { // header + continue + } + pct, err := strconv.Atoi(strings.TrimSuffix(fields[1], "%")) + if err != nil { + continue + } + rows = append(rows, Mount{Target: fields[0], UsedPct: pct, Avail: fields[2]}) + } + return rows +} + +// ParseFailedUnits parses `systemctl --failed --no-legend --plain` +// into unit names. +func ParseFailedUnits(out string) []string { + var units []string + for _, line := range strings.Split(out, "\n") { + fields := strings.Fields(strings.TrimPrefix(strings.TrimSpace(line), "● ")) + if len(fields) == 0 { + continue + } + u := fields[0] + if strings.Contains(u, ".") { + units = append(units, u) + } + } + return units +} + +// Finding is one thing worth attention. +type Finding struct { + Host string + Text string +} + +func (f Finding) String() string { return f.Host + ": " + f.Text } + +// Evaluate turns parsed data into findings. diskWarnPct is the fill +// grade that counts as a problem. +func Evaluate(host string, containers []Container, mounts []Mount, failedSys, failedUser []string, diskWarnPct int) []Finding { + var out []Finding + for _, c := range containers { + if !c.Healthy() { + out = append(out, Finding{host, fmt.Sprintf("container %s: %s (%s)", c.Name, c.State, c.Status)}) + } + } + for _, m := range mounts { + if m.UsedPct >= diskWarnPct { + out = append(out, Finding{host, fmt.Sprintf("disk %s at %d%% (%s left)", m.Target, m.UsedPct, m.Avail)}) + } + } + for _, u := range failedSys { + out = append(out, Finding{host, "failed unit: " + u}) + } + for _, u := range failedUser { + out = append(out, Finding{host, "failed user unit: " + u}) + } + return out +} diff --git a/fleet/check/check_test.go b/fleet/check/check_test.go new file mode 100644 index 0000000..71abe81 --- /dev/null +++ b/fleet/check/check_test.go @@ -0,0 +1,78 @@ +package check + +import ( + "strings" + "testing" +) + +const psOut = `GITEA-ACTIONS-TASK-120-WORKFLOW-x running Up 2 minutes catthehacker/ubuntu:act-latest +helm-hub running Up 58 minutes localhost:5000/helmd:latest +gitea-d running Up 27 hours (healthy) gitea/gitea:1.24 +deluge exited Exited (1) 3 days ago linuxserver/deluge +kuma running Up 4 days (unhealthy) louislam/uptime-kuma:1` + +func TestParseDockerPS(t *testing.T) { + rows := ParseDockerPS(psOut) + if len(rows) != 4 { + t.Fatalf("got %d rows, want 4 (CI worker skipped)", len(rows)) + } + if rows[0].Name != "helm-hub" || !rows[0].Healthy() { + t.Errorf("helm-hub should be healthy: %+v", rows[0]) + } + for _, r := range rows { + switch r.Name { + case "deluge": + if r.Healthy() { + t.Error("exited container marked healthy") + } + case "kuma": + if r.Healthy() { + t.Error("unhealthy container marked healthy") + } + } + } +} + +const dfOut = `Mounted on Use% Avail +/ 81% 82G +/srv/storage1 58% 1.2T` + +func TestParseDF(t *testing.T) { + rows := ParseDF(dfOut) + if len(rows) != 2 || rows[0].Target != "/" || rows[0].UsedPct != 81 || rows[1].Avail != "1.2T" { + t.Errorf("df parse wrong: %+v", rows) + } +} + +func TestParseFailedUnits(t *testing.T) { + units := ParseFailedUnits("● backup.service loaded failed failed Nightly backup\nfoo.timer loaded failed failed X\n\n") + if len(units) != 2 || units[0] != "backup.service" || units[1] != "foo.timer" { + t.Errorf("failed units: %v", units) + } + if len(ParseFailedUnits("")) != 0 { + t.Error("empty output should give no units") + } +} + +func TestEvaluate(t *testing.T) { + findings := Evaluate("pi5", + ParseDockerPS(psOut), + ParseDF(dfOut), + []string{"backup.service"}, nil, 80) + var text []string + for _, f := range findings { + text = append(text, f.String()) + } + joined := strings.Join(text, "\n") + for _, want := range []string{"deluge", "kuma", "disk / at 81%", "backup.service"} { + if !strings.Contains(joined, want) { + t.Errorf("findings missing %q:\n%s", want, joined) + } + } + if strings.Contains(joined, "storage1") { + t.Error("58% disk should not be a finding at threshold 80") + } + if len(findings) != 4 { + t.Errorf("got %d findings, want 4", len(findings)) + } +} diff --git a/fleet/go.mod b/fleet/go.mod new file mode 100644 index 0000000..b864c67 --- /dev/null +++ b/fleet/go.mod @@ -0,0 +1,3 @@ +module gitea.brasse-pc.eu/brasse/agent-tools/fleet + +go 1.24 diff --git a/fleet/main.go b/fleet/main.go new file mode 100644 index 0000000..32b17a5 --- /dev/null +++ b/fleet/main.go @@ -0,0 +1,191 @@ +// fleet takes a one-shot health snapshot of the homelab: container +// states, disk fill and failed systemd units per host — the loop every +// runbook used to hand-roll, done once and properly. Read-only by +// construction. See doc/tool-parity.md §4.2. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "gitea.brasse-pc.eu/brasse/agent-tools/fleet/check" +) + +var version = "dev" + +const usage = `fleet - one-shot homelab health snapshot + +Usage: + fleet status [--host NAME] [--all] containers, disks, failed units + fleet version + +Exit codes: 0 = everything healthy, 2 = findings, 1 = error. + +Hosts come from ~/.config/fleet/config.json (created on first run; +FLEET_CONFIG overrides). Default: the Pi5 through the read-only +claude-docker wrapper, and this workstation locally. All commands are +read-only (ps/df/systemctl status) — mutations stay in the supervised +tmux flow. +` + +type host struct { + Name string `json:"name"` + SSH string `json:"ssh"` // ssh host alias, "" = run locally + DockerCmd string `json:"docker_cmd"` // e.g. "sudo claude-docker" or "docker" + DFTargets string `json:"df_targets"` // space-separated mount points + UserUnits bool `json:"user_units"` // also check systemctl --user --failed +} + +type config struct { + DiskWarnPct int `json:"disk_warn_pct"` + Hosts []host `json:"hosts"` +} + +func defaultConfig() *config { + return &config{ + DiskWarnPct: 85, + Hosts: []host{ + {Name: "pi5", SSH: "pi5-claude", DockerCmd: "sudo claude-docker", DFTargets: "/ /srv/storage1", UserUnits: false}, + {Name: "brasse-linux01", SSH: "", DockerCmd: "docker", DFTargets: "/ /home", UserUnits: true}, + }, + } +} + +func configPath() string { + if p := os.Getenv("FLEET_CONFIG"); p != "" { + return p + } + dir, err := os.UserConfigDir() + if err != nil { + dir = "." + } + return filepath.Join(dir, "fleet", "config.json") +} + +func loadConfig() *config { + cfg := defaultConfig() + path := configPath() + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err == nil { + out, _ := json.MarshalIndent(cfg, "", " ") + os.WriteFile(path, append(out, '\n'), 0o600) + fmt.Fprintf(os.Stderr, "fleet: created %s\n", path) + } + return cfg + } + if err == nil { + json.Unmarshal(data, cfg) + } + return cfg +} + +func main() { + if len(os.Args) < 2 { + fmt.Print(usage) + os.Exit(2) + } + switch os.Args[1] { + case "status": + cmdStatus(os.Args[2:]) + case "version", "--version", "-v": + fmt.Println("fleet", version) + case "help", "--help", "-h": + fmt.Print(usage) + default: + die("unknown command %q — run 'fleet help'", os.Args[1]) + } +} + +// run executes a read-only command locally or over ssh, with a +// timeout so one dead host cannot hang the snapshot. +func run(h host, cmdline string) (string, error) { + var c *exec.Cmd + if h.SSH != "" { + c = exec.Command("ssh", "-o", "ConnectTimeout=10", h.SSH, cmdline) + } else { + c = exec.Command("sh", "-c", cmdline) + } + done := make(chan struct{}) + var out []byte + var err error + go func() { out, err = c.CombinedOutput(); close(done) }() + select { + case <-done: + case <-time.After(45 * time.Second): + c.Process.Kill() + return "", fmt.Errorf("timeout") + } + return string(out), err +} + +func cmdStatus(args []string) { + fs := flag.NewFlagSet("status", flag.ExitOnError) + hostFilter := fs.String("host", "", "only this host (default: all)") + fs.Parse(args) + cfg := loadConfig() + + var allFindings []check.Finding + checked := 0 + for _, h := range cfg.Hosts { + if *hostFilter != "" && h.Name != *hostFilter { + continue + } + checked++ + fmt.Printf("== %s ==\n", h.Name) + + psOut, err := run(h, h.DockerCmd+` ps -a --format '{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Image}}'`) + if err != nil { + fmt.Printf(" docker: UNREACHABLE (%v)\n", err) + allFindings = append(allFindings, check.Finding{Host: h.Name, Text: "docker unreachable: " + strings.TrimSpace(psOut)}) + } + containers := check.ParseDockerPS(psOut) + up := 0 + for _, c := range containers { + if c.Healthy() { + up++ + } + } + fmt.Printf(" containers: %d/%d healthy\n", up, len(containers)) + + dfOut, _ := run(h, "df -h --output=target,pcent,avail "+h.DFTargets) + mounts := check.ParseDF(dfOut) + for _, m := range mounts { + fmt.Printf(" disk %-16s %3d%% used, %s free\n", m.Target, m.UsedPct, m.Avail) + } + + sysOut, _ := run(h, "systemctl --failed --no-legend --plain") + failedSys := check.ParseFailedUnits(sysOut) + var failedUser []string + if h.UserUnits { + userOut, _ := run(h, "systemctl --user --failed --no-legend --plain") + failedUser = check.ParseFailedUnits(userOut) + } + fmt.Printf(" failed units: %d system, %d user\n", len(failedSys), len(failedUser)) + + allFindings = append(allFindings, check.Evaluate(h.Name, containers, mounts, failedSys, failedUser, cfg.DiskWarnPct)...) + } + if checked == 0 { + die("no host named %q in the config", *hostFilter) + } + if len(allFindings) == 0 { + fmt.Println("\nall healthy") + return + } + fmt.Printf("\n%d finding(s):\n", len(allFindings)) + for _, f := range allFindings { + fmt.Println(" " + f.String()) + } + os.Exit(2) +} + +func die(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "fleet: "+format+"\n", args...) + os.Exit(1) +}