agent-tools/fleet/main.go

// fleet takes a one-shot health snapshot of the homelab: container
// states, disk fill and failed systemd units per host — the loop every
// runbook used to hand-roll, done once and properly. Read-only by
// construction. See doc/tool-parity.md §4.2.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"gitea.brasse-pc.eu/brasse/agent-tools/fleet/check"
)

var version = "dev"

const usage = `fleet - one-shot homelab health snapshot

Usage:
  fleet status [--host NAME] [--all]      containers, disks, failed units
  fleet version

Exit codes: 0 = everything healthy, 2 = findings, 1 = error.

Hosts come from ~/.config/fleet/config.json (created on first run;
FLEET_CONFIG overrides). Default: the Pi5 through the read-only
claude-docker wrapper, and this workstation locally. All commands are
read-only (ps/df/systemctl status) — mutations stay in the supervised
tmux flow.
`

type host struct {
	Name      string `json:"name"`
	SSH       string `json:"ssh"`        // ssh host alias, "" = run locally
	DockerCmd string `json:"docker_cmd"` // e.g. "sudo claude-docker" or "docker"
	DFTargets string `json:"df_targets"` // space-separated mount points
	UserUnits bool   `json:"user_units"` // also check systemctl --user --failed
}

type config struct {
	DiskWarnPct int    `json:"disk_warn_pct"`
	Hosts       []host `json:"hosts"`
}

func defaultConfig() *config {
	return &config{
		DiskWarnPct: 85,
		Hosts: []host{
			{Name: "pi5", SSH: "pi5-claude", DockerCmd: "sudo claude-docker", DFTargets: "/ /srv/storage1", UserUnits: false},
			{Name: "brasse-linux01", SSH: "", DockerCmd: "docker", DFTargets: "/ /home", UserUnits: true},
		},
	}
}

func configPath() string {
	if p := os.Getenv("FLEET_CONFIG"); p != "" {
		return p
	}
	dir, err := os.UserConfigDir()
	if err != nil {
		dir = "."
	}
	return filepath.Join(dir, "fleet", "config.json")
}

func loadConfig() *config {
	cfg := defaultConfig()
	path := configPath()
	data, err := os.ReadFile(path)
	if os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(path), 0o755); err == nil {
			out, _ := json.MarshalIndent(cfg, "", "  ")
			os.WriteFile(path, append(out, '\n'), 0o600)
			fmt.Fprintf(os.Stderr, "fleet: created %s\n", path)
		}
		return cfg
	}
	if err == nil {
		json.Unmarshal(data, cfg)
	}
	return cfg
}

func main() {
	if len(os.Args) < 2 {
		fmt.Print(usage)
		os.Exit(2)
	}
	switch os.Args[1] {
	case "status":
		cmdStatus(os.Args[2:])
	case "version", "--version", "-v":
		fmt.Println("fleet", version)
	case "help", "--help", "-h":
		fmt.Print(usage)
	default:
		die("unknown command %q — run 'fleet help'", os.Args[1])
	}
}

// run executes a read-only command locally or over ssh, with a
// timeout so one dead host cannot hang the snapshot.
func run(h host, cmdline string) (string, error) {
	var c *exec.Cmd
	if h.SSH != "" {
		c = exec.Command("ssh", "-o", "ConnectTimeout=10", h.SSH, cmdline)
	} else {
		c = exec.Command("sh", "-c", cmdline)
	}
	done := make(chan struct{})
	var out []byte
	var err error
	go func() { out, err = c.CombinedOutput(); close(done) }()
	select {
	case <-done:
	case <-time.After(45 * time.Second):
		c.Process.Kill()
		return "", fmt.Errorf("timeout")
	}
	return string(out), err
}

func cmdStatus(args []string) {
	fs := flag.NewFlagSet("status", flag.ExitOnError)
	hostFilter := fs.String("host", "", "only this host (default: all)")
	fs.Parse(args)
	cfg := loadConfig()

	var allFindings []check.Finding
	checked := 0
	for _, h := range cfg.Hosts {
		if *hostFilter != "" && h.Name != *hostFilter {
			continue
		}
		checked++
		fmt.Printf("== %s ==\n", h.Name)

		psOut, err := run(h, h.DockerCmd+` ps -a --format '{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Image}}'`)
		if err != nil {
			fmt.Printf("  docker: UNREACHABLE (%v)\n", err)
			allFindings = append(allFindings, check.Finding{Host: h.Name, Text: "docker unreachable: " + strings.TrimSpace(psOut)})
		}
		containers := check.ParseDockerPS(psOut)
		up := 0
		for _, c := range containers {
			if c.Healthy() {
				up++
			}
		}
		fmt.Printf("  containers: %d/%d healthy\n", up, len(containers))

		dfOut, _ := run(h, "df -h --output=target,pcent,avail "+h.DFTargets)
		mounts := check.ParseDF(dfOut)
		for _, m := range mounts {
			fmt.Printf("  disk %-16s %3d%% used, %s free\n", m.Target, m.UsedPct, m.Avail)
		}

		sysOut, _ := run(h, "systemctl --failed --no-legend --plain")
		failedSys := check.ParseFailedUnits(sysOut)
		var failedUser []string
		if h.UserUnits {
			userOut, _ := run(h, "systemctl --user --failed --no-legend --plain")
			failedUser = check.ParseFailedUnits(userOut)
		}
		fmt.Printf("  failed units: %d system, %d user\n", len(failedSys), len(failedUser))

		allFindings = append(allFindings, check.Evaluate(h.Name, containers, mounts, failedSys, failedUser, cfg.DiskWarnPct)...)
	}
	if checked == 0 {
		die("no host named %q in the config", *hostFilter)
	}
	if len(allFindings) == 0 {
		fmt.Println("\nall healthy")
		return
	}
	fmt.Printf("\n%d finding(s):\n", len(allFindings))
	for _, f := range allFindings {
		fmt.Println("  " + f.String())
	}
	os.Exit(2)
}

func die(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "fleet: "+format+"\n", args...)
	os.Exit(1)
}