From f8cc6f6419ffef069907ca9adfbbe520c2d38574 Mon Sep 17 00:00:00 2001 From: John Gatward Date: Fri, 23 Oct 2026 10:43:27 +0100 Subject: [PATCH] [main]: Add notifications --- docs/home-server/backups.md | 13 +- docs/home-server/ideas-next-steps.md | 229 +++++++++++++++++++++++++++ docs/home-server/index.md | 6 +- docs/home-server/mercury.md | 20 ++- docs/home-server/operations.md | 27 +++- zensical.toml | 1 + 6 files changed, 288 insertions(+), 8 deletions(-) create mode 100644 docs/home-server/ideas-next-steps.md diff --git a/docs/home-server/backups.md b/docs/home-server/backups.md index 1a228b7..d226b7c 100644 --- a/docs/home-server/backups.md +++ b/docs/home-server/backups.md @@ -4,7 +4,7 @@ id: home-server.backups type: reference scope: [jupiter, backrest, restic, google-drive, recovery] sensitivity: private-infrastructure -last_reviewed: "2026-10-16" +last_reviewed: "2026-10-23" sources: - kind: owner-report reference: "Backrest plan JSON, destination, key custody and no restore test, 2026-10-06" @@ -14,6 +14,8 @@ sources: reference: "Only restic key has independent custody; no independent credential recovery or backup/restore evidence, 2026-10-06" - kind: owner-report reference: "Updated six-directory Backrest plan including Authentik and Wardrowbe, deployed Authentik hook container names confirmed; independent recovery custody unchanged, 2026-10-16" + - kind: owner-report + reference: "Backrest notifications received in Discord server Jupiter, channel #backrest-alerts, 2026-10-23" - kind: repository repository: jupiter-stacks revision: d09df456ad331b8beb9a40615114ea8aae2dced5 @@ -45,6 +47,15 @@ unknowns: | Backup / integrity evidence | None supplied | | Restore test | Never performed | +## Notifications + +Backrest notifications are delivered to **Jupiter** (Discord server) in +`#backrest-alerts`; receipt confirmed by the owner on 2026-10-23. +Exact enabled hook conditions have not been recorded. Delivery alone does not +confirm backup integrity, successful application restarts or detection of a +backup that never runs. See [Observability](operations.md#observability) for +the shared notification destinations and remaining coverage gaps. + ## Coverage Backrest bind-mounts host `/data` at container `/userdata:ro`. diff --git a/docs/home-server/ideas-next-steps.md b/docs/home-server/ideas-next-steps.md new file mode 100644 index 0000000..a54264b --- /dev/null +++ b/docs/home-server/ideas-next-steps.md @@ -0,0 +1,229 @@ +--- +schema_version: 1 +id: home-server.ideas-next-steps +type: plan +scope: [jupiter, mercury, private-access, recovery, downloads, power, services] +sensitivity: private-infrastructure +last_reviewed: "2026-10-22" +sources: + - kind: owner-report + reference: "Access, device-loss recovery, hardware constraints, broadband and VPN preferences interview, 2026-10-22" + - kind: documentation + reference: "Planning discussion based on jupiter-stacks and home-server reference notes, 2026-10-22; proposals only, no live inspection or deployment" +related: + - home-server.reference + - home-server.networking + - home-server.authentication + - home-server.backups + - home-server.operations + - home-server.seedbox + - home-server.storage + - home-server.host +update_triggers: [implementation, priority-change, access-requirements-change, pricing-change, power-measurement] +unknowns: + - live-routing-firewall-and-wireguard-configuration + - work-laptop-local-dns-and-lan-access-policy + - measured-wall-power-and-electricity-tariff + - current-vpn-plan-features-and-pricing + - immich-original-photo-backup-coverage +--- + +# Ideas / next steps + +Planning backlog from the 2026-10-22 discussion. These are recommendations and +owner preferences, **not implemented changes or deployment evidence**. +Existing reference pages describe the documented configuration; update them +when changes are actually deployed. + +## Requirements confirmed with the owner + +| Area | Requirement / preference | +| --- | --- | +| Private administration | Portainer, Beszel, Dozzle, Backrest, Prowlarr, Radarr and Sonarr should stop being publicly reachable | +| Public applications | Keep normal user-facing applications public, including Vaultwarden | +| Personal devices | Owner-only admin access; VPN clients or certificates can be installed on personal phone/laptop | +| Work laptop | Browser-only admin access needed on home Wi-Fi; cannot install VPN clients or certificates | +| Device loss | Vaultwarden access remains essential; admin tools can wait until returning home | +| Vaultwarden recovery | Owner knows the master password; TOTP is backed up/restorable through Microsoft, with independently recoverable Microsoft access; owner report, not an observed recovery exercise | +| Private-access tooling | Prefer minimal infrastructure and no vendor-hosted coordination service; existing Cloudflare DNS/SSH use is acceptable | +| Hardware | Software/BIOS tuning only; no hardware purchases or removals | +| Broadband | Reported 500 Mbps download and upload; public trackers only | +| VPN candidates | Proton VPN or AirVPN; owner quotes GBP 2.50/month without upfront commitment; exact plan/features/pricing not independently checked | +| New services | Interested in reliability, everyday tools, media, development and privacy | + +## Priority order + +1. Prepare independent recovery access, then introduce private admin ingress. +2. Make backups recoverable and add failure notifications. +3. Replace the seedbox with VPN-isolated local downloads if the trial succeeds. +4. Measure and tune power use. +5. Add services that solve an actual day-to-day problem. + +## Private admin access + +### Preferred approach: plain WireGuard through Mercury + +Reuse Mercury's public address and existing WireGuard connection to Jupiter. +Give each personal device its own peer/key, narrowly scoped routes and firewall +permissions. No Tailscale account or self-hosted coordination service is needed. + +| Client | Intended access path | +| --- | --- | +| Work laptop on home Wi-Fi | LAN directly to private admin ingress; browser only | +| Personal device away from home | WireGuard to Mercury, then routed private access to Jupiter | +| Public internet | Mercury nginx to explicitly public applications only | + +Use split DNS: admin hostnames resolve to private addresses on the LAN and +through the VPN. Keep HTTPS certificates using the existing DNS-based challenge; +certificate issuance does not require a public website. + +Before choosing exact addresses/listeners, inspect live WireGuard `AllowedIPs`, +routing, firewall rules and Docker port publication. Check that the work laptop's +corporate DNS/proxy policy permits local access; do not bypass employer controls. + +### Implementation checklist + +- [ ] Prepare recovery access and a rollback route before restricting ingress. +- [ ] Add per-device WireGuard peers and a way to revoke a lost device's key. +- [ ] Configure LAN/VPN DNS for the private admin hostnames. +- [ ] Create private-only Traefik ingress; public requests must not reach admin + backends simply by supplying their hostnames. +- [ ] Replace or constrain Mercury's wildcard forwarding in + `jupiter-stacks/mercury/nginx/umbra.mom`, retaining all intended public apps. +- [ ] Apply the same boundary to Authentik's separate outpost callback routes + for Backrest and the Arr applications. +- [ ] Remove or restrict Portainer's direct `9123:9000` publication in + `stacks/portainer/portainer_start.sh`. +- [ ] Retain application authentication and Authentik MFA behind the private + boundary; VPN connectivity must not imply application administrator rights. +- [ ] Confirm private apps and authentication callbacks work from LAN and VPN, + but cannot be reached through public ingress or alternative published ports. +- [ ] Confirm public applications, especially Vaultwarden, remain accessible. + +Removing public DNS records or adding a login page is not sufficient isolation. +The network/proxy boundary must enforce the restriction. + +### Alternatives + +| Option | Assessment | +| --- | --- | +| Plain WireGuard | Preferred: reuse existing infrastructure, few devices, no hosted control service | +| Tailscale | Convenient, but adds the vendor dependency the owner prefers to avoid | +| Headscale | Self-hosted coordination, but more infrastructure to maintain than currently needed | +| mTLS | Device-certificate authentication; adds certificate distribution/recovery and cannot be used directly on the work laptop | + +mTLS rejects clients without a certificate but leaves the HTTPS endpoint +internet-facing. Mercury terminates public TLS, so browser client-certificate +enforcement would need to happen there, not merely between Mercury and Jupiter. + +## Recovery and reliability + +Keep Vaultwarden public with application login and MFA. Do not make access +depend on the new admin VPN or Authentik. + +- [ ] Maintain an encrypted recovery pack outside Jupiter: Mercury provider + console/SSH access, WireGuard re-enrolment instructions, Authentik recovery, + retained local admin credentials, backup credentials, restic key, stack + secrets and essential host configuration. +- [ ] Ensure retrieving/decrypting that pack does not depend on Jupiter, + Portainer or secrets available only inside Vaultwarden. +- [ ] Retain the existing Cloudflare SSH route during migration, with its + effective Access policy reviewed. It still depends on Jupiter's connector + and networking; it is not an out-of-band console. +- [ ] Restore an important service into an isolated location and record the + dependencies, time taken and acceptable data loss. +- [ ] Review backup coverage for Backrest configuration, host configuration, + Docker named volumes and Mercury. +- [ ] Establish whether Immich contains irreplaceable originals and where those + originals and the associated database are backed up. +- [ ] Make failed backup restart hooks visible; a successful snapshot must not + hide stopped Authentik, Portainer or Wardrowbe services. +- [ ] Add service, disk-health, capacity and backup-failure notifications using + Beszel and the existing Mercury uptime service where suitable. +- [ ] Add an external missed-backup heartbeat so Jupiter going offline cannot + prevent notification of its own failure. + +Private connectivity does not fix Authentik downtime. The documented backup plan +stops Authentik and ignores some restart failures; independent SSH/local recovery +remains necessary. See [Backups](backups.md) and [Operations](operations.md). + +## Local downloads instead of the seedbox + +At the owner's quoted prices, replacing the GBP 4.50/month seedbox with a +GBP 2.50/month VPN saves **GBP 2/month or GBP 24/year before extra electricity**. +Confirm actual plan terms before purchasing. The simpler local storage/import +path may be more valuable than the modest subscription saving. + +### Proposed arrangement + +- [ ] Check the chosen Proton VPN/AirVPN plan supports P2P and the intended + manual/container connection; assess port forwarding for peer connectivity. +- [ ] Run qBittorrent behind a VPN gateway container such as Gluetun. +- [ ] Block torrent egress whenever the VPN is unavailable, including startup; + use qBittorrent interface binding as additional protection. +- [ ] Keep the qBittorrent Web UI private and ensure DNS/IPv6 cannot bypass + the intended VPN path. +- [ ] Keep the host, public applications and admin-access WireGuard routes on + their normal connections. The commercial download VPN is a separate job. +- [ ] Use a common container mount layout for downloads and media on the HDD + filesystem so Radarr/Sonarr can hardlink imports while torrents seed. +- [ ] Confirm imports, seeding, removal behaviour and VPN-failure isolation + before cancelling the seedbox. +- [ ] Migrate any retained downloads and remove the rclone/FUSE dependency only + after the replacement workflow works. + +A hardlink gives the same file data two names without storing a second full +copy. The current separate download/library mounts need review; sharing physical +disks alone is not enough. + +The [Seedbox](seedbox.md) notes record an expiry of 2026-10-25. Confirm renewal +status rather than assuming this snapshot is still the billing state. + +## Electricity and compute efficiency + +Reported average CPU usage is approximately 1%, but CPU utilisation does not +measure wall-socket power. No power measurement or actual electricity tariff +was supplied. + +Illustrative costs at **25p/kWh**, using 365 days/year divided by 12: + +| Continuous average draw | Approximate monthly cost | +| --- | --- | +| 40 W | GBP 7.30 | +| 60 W | GBP 10.95 | +| 80 W | GBP 14.60 | + +At that illustrative tariff, saving 10 W continuously saves about GBP 18/year. +An extra average draw of about 11 W would consume the GBP 2/month seedbox saving. +Count the incremental download workload, not the server's existing baseline. + +- [ ] Measure wall power and obtain the actual unit rate when practical. +- [ ] Inspect CPU idle states and frequency-scaling policy before BIOS tuning; + low CPU limits do not automatically save energy. +- [ ] Inspect RAM pressure and swapping during Immich processing and Wardrowbe + Ollama jobs; the host has 16 GB RAM. +- [ ] Schedule heavy jobs where useful and avoid unnecessary repeated scans. +- [ ] Review effective container log rotation; daemon defaults require activation + and recreation, and explicit Compose settings override them. +- [ ] Avoid aggressive HDD spindown while `/data` hosts active databases. +- [ ] Avoid consolidating unrelated application databases solely to reduce + container count; shared failures/upgrades may outweigh uncertain savings. + +No hardware changes are proposed. Leave spare CPU capacity idle unless a useful +workload justifies the added power and maintenance. + +## Optional service ideas + +These are candidates, not a deployment list. + +| Area | Idea | Constraint / reason | +| --- | --- | --- | +| Reliability | Notifications and external backup heartbeats | Start with existing monitoring rather than a second metrics stack | +| Privacy | AdGuard Home or Pi-hole | Choose one; household DNS availability becomes an operational dependency | +| Files | Syncthing for selected folders | Sync is not backup; check client support on the owner's devices | +| Bookmarks / reading | Linkding; FreshRSS | Small tools with a clear everyday purpose | +| Development | Renovate automation for existing Gitea workflows | Propose updates; avoid unattended production upgrades | +| Media | Profilarr for supported Arr profiles/custom formats | Add only if manual profile maintenance is a real problem | + +Defer Kubernetes, a large metrics stack, another identity provider or a general +cloud suite unless a concrete need emerges. diff --git a/docs/home-server/index.md b/docs/home-server/index.md index 267a8df..2c681bb 100644 --- a/docs/home-server/index.md +++ b/docs/home-server/index.md @@ -4,8 +4,10 @@ id: home-server.reference type: reference scope: [jupiter, mercury, seedbox, shared-infrastructure] sensitivity: private-infrastructure -last_reviewed: "2026-10-16" +last_reviewed: "2026-10-22" sources: + - kind: owner-report + reference: "Added ideas/next-steps planning page from owner discussion, 2026-10-22" - kind: owner-report reference: "Owner interview, 2026-10-06" - kind: owner-report @@ -22,6 +24,7 @@ related: - home-server.storage - home-server.authentication - home-server.backups + - home-server.ideas-next-steps update_triggers: [topology-change, source-change, infrastructure-change] unknowns: [] --- @@ -41,6 +44,7 @@ unknowns: [] | [Storage](storage.md) | SSD; RAID 0; mounts; volumes | | [Authentication](authentication.md) | Authentik; OIDC; access groups; forward-auth; secrets | | [Backups](backups.md) | Backrest; coverage; retention; recovery | +| [Ideas / next steps](ideas-next-steps.md) | Proposed private access, recovery, savings and optional services; not deployed state | ## Evidence / agent contract diff --git a/docs/home-server/mercury.md b/docs/home-server/mercury.md index 3b2d28d..2923c4a 100644 --- a/docs/home-server/mercury.md +++ b/docs/home-server/mercury.md @@ -4,20 +4,23 @@ id: home-server.mercury type: reference scope: [mercury, vps, hosting, nginx, wireguard, firewall] sensitivity: private-infrastructure -last_reviewed: "2026-10-06" +last_reviewed: "2026-10-23" sources: - kind: owner-report reference: "Fasthosts plan, fastfetch, nginx sites, iptables and fail2ban output, 2026-10-06" - kind: owner-report reference: "nginx/WireGuard apt management and public IPv4/CGNAT rationale, 2026-10-06" + - kind: owner-report + reference: "Uptime service confirmed as Uptime Kuma; notifications received in Discord server Jupiter, channel #uptimekuma-alerts, 2026-10-23" related: [home-server.reference, home-server.host, home-server.networking, home-server.operations] -update_triggers: [hosting-renewal, vps-resize, os-upgrade, proxy-change, firewall-change, certificate-change] +update_triggers: [hosting-renewal, vps-resize, os-upgrade, proxy-change, firewall-change, certificate-change, alerting-change] unknowns: - renewal-price-and-renewal-date - creation-timestamp-timezone - certificate-renewal-method - wireguard-allowedips-and-keepalive - backup-and-restore-policy + - uptime-kuma-monitor-targets-and-outage-coverage --- # Mercury VPS @@ -64,6 +67,19 @@ home connection uses CGNAT. Residential-IP exposure is avoided by design. forwards WebSocket upgrade headers. - Gitea `/v2/`: unlimited request body; proxy timeouts `900s`. +## Monitoring / notifications + +Uptime Kuma runs on Mercury, served at `https://uptime.umbra.mom` through +the local upstream `http://127.0.0.1:3001`. Notifications are delivered to +**Jupiter** (Discord server) in `#uptimekuma-alerts`; receipt confirmed by +the owner on 2026-10-23. + +Monitor targets and outage coverage have not been recorded. Running Kuma on +Mercury allows monitoring independently of Jupiter, but does not establish that +Jupiter or its containers are currently monitored. Mercury cannot report its +own complete outage through this instance; an external monitor is not yet +confirmed. See [Observability](operations.md#observability). + ## Firewall / fail2ban - UFW-managed iptables: INPUT/FORWARD `DROP`; OUTPUT `ACCEPT`. diff --git a/docs/home-server/operations.md b/docs/home-server/operations.md index 3132961..bb32549 100644 --- a/docs/home-server/operations.md +++ b/docs/home-server/operations.md @@ -4,7 +4,7 @@ id: home-server.operations type: reference scope: [jupiter, mercury, design-decisions, monitoring, recovery] sensitivity: private-infrastructure -last_reviewed: "2026-10-16" +last_reviewed: "2026-10-23" sources: - kind: owner-report reference: "RAID/media tradeoff, CGNAT/public IPv4 rationale, no alerting and running monitoring/log containers, 2026-10-06" @@ -12,6 +12,8 @@ sources: reference: "Outages over one day catastrophic; no independent credentials except restic key, recovery evidence, host-config archive or operational rules, 2026-10-06" - kind: owner-report reference: "OIDC and three-group motivation; Authentik fully deployed and Authelia retired; updated six-directory backup plan; independent recovery custody unchanged, 2026-10-16" + - kind: owner-report + reference: "Notifications received in Discord server Jupiter: Beszel in #beszel-alerts, Backrest in #backrest-alerts, and Mercury Uptime Kuma in #uptimekuma-alerts, 2026-10-23" related: [home-server.reference, home-server.host, home-server.mercury, home-server.networking, home-server.storage, home-server.authentication, home-server.backups, home-server.deployment] update_triggers: [data-priority-change, architecture-change, alerting-change, recovery-test] unknowns: @@ -19,6 +21,7 @@ unknowns: - data-criticality-outside-the-six-backed-up-directories - tested-full-host-recovery-order - monitoring-review-frequency + - configured-alert-rules-and-uptime-monitor-coverage --- # Operations / decisions @@ -49,10 +52,26 @@ administrator privileges or Authentik superuser status. | --- | --- | | Host/container monitoring | Beszel hub and host-network agent present | | Container logs | Dozzle with Docker socket proxy present | -| Automatic alerting | None configured | +| Uptime monitoring | Uptime Kuma on Mercury at `uptime.umbra.mom` | +| Automatic alerting | Discord delivery configured for Beszel, Backrest and Uptime Kuma; receipt confirmed by owner | + +### Discord notifications + +The owner confirms receiving notifications in the Discord server **Jupiter** +as of 2026-10-23: + +| Source | Channel | +| --- | --- | +| Beszel | `#beszel-alerts` | +| Backrest | `#backrest-alerts` | +| Uptime Kuma on Mercury | `#uptimekuma-alerts` | + +Notification receipt does not establish complete monitoring coverage. Exact +enabled alert rules, thresholds, Backrest hook conditions and Uptime Kuma +monitor targets have not been recorded. Per-container outage detection, +whole-Jupiter outage detection, external monitoring of Mercury and missed-backup +alerts are not yet confirmed. No webhook URLs or tokens are stored in these notes. -Monitoring does not imply notification delivery. Backup-hook failures, -stopped services and storage failures require manual detection. The backup plan stops Authentik, interrupting new OIDC logins and forward-auth access. Authentik/Wardrowbe restart hooks ignore errors, so backup completion alone does not establish that authentication or applications recovered. diff --git a/zensical.toml b/zensical.toml index f502083..ed4e4db 100644 --- a/zensical.toml +++ b/zensical.toml @@ -23,6 +23,7 @@ nav = [ { "Storage" = "home-server/storage.md" }, { "Authentication" = "home-server/authentication.md" }, { "Backups" = "home-server/backups.md" }, + { "Ideas / next steps" = "home-server/ideas-next-steps.md" }, ] }, { "Books" = [ { "Designing Data-Intensive Applications" = [