diff --git a/CLAUDE.md b/CLAUDE.md index 9dc3781..95a1ad5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,8 +26,9 @@ host, Docker Compose per site, no orchestrator. - ✅ Phase 2 — platform services (Traefik, MariaDB, Forgejo) in [platform-infra/stacks/](platform-infra/stacks/README.md) - ✅ Phase 3 — site templates & base images in [site-templates/](site-templates/README.md) - ✅ Phase 4 — provisioning CLI `heleosctl` in [control-panel/](control-panel/README.md) (provision/deprovision/list/render/backup/restore; `--dry-run`; 22 tests) -- 🚧 Phase 5 — backup/DR automation (ZFS snapshot/`send` + automysqlbackup jobs, offsite, restore drill) -- ⬜ Phases 6–8 — observability, migration, panel +- ✅ Phase 5 — backup/DR automation in the `backup` Ansible role (sanoid snapshots, per-DB dump timer, optional syncoid/rsync offsite) +- 🚧 Phase 6 — observability (Prometheus/Grafana + cAdvisor/node_exporter/Traefik metrics, Loki/Promtail, Uptime-Kuma) +- ⬜ Phases 7–8 — migration, customer panel ### Phase 4 follow-ups (not yet done) - `reconfigure` + `rotate-secret` commands; git-commit of `deployments/` on provision; php-fpm umask for two-way SFTP editing (see control-panel/README.md). diff --git a/docs/06-backup-and-dr.md b/docs/06-backup-and-dr.md index 10f2385..61862ba 100644 --- a/docs/06-backup-and-dr.md +++ b/docs/06-backup-and-dr.md @@ -32,6 +32,21 @@ config) is backed up with the same ZFS mechanism from `tank/platform/*`. - **DB dumps:** rsync to the same or another offsite location (delta transfer). - Both offsite copies are the recovery source if the primary host is lost. +## 2b. Implementation (Phase 5) + +Automated by the `backup` Ansible role (`platform-infra/ansible/roles/backup`): + +- **Files:** [sanoid](https://github.com/jimsalterjrs/sanoid) takes and prunes + snapshots per policy (`sanoid.conf`, driven by `sanoid_datasets`), on the + packaged `sanoid.timer`. **syncoid** replicates offsite + (`heleos-zfs-offsite`), enabled only when `zfs_offsite_target` is set. +- **DB:** `heleos-db-backup` (systemd timer, nightly) walks the deployments dir, + dumps each DB-backed site via `docker exec … mariadb-dump` into + `db-backups///{daily,weekly,monthly}` with rotation, then + `heleos-db-offsite` rsyncs offsite (enabled only when `db_offsite_target` set). +- Toggle streams with `backup_snapshots_enabled` / `db_backup_enabled` and the + offsite target vars in `group_vars/all.yml`. + ## 3. Restore — single site (the critical drill) Two coordinated steps: diff --git a/platform-infra/ansible/README.md b/platform-infra/ansible/README.md index e732ea7..345b081 100644 --- a/platform-infra/ansible/README.md +++ b/platform-infra/ansible/README.md @@ -2,7 +2,8 @@ Idempotent host configuration for the heleos platform, targeting **Ubuntu 24.04 LTS**. Roles: base packages → ZFS pool/datasets → Docker (data-root on ZFS) → -nftables firewall + container egress filter → SSH hardening. +nftables firewall + container egress filter → SSH hardening → backup/DR +(sanoid snapshots, per-DB dumps, offsite). ## Prerequisites @@ -67,6 +68,9 @@ Run a single layer with tags: `--tags zfs`, `--tags docker`, `--tags firewall`, ICMP); outbound SMTP blocked from containers. - Hardened SSH (key-first, root prohibit-password) and the `sftponly` group that per-customer SFTP accounts will join in Phase 4. +- Backup/DR (Phase 5): sanoid snapshot policy + timer, nightly per-database dumps + with rotation, and optional offsite `zfs send` (syncoid) / rsync — enable + offsite by setting `zfs_offsite_target` / `db_offsite_target`. ## Verify after running diff --git a/platform-infra/ansible/group_vars/all.yml b/platform-infra/ansible/group_vars/all.yml index 50f671e..81fd706 100644 --- a/platform-infra/ansible/group_vars/all.yml +++ b/platform-infra/ansible/group_vars/all.yml @@ -54,3 +54,29 @@ smtp_relay_host: "" # optional: allow SMTP only to this hos admin_user: "{{ ansible_user }}" admin_authorized_keys: [] # list of public key strings ssh_disable_password_auth: false # flip to true once key login is verified + +# --- Backups / DR (Phase 5) ------------------------------------------------- +mariadb_container: mariadb +mariadb_env_file: /opt/heleos/platform-infra/stacks/mariadb/.env +deployments_dir: /opt/heleos/deployments + +# ZFS snapshots via sanoid (files stream). Retention is per policy below. +backup_snapshots_enabled: true +sanoid_datasets: + - { name: "{{ zfs_pool_name }}/customers", recursive: true, hourly: 36, daily: 30, weekly: 8, monthly: 6 } + - { name: "{{ zfs_pool_name }}/platform", recursive: true, hourly: 0, daily: 14, weekly: 4, monthly: 3 } + +# Per-database dumps (DB stream), automysqlbackup-style rotation via docker exec. +db_backup_enabled: true +db_backup_dir: "/{{ zfs_pool_name }}/platform/db-backups" +db_backup_oncalendar: "*-*-* 01:30:00" +db_backup_keep_daily: 14 +db_backup_keep_weekly: 8 +db_backup_keep_monthly: 6 + +# Offsite. Leave the targets empty to disable that stream. +zfs_offsite_target: "" # e.g. "user@backup-host:backup/heleos" +zfs_offsite_datasets: ["{{ zfs_pool_name }}/customers", "{{ zfs_pool_name }}/platform"] +zfs_offsite_oncalendar: "*-*-* 03:00:00" +db_offsite_target: "" # e.g. "user@backup-host:/srv/heleos/db-backups" +db_offsite_oncalendar: "*-*-* 03:30:00" diff --git a/platform-infra/ansible/roles/backup/handlers/main.yml b/platform-infra/ansible/roles/backup/handlers/main.yml new file mode 100644 index 0000000..c28484f --- /dev/null +++ b/platform-infra/ansible/roles/backup/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/platform-infra/ansible/roles/backup/tasks/main.yml b/platform-infra/ansible/roles/backup/tasks/main.yml new file mode 100644 index 0000000..406f5d6 --- /dev/null +++ b/platform-infra/ansible/roles/backup/tasks/main.yml @@ -0,0 +1,175 @@ +--- +# ── ZFS snapshots (files stream) via sanoid ───────────────────────────────── +- name: Install sanoid (snapshots + syncoid replication) + ansible.builtin.apt: + name: sanoid + state: present + when: backup_snapshots_enabled + +- name: Configure sanoid snapshot policy + ansible.builtin.template: + src: sanoid.conf.j2 + dest: /etc/sanoid/sanoid.conf + owner: root + group: root + mode: "0644" + when: backup_snapshots_enabled + +- name: Enable the sanoid timer + ansible.builtin.systemd: + name: sanoid.timer + enabled: true + state: started + when: backup_snapshots_enabled + +# ── Per-database dumps (DB stream) ────────────────────────────────────────── +- name: Install the DB backup script + ansible.builtin.template: + src: heleos-db-backup.sh.j2 + dest: /usr/local/sbin/heleos-db-backup.sh + owner: root + group: root + mode: "0755" + when: db_backup_enabled + +- name: Install the DB backup service + timer + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ item.name }}" + owner: root + group: root + mode: "0644" + content: "{{ item.content }}" + loop: + - name: heleos-db-backup.service + content: | + [Unit] + Description=heleos per-database dumps (rotated) + After=docker.service + Requires=docker.service + + [Service] + Type=oneshot + ExecStart=/usr/local/sbin/heleos-db-backup.sh + - name: heleos-db-backup.timer + content: | + [Unit] + Description=Run heleos DB backups on a schedule + + [Timer] + OnCalendar={{ db_backup_oncalendar }} + Persistent=true + + [Install] + WantedBy=timers.target + loop_control: + label: "{{ item.name }}" + when: db_backup_enabled + notify: Reload systemd + +- name: Enable the DB backup timer + ansible.builtin.systemd: + name: heleos-db-backup.timer + enabled: true + daemon_reload: true + state: started + when: db_backup_enabled + +# ── Offsite: files (syncoid / zfs send) ───────────────────────────────────── +- name: Install the ZFS offsite script + ansible.builtin.template: + src: heleos-zfs-offsite.sh.j2 + dest: /usr/local/sbin/heleos-zfs-offsite.sh + owner: root + group: root + mode: "0755" + when: zfs_offsite_target | length > 0 + +- name: Install the ZFS offsite service + timer + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ item.name }}" + owner: root + group: root + mode: "0644" + content: "{{ item.content }}" + loop: + - name: heleos-zfs-offsite.service + content: | + [Unit] + Description=heleos offsite ZFS replication (syncoid) + + [Service] + Type=oneshot + ExecStart=/usr/local/sbin/heleos-zfs-offsite.sh + - name: heleos-zfs-offsite.timer + content: | + [Unit] + Description=Run heleos offsite ZFS replication on a schedule + + [Timer] + OnCalendar={{ zfs_offsite_oncalendar }} + Persistent=true + + [Install] + WantedBy=timers.target + loop_control: + label: "{{ item.name }}" + when: zfs_offsite_target | length > 0 + notify: Reload systemd + +- name: Enable the ZFS offsite timer + ansible.builtin.systemd: + name: heleos-zfs-offsite.timer + enabled: true + daemon_reload: true + state: started + when: zfs_offsite_target | length > 0 + +# ── Offsite: DB dumps (rsync) ─────────────────────────────────────────────── +- name: Install the DB offsite script + ansible.builtin.template: + src: heleos-db-offsite.sh.j2 + dest: /usr/local/sbin/heleos-db-offsite.sh + owner: root + group: root + mode: "0755" + when: db_offsite_target | length > 0 + +- name: Install the DB offsite service + timer + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ item.name }}" + owner: root + group: root + mode: "0644" + content: "{{ item.content }}" + loop: + - name: heleos-db-offsite.service + content: | + [Unit] + Description=heleos offsite DB dump sync (rsync) + + [Service] + Type=oneshot + ExecStart=/usr/local/sbin/heleos-db-offsite.sh + - name: heleos-db-offsite.timer + content: | + [Unit] + Description=Run heleos offsite DB sync on a schedule + + [Timer] + OnCalendar={{ db_offsite_oncalendar }} + Persistent=true + + [Install] + WantedBy=timers.target + loop_control: + label: "{{ item.name }}" + when: db_offsite_target | length > 0 + notify: Reload systemd + +- name: Enable the DB offsite timer + ansible.builtin.systemd: + name: heleos-db-offsite.timer + enabled: true + daemon_reload: true + state: started + when: db_offsite_target | length > 0 diff --git a/platform-infra/ansible/roles/backup/templates/heleos-db-backup.sh.j2 b/platform-infra/ansible/roles/backup/templates/heleos-db-backup.sh.j2 new file mode 100755 index 0000000..627aa45 --- /dev/null +++ b/platform-infra/ansible/roles/backup/templates/heleos-db-backup.sh.j2 @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# heleos per-database dumps — automysqlbackup-style rotation, adapted for the +# containerized shared MariaDB. Managed by Ansible. Iterates the deployments dir +# (source of truth) and dumps each DB-backed site into +# db-backups///{daily,weekly,monthly}. +set -euo pipefail + +CONTAINER="{{ mariadb_container }}" +BASE="{{ db_backup_dir }}" +DEPLOYMENTS="{{ deployments_dir }}" +KEEP_DAILY={{ db_backup_keep_daily }} +KEEP_WEEKLY={{ db_backup_keep_weekly }} +KEEP_MONTHLY={{ db_backup_keep_monthly }} + +# Root password: env var wins, else read from the mariadb stack .env. +ROOTPW="${MARIADB_ROOT_PASSWORD:-}" +if [ -z "$ROOTPW" ] && [ -f "{{ mariadb_env_file }}" ]; then + ROOTPW="$(grep -E '^MARIADB_ROOT_PASSWORD=' "{{ mariadb_env_file }}" | cut -d= -f2-)" +fi +if [ -z "$ROOTPW" ]; then + echo "no MariaDB root password (env or {{ mariadb_env_file }})" >&2 + exit 1 +fi + +dow="$(date +%u)" # 7 = Sunday +dom="$(date +%d)" # 01 = first of month +ts="$(date +%Y%m%d-%H%M%S)" + +shopt -s nullglob +for meta in "$DEPLOYMENTS"/*/*/site.yaml; do + d="$(dirname "$meta")" + site="$(basename "$d")" + customer="$(basename "$(dirname "$d")")" + grep -qiE '^database:[[:space:]]*true' "$meta" || continue + + db="db_${customer//-/_}_${site//-/_}" + dest="$BASE/$customer/$site" + mkdir -p "$dest/daily" "$dest/weekly" "$dest/monthly" + + out="$dest/daily/${db}-${ts}.sql.gz" + # MYSQL_PWD keeps the password out of the container's process list. + docker exec -e MYSQL_PWD="$ROOTPW" "$CONTAINER" \ + mariadb-dump --single-transaction --databases "$db" -uroot | gzip > "$out" + + [ "$dow" = "7" ] && cp -f "$out" "$dest/weekly/" + [ "$dom" = "01" ] && cp -f "$out" "$dest/monthly/" + + find "$dest/daily" -name '*.sql.gz' -type f -mtime +"$KEEP_DAILY" -delete + find "$dest/weekly" -name '*.sql.gz' -type f -mtime +"$(( KEEP_WEEKLY * 7 ))" -delete + find "$dest/monthly" -name '*.sql.gz' -type f -mtime +"$(( KEEP_MONTHLY * 31 ))" -delete +done diff --git a/platform-infra/ansible/roles/backup/templates/heleos-db-offsite.sh.j2 b/platform-infra/ansible/roles/backup/templates/heleos-db-offsite.sh.j2 new file mode 100755 index 0000000..250230d --- /dev/null +++ b/platform-infra/ansible/roles/backup/templates/heleos-db-offsite.sh.j2 @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# heleos offsite DB dump sync via rsync. Managed by Ansible. Only installed/ +# enabled when db_offsite_target is set. Dumps are already rotated locally. +set -euo pipefail + +rsync -a --delete "{{ db_backup_dir }}/" "{{ db_offsite_target }}/" diff --git a/platform-infra/ansible/roles/backup/templates/heleos-zfs-offsite.sh.j2 b/platform-infra/ansible/roles/backup/templates/heleos-zfs-offsite.sh.j2 new file mode 100755 index 0000000..315239a --- /dev/null +++ b/platform-infra/ansible/roles/backup/templates/heleos-zfs-offsite.sh.j2 @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# heleos offsite ZFS replication via syncoid (incremental zfs send). Managed by +# Ansible. Only installed/enabled when zfs_offsite_target is set. +set -euo pipefail + +TARGET="{{ zfs_offsite_target }}" + +for ds in {{ zfs_offsite_datasets | join(' ') }}; do + # DEST mirrors the last path component under the offsite base, e.g. + # tank/customers -> /customers + dest="${TARGET}/${ds##*/}" + echo "==> syncoid ${ds} -> ${dest}" + syncoid --recursive --no-sync-snap "${ds}" "${dest}" +done diff --git a/platform-infra/ansible/roles/backup/templates/sanoid.conf.j2 b/platform-infra/ansible/roles/backup/templates/sanoid.conf.j2 new file mode 100644 index 0000000..9867ae2 --- /dev/null +++ b/platform-infra/ansible/roles/backup/templates/sanoid.conf.j2 @@ -0,0 +1,17 @@ +# heleos sanoid snapshot policy — managed by Ansible. +# Files stream only; DB dumps are handled separately (heleos-db-backup). +{% for d in sanoid_datasets %} +[{{ d.name }}] + use_template = {{ d.name | replace('/', '_') }} + recursive = {{ 'yes' if d.recursive else 'no' }} +{% endfor %} + +{% for d in sanoid_datasets %} +[template_{{ d.name | replace('/', '_') }}] + hourly = {{ d.hourly }} + daily = {{ d.daily }} + weekly = {{ d.weekly }} + monthly = {{ d.monthly }} + autosnap = yes + autoprune = yes +{% endfor %} diff --git a/platform-infra/ansible/site.yml b/platform-infra/ansible/site.yml index c5b3918..53c2b72 100644 --- a/platform-infra/ansible/site.yml +++ b/platform-infra/ansible/site.yml @@ -20,3 +20,5 @@ tags: [firewall] - role: ssh_hardening tags: [ssh] + - role: backup + tags: [backup]