diff --git a/.gitignore b/.gitignore index 1f985ac..9149e6e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ __pycache__/ *.pyc .venv/ venv/ +*.egg-info/ +.pytest_cache/ +control-panel/config.yaml diff --git a/CLAUDE.md b/CLAUDE.md index 0c83290..9dc3781 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,8 +25,12 @@ host, Docker Compose per site, no orchestrator. - ✅ Phase 1 — host baseline Ansible ([platform-infra/ansible/](platform-infra/ansible/README.md)) - ✅ Phase 2 — platform services (Traefik, MariaDB, Forgejo) in [platform-infra/stacks/](platform-infra/stacks/README.md) - ✅ Phase 3 — site templates & base images in [site-templates/](site-templates/README.md) -- 🚧 Phase 4 — provisioning CLI in `control-panel/` (renders templates → deployments/, creates ZFS+DB+SFTP, `compose up`) -- ⬜ Phases 5–8 — backup/DR, observability, migration, panel +- ✅ Phase 4 — provisioning CLI `heleosctl` in [control-panel/](control-panel/README.md) (provision/deprovision/list/render/backup/restore; `--dry-run`; 22 tests) +- 🚧 Phase 5 — backup/DR automation (ZFS snapshot/`send` + automysqlbackup jobs, offsite, restore drill) +- ⬜ Phases 6–8 — observability, migration, panel + +### Phase 4 follow-ups (not yet done) +- `reconfigure` + `rotate-secret` commands; git-commit of `deployments/` on provision; php-fpm umask for two-way SFTP editing (see control-panel/README.md). ## Repo map | Path | Purpose | diff --git a/control-panel/README.md b/control-panel/README.md index 0a207e7..ac0deb2 100644 --- a/control-panel/README.md +++ b/control-panel/README.md @@ -1,18 +1,87 @@ -# control-panel +# control-panel — heleosctl -The provisioning CLI (Phase 4), later the customer-facing web panel (Phase 8). -Empty until Phase 4. +Provisioning CLI for the platform. Reads a site's `site.yaml`, renders the +[site-templates](../site-templates/README.md) into +[deployments/](../deployments/README.md), creates the ZFS dataset + database + +SFTP account, and runs `docker compose up`. Implements the flow in +[docs/05-provisioning-workflow.md](../docs/05-provisioning-workflow.md). -**Planned CLI surface** (see -[../docs/05-provisioning-workflow.md](../docs/05-provisioning-workflow.md)): +> Runs **on the host** (Linux), typically as root, since it drives `zfs`, +> `docker`, `useradd`, and `systemctl`. Use `--dry-run` to preview every action +> first — nothing touches the system until you drop the flag. -| Command | Action | -|---------|--------| -| `provision` | Create a new site end-to-end (ZFS + DB + compose + route + SFTP). | -| `reconfigure` | Apply changes from an edited `site.yaml`. | -| `deprovision` | Tear down a site with gated, backed-up deletion. | -| `list` | Show all sites, profiles, status. | -| `backup` / `restore` | Ad-hoc backup; two-step (files + DB) restore. | -| `rotate-secret` | Regenerate DB/SFTP credentials. | +## Install +```bash +cd control-panel +python -m venv .venv && . .venv/bin/activate +pip install -e ".[dev]" # installs the `heleosctl` command + pytest +cp config.example.yaml config.yaml && $EDITOR config.yaml +``` +Config is found via `--config`, `$HELEOS_CONFIG`, `./config.yaml`, or +`/etc/heleos/config.yaml`. -Idempotent, declarative (`site.yaml` is source of truth), logs to Loki. +## Usage +```bash +# Preview everything (no changes) +heleosctl --dry-run provision -f examples/acme-shop.site.yaml + +# Provision for real +heleosctl provision -f examples/acme-shop.site.yaml + +# Inspect / operate on an existing site (by customer+site or by file) +heleosctl list +heleosctl render -c acme -s shop # print rendered config +heleosctl backup -c acme -s shop # snapshot + DB dump +heleosctl restore -c acme -s shop --snapshot auto-20260101-0300 --db /path/dump.sql +heleosctl deprovision -c acme -s shop # stop + drop DB, KEEP data +heleosctl deprovision -c acme -s shop --purge # also destroy the dataset (after a final backup) +``` + +## What each command does +| Command | Actions | +|---------|---------| +| `provision` | ZFS web dataset + ownership → DB + user (if any) → `.env` + SOPS encrypt → render compose/nginx/site.yaml → per-customer SFTP account → `compose up -d`. | +| `deprovision` | `compose down` → final backup → drop DB/user → (with `--purge`) destroy dataset + remove config. | +| `render` | Print the rendered files without writing (debugging). | +| `backup` | ZFS snapshot + `mariadb-dump` into `db-backups//`. | +| `restore` | ZFS rollback + import a DB dump. | +| `list` | Table of provisioned sites found under the deployments dir. | + +## Design notes +- **Idempotent & declarative:** `site.yaml` is the source of truth; steps use + `zfs create -p`, `CREATE ... IF NOT EXISTS`, and existence checks so re-running + converges. A customer's SFTP password is set only when the account is first + created. +- **Secrets:** the DB password is generated per site, written to a git-ignored + `.env`, and encrypted to `secrets.enc.yaml` via SOPS/age (set + `sops_age_recipient`). Passwords never appear in command logs (`--redacted`). +- **Safety:** all host mutations go through one runner supporting `--dry-run`; + data is destroyed only with `--purge`, always after a final backup. + +## Layout +``` +src/heleos/ + cli.py # click commands + config.py # Config + Site loading/validation + naming.py # slug / db / sftp identifier rules (docs/03) + context.py # Site -> template variables + render.py # Jinja2 render of a profile + runner.py # command + file runner with --dry-run + zfs.py database.py secrets.py sftp.py compose.py # host ops + provision.py # provision / deprovision orchestration + backup.py # snapshot/dump + restore +tests/ # pure-logic unit tests (naming, config, render) +``` + +## Test +```bash +pip install -e ".[dev]" && pytest # or: PYTHONPATH=src pytest +``` + +## Known follow-ups +- **Web-root umask for php-fpm:** SFTP writes with umask 0002 (group-writable); + php-fpm-created files may need the same for two-way SFTP editing. See the note + in [site-templates/README.md](../site-templates/README.md). +- **git commit of `deployments/`** on provision (GitOps audit trail) — currently + left to the operator; wire into the flow when the deployments repo is set up. +- **`reconfigure` / `rotate-secret`** commands (documented in docs/05) — to add. diff --git a/control-panel/config.example.yaml b/control-panel/config.example.yaml new file mode 100644 index 0000000..ec0d487 --- /dev/null +++ b/control-panel/config.example.yaml @@ -0,0 +1,34 @@ +# heleos control-panel config. Copy to config.yaml (or /etc/heleos/config.yaml, +# or point at it with HELEOS_CONFIG) and adjust for your host. + +pool: tank +customers_root: /tank/customers + +# Where rendered per-site configs are written (the deployments repo/dir). +deployments_dir: /opt/heleos/deployments +# Where the profile templates live (this monorepo's site-templates/profiles). +templates_dir: /opt/heleos/site-templates/profiles + +# Container image registry + image reference templates. +registry: git.example.com/heleos +images: + nginx: "{registry}/nginx:latest" + php_fpm: "{registry}/php-fpm:{php_version}" + wordpress: "wordpress:{php_version}-fpm-alpine" + +# Shared MariaDB. Root password is read from this .env (or the +# MARIADB_ROOT_PASSWORD environment variable). +mariadb_container: mariadb +mariadb_env_file: /opt/heleos/platform-infra/stacks/mariadb/.env + +# SOPS/age recipient (public key) used to encrypt each site's .env into +# secrets.enc.yaml. Leave empty to skip encryption (dev only). +sops_age_recipient: "" + +# fpm user/group the web root is owned by (matches the alpine images: www-data). +fpm_uid: 82 +fpm_gid: 82 + +default_resources: + cpu: "1.0" + memory: "512m" diff --git a/control-panel/examples/acme-shop.site.yaml b/control-panel/examples/acme-shop.site.yaml new file mode 100644 index 0000000..bcc0685 --- /dev/null +++ b/control-panel/examples/acme-shop.site.yaml @@ -0,0 +1,12 @@ +# Example site definition (custom-php with a database). +customer: acme +site: shop +profile: custom-php +domains: + - shop.acme.com + - www.shop.acme.com +php_version: "8.3" +database: true +resources: + cpu: "1.0" + memory: "512m" diff --git a/control-panel/pyproject.toml b/control-panel/pyproject.toml new file mode 100644 index 0000000..3eb06a3 --- /dev/null +++ b/control-panel/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "heleos-control-panel" +version = "0.1.0" +description = "Provisioning CLI for the heleos multi-tenant hosting platform" +requires-python = ">=3.10" +dependencies = [ + "click>=8.1", + "jinja2>=3.1", + "pyyaml>=6.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0"] + +[project.scripts] +heleosctl = "heleos.cli:main" + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/control-panel/src/heleos/__init__.py b/control-panel/src/heleos/__init__.py new file mode 100644 index 0000000..e5cacb2 --- /dev/null +++ b/control-panel/src/heleos/__init__.py @@ -0,0 +1,3 @@ +"""heleos control-panel — provisioning CLI for the multi-tenant hosting platform.""" + +__version__ = "0.1.0" diff --git a/control-panel/src/heleos/backup.py b/control-panel/src/heleos/backup.py new file mode 100644 index 0000000..571bfe0 --- /dev/null +++ b/control-panel/src/heleos/backup.py @@ -0,0 +1,63 @@ +"""Backup/restore for a single site: ZFS snapshot for files, mariadb-dump for DB. + +Two decoupled streams (see docs/06-backup-and-dr.md). Offsite `zfs send` / rsync +are scheduled separately (Phase 5); this module covers the on-host operations the +CLI drives. +""" +from __future__ import annotations + +from datetime import datetime +from pathlib import Path + +from . import database, naming, zfs +from .config import Config, Site +from .runner import Runner + + +def db_backup_dir(cfg: Config, site: Site) -> str: + return f"/{cfg.pool}/platform/db-backups/{site.customer}/{site.site}" + + +def dump_database(runner: Runner, cfg: Config, site: Site) -> str | None: + if not site.database: + return None + db = naming.database_name(site.customer, site.site) + ts = datetime.now().strftime("%Y%m%d-%H%M%S") + out_dir = db_backup_dir(cfg, site) + dest = f"{out_dir}/{db}-{ts}.sql" + runner.mkdir(out_dir) + pw = "" if runner.dry_run else database.root_password(cfg) + sql = runner.run( + ["docker", "exec", cfg.mariadb_container, "mariadb-dump", + "--single-transaction", "--databases", db, "-uroot", f"-p{pw}"], + capture=True, secret=True, + ) + if not runner.dry_run: + runner.write_file(dest, sql) + else: + runner._echo(f"DRY write {dest}") + return dest + + +def backup_site(runner: Runner, cfg: Config, site: Site) -> dict: + snap = zfs.snapshot(runner, zfs.web_dataset(cfg, site)) + dump = dump_database(runner, cfg, site) + return {"snapshot": snap, "dump": dump} + + +def restore_files(runner: Runner, cfg: Config, site: Site, snapshot: str) -> None: + ds = zfs.web_dataset(cfg, site) + name = snapshot if "@" in snapshot else f"{ds}@{snapshot}" + # -r discards snapshots newer than the target; acceptable for a deliberate restore. + runner.run(["zfs", "rollback", "-r", name]) + + +def restore_database(runner: Runner, cfg: Config, site: Site, dump_path: str) -> None: + if not site.database: + return + pw = "" if runner.dry_run else database.root_password(cfg) + data = "" if runner.dry_run else Path(dump_path).read_text(encoding="utf-8") + runner.run( + ["docker", "exec", "-i", cfg.mariadb_container, "mariadb", "-uroot", f"-p{pw}"], + input=data, secret=True, + ) diff --git a/control-panel/src/heleos/cli.py b/control-panel/src/heleos/cli.py new file mode 100644 index 0000000..3ff04d7 --- /dev/null +++ b/control-panel/src/heleos/cli.py @@ -0,0 +1,149 @@ +"""heleosctl — command-line entry point.""" +from __future__ import annotations + +import sys +from pathlib import Path + +import click +import yaml + +from . import provision as provision_mod +from . import backup as backup_mod +from . import render as render_mod +from .config import Config, Site +from .errors import HeleosError +from .runner import Runner + + +def _load_site(cfg: Config, file: str | None, customer: str | None, site: str | None) -> Site: + if file: + return Site.load(file) + if customer and site: + path = cfg.deployments_dir / customer / site / "site.yaml" + if not path.is_file(): + raise HeleosError(f"no site.yaml at {path}") + return Site.load(path) + raise HeleosError("provide --file, or both --customer and --site") + + +@click.group() +@click.option("--config", "config_path", default=None, help="Path to config.yaml.") +@click.option("--dry-run", is_flag=True, help="Print actions without executing them.") +@click.option("--quiet", is_flag=True, help="Suppress the command log.") +@click.pass_context +def cli(ctx: click.Context, config_path, dry_run, quiet): + """Provisioning CLI for the heleos hosting platform.""" + cfg = Config.load(config_path) + ctx.obj = { + "cfg": cfg, + "runner": Runner(dry_run=dry_run, verbose=not quiet), + } + + +# Shared site-selection options. +def _site_opts(f): + f = click.option("-f", "--file", default=None, help="Path to a site.yaml.")(f) + f = click.option("-c", "--customer", default=None)(f) + f = click.option("-s", "--site", default=None)(f) + return f + + +@cli.command() +@_site_opts +@click.pass_context +def provision(ctx, file, customer, site): + """Provision a site from its site.yaml.""" + cfg, runner = ctx.obj["cfg"], ctx.obj["runner"] + s = _load_site(cfg, file, customer, site) + summary = provision_mod.provision(cfg, s, runner) + click.echo(yaml.safe_dump({"provisioned": summary}, sort_keys=False)) + + +@cli.command() +@_site_opts +@click.option("--purge", is_flag=True, help="Destroy the site's data (after a final backup).") +@click.option("--no-backup", is_flag=True, help="Skip the final backup on deprovision.") +@click.pass_context +def deprovision(ctx, file, customer, site, purge, no_backup): + """Tear down a site. Data is kept unless --purge is given.""" + cfg, runner = ctx.obj["cfg"], ctx.obj["runner"] + s = _load_site(cfg, file, customer, site) + summary = provision_mod.deprovision(cfg, s, runner, purge=purge, final_backup=not no_backup) + click.echo(yaml.safe_dump({"deprovisioned": summary}, sort_keys=False)) + + +@cli.command() +@_site_opts +@click.pass_context +def render(ctx, file, customer, site): + """Render a site's templates to stdout (no writes).""" + cfg = ctx.obj["cfg"] + s = _load_site(cfg, file, customer, site) + for name, content in render_mod.render(cfg, s).items(): + click.echo(f"# ===== {name} =====") + click.echo(content) + + +@cli.command() +@_site_opts +@click.pass_context +def backup(ctx, file, customer, site): + """Snapshot a site's files + dump its database.""" + cfg, runner = ctx.obj["cfg"], ctx.obj["runner"] + s = _load_site(cfg, file, customer, site) + result = backup_mod.backup_site(runner, cfg, s) + click.echo(yaml.safe_dump({"backup": result}, sort_keys=False)) + + +@cli.command() +@_site_opts +@click.option("--snapshot", required=True, help="Snapshot name or full dataset@snap.") +@click.option("--db", "dump", default=None, help="Path to a DB dump to restore.") +@click.pass_context +def restore(ctx, file, customer, site, snapshot, dump): + """Restore a site's files (from a snapshot) and DB (from a dump).""" + cfg, runner = ctx.obj["cfg"], ctx.obj["runner"] + s = _load_site(cfg, file, customer, site) + backup_mod.restore_files(runner, cfg, s, snapshot) + if dump: + backup_mod.restore_database(runner, cfg, s, dump) + click.echo("restore complete") + + +@cli.command(name="list") +@click.pass_context +def list_sites(ctx): + """List provisioned sites found under the deployments directory.""" + cfg = ctx.obj["cfg"] + rows = [] + for meta in sorted(cfg.deployments_dir.glob("*/*/site.yaml")): + data = yaml.safe_load(meta.read_text(encoding="utf-8")) or {} + rows.append((data.get("customer", "?"), data.get("site", "?"), + data.get("profile", "?"), ",".join(data.get("domains", [])))) + if not rows: + click.echo("no sites provisioned") + return + width = [max(len(r[i]) for r in rows + [("customer", "site", "profile", "domains")]) for i in range(4)] + header = ("customer", "site", "profile", "domains") + click.echo(" ".join(h.ljust(width[i]) for i, h in enumerate(header))) + for r in rows: + click.echo(" ".join(r[i].ljust(width[i]) for i in range(4))) + + +def main() -> int: + try: + cli(standalone_mode=False) + return 0 + except HeleosError as e: + click.echo(f"error: {e}", err=True) + return 1 + except click.ClickException as e: + e.show() + return e.exit_code + except click.exceptions.Abort: + click.echo("aborted", err=True) + return 130 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/control-panel/src/heleos/compose.py b/control-panel/src/heleos/compose.py new file mode 100644 index 0000000..7f89f37 --- /dev/null +++ b/control-panel/src/heleos/compose.py @@ -0,0 +1,23 @@ +"""Docker Compose bring-up / tear-down for a site's deployment directory.""" +from __future__ import annotations + +from pathlib import Path + +from .runner import Runner + + +def _base(deploy_dir: Path) -> list[str]: + return ["docker", "compose", + "--project-directory", str(deploy_dir), + "-f", str(deploy_dir / "docker-compose.yml")] + + +def up(runner: Runner, deploy_dir: Path) -> None: + runner.run(_base(deploy_dir) + ["up", "-d"]) + + +def down(runner: Runner, deploy_dir: Path, *, remove_orphans: bool = True) -> None: + cmd = _base(deploy_dir) + ["down"] + if remove_orphans: + cmd.append("--remove-orphans") + runner.run(cmd) diff --git a/control-panel/src/heleos/config.py b/control-panel/src/heleos/config.py new file mode 100644 index 0000000..7c5674f --- /dev/null +++ b/control-panel/src/heleos/config.py @@ -0,0 +1,106 @@ +"""Platform + site configuration loading and validation.""" +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +from . import naming +from .errors import ValidationError + +PROFILES = {"static", "redirect", "custom-php", "wordpress"} +CONFIG_ENV = "HELEOS_CONFIG" +DEFAULT_CONFIG_PATHS = ("./config.yaml", "/etc/heleos/config.yaml") + + +@dataclass +class Config: + pool: str = "tank" + customers_root: str = "/tank/customers" + deployments_dir: Path = Path("deployments") + templates_dir: Path = Path("site-templates/profiles") + registry: str = "git.example.com/heleos" + images: dict = field(default_factory=lambda: { + "nginx": "{registry}/nginx:latest", + "php_fpm": "{registry}/php-fpm:{php_version}", + "wordpress": "wordpress:{php_version}-fpm-alpine", + }) + mariadb_container: str = "mariadb" + mariadb_env_file: str = "platform-infra/stacks/mariadb/.env" + sops_age_recipient: str = "" + default_resources: dict = field(default_factory=lambda: {"cpu": "1.0", "memory": "512m"}) + fpm_uid: int = 82 + fpm_gid: int = 82 + + @classmethod + def load(cls, path: str | None = None) -> "Config": + chosen = path or os.environ.get(CONFIG_ENV) + if not chosen: + chosen = next((p for p in DEFAULT_CONFIG_PATHS if Path(p).is_file()), None) + data: dict = {} + if chosen and Path(chosen).is_file(): + data = yaml.safe_load(Path(chosen).read_text(encoding="utf-8")) or {} + cfg = cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__}) + cfg.deployments_dir = Path(cfg.deployments_dir) + cfg.templates_dir = Path(cfg.templates_dir) + return cfg + + def image(self, key: str, **fmt) -> str: + return self.images[key].format(registry=self.registry, **fmt) + + +@dataclass +class Site: + customer: str + site: str + profile: str + domains: list[str] + php_version: str = "8.3" + database: bool = False + redirect_to: str | None = None + redirect_code: int = 301 + resources: dict | None = None + + @classmethod + def load(cls, path: str | Path) -> "Site": + data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} + return cls.from_dict(data) + + @classmethod + def from_dict(cls, data: dict) -> "Site": + known = {k: v for k, v in data.items() if k in cls.__dataclass_fields__} + try: + site = cls(**known) + except TypeError as e: + raise ValidationError(f"invalid site definition: {e}") from e + site.validate() + return site + + def validate(self) -> None: + naming.validate_customer(self.customer) + naming.validate_site(self.site) + if self.profile not in PROFILES: + raise ValidationError( + f"unknown profile {self.profile!r}; must be one of {sorted(PROFILES)}" + ) + # WordPress always needs a database. + if self.profile == "wordpress": + self.database = True + # Every profile needs at least one (source) domain for the Host rule. + if not self.domains: + raise ValidationError(f"{self.profile} site requires at least one domain") + # Redirect additionally needs a target URL. + if self.profile == "redirect" and not self.redirect_to: + raise ValidationError("redirect profile requires redirect_to") + if self.profile in {"static", "redirect"} and self.database: + raise ValidationError(f"{self.profile} profile cannot have a database") + + @property + def slug(self) -> str: + return naming.slug(self.customer, self.site) + + @property + def slug_underscored(self) -> str: + return naming.slug_underscored(self.customer, self.site) diff --git a/control-panel/src/heleos/context.py b/control-panel/src/heleos/context.py new file mode 100644 index 0000000..4c60bdf --- /dev/null +++ b/control-panel/src/heleos/context.py @@ -0,0 +1,33 @@ +"""Build the render context (template variables) from Config + Site.""" +from __future__ import annotations + +from .config import Config, Site + + +def webroot(cfg: Config, site: Site) -> str: + return f"{cfg.customers_root}/{site.customer}/{site.site}/web" + + +def build(cfg: Config, site: Site) -> dict: + """Return the variable dict used to render a profile's templates. + + Matches the render-context table in site-templates/README.md. + """ + resources = site.resources or cfg.default_resources + return { + "customer": site.customer, + "site": site.site, + "slug": site.slug, + "slug_underscored": site.slug_underscored, + "profile": site.profile, + "domains": list(site.domains), + "webroot": webroot(cfg, site), + "database": site.database, + "redirect_to": site.redirect_to, + "redirect_code": site.redirect_code, + "php_version": site.php_version, + "nginx_image": cfg.image("nginx"), + "php_image": cfg.image("php_fpm", php_version=site.php_version), + "wordpress_image": cfg.image("wordpress", php_version=site.php_version), + "resources": resources, + } diff --git a/control-panel/src/heleos/database.py b/control-panel/src/heleos/database.py new file mode 100644 index 0000000..f68b3d4 --- /dev/null +++ b/control-panel/src/heleos/database.py @@ -0,0 +1,62 @@ +"""Shared-MariaDB operations: create/drop per-site database + least-priv user. + +SQL is piped over stdin to ``docker exec -i mariadb mariadb`` so the root +password never appears in the process list; the command itself is logged redacted. +""" +from __future__ import annotations + +import os +from pathlib import Path + +from . import naming +from .config import Config, Site +from .errors import CommandError +from .runner import Runner + + +def root_password(cfg: Config) -> str: + val = os.environ.get("MARIADB_ROOT_PASSWORD") + if val: + return val + p = Path(cfg.mariadb_env_file) + if p.is_file(): + for line in p.read_text(encoding="utf-8").splitlines(): + if line.startswith("MARIADB_ROOT_PASSWORD="): + return line.split("=", 1)[1].strip() + raise CommandError( + "MARIADB_ROOT_PASSWORD not set (env var or mariadb .env at " + f"{cfg.mariadb_env_file})" + ) + + +def _exec_sql(runner: Runner, cfg: Config, sql: str) -> None: + pw = "" if runner.dry_run else root_password(cfg) + runner.run( + ["docker", "exec", "-i", cfg.mariadb_container, "mariadb", "-uroot", f"-p{pw}"], + input=sql, secret=True, + ) + + +def create_db_user(runner: Runner, cfg: Config, site: Site, password: str) -> None: + db = naming.database_name(site.customer, site.site) + user = naming.database_user(site.customer, site.site) + sql = ( + f"CREATE DATABASE IF NOT EXISTS `{db}` " + "CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;\n" + f"CREATE USER IF NOT EXISTS '{user}'@'%' IDENTIFIED BY '{password}';\n" + f"ALTER USER '{user}'@'%' IDENTIFIED BY '{password}';\n" + f"GRANT ALL PRIVILEGES ON `{db}`.* TO '{user}'@'%';\n" + "FLUSH PRIVILEGES;\n" + ) + _exec_sql(runner, cfg, sql) + + +def drop_db_user(runner: Runner, cfg: Config, site: Site) -> None: + db = naming.database_name(site.customer, site.site) + user = naming.database_user(site.customer, site.site) + sql = ( + f"DROP USER IF EXISTS '{user}'@'%';\n" + f"DROP DATABASE IF EXISTS `{db}`;\n" + "FLUSH PRIVILEGES;\n" + ) + _exec_sql(runner, cfg, sql) diff --git a/control-panel/src/heleos/errors.py b/control-panel/src/heleos/errors.py new file mode 100644 index 0000000..d6ce558 --- /dev/null +++ b/control-panel/src/heleos/errors.py @@ -0,0 +1,13 @@ +"""Typed errors so the CLI can report clean messages instead of tracebacks.""" + + +class HeleosError(Exception): + """Base class for expected, user-facing errors.""" + + +class ValidationError(HeleosError): + """Invalid site.yaml, naming, or configuration.""" + + +class CommandError(HeleosError): + """A shelled-out command failed.""" diff --git a/control-panel/src/heleos/naming.py b/control-panel/src/heleos/naming.py new file mode 100644 index 0000000..bd8f021 --- /dev/null +++ b/control-panel/src/heleos/naming.py @@ -0,0 +1,61 @@ +"""Naming rules — the single source of the slug/identifier conventions. + +Mirrors docs/03-naming-conventions.md. The slug (``-``) is the +join key across Docker, the database, and deployment paths. +""" +from __future__ import annotations + +import re + +from .errors import ValidationError + +CUSTOMER_RE = re.compile(r"^[a-z][a-z0-9-]{1,31}$") # 2–32 chars, starts alpha +SITE_RE = re.compile(r"^[a-z][a-z0-9-]{1,39}$") # 2–40 chars, starts alpha + +# Names that would collide with platform resources (see docs/03 §8). +RESERVED = { + "proxy", "platform", "traefik", "mariadb", "forgejo", "monitoring", "docker", + "customers", "platform-infra", "site-templates", "deployments", "control-panel", +} + + +def validate_customer(customer: str) -> str: + if not CUSTOMER_RE.match(customer): + raise ValidationError( + f"invalid customer id {customer!r}: must match {CUSTOMER_RE.pattern}" + ) + if customer in RESERVED: + raise ValidationError(f"customer id {customer!r} is reserved") + return customer + + +def validate_site(site: str) -> str: + if not SITE_RE.match(site): + raise ValidationError( + f"invalid site id {site!r}: must match {SITE_RE.pattern}" + ) + if site in RESERVED: + raise ValidationError(f"site id {site!r} is reserved") + return site + + +def slug(customer: str, site: str) -> str: + """Globally-unique flattened id used for Docker/router names.""" + return f"{customer}-{site}" + + +def slug_underscored(customer: str, site: str) -> str: + """MySQL-safe identifier stem (hyphens → underscores).""" + return slug(customer, site).replace("-", "_") + + +def database_name(customer: str, site: str) -> str: + return f"db_{slug_underscored(customer, site)}" + + +def database_user(customer: str, site: str) -> str: + return f"u_{slug_underscored(customer, site)}" + + +def sftp_user(customer: str) -> str: + return f"sftp_{customer}" diff --git a/control-panel/src/heleos/provision.py b/control-panel/src/heleos/provision.py new file mode 100644 index 0000000..bd6d3bd --- /dev/null +++ b/control-panel/src/heleos/provision.py @@ -0,0 +1,84 @@ +"""Provisioning orchestrator: ties ZFS, DB, secrets, render, SFTP and compose +into the provision / deprovision flows described in docs/05-provisioning-workflow.md. +""" +from __future__ import annotations + +from pathlib import Path + +import yaml + +from . import backup, compose, database, render, secrets, sftp, zfs +from .config import Config, Site +from .runner import Runner + +SITE_FIELDS = ("customer", "site", "profile", "domains", "php_version", + "database", "redirect_to", "redirect_code", "resources") + + +def deployment_dir(cfg: Config, site: Site) -> Path: + return cfg.deployments_dir / site.customer / site.site + + +def _site_to_dict(site: Site) -> dict: + return {f: getattr(site, f) for f in SITE_FIELDS} + + +def provision(cfg: Config, site: Site, runner: Runner) -> dict: + """Provision a site end-to-end. Idempotent where the underlying steps are.""" + deploy_dir = deployment_dir(cfg, site) + summary: dict = {"slug": site.slug, "deploy_dir": str(deploy_dir)} + + # 1. Storage + zfs.create_web(runner, cfg, site) + sftp.set_web_ownership(runner, cfg, site) + + # 2. Database + secrets (only for DB profiles) + if site.database: + db_password = secrets.generate_password() + database.create_db_user(runner, cfg, site, db_password) + secrets.write_env(runner, deploy_dir, {secrets.db_env_key(site): db_password}) + secrets.encrypt_env(runner, cfg, deploy_dir) + + # 3. Render compose + nginx config + persist site.yaml (source of truth) + for name, content in render.render(cfg, site).items(): + runner.write_file(deploy_dir / name, content) + runner.write_file(deploy_dir / "site.yaml", + yaml.safe_dump(_site_to_dict(site), sort_keys=False)) + + # 4. SFTP (per customer; password only set when the account is first created) + sftp_password = secrets.generate_password() + created = sftp.ensure_customer_account(runner, cfg, site.customer, sftp_password) + summary["sftp_user"] = f"sftp_{site.customer}" + summary["sftp_password"] = sftp_password if created else None + + # 5. Deploy + compose.up(runner, deploy_dir) + return summary + + +def deprovision(cfg: Config, site: Site, runner: Runner, *, + purge: bool = False, final_backup: bool = True) -> dict: + """Tear down a site. Data is destroyed only with ``purge=True`` and always + after a final backup. + """ + deploy_dir = deployment_dir(cfg, site) + summary: dict = {"slug": site.slug, "purged": purge} + + compose.down(runner, deploy_dir) + + if final_backup: + summary["final_backup"] = backup.backup_site(runner, cfg, site) + + if site.database: + database.drop_db_user(runner, cfg, site) + + if purge: + # Destroy the site dataset (its /web child included) but keep the customer + # dataset + SFTP account — other sites may still belong to this customer. + zfs.destroy(runner, f"{cfg.pool}/customers/{site.customer}/{site.site}") + runner.run(["rm", "-rf", str(deploy_dir)]) + summary["data_destroyed"] = True + else: + summary["data_destroyed"] = False + summary["note"] = "config/containers removed; data retained. Re-run with purge to delete." + return summary diff --git a/control-panel/src/heleos/render.py b/control-panel/src/heleos/render.py new file mode 100644 index 0000000..7c7152d --- /dev/null +++ b/control-panel/src/heleos/render.py @@ -0,0 +1,47 @@ +"""Render a profile's Jinja2 templates into a site's deployment directory.""" +from __future__ import annotations + +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +from . import context +from .config import Config, Site +from .errors import ValidationError + + +def _env(profile_dir: Path) -> Environment: + return Environment( + loader=FileSystemLoader(str(profile_dir)), + undefined=StrictUndefined, + keep_trailing_newline=True, + ) + + +def render(cfg: Config, site: Site) -> dict[str, str]: + """Render all ``*.j2`` files for the site's profile. + + Returns a mapping of output filename (``.j2`` stripped) → rendered content. + Pure: does not write to disk. + """ + profile_dir = cfg.templates_dir / site.profile + if not profile_dir.is_dir(): + raise ValidationError(f"no templates for profile {site.profile!r} at {profile_dir}") + ctx = context.build(cfg, site) + env = _env(profile_dir) + out: dict[str, str] = {} + for tpl in sorted(profile_dir.glob("*.j2")): + rendered = env.get_template(tpl.name).render(**ctx) + out[tpl.name[:-len(".j2")]] = rendered + return out + + +def write(rendered: dict[str, str], out_dir: Path) -> list[Path]: + """Write rendered files into out_dir, returning the paths written.""" + out_dir.mkdir(parents=True, exist_ok=True) + written = [] + for name, content in rendered.items(): + p = out_dir / name + p.write_text(content, encoding="utf-8", newline="\n") + written.append(p) + return written diff --git a/control-panel/src/heleos/runner.py b/control-panel/src/heleos/runner.py new file mode 100644 index 0000000..d736f5b --- /dev/null +++ b/control-panel/src/heleos/runner.py @@ -0,0 +1,65 @@ +"""Thin command runner. Every host mutation goes through here so we get uniform +logging and a real ``--dry-run`` that prints commands instead of running them.""" +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +from pathlib import Path + +from .errors import CommandError + + +class Runner: + def __init__(self, dry_run: bool = False, verbose: bool = True): + self.dry_run = dry_run + self.verbose = verbose + + def _echo(self, msg: str) -> None: + if self.verbose: + print(msg, file=sys.stderr) + + def run(self, cmd: list[str], *, input: str | None = None, check: bool = True, + capture: bool = False, secret: bool = False) -> str: + """Run a command. In dry-run mode, print and return "". + + ``secret=True`` redacts the command in logs (e.g. contains a password). + """ + shown = "" if secret else " ".join(shlex.quote(c) for c in cmd) + prefix = "DRY " if self.dry_run else "RUN " + self._echo(prefix + shown) + if self.dry_run: + return "" + try: + proc = subprocess.run( + cmd, input=input, text=True, check=check, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + ) + except FileNotFoundError as e: + raise CommandError(f"command not found: {cmd[0]}") from e + except subprocess.CalledProcessError as e: + detail = (e.stderr or "").strip() if capture else "" + raise CommandError( + f"command failed ({e.returncode}): {shown}" + + (f"\n{detail}" if detail else "") + ) from e + return (proc.stdout or "") if capture else "" + + def mkdir(self, path: str | Path) -> None: + self._echo(("DRY " if self.dry_run else "RUN ") + f"mkdir -p {path}") + if not self.dry_run: + Path(path).mkdir(parents=True, exist_ok=True) + + def write_file(self, path: str | Path, content: str, *, mode: int | None = None, + secret: bool = False) -> None: + note = f"write {path}" + (" " if secret else "") + self._echo(("DRY " if self.dry_run else "RUN ") + note) + if self.dry_run: + return + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content, encoding="utf-8", newline="\n") + if mode is not None: + os.chmod(p, mode) diff --git a/control-panel/src/heleos/secrets.py b/control-panel/src/heleos/secrets.py new file mode 100644 index 0000000..eba2cc8 --- /dev/null +++ b/control-panel/src/heleos/secrets.py @@ -0,0 +1,39 @@ +"""Per-site secret generation + SOPS/age encryption of the .env file.""" +from __future__ import annotations + +import secrets as _secrets +from pathlib import Path + +from .config import Config, Site +from .runner import Runner + + +def generate_password(nbytes: int = 24) -> str: + """URL-safe token (~32 chars); no shell-special characters.""" + return _secrets.token_urlsafe(nbytes) + + +def db_env_key(site: Site) -> str: + return "WORDPRESS_DB_PASSWORD" if site.profile == "wordpress" else "DB_PASSWORD" + + +def write_env(runner: Runner, out_dir: Path, mapping: dict[str, str]) -> None: + body = "".join(f"{k}={v}\n" for k, v in mapping.items()) + runner.write_file(out_dir / ".env", body, mode=0o600, secret=True) + + +def encrypt_env(runner: Runner, cfg: Config, out_dir: Path) -> None: + """Encrypt .env → secrets.enc.yaml with SOPS/age (the committed artifact).""" + if not cfg.sops_age_recipient: + runner._echo("WARN no sops_age_recipient configured; skipping encryption " + "(.env is git-ignored but secrets.enc.yaml will not be created)") + return + if runner.dry_run: + runner._echo(f"DRY sops --encrypt {out_dir/'.env'} > {out_dir/'secrets.enc.yaml'}") + return + out = runner.run( + ["sops", "--encrypt", "--age", cfg.sops_age_recipient, + "--input-type", "dotenv", "--output-type", "yaml", str(out_dir / ".env")], + capture=True, + ) + runner.write_file(out_dir / "secrets.enc.yaml", out) diff --git a/control-panel/src/heleos/sftp.py b/control-panel/src/heleos/sftp.py new file mode 100644 index 0000000..a068477 --- /dev/null +++ b/control-panel/src/heleos/sftp.py @@ -0,0 +1,73 @@ +"""Per-customer SFTP account + web-root ownership. + +One chrooted SFTP user per customer (``sftp_``), chrooted to the +customer directory so they see each of their sites' web roots as sub-folders. + +Ownership scheme (see site-templates/README.md "provisioning note"): +- The customer dir (chroot target) stays root-owned 755 — an sshd requirement. +- Each site's web dir is chowned to the fpm uid/gid and set 2775 (setgid) so + files created there stay group-owned and group-writable. +- The SFTP user's primary group is the fpm group, so php-fpm and SFTP share + group access. SFTP writes with umask 0002 (set via ForceCommand) to keep new + files group-writable. (php-fpm umask hardening is a documented follow-up.) +""" +from __future__ import annotations + +from . import context, naming +from .config import Config, Site +from .runner import Runner + +SSHD_DROPIN = "/etc/ssh/sshd_config.d/sftp-{customer}.conf" + + +def _user_exists(runner: Runner, user: str) -> bool: + if runner.dry_run: + return False + out = runner.run(["id", "-u", user], check=False, capture=True) + return out.strip().isdigit() + + +def set_web_ownership(runner: Runner, cfg: Config, site: Site) -> None: + web = context.webroot(cfg, site) + runner.run(["chown", "-R", f"{cfg.fpm_uid}:{cfg.fpm_gid}", web]) + runner.run(["chmod", "2775", web]) + + +def ensure_customer_account(runner: Runner, cfg: Config, customer: str, + password: str) -> bool: + """Ensure the customer's SFTP account + chroot config exist. Idempotent: + the password is set only when the account is first created (so provisioning a + second site for an existing customer doesn't reset their password). + + Returns True if the account was newly created. + """ + user = naming.sftp_user(customer) + home = f"{cfg.customers_root}/{customer}" + created = not _user_exists(runner, user) + if created: + runner.run([ + "useradd", "-M", "-N", + "-g", str(cfg.fpm_gid), # primary group = fpm group + "-G", "sftponly", # supplementary: enables the Match block + "-s", "/usr/sbin/nologin", + "-d", home, + user, + ]) + runner.run(["chpasswd"], input=f"{user}:{password}\n", secret=True) + + dropin = SSHD_DROPIN.format(customer=customer) + content = ( + f"Match User {user}\n" + f" ChrootDirectory {home}\n" + f" ForceCommand internal-sftp -u 0002\n" + ) + runner.write_file(dropin, content, mode=0o644) + runner.run(["systemctl", "reload", "ssh"]) + return created + + +def remove_customer_account(runner: Runner, cfg: Config, customer: str) -> None: + user = naming.sftp_user(customer) + runner.run(["userdel", user], check=False) + runner.run(["rm", "-f", SSHD_DROPIN.format(customer=customer)]) + runner.run(["systemctl", "reload", "ssh"]) diff --git a/control-panel/src/heleos/zfs.py b/control-panel/src/heleos/zfs.py new file mode 100644 index 0000000..fa3846c --- /dev/null +++ b/control-panel/src/heleos/zfs.py @@ -0,0 +1,47 @@ +"""ZFS operations for a site's web dataset.""" +from __future__ import annotations + +from datetime import datetime + +from .config import Config, Site +from .runner import Runner + + +def web_dataset(cfg: Config, site: Site) -> str: + return f"{cfg.pool}/customers/{site.customer}/{site.site}/web" + + +def customer_dataset(cfg: Config, customer: str) -> str: + return f"{cfg.pool}/customers/{customer}" + + +def exists(runner: Runner, dataset: str) -> bool: + if runner.dry_run: + return False + out = runner.run(["zfs", "list", "-H", "-o", "name", dataset], + check=False, capture=True) + return dataset in out.split() + + +def create_web(runner: Runner, cfg: Config, site: Site) -> str: + ds = web_dataset(cfg, site) + if not exists(runner, ds): + # -p creates the customer/site parent datasets too (root-owned, 755 — + # suitable as the SFTP chroot). The web child is chowned by the sftp step. + runner.run(["zfs", "create", "-p", ds]) + return ds + + +def snapshot(runner: Runner, dataset: str, tag: str | None = None) -> str: + tag = tag or "auto-" + datetime.now().strftime("%Y%m%d-%H%M%S") + name = f"{dataset}@{tag}" + runner.run(["zfs", "snapshot", name]) + return name + + +def destroy(runner: Runner, dataset: str, *, recursive: bool = True) -> None: + cmd = ["zfs", "destroy"] + if recursive: + cmd.append("-r") + cmd.append(dataset) + runner.run(cmd) diff --git a/control-panel/tests/test_config.py b/control-panel/tests/test_config.py new file mode 100644 index 0000000..cf68883 --- /dev/null +++ b/control-panel/tests/test_config.py @@ -0,0 +1,34 @@ +import pytest + +from heleos.config import Site +from heleos.errors import ValidationError + + +def test_wordpress_forces_database(): + s = Site.from_dict({"customer": "acme", "site": "blog", "profile": "wordpress", + "domains": ["blog.acme.com"], "database": False}) + assert s.database is True + + +def test_redirect_requires_target(): + with pytest.raises(ValidationError): + Site.from_dict({"customer": "acme", "site": "old", "profile": "redirect", + "domains": ["old.acme.com"]}) + + +def test_redirect_needs_source_domain(): + with pytest.raises(ValidationError): + Site.from_dict({"customer": "acme", "site": "old", "profile": "redirect", + "domains": [], "redirect_to": "https://acme.com"}) + + +def test_static_cannot_have_database(): + with pytest.raises(ValidationError): + Site.from_dict({"customer": "acme", "site": "site", "profile": "static", + "domains": ["acme.com"], "database": True}) + + +def test_unknown_profile_rejected(): + with pytest.raises(ValidationError): + Site.from_dict({"customer": "acme", "site": "x", "profile": "django", + "domains": ["acme.com"]}) diff --git a/control-panel/tests/test_naming.py b/control-panel/tests/test_naming.py new file mode 100644 index 0000000..bd0b38d --- /dev/null +++ b/control-panel/tests/test_naming.py @@ -0,0 +1,31 @@ +import pytest + +from heleos import naming +from heleos.errors import ValidationError + + +def test_slug_and_db_identifiers(): + assert naming.slug("acme", "shop") == "acme-shop" + assert naming.slug_underscored("acme", "my-shop") == "acme_my_shop" + assert naming.database_name("acme", "shop") == "db_acme_shop" + assert naming.database_user("acme", "shop") == "u_acme_shop" + assert naming.sftp_user("acme") == "sftp_acme" + + +@pytest.mark.parametrize("bad", ["", "A", "1abc", "ab_cd", "-abc", "x" * 40]) +def test_invalid_customer(bad): + with pytest.raises(ValidationError): + naming.validate_customer(bad) + + +@pytest.mark.parametrize("name", ["proxy", "platform", "mariadb", "deployments"]) +def test_reserved_names_rejected(name): + with pytest.raises(ValidationError): + naming.validate_customer(name) + with pytest.raises(ValidationError): + naming.validate_site(name) + + +def test_valid_ids(): + assert naming.validate_customer("acme") == "acme" + assert naming.validate_site("blog-2") == "blog-2" diff --git a/control-panel/tests/test_render.py b/control-panel/tests/test_render.py new file mode 100644 index 0000000..1b5b863 --- /dev/null +++ b/control-panel/tests/test_render.py @@ -0,0 +1,66 @@ +from pathlib import Path + +import yaml + +from heleos import context, render +from heleos.config import Config, Site + +REPO_ROOT = Path(__file__).resolve().parents[2] +TEMPLATES = REPO_ROOT / "site-templates" / "profiles" + + +def _cfg(): + return Config(templates_dir=TEMPLATES, registry="reg.example.com/heleos") + + +def test_context_resolves_images_and_paths(): + cfg = _cfg() + s = Site.from_dict({"customer": "acme", "site": "shop", "profile": "custom-php", + "domains": ["shop.acme.com"], "database": True}) + ctx = context.build(cfg, s) + assert ctx["slug"] == "acme-shop" + assert ctx["slug_underscored"] == "acme_shop" + assert ctx["webroot"] == "/tank/customers/acme/shop/web" + assert ctx["php_image"] == "reg.example.com/heleos/php-fpm:8.3" + assert ctx["nginx_image"] == "reg.example.com/heleos/nginx:latest" + + +def test_custom_php_renders_valid_compose_with_db(): + cfg = _cfg() + s = Site.from_dict({"customer": "acme", "site": "shop", "profile": "custom-php", + "domains": ["shop.acme.com", "www.shop.acme.com"], "database": True}) + out = render.render(cfg, s) + doc = yaml.safe_load(out["docker-compose.yml"]) + assert doc["name"] == "acme-shop" + assert set(doc["services"]) == {"nginx", "fpm"} + # fpm reaches the DB, so it is on the platform network. + assert "platform" in doc["services"]["fpm"]["networks"] + labels = doc["services"]["nginx"]["labels"] + rule = [l for l in labels if ".rule=" in l][0] + assert "Host(`shop.acme.com`) || Host(`www.shop.acme.com`)" in rule + + +def test_custom_php_without_db_has_no_platform_network(): + cfg = _cfg() + s = Site.from_dict({"customer": "acme", "site": "api", "profile": "custom-php", + "domains": ["api.acme.com"], "database": False}) + doc = yaml.safe_load(render.render(cfg, s)["docker-compose.yml"]) + assert doc["services"]["fpm"]["networks"] == ["site"] + assert "platform" not in doc.get("networks", {}) + + +def test_static_profile_is_nginx_only(): + cfg = _cfg() + s = Site.from_dict({"customer": "acme", "site": "www", "profile": "static", + "domains": ["acme.com"]}) + doc = yaml.safe_load(render.render(cfg, s)["docker-compose.yml"]) + assert list(doc["services"]) == ["nginx"] + + +def test_redirect_profile_renders_redirect_conf(): + cfg = _cfg() + s = Site.from_dict({"customer": "acme", "site": "old", "profile": "redirect", + "domains": ["old.acme.com"], "redirect_to": "https://acme.com", + "redirect_code": 302}) + out = render.render(cfg, s) + assert "return 302 https://acme.com$request_uri;" in out["nginx-redirect.conf"]