From f53ccbab4a97be67d8a8dad52efc86ee115ee362 Mon Sep 17 00:00:00 2001 From: ml Date: Sun, 28 Jun 2026 16:20:47 +0200 Subject: [PATCH] Add public download server at dl.ladkau.de and improve vault validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New dl role — nginx serves files publicly over HTTPS with directory listing; atmoz/sftp on port 2223 for key-only uploads; both containers share /opt/dl/files volume - check-vault.sh gains a third tier: optional secrets are validated when present — vaultwarden_sso_client_secret must be non-empty, dl_sftp_authorized_keys must begin with a recognised SSH public key prefix - dl.ladkau.de added to DNS table, check-services.sh, and status dashboard - Configuration runbook section 7: deploy key generation, Gitea Actions scp workflow example, and SFTP client connection settings - Provisioning runbook documents the three-tier vault validation behaviour --- ansible/group_vars/all/vars.yml | 9 +++ ansible/roles/dashboard/templates/app.py.j2 | 1 + ansible/roles/dl/handlers/main.yml | 7 +++ ansible/roles/dl/tasks/main.yml | 62 +++++++++++++++++++ .../roles/dl/templates/docker-compose.yml.j2 | 34 ++++++++++ ansible/roles/dl/templates/nginx.conf.j2 | 19 ++++++ ansible/site.yml | 1 + docs/runbook-configuration.md | 54 +++++++++++++++- docs/runbook-provisioning.md | 26 ++++++-- scripts/check-services.sh | 1 + scripts/check-vault.sh | 40 +++++++++--- 11 files changed, 240 insertions(+), 14 deletions(-) create mode 100644 ansible/roles/dl/handlers/main.yml create mode 100644 ansible/roles/dl/tasks/main.yml create mode 100644 ansible/roles/dl/templates/docker-compose.yml.j2 create mode 100644 ansible/roles/dl/templates/nginx.conf.j2 diff --git a/ansible/group_vars/all/vars.yml b/ansible/group_vars/all/vars.yml index e523fef..cfe72e5 100644 --- a/ansible/group_vars/all/vars.yml +++ b/ansible/group_vars/all/vars.yml @@ -9,6 +9,7 @@ domain_mail: "mail.{{ domain_base }}" domain_registry: "cr.{{ domain_base }}" domain_k8s: "k8s.{{ domain_base }}" domain_vault: "vault.{{ domain_base }}" +domain_dl: "dl.{{ domain_base }}" # Let's Encrypt acme_email: matthias.ladkau@gmail.com @@ -105,5 +106,13 @@ vaultwarden_data_dir: /opt/vaultwarden # vaultwarden_admin_token: "" # generate: openssl rand -hex 32 # vaultwarden_sso_client_secret: "" # added after Keycloak is configured — see runbook-configuration.md step 2.6 +# Download server (nginx HTTPS + SFTP) +dl_data_dir: /opt/dl +dl_upload_user: uploader +dl_sftp_port: 2223 +# Secrets — store values in ansible/group_vars/all/vault.yml (Ansible Vault) +# dl_sftp_authorized_keys: | +# ssh-ed25519 AAAA... gitea-actions + # Status dashboard (public — cloud.ladkau.de root) dashboard_data_dir: /opt/dashboard diff --git a/ansible/roles/dashboard/templates/app.py.j2 b/ansible/roles/dashboard/templates/app.py.j2 index 7a6c9b5..841ae47 100644 --- a/ansible/roles/dashboard/templates/app.py.j2 +++ b/ansible/roles/dashboard/templates/app.py.j2 @@ -24,6 +24,7 @@ SERVICES = [ ("Registry", "https://{{ domain_registry }}/v2/", 401, "https://{{ domain_registry }}"), ("k8s", "https://{{ domain_k8s }}", 200, "https://{{ domain_k8s }}"), ("Vaultwarden", "https://{{ domain_vault }}", 200, "https://{{ domain_vault }}"), + ("Downloads", "https://{{ domain_dl }}", 200, "https://{{ domain_dl }}"), ] {% raw %} diff --git a/ansible/roles/dl/handlers/main.yml b/ansible/roles/dl/handlers/main.yml new file mode 100644 index 0000000..133a977 --- /dev/null +++ b/ansible/roles/dl/handlers/main.yml @@ -0,0 +1,7 @@ +--- +- name: Restart dl + community.docker.docker_compose_v2: + project_src: "{{ dl_data_dir }}" + state: present + pull: missing + recreate: always diff --git a/ansible/roles/dl/tasks/main.yml b/ansible/roles/dl/tasks/main.yml new file mode 100644 index 0000000..e5f1240 --- /dev/null +++ b/ansible/roles/dl/tasks/main.yml @@ -0,0 +1,62 @@ +--- +- name: Create dl data directory + ansible.builtin.file: + path: "{{ dl_data_dir }}" + state: directory + owner: root + group: root + mode: "0755" + tags: dl + +- name: Create dl files directory (sftp user UID 1001) + ansible.builtin.file: + path: "{{ dl_data_dir }}/files" + state: directory + owner: "1001" + group: "1001" + mode: "0755" + tags: dl + +- name: Deploy SFTP authorized keys + ansible.builtin.copy: + content: "{{ dl_sftp_authorized_keys }}" + dest: "{{ dl_data_dir }}/authorized_keys" + owner: root + group: root + mode: "0644" + notify: Restart dl + tags: dl + +- name: Deploy nginx config + ansible.builtin.template: + src: nginx.conf.j2 + dest: "{{ dl_data_dir }}/nginx.conf" + owner: root + group: root + mode: "0644" + notify: Restart dl + tags: dl + +- name: Deploy Docker Compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ dl_data_dir }}/docker-compose.yml" + owner: root + group: root + mode: "0644" + notify: Restart dl + tags: dl + +- name: Allow SFTP port through firewall + community.general.ufw: + rule: allow + port: "{{ dl_sftp_port }}" + proto: tcp + tags: dl + +- name: Start dl stack + community.docker.docker_compose_v2: + project_src: "{{ dl_data_dir }}" + state: present + pull: missing + tags: dl diff --git a/ansible/roles/dl/templates/docker-compose.yml.j2 b/ansible/roles/dl/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..b84b98b --- /dev/null +++ b/ansible/roles/dl/templates/docker-compose.yml.j2 @@ -0,0 +1,34 @@ +# Managed by Ansible — do not edit manually +services: + dl-web: + image: nginx:alpine + container_name: dl-web + restart: unless-stopped + volumes: + - {{ dl_data_dir }}/nginx.conf:/etc/nginx/conf.d/default.conf:ro + - {{ dl_data_dir }}/files:/var/www/dl:ro + networks: + - traefik_public + labels: + - "traefik.enable=true" + - "traefik.http.routers.dl.rule=Host(`{{ domain_dl }}`)" + - "traefik.http.routers.dl.entrypoints=websecure" + - "traefik.http.routers.dl.tls.certresolver=letsencrypt" + - "traefik.http.services.dl.loadbalancer.server.port=80" + - "traefik.http.routers.dl.middlewares=rate-limit@docker" + + dl-sftp: + image: atmoz/sftp + container_name: dl-sftp + restart: unless-stopped + volumes: + - {{ dl_data_dir }}/files:/home/{{ dl_upload_user }}/files + - {{ dl_data_dir }}/authorized_keys:/home/{{ dl_upload_user }}/.ssh/keys/authorized_keys:ro + ports: + - "{{ dl_sftp_port }}:22" + # Empty password disables password auth — key auth only + command: "{{ dl_upload_user }}::1001:1001:files" + +networks: + traefik_public: + external: true diff --git a/ansible/roles/dl/templates/nginx.conf.j2 b/ansible/roles/dl/templates/nginx.conf.j2 new file mode 100644 index 0000000..3834d5d --- /dev/null +++ b/ansible/roles/dl/templates/nginx.conf.j2 @@ -0,0 +1,19 @@ +# Managed by Ansible — do not edit manually +server { + listen 80; + server_name _; + + root /var/www/dl; + charset utf-8; + + location / { + autoindex on; + autoindex_exact_size off; + autoindex_localtime on; + } + + # Deny access to hidden files + location ~ /\. { + deny all; + } +} diff --git a/ansible/site.yml b/ansible/site.yml index 7ab29d5..2e202ff 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -13,4 +13,5 @@ - registry - k8s - vaultwarden + - dl - dashboard diff --git a/docs/runbook-configuration.md b/docs/runbook-configuration.md index 05d854f..8b6feb2 100644 --- a/docs/runbook-configuration.md +++ b/docs/runbook-configuration.md @@ -24,6 +24,7 @@ The script checks these endpoints and verifies the expected HTTP status code: | `https://cr.ladkau.de/v2/` | 401 | Registry API — auth required, correct without credentials | | `https://k8s.ladkau.de` | 200 | Placeholder page | | `https://vault.ladkau.de` | 200 | Vaultwarden web vault | +| `https://dl.ladkau.de` | 200 | Public download server | A `000` result means the connection was refused or timed out — a container that did not start. Keycloak and Nextcloud may return `502` for up to 90 seconds on @@ -234,7 +235,58 @@ automatically (public self-registration is otherwise disabled). In any official Bitwarden client, set the **Server URL** to `https://vault.ladkau.de` before logging in. -## 7. Container registry +## 7. Download server + +Files placed under `dl.ladkau.de` are publicly browsable and downloadable over +HTTPS. Upload is via SFTP on port 2223, key auth only. + +### 7.1 Generate the deploy key + +Run this once locally and keep both files: + +```bash +ssh-keygen -t ed25519 -f dl_deploy_key -N "" -C "gitea-actions" +``` + +- Add the contents of `dl_deploy_key.pub` to the vault as `dl_sftp_authorized_keys` + and redeploy (`--tags dl`) to authorise the key. +- Store `dl_deploy_key` (the private key) as a Gitea Actions secret named + `DL_SSH_KEY` in any repository that needs to publish releases. + +### 7.2 Upload from Gitea Actions + +Add a step to your workflow after building the binary: + +```yaml +- name: Upload release + run: | + echo "${{ secrets.DL_SSH_KEY }}" > /tmp/deploy_key + chmod 600 /tmp/deploy_key + ssh -i /tmp/deploy_key -p 2223 \ + -o StrictHostKeyChecking=no \ + uploader@dl.ladkau.de \ + "mkdir -p files/releases/${{ gitea.ref_name }}" + scp -i /tmp/deploy_key -P 2223 \ + -o StrictHostKeyChecking=no \ + dist/myapp-linux-amd64 \ + uploader@dl.ladkau.de:files/releases/${{ gitea.ref_name }}/ + rm /tmp/deploy_key +``` + +The file is then available at: +`https://dl.ladkau.de/releases/v1.2.3/myapp-linux-amd64` + +### 7.3 Connect an SFTP client + +| Setting | Value | +|-----------|--------------------| +| Host | `dl.ladkau.de` | +| Port | `2223` | +| User | `uploader` | +| Auth | SSH key | +| Root path | `files/` | + +## 8. Container registry ```bash # Login diff --git a/docs/runbook-provisioning.md b/docs/runbook-provisioning.md index a5ca30d..4f3520e 100644 --- a/docs/runbook-provisioning.md +++ b/docs/runbook-provisioning.md @@ -79,6 +79,7 @@ Let's Encrypt certificates on first start and DNS must resolve at that point. | cr.ladkau.de | A → server IP | | k8s.ladkau.de | A → server IP | | vault.ladkau.de | A → server IP | +| dl.ladkau.de | A → server IP | ### 4. Create the vault and populate secrets @@ -124,6 +125,12 @@ vaultwarden_admin_token: "" # generate: openssl rand -hex 32 # vaultwarden_sso_client_secret is added after Keycloak is configured — see # step 2.6 of runbook-configuration.md +# Download server — SFTP upload key +# Generate: ssh-keygen -t ed25519 -f dl_deploy_key -N "" -C "gitea-actions" +# Paste the contents of dl_deploy_key.pub here; store dl_deploy_key as a Gitea Actions secret +dl_sftp_authorized_keys: | + ssh-ed25519 AAAA... + # Dovecot IMAP users dovecot_users: - username: alice @@ -147,6 +154,16 @@ Verify all secrets are present and non-empty: bash scripts/check-vault.sh ``` +The script enforces three tiers: +- **Required scalars** — must be present and non-empty (all secrets above except + `vaultwarden_sso_client_secret` and `dl_sftp_authorized_keys`) +- **Required lists** — must be present and contain at least one entry + (`traefik_dashboard_users`, `registry_users`, `dovecot_users`) +- **Optional but validated when present** — if the key exists in the vault it + must pass a format check: + - `vaultwarden_sso_client_secret` — non-empty string (added after Keycloak is configured) + - `dl_sftp_authorized_keys` — must begin with a recognised SSH public key prefix + To edit the vault later: ```bash @@ -165,8 +182,8 @@ The script runs the following steps in order: 1. Verifies all four key files exist under `keys/` (both root and deploy key pairs) 2. Verifies `scripts/bootstrap-deploy-user.sh` exists 3. Sets `chmod 600` on the private key files (SSH refuses keys with open permissions) -4. Runs `scripts/check-vault.sh` — decrypts the vault and confirms all 11 required - secrets are present and non-empty +4. Runs `scripts/check-vault.sh` — decrypts the vault, confirms all required + secrets are present and non-empty, and validates optional secrets when present **Remote actions** 5. Opens a test SSH connection as `root` to confirm the root key works @@ -197,7 +214,8 @@ This runs all roles in order: | 8 | `registry` | Docker Registry v2 with htpasswd auth | | 9 | `k8s` | Placeholder page at k8s.ladkau.de | | 10 | `vaultwarden` | Vaultwarden password vault at vault.ladkau.de | -| 11 | `dashboard` | Public status dashboard at cloud.ladkau.de — service health and server stats | +| 11 | `dl` | Public download server at dl.ladkau.de — nginx HTTPS + SFTP upload | +| 12 | `dashboard` | Public status dashboard at cloud.ladkau.de — service health and server stats | To apply a single role: @@ -211,5 +229,5 @@ ansible-playbook -i ansible/inventory.ini ansible/site.yml --tags --ask-v |--------|---------| | `scripts/run-bootstrap.sh` | Copy and run the deploy user bootstrap on a fresh server | | `scripts/bootstrap-deploy-user.sh` | Runs on the server as root — creates the deploy user | -| `scripts/check-vault.sh` | Verify vault.yml exists and all required secrets are non-empty | +| `scripts/check-vault.sh` | Verify required secrets are present; validate optional secrets (`vaultwarden_sso_client_secret`, `dl_sftp_authorized_keys`) when present | | `scripts/check-services.sh` | Verify all service endpoints are reachable (run after provisioning) | diff --git a/scripts/check-services.sh b/scripts/check-services.sh index c02720d..ecf0d23 100644 --- a/scripts/check-services.sh +++ b/scripts/check-services.sh @@ -17,6 +17,7 @@ CHECKS=( "https://cr.ladkau.de/v2/ 401 10" "https://k8s.ladkau.de 200 10" "https://vault.ladkau.de 200 10" + "https://dl.ladkau.de 200 10" ) if ! command -v curl &>/dev/null; then diff --git a/scripts/check-vault.sh b/scripts/check-vault.sh index 8e8c726..948d211 100755 --- a/scripts/check-vault.sh +++ b/scripts/check-vault.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Decrypts ansible/group_vars/vault.yml and checks all required secrets are present -# and non-empty. Exits non-zero if any are missing. +# and non-empty, and validates optional secrets when they are present. +# Exits non-zero if any check fails. # Usage: bash scripts/check-vault.sh set -euo pipefail @@ -40,6 +41,7 @@ required_scalars = [ "roundcube_db_password", "roundcube_des_key", "vaultwarden_admin_token", + "dl_sftp_authorized_keys", ] # Required non-empty lists (must contain at least one entry) @@ -49,26 +51,46 @@ required_lists = [ "dovecot_users", ] -missing = [] +# Optional scalars: validated only when present — (key, validator_fn, hint) +SSH_KEY_PREFIXES = ("ssh-rsa", "ssh-ed25519", "ssh-ecdsa", "ecdsa-sha2-", "sk-ssh-") + +def is_nonempty(val): + return bool(str(val).strip()) + +def is_ssh_pubkey(val): + return any(str(val).strip().startswith(p) for p in SSH_KEY_PREFIXES) + +optional_scalars = [ + ("vaultwarden_sso_client_secret", is_nonempty, "must be a non-empty string"), + ("dl_sftp_authorized_keys", is_ssh_pubkey, "must be a valid SSH public key (ssh-ed25519 / ssh-rsa / ecdsa-sha2-*)"), +] + +errors = [] for key in required_scalars: val = data.get(key, "") if not val or str(val).strip() in ("", '""', "''"): - missing.append(key) + errors.append(f"{key}: missing or empty (required)") for key in required_lists: val = data.get(key) if not isinstance(val, list) or len(val) == 0: - missing.append(key) + errors.append(f"{key}: missing or empty list (required)") -if missing: +for key, validator, hint in optional_scalars: + if key in data: + val = data[key] + if not val or not validator(str(val).strip()): + errors.append(f"{key}: present but invalid — {hint}") + +if errors: print("", file=sys.stderr) - print("ERROR: the following secrets are missing or empty in vault.yml:", file=sys.stderr) - for key in missing: - print(f" - {key}", file=sys.stderr) + print("ERROR: vault.yml has the following issues:", file=sys.stderr) + for msg in errors: + print(f" - {msg}", file=sys.stderr) print("", file=sys.stderr) print("Edit the vault with: ansible-vault edit ansible/group_vars/all/vault.yml", file=sys.stderr) sys.exit(1) -print(" OK — all required secrets are present") +print(" OK — all secrets are present and valid") PYEOF