From b15754ba2f8c0f66d6d554bc542de4b505498edb Mon Sep 17 00:00:00 2001 From: ml Date: Sun, 28 Jun 2026 08:02:45 +0200 Subject: [PATCH] Harden against bot floods and add post-provisioning service check - Traefik rate-limits all routes (60 req/min standard, 300 for Nextcloud) and writes access logs to /var/log/traefik/access.log - Fail2ban watches Traefik access logs and bans IPs after 10 x 403/404 within 60 s for 24 h - Nextcloud html directory created as www-data (UID 33) so Apache can process .htaccess and serve requests correctly - scripts/check-services.sh verifies all seven endpoints return the expected HTTP status after provisioning --- .../roles/base/files/filter.d/traefik.conf | 6 ++ ansible/roles/base/files/jail.d/traefik.conf | 8 +++ ansible/roles/base/handlers/main.yml | 5 ++ ansible/roles/base/tasks/main.yml | 20 +++++++ .../gitea/templates/docker-compose.yml.j2 | 1 + .../roles/k8s/templates/docker-compose.yml.j2 | 1 + .../mail/templates/docker-compose.yml.j2 | 1 + ansible/roles/nextcloud/tasks/main.yml | 12 +++- .../nextcloud/templates/docker-compose.yml.j2 | 2 +- .../registry/templates/docker-compose.yml.j2 | 2 +- .../roles/sso/templates/docker-compose.yml.j2 | 1 + ansible/roles/traefik/tasks/main.yml | 9 +++ .../traefik/templates/docker-compose.yml.j2 | 10 ++++ .../roles/traefik/templates/traefik.yml.j2 | 4 +- docs/runbook-configuration.md | 29 +++++++++ docs/runbook-provisioning.md | 1 + scripts/check-services.sh | 60 +++++++++++++++++++ 17 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 ansible/roles/base/files/filter.d/traefik.conf create mode 100644 ansible/roles/base/files/jail.d/traefik.conf create mode 100644 scripts/check-services.sh diff --git a/ansible/roles/base/files/filter.d/traefik.conf b/ansible/roles/base/files/filter.d/traefik.conf new file mode 100644 index 0000000..e380df3 --- /dev/null +++ b/ansible/roles/base/files/filter.d/traefik.conf @@ -0,0 +1,6 @@ +[Definition] +# Matches Traefik CLF access log lines where the response status is 403 or 404. +# Banning on these two codes catches vulnerability scanners (403) and path probers (404) +# without touching legitimate 401 auth challenges or 429 rate-limit responses. +failregex = ^ \S+ \S+ \[.*?\] ".*?" (?:403|404) .*$ +ignoreregex = diff --git a/ansible/roles/base/files/jail.d/traefik.conf b/ansible/roles/base/files/jail.d/traefik.conf new file mode 100644 index 0000000..d8a4cda --- /dev/null +++ b/ansible/roles/base/files/jail.d/traefik.conf @@ -0,0 +1,8 @@ +[traefik] +enabled = true +port = http,https +filter = traefik +logpath = /var/log/traefik/access.log +maxretry = 10 +findtime = 60 +bantime = 86400 diff --git a/ansible/roles/base/handlers/main.yml b/ansible/roles/base/handlers/main.yml index cad74d8..bfd5d2c 100644 --- a/ansible/roles/base/handlers/main.yml +++ b/ansible/roles/base/handlers/main.yml @@ -3,3 +3,8 @@ ansible.builtin.service: name: ssh state: restarted + +- name: Restart fail2ban + ansible.builtin.service: + name: fail2ban + state: restarted diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml index 0c7aeaa..26ffd5f 100644 --- a/ansible/roles/base/tasks/main.yml +++ b/ansible/roles/base/tasks/main.yml @@ -126,6 +126,26 @@ # --- Fail2ban --- +- name: Deploy fail2ban filter for Traefik + ansible.builtin.copy: + src: filter.d/traefik.conf + dest: /etc/fail2ban/filter.d/traefik.conf + owner: root + group: root + mode: "0644" + notify: Restart fail2ban + tags: base + +- name: Deploy fail2ban jail for Traefik + ansible.builtin.copy: + src: jail.d/traefik.conf + dest: /etc/fail2ban/jail.d/traefik.conf + owner: root + group: root + mode: "0644" + notify: Restart fail2ban + tags: base + - name: Enable and start fail2ban ansible.builtin.service: name: fail2ban diff --git a/ansible/roles/gitea/templates/docker-compose.yml.j2 b/ansible/roles/gitea/templates/docker-compose.yml.j2 index c295fc5..6ad5a94 100644 --- a/ansible/roles/gitea/templates/docker-compose.yml.j2 +++ b/ansible/roles/gitea/templates/docker-compose.yml.j2 @@ -35,6 +35,7 @@ services: - "traefik.http.routers.gitea.entrypoints=websecure" - "traefik.http.routers.gitea.tls.certresolver=letsencrypt" - "traefik.http.services.gitea.loadbalancer.server.port=3000" + - "traefik.http.routers.gitea.middlewares=rate-limit@docker" depends_on: gitea-db: condition: service_healthy diff --git a/ansible/roles/k8s/templates/docker-compose.yml.j2 b/ansible/roles/k8s/templates/docker-compose.yml.j2 index f86e3f1..e933172 100644 --- a/ansible/roles/k8s/templates/docker-compose.yml.j2 +++ b/ansible/roles/k8s/templates/docker-compose.yml.j2 @@ -14,6 +14,7 @@ services: - "traefik.http.routers.k8s.entrypoints=websecure" - "traefik.http.routers.k8s.tls.certresolver=letsencrypt" - "traefik.http.services.k8s.loadbalancer.server.port=80" + - "traefik.http.routers.k8s.middlewares=rate-limit@docker" networks: traefik_public: diff --git a/ansible/roles/mail/templates/docker-compose.yml.j2 b/ansible/roles/mail/templates/docker-compose.yml.j2 index 3f3a131..78b3299 100644 --- a/ansible/roles/mail/templates/docker-compose.yml.j2 +++ b/ansible/roles/mail/templates/docker-compose.yml.j2 @@ -49,6 +49,7 @@ services: - "traefik.http.routers.mail.entrypoints=websecure" - "traefik.http.routers.mail.tls.certresolver=letsencrypt" - "traefik.http.services.mail.loadbalancer.server.port=80" + - "traefik.http.routers.mail.middlewares=rate-limit@docker" depends_on: mail-db: condition: service_healthy diff --git a/ansible/roles/nextcloud/tasks/main.yml b/ansible/roles/nextcloud/tasks/main.yml index 4e58c87..8a4c05d 100644 --- a/ansible/roles/nextcloud/tasks/main.yml +++ b/ansible/roles/nextcloud/tasks/main.yml @@ -8,10 +8,20 @@ mode: "0755" loop: - "{{ nextcloud_data_dir }}" - - "{{ nextcloud_data_dir }}/html" - "{{ nextcloud_data_dir }}/redis" tags: nextcloud +# www-data inside the container is UID 33. The html directory must be owned by +# this user so Apache can traverse it and process .htaccess files. +- name: Create Nextcloud html directory (www-data UID 33) + ansible.builtin.file: + path: "{{ nextcloud_data_dir }}/html" + state: directory + owner: "33" + group: "33" + mode: "0755" + tags: nextcloud + - name: Create Nextcloud database directory (postgres UID 999) ansible.builtin.file: path: "{{ nextcloud_data_dir }}/db" diff --git a/ansible/roles/nextcloud/templates/docker-compose.yml.j2 b/ansible/roles/nextcloud/templates/docker-compose.yml.j2 index 5be79e6..b052d23 100644 --- a/ansible/roles/nextcloud/templates/docker-compose.yml.j2 +++ b/ansible/roles/nextcloud/templates/docker-compose.yml.j2 @@ -68,7 +68,7 @@ services: - "traefik.http.routers.nextcloud.entrypoints=websecure" - "traefik.http.routers.nextcloud.tls.certresolver=letsencrypt" - "traefik.http.services.nextcloud.loadbalancer.server.port=80" - - "traefik.http.routers.nextcloud.middlewares=nextcloud-wellknown,nextcloud-headers" + - "traefik.http.routers.nextcloud.middlewares=nextcloud-wellknown,nextcloud-headers,rate-limit-lax@docker" # Redirect .well-known CalDAV/CardDAV to the proper Nextcloud endpoint - "traefik.http.middlewares.nextcloud-wellknown.redirectregex.regex=^https://([^/]*)/.well-known/(card|cal)dav" - "traefik.http.middlewares.nextcloud-wellknown.redirectregex.replacement=https://$$1/remote.php/dav/" diff --git a/ansible/roles/registry/templates/docker-compose.yml.j2 b/ansible/roles/registry/templates/docker-compose.yml.j2 index 7a565f8..f866d38 100644 --- a/ansible/roles/registry/templates/docker-compose.yml.j2 +++ b/ansible/roles/registry/templates/docker-compose.yml.j2 @@ -20,7 +20,7 @@ services: - "traefik.http.routers.registry.entrypoints=websecure" - "traefik.http.routers.registry.tls.certresolver=letsencrypt" - "traefik.http.services.registry.loadbalancer.server.port=5000" - - "traefik.http.routers.registry.middlewares=registry-buffering" + - "traefik.http.routers.registry.middlewares=registry-buffering,rate-limit@docker" # Remove body size limit so large image layers can be pushed - "traefik.http.middlewares.registry-buffering.buffering.maxRequestBodyBytes=0" diff --git a/ansible/roles/sso/templates/docker-compose.yml.j2 b/ansible/roles/sso/templates/docker-compose.yml.j2 index 8c581e2..dcebe50 100644 --- a/ansible/roles/sso/templates/docker-compose.yml.j2 +++ b/ansible/roles/sso/templates/docker-compose.yml.j2 @@ -47,6 +47,7 @@ services: - "traefik.http.routers.sso.entrypoints=websecure" - "traefik.http.routers.sso.tls.certresolver=letsencrypt" - "traefik.http.services.sso.loadbalancer.server.port=8080" + - "traefik.http.routers.sso.middlewares=rate-limit@docker" healthcheck: test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/9000"] interval: 30s diff --git a/ansible/roles/traefik/tasks/main.yml b/ansible/roles/traefik/tasks/main.yml index 8a8f675..4f56b30 100644 --- a/ansible/roles/traefik/tasks/main.yml +++ b/ansible/roles/traefik/tasks/main.yml @@ -1,4 +1,13 @@ --- +- name: Create Traefik log directory + ansible.builtin.file: + path: /var/log/traefik + state: directory + owner: root + group: root + mode: "0755" + tags: traefik + - name: Create Traefik data directory ansible.builtin.file: path: "{{ traefik_data_dir }}" diff --git a/ansible/roles/traefik/templates/docker-compose.yml.j2 b/ansible/roles/traefik/templates/docker-compose.yml.j2 index 3f7ca37..c5a659b 100644 --- a/ansible/roles/traefik/templates/docker-compose.yml.j2 +++ b/ansible/roles/traefik/templates/docker-compose.yml.j2 @@ -11,6 +11,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro - {{ traefik_data_dir }}/traefik.yml:/traefik.yml:ro - {{ traefik_data_dir }}/acme.json:/acme.json + - /var/log/traefik:/var/log/traefik networks: - traefik_public labels: @@ -22,6 +23,15 @@ services: - "traefik.http.routers.dashboard.service=api@internal" - "traefik.http.routers.dashboard.middlewares=dashboard-auth" - "traefik.http.middlewares.dashboard-auth.basicauth.users={{ traefik_dashboard_users }}" + # Rate-limit middlewares — referenced by services as rate-limit@docker / rate-limit-lax@docker + # Standard: 60 req/min per IP, burst 20 — protects all services from bot floods + - "traefik.http.middlewares.rate-limit.rateLimit.average=60" + - "traefik.http.middlewares.rate-limit.rateLimit.period=1m" + - "traefik.http.middlewares.rate-limit.rateLimit.burst=20" + # Lax: 300 req/min per IP, burst 100 — for Nextcloud sync clients making many small requests + - "traefik.http.middlewares.rate-limit-lax.rateLimit.average=300" + - "traefik.http.middlewares.rate-limit-lax.rateLimit.period=1m" + - "traefik.http.middlewares.rate-limit-lax.rateLimit.burst=100" networks: traefik_public: diff --git a/ansible/roles/traefik/templates/traefik.yml.j2 b/ansible/roles/traefik/templates/traefik.yml.j2 index e09fd07..708fffe 100644 --- a/ansible/roles/traefik/templates/traefik.yml.j2 +++ b/ansible/roles/traefik/templates/traefik.yml.j2 @@ -9,7 +9,9 @@ api: log: level: INFO -accessLog: {} +accessLog: + filePath: /var/log/traefik/access.log + bufferingSize: 100 entryPoints: web: diff --git a/docs/runbook-configuration.md b/docs/runbook-configuration.md index 516ddce..311e0a3 100644 --- a/docs/runbook-configuration.md +++ b/docs/runbook-configuration.md @@ -3,6 +3,35 @@ First-run configuration steps to perform after the Ansible playbook has provisioned the server. See `runbook-provisioning.md` for the provisioning steps. +## 1. Verify all services are reachable + +Before configuring individual services, confirm every HTTPS endpoint is up and +TLS certificates are valid. Run this from your local machine: + +```bash +bash scripts/check-services.sh +``` + +The script checks these endpoints and verifies the expected HTTP status code: + +| URL | Expected | Notes | +|-----|----------|-------| +| `https://cloud.ladkau.de/dashboard/` | 401 | Basic-auth prompt — correct without credentials | +| `https://gitea.ladkau.de` | 200 | Gitea sign-in page | +| `https://nextcloud.ladkau.de` | 200 | First visit triggers setup and takes 1–2 min | +| `https://sso.ladkau.de/realms/master` | 200 | Keycloak realm JSON — first start takes ~90 s | +| `https://mail.ladkau.de` | 200 | Roundcube webmail login | +| `https://cr.ladkau.de/v2/` | 401 | Registry API — auth required, correct without credentials | +| `https://k8s.ladkau.de` | 200 | Placeholder page | + +A `000` result means the connection was refused or timed out — a container that +did not start. Keycloak and Nextcloud may return `502` for up to 90 seconds on +first boot; wait and retry before investigating. + +Once the script passes, open `https://cloud.ladkau.de/dashboard/` in a browser +and authenticate with the `traefik_dashboard_users` credentials to verify the +dashboard loads correctly. + ## Gitea `gitea_disable_registration` defaults to `true`. For the first deploy, override diff --git a/docs/runbook-provisioning.md b/docs/runbook-provisioning.md index 0aa9976..de7b218 100644 --- a/docs/runbook-provisioning.md +++ b/docs/runbook-provisioning.md @@ -189,3 +189,4 @@ ansible-playbook -i ansible/inventory.ini ansible/site.yml --tags --ask-v | `scripts/run-bootstrap.sh` | Copy and run the deploy user bootstrap on a fresh server | | `scripts/bootstrap-deploy-user.sh` | Runs on the server as root — creates the deploy user | | `scripts/check-vault.sh` | Verify vault.yml exists and all required secrets are non-empty | +| `scripts/check-services.sh` | Verify all service endpoints are reachable (run after provisioning) | diff --git a/scripts/check-services.sh b/scripts/check-services.sh new file mode 100644 index 0000000..54bf5ea --- /dev/null +++ b/scripts/check-services.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Checks that all service endpoints are reachable and return the expected HTTP +# status code. Run this after provisioning, before configuring individual services. +# Usage: bash scripts/check-services.sh + +set -uo pipefail + +# "url expected_status timeout_seconds" tuples +# Nextcloud runs its first-time installation on the initial request (1-2 min), +# so it gets a much longer timeout than the other services. +CHECKS=( + "https://cloud.ladkau.de/dashboard/ 401 10" + "https://gitea.ladkau.de 200 10" + "https://nextcloud.ladkau.de 200 180" + "https://sso.ladkau.de/realms/master 200 10" + "https://mail.ladkau.de 200 10" + "https://cr.ladkau.de/v2/ 401 10" + "https://k8s.ladkau.de 200 10" +) + +if ! command -v curl &>/dev/null; then + echo "ERROR: curl not found — install with: apt install curl or brew install curl" >&2 + exit 1 +fi + +echo "==> Checking service endpoints" + +FAILED=() +for check in "${CHECKS[@]}"; do + read -r url expected timeout <<< "$check" + + if [ "$timeout" -gt 10 ]; then + printf " ... %s (first-run setup may take up to %d s)\n" "$url" "$timeout" + fi + + actual=$(curl -sL -o /dev/null -w "%{http_code}" --max-time "$timeout" "$url" 2>/dev/null) || actual="000" + + if [ "$actual" = "$expected" ]; then + printf " OK %s (%s)\n" "$url" "$actual" + else + printf " FAIL %s (expected %s, got %s)\n" "$url" "$expected" "$actual" + FAILED+=("$url") + fi +done + +echo "" + +if [ ${#FAILED[@]} -gt 0 ]; then + echo "ERROR: ${#FAILED[@]} endpoint(s) did not return the expected status:" >&2 + for url in "${FAILED[@]}"; do + echo " - ${url}" >&2 + done + echo "" >&2 + echo " Check container status on the server: docker ps" >&2 + echo " Check container logs: docker logs " >&2 + echo " Note: Keycloak and Nextcloud may take longer >90s on first boot — wait and retry." >&2 + exit 1 +fi + +echo " All services are reachable."