Harden against bot floods and add post-provisioning service check

- Traefik rate-limits all routes (60 req/min standard, 300 for Nextcloud)
  and writes access logs to /var/log/traefik/access.log
- Fail2ban watches Traefik access logs and bans IPs after 10 x 403/404
  within 60 s for 24 h
- Nextcloud html directory created as www-data (UID 33) so Apache can
  process .htaccess and serve requests correctly
- scripts/check-services.sh verifies all seven endpoints return the
  expected HTTP status after provisioning

ansible/roles/base/files/filter.d/traefik.conf

@@ -0,0 +1,6 @@
+[Definition]
+# Matches Traefik CLF access log lines where the response status is 403 or 404.
+# Banning on these two codes catches vulnerability scanners (403) and path probers (404)
+# without touching legitimate 401 auth challenges or 429 rate-limit responses.
+failregex = ^<HOST> \S+ \S+ \[.*?\] ".*?" (?:403|404) .*$
+ignoreregex =

ansible/roles/base/files/jail.d/traefik.conf

@@ -0,0 +1,8 @@
+[traefik]
+enabled  = true
+port     = http,https
+filter   = traefik
+logpath  = /var/log/traefik/access.log
+maxretry = 10
+findtime = 60
+bantime  = 86400

ansible/roles/base/handlers/main.yml

@@ -3,3 +3,8 @@
   ansible.builtin.service:
     name: ssh
     state: restarted
+
+- name: Restart fail2ban
+  ansible.builtin.service:
+    name: fail2ban
+    state: restarted

ansible/roles/base/tasks/main.yml

@@ -126,6 +126,26 @@
 
 # --- Fail2ban ---
 
+- name: Deploy fail2ban filter for Traefik
+  ansible.builtin.copy:
+    src: filter.d/traefik.conf
+    dest: /etc/fail2ban/filter.d/traefik.conf
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart fail2ban
+  tags: base
+
+- name: Deploy fail2ban jail for Traefik
+  ansible.builtin.copy:
+    src: jail.d/traefik.conf
+    dest: /etc/fail2ban/jail.d/traefik.conf
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart fail2ban
+  tags: base
+
 - name: Enable and start fail2ban
   ansible.builtin.service:
     name: fail2ban

ansible/roles/gitea/templates/docker-compose.yml.j2

@@ -35,6 +35,7 @@ services:
       - "traefik.http.routers.gitea.entrypoints=websecure"
       - "traefik.http.routers.gitea.tls.certresolver=letsencrypt"
       - "traefik.http.services.gitea.loadbalancer.server.port=3000"
+      - "traefik.http.routers.gitea.middlewares=rate-limit@docker"
     depends_on:
       gitea-db:
         condition: service_healthy

ansible/roles/k8s/templates/docker-compose.yml.j2

@@ -14,6 +14,7 @@ services:
       - "traefik.http.routers.k8s.entrypoints=websecure"
       - "traefik.http.routers.k8s.tls.certresolver=letsencrypt"
       - "traefik.http.services.k8s.loadbalancer.server.port=80"
+      - "traefik.http.routers.k8s.middlewares=rate-limit@docker"
 
 networks:
   traefik_public:

ansible/roles/mail/templates/docker-compose.yml.j2

@@ -49,6 +49,7 @@ services:
       - "traefik.http.routers.mail.entrypoints=websecure"
       - "traefik.http.routers.mail.tls.certresolver=letsencrypt"
       - "traefik.http.services.mail.loadbalancer.server.port=80"
+      - "traefik.http.routers.mail.middlewares=rate-limit@docker"
     depends_on:
       mail-db:
         condition: service_healthy

ansible/roles/nextcloud/tasks/main.yml

@@ -8,10 +8,20 @@
     mode: "0755"
   loop:
     - "{{ nextcloud_data_dir }}"
-    - "{{ nextcloud_data_dir }}/html"
     - "{{ nextcloud_data_dir }}/redis"
   tags: nextcloud
 
+# www-data inside the container is UID 33. The html directory must be owned by
+# this user so Apache can traverse it and process .htaccess files.
+- name: Create Nextcloud html directory (www-data UID 33)
+  ansible.builtin.file:
+    path: "{{ nextcloud_data_dir }}/html"
+    state: directory
+    owner: "33"
+    group: "33"
+    mode: "0755"
+  tags: nextcloud
+
 - name: Create Nextcloud database directory (postgres UID 999)
   ansible.builtin.file:
     path: "{{ nextcloud_data_dir }}/db"

ansible/roles/nextcloud/templates/docker-compose.yml.j2

@@ -68,7 +68,7 @@ services:
       - "traefik.http.routers.nextcloud.entrypoints=websecure"
       - "traefik.http.routers.nextcloud.tls.certresolver=letsencrypt"
       - "traefik.http.services.nextcloud.loadbalancer.server.port=80"
-      - "traefik.http.routers.nextcloud.middlewares=nextcloud-wellknown,nextcloud-headers"
+      - "traefik.http.routers.nextcloud.middlewares=nextcloud-wellknown,nextcloud-headers,rate-limit-lax@docker"
       # Redirect .well-known CalDAV/CardDAV to the proper Nextcloud endpoint
       - "traefik.http.middlewares.nextcloud-wellknown.redirectregex.regex=^https://([^/]*)/.well-known/(card|cal)dav"
       - "traefik.http.middlewares.nextcloud-wellknown.redirectregex.replacement=https://$$1/remote.php/dav/"

ansible/roles/registry/templates/docker-compose.yml.j2

@@ -20,7 +20,7 @@ services:
       - "traefik.http.routers.registry.entrypoints=websecure"
       - "traefik.http.routers.registry.tls.certresolver=letsencrypt"
       - "traefik.http.services.registry.loadbalancer.server.port=5000"
-      - "traefik.http.routers.registry.middlewares=registry-buffering"
+      - "traefik.http.routers.registry.middlewares=registry-buffering,rate-limit@docker"
       # Remove body size limit so large image layers can be pushed
       - "traefik.http.middlewares.registry-buffering.buffering.maxRequestBodyBytes=0"
 

ansible/roles/sso/templates/docker-compose.yml.j2

@@ -47,6 +47,7 @@ services:
       - "traefik.http.routers.sso.entrypoints=websecure"
       - "traefik.http.routers.sso.tls.certresolver=letsencrypt"
       - "traefik.http.services.sso.loadbalancer.server.port=8080"
+      - "traefik.http.routers.sso.middlewares=rate-limit@docker"
     healthcheck:
       test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/9000"]
       interval: 30s

ansible/roles/traefik/tasks/main.yml

@@ -1,4 +1,13 @@
 ---
+- name: Create Traefik log directory
+  ansible.builtin.file:
+    path: /var/log/traefik
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  tags: traefik
+
 - name: Create Traefik data directory
   ansible.builtin.file:
     path: "{{ traefik_data_dir }}"

ansible/roles/traefik/templates/docker-compose.yml.j2

@@ -11,6 +11,7 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - {{ traefik_data_dir }}/traefik.yml:/traefik.yml:ro
       - {{ traefik_data_dir }}/acme.json:/acme.json
+      - /var/log/traefik:/var/log/traefik
     networks:
       - traefik_public
     labels:
@@ -22,6 +23,15 @@ services:
       - "traefik.http.routers.dashboard.service=api@internal"
       - "traefik.http.routers.dashboard.middlewares=dashboard-auth"
       - "traefik.http.middlewares.dashboard-auth.basicauth.users={{ traefik_dashboard_users }}"
+      # Rate-limit middlewares — referenced by services as rate-limit@docker / rate-limit-lax@docker
+      # Standard: 60 req/min per IP, burst 20 — protects all services from bot floods
+      - "traefik.http.middlewares.rate-limit.rateLimit.average=60"
+      - "traefik.http.middlewares.rate-limit.rateLimit.period=1m"
+      - "traefik.http.middlewares.rate-limit.rateLimit.burst=20"
+      # Lax: 300 req/min per IP, burst 100 — for Nextcloud sync clients making many small requests
+      - "traefik.http.middlewares.rate-limit-lax.rateLimit.average=300"
+      - "traefik.http.middlewares.rate-limit-lax.rateLimit.period=1m"
+      - "traefik.http.middlewares.rate-limit-lax.rateLimit.burst=100"
 
 networks:
   traefik_public:

ansible/roles/traefik/templates/traefik.yml.j2

@@ -9,7 +9,9 @@ api:
 log:
   level: INFO
 
-accessLog: {}
+accessLog:
+  filePath: /var/log/traefik/access.log
+  bufferingSize: 100
 
 entryPoints:
   web:

docs/runbook-configuration.md

@@ -3,6 +3,35 @@
 First-run configuration steps to perform after the Ansible playbook has
 provisioned the server. See `runbook-provisioning.md` for the provisioning steps.
 
+## 1. Verify all services are reachable
+
+Before configuring individual services, confirm every HTTPS endpoint is up and
+TLS certificates are valid. Run this from your local machine:
+
+```bash
+bash scripts/check-services.sh
+```
+
+The script checks these endpoints and verifies the expected HTTP status code:
+
+| URL | Expected | Notes |
+|-----|----------|-------|
+| `https://cloud.ladkau.de/dashboard/` | 401 | Basic-auth prompt — correct without credentials |
+| `https://gitea.ladkau.de` | 200 | Gitea sign-in page |
+| `https://nextcloud.ladkau.de` | 200 | First visit triggers setup and takes 1–2 min |
+| `https://sso.ladkau.de/realms/master` | 200 | Keycloak realm JSON — first start takes ~90 s |
+| `https://mail.ladkau.de` | 200 | Roundcube webmail login |
+| `https://cr.ladkau.de/v2/` | 401 | Registry API — auth required, correct without credentials |
+| `https://k8s.ladkau.de` | 200 | Placeholder page |
+
+A `000` result means the connection was refused or timed out — a container that
+did not start. Keycloak and Nextcloud may return `502` for up to 90 seconds on
+first boot; wait and retry before investigating.
+
+Once the script passes, open `https://cloud.ladkau.de/dashboard/` in a browser
+and authenticate with the `traefik_dashboard_users` credentials to verify the
+dashboard loads correctly.
+
 ## Gitea
 
 `gitea_disable_registration` defaults to `true`. For the first deploy, override

docs/runbook-provisioning.md

@@ -189,3 +189,4 @@ ansible-playbook -i ansible/inventory.ini ansible/site.yml --tags <role> --ask-v
 | `scripts/run-bootstrap.sh` | Copy and run the deploy user bootstrap on a fresh server |
 | `scripts/bootstrap-deploy-user.sh` | Runs on the server as root — creates the deploy user |
 | `scripts/check-vault.sh` | Verify vault.yml exists and all required secrets are non-empty |
+| `scripts/check-services.sh` | Verify all service endpoints are reachable (run after provisioning) |

scripts/check-services.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Checks that all service endpoints are reachable and return the expected HTTP
+# status code. Run this after provisioning, before configuring individual services.
+# Usage: bash scripts/check-services.sh
+
+set -uo pipefail
+
+# "url expected_status timeout_seconds" tuples
+# Nextcloud runs its first-time installation on the initial request (1-2 min),
+# so it gets a much longer timeout than the other services.
+CHECKS=(
+  "https://cloud.ladkau.de/dashboard/ 401 10"
+  "https://gitea.ladkau.de 200 10"
+  "https://nextcloud.ladkau.de 200 180"
+  "https://sso.ladkau.de/realms/master 200 10"
+  "https://mail.ladkau.de 200 10"
+  "https://cr.ladkau.de/v2/ 401 10"
+  "https://k8s.ladkau.de 200 10"
+)
+
+if ! command -v curl &>/dev/null; then
+  echo "ERROR: curl not found — install with: apt install curl  or  brew install curl" >&2
+  exit 1
+fi
+
+echo "==> Checking service endpoints"
+
+FAILED=()
+for check in "${CHECKS[@]}"; do
+  read -r url expected timeout <<< "$check"
+
+  if [ "$timeout" -gt 10 ]; then
+    printf "  ...   %s  (first-run setup may take up to %d s)\n" "$url" "$timeout"
+  fi
+
+  actual=$(curl -sL -o /dev/null -w "%{http_code}" --max-time "$timeout" "$url" 2>/dev/null) || actual="000"
+
+  if [ "$actual" = "$expected" ]; then
+    printf "    OK  %s  (%s)\n" "$url" "$actual"
+  else
+    printf "  FAIL  %s  (expected %s, got %s)\n" "$url" "$expected" "$actual"
+    FAILED+=("$url")
+  fi
+done
+
+echo ""
+
+if [ ${#FAILED[@]} -gt 0 ]; then
+  echo "ERROR: ${#FAILED[@]} endpoint(s) did not return the expected status:" >&2
+  for url in "${FAILED[@]}"; do
+    echo "       - ${url}" >&2
+  done
+  echo "" >&2
+  echo "       Check container status on the server: docker ps" >&2
+  echo "       Check container logs: docker logs <container>" >&2
+  echo "       Note: Keycloak and Nextcloud may take longer >90s on first boot — wait and retry." >&2
+  exit 1
+fi
+
+echo "    All services are reachable."