Add adaptive preprocessing escalation and single-receipt test scripts

Preprocessing now tries progressively heavier variants per receipt (plain -> high-pass -> edge-enhance -> both), stopping as soon as a cheap LLM-free arithmetic check on the OCR text looks plausible, so the extra processing only applies to receipts that need it. A flat version of the same heavier processing regressed the full batch by helping hard receipts but hurting easy ones; the adaptive version improves date-matching 37/52 -> 40/52 with no regression (see testing/2026_10_04_gimp_highpass_nlfilter_experiment.md). Also adds test_ocr.sh and test_single.sh for checking a single receipt's OCR text or full pipeline result without a batch run.
2026-10-04 10:25:41 +02:00
parent 9bb1844cf4
commit ca85530b5a
11 changed files with 694 additions and 52 deletions
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Combines steps 1+2 (preprocess + OCR) adaptively: tries preprocess.py's
+# variants cheapest-first (0=baseline, same as 1_preprocess_image.sh's
+# default, up to 3=most elaborate), scoring each variant's OCR text with
+# tooling-container/src/plausibility.py and stopping at the first one
+# that looks receipt-consistent - no LLM call involved, so this costs
+# nothing extra over the single-variant path for the receipts the
+# baseline already reads correctly. See tooling-container/src/adaptive.py
+# and testing/2026_10_04_gimp_highpass_nlfilter_experiment.md for why.
+#
+# Writes <out>/variant<N>/{01_gray,...,0X_final}.png and its own ocr.txt
+# for every variant tried (for inspection - which variant was kept is in
+# the printed summary) and <out>/ocr.txt (the winning variant's OCR
+# text, ready for ./3_extract_data.sh).
+#
+# Usage: ./12_adaptive_preprocess_ocr.sh -in <photo> -out <scratch-dir>
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+ROOT="$(pwd)"
+
+fail() { echo "FAIL: $*" >&2; exit 1; }
+
+IN_PATH=""
+OUT_DIR=""
+while [ $# -gt 0 ]; do
+    case "$1" in
+        -in) IN_PATH="${2:-}"; shift 2 ;;
+        -out) OUT_DIR="${2:-}"; shift 2 ;;
+        *) fail "unknown argument: $1" ;;
+    esac
+done
+[ -n "$IN_PATH" ] && [ -n "$OUT_DIR" ] \
+    || fail "usage: $0 -in <photo> -out <scratch-dir>"
+[ -f "$IN_PATH" ] || fail "input file not found: $IN_PATH"
+
+command -v docker >/dev/null 2>&1 || fail "docker not found in PATH"
+
+IMAGE="${REGISTRY_IMAGE:-fuelish/tools}"
+if [ -f "$ROOT/registry.env" ]; then
+    # shellcheck disable=SC1091
+    source "$ROOT/registry.env"
+    IMAGE="${REGISTRY_IMAGE:-$IMAGE}"
+fi
+VERSION="$(<"$ROOT/tooling-container/VERSION")"
+IMAGE="$IMAGE:$VERSION"
+docker image inspect "$IMAGE" >/dev/null 2>&1 \
+    || fail "$IMAGE not found locally - run ./build-image.sh first"
+
+IN_ABS="$(realpath "$IN_PATH")"
+IN_DIR="$(dirname "$IN_ABS")"
+IN_FILE="$(basename "$IN_ABS")"
+mkdir -p "$OUT_DIR"
+OUT_ABS="$(realpath "$OUT_DIR")"
+
+GROUP_ADD_ARGS=(--group-add "$(stat -c %g "$IN_DIR")")
+OUT_GID="$(stat -c %g "$OUT_ABS")"
+[ "$OUT_GID" = "$(stat -c %g "$IN_DIR")" ] || GROUP_ADD_ARGS+=(--group-add "$OUT_GID")
+
+echo "== Adaptively preprocessing + OCRing $IN_FILE =="
+docker run --rm \
+    --user "$(id -u):$(id -g)" \
+    "${GROUP_ADD_ARGS[@]}" \
+    -v "$IN_DIR:/data/in:ro" \
+    -v "$OUT_ABS:/data/out" \
+    "$IMAGE" \
+    python3 /opt/app/src/adaptive.py -in "/data/in/$IN_FILE" -out /data/out
+
+echo "== Done =="
+echo "Next: ./3_extract_data.sh -in $OUT_ABS/ocr.txt -out $OUT_ABS"