Add adaptive preprocessing escalation and single-receipt test scripts

Preprocessing now tries progressively heavier variants per receipt (plain -> high-pass -> edge-enhance -> both), stopping as soon as a cheap LLM-free arithmetic check on the OCR text looks plausible, so the extra processing only applies to receipts that need it. A flat version of the same heavier processing regressed the full batch by helping hard receipts but hurting easy ones; the adaptive version improves date-matching 37/52 -> 40/52 with no regression (see testing/2026_10_04_gimp_highpass_nlfilter_experiment.md). Also adds test_ocr.sh and test_single.sh for checking a single receipt's OCR text or full pipeline result without a batch run.
2026-10-04 10:25:41 +02:00
parent 9bb1844cf4
commit ca85530b5a
11 changed files with 694 additions and 52 deletions
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+# Test-only tool: runs the full container pipeline
+# (12_adaptive_preprocess_ocr.sh, 3_extract_data.sh) on a single receipt
+# photo - the test_batch.sh equivalent for re-checking one receipt (e.g.
+# a known failure, after a preprocessing/prompt tweak) without re-running
+# a whole directory. If a ground-truth spreadsheet (*.ods) is found next
+# to the photo, compares the result against it the same way
+# test_batch.sh does; otherwise just prints the extracted JSON.
+# Everything - preprocessing, OCR, structuring, and the comparison itself
+# - runs inside the tooling container; nothing here depends on a host
+# Python environment.
+#
+# For an input photo <path>/<name>.jpg, two sibling directories are
+# created (mirroring test_batch.sh's layout, just for one receipt):
+#   <path>/<name>-scratch/<name>/  - intermediate step output: one
+#                                    variant<N>/ subdir per preprocessing
+#                                    variant tried, plus ocr.txt for the
+#                                    winning variant
+#   <path>/<name>-result/<name>/   - extracted.json, plus summary.txt
+#                                    if a ground-truth .ods was found
+#
+# Usage: ./test_single.sh -in <photo.jpg>
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+ROOT="$(pwd)"
+
+fail() { echo "FAIL: $*" >&2; exit 1; }
+
+IN_PATH=""
+while [ $# -gt 0 ]; do
+    case "$1" in
+        -in) IN_PATH="${2:-}"; shift 2 ;;
+        *) fail "unknown argument: $1" ;;
+    esac
+done
+[ -n "$IN_PATH" ] || fail "usage: $0 -in <photo.jpg>"
+[ -f "$IN_PATH" ] || fail "input file not found: $IN_PATH"
+
+IN_ABS="$(realpath "$IN_PATH")"
+IN_DIR="$(dirname "$IN_ABS")"
+NAME="$(basename "${IN_ABS%.jpg}")"
+
+SCRATCH_DIR="$IN_DIR/$NAME-scratch"
+RESULT_DIR="$IN_DIR/$NAME-result"
+receipt_scratch="$SCRATCH_DIR/$NAME"
+receipt_result="$RESULT_DIR/$NAME"
+mkdir -p "$receipt_scratch" "$receipt_result"
+
+echo "== Running pipeline on $NAME =="
+./12_adaptive_preprocess_ocr.sh -in "$IN_ABS" -out "$receipt_scratch" >/dev/null
+./3_extract_data.sh -in "$receipt_scratch/ocr.txt" -out "$receipt_result" >/dev/null
+
+echo
+echo "== Extracted =="
+cat "$receipt_result/extracted.json"
+echo
+
+ODS_PATH="$(find "$IN_DIR" -maxdepth 1 -name '*.ods' | head -n1)"
+if [ -n "$ODS_PATH" ]; then
+    echo
+    echo "== Comparing against ground truth =="
+
+    IMAGE="${REGISTRY_IMAGE:-fuelish/tools}"
+    if [ -f "$ROOT/registry.env" ]; then
+        # shellcheck disable=SC1091
+        source "$ROOT/registry.env"
+        IMAGE="${REGISTRY_IMAGE:-$IMAGE}"
+    fi
+    VERSION="$(<"$ROOT/tooling-container/VERSION")"
+    IMAGE="$IMAGE:$VERSION"
+
+    ODS_DIR="$(dirname "$ODS_PATH")"
+    ODS_FILE="$(basename "$ODS_PATH")"
+    GROUP_ADD_ARGS=(--group-add "$(stat -c %g "$ODS_DIR")")
+    RESULT_GID="$(stat -c %g "$RESULT_DIR")"
+    [ "$RESULT_GID" = "$(stat -c %g "$ODS_DIR")" ] || GROUP_ADD_ARGS+=(--group-add "$RESULT_GID")
+
+    docker run --rm \
+        --user "$(id -u):$(id -g)" \
+        "${GROUP_ADD_ARGS[@]}" \
+        -v "$ODS_DIR:/data/gt:ro" \
+        -v "$RESULT_DIR:/data/results:ro" \
+        "$IMAGE" \
+        python3 /opt/app/src/compare_ground_truth.py -gt "/data/gt/$ODS_FILE" -results /data/results \
+        | tee "$RESULT_DIR/summary.txt"
+else
+    echo "(no ground-truth .ods found next to $IN_PATH - skipping comparison)"
+fi
+
+echo
+echo "== Done =="
+echo "Intermediate step output: $receipt_scratch/"
+echo "Final result:             $receipt_result/extracted.json"