Add adaptive preprocessing escalation and single-receipt test scripts

Preprocessing now tries progressively heavier variants per receipt
(plain -> high-pass -> edge-enhance -> both), stopping as soon as a
cheap LLM-free arithmetic check on the OCR text looks plausible, so
the extra processing only applies to receipts that need it. A flat
version of the same heavier processing regressed the full batch by
helping hard receipts but hurting easy ones; the adaptive version
improves date-matching 37/52 -> 40/52 with no regression (see
testing/2026_10_04_gimp_highpass_nlfilter_experiment.md).

Also adds test_ocr.sh and test_single.sh for checking a single
receipt's OCR text or full pipeline result without a batch run.

tooling-container/src/adaptive.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Adaptive preprocessing: tries preprocess.py's variants (0=cheapest,
+up to VARIANT_COUNT-1=most elaborate) in order, OCRing each and scoring
+the result with plausibility.best_residual() - no LLM call - stopping at
+the first variant whose OCR text already looks receipt-consistent.
+
+Exists because always applying the heavier variants regressed the full
+test batch even though they clearly help the specific hard receipts they
+were designed for (see
+testing/2026_10_04_gimp_highpass_nlfilter_experiment.md): escalating only
+when the cheap variant already looks implausible avoids that cost - and
+that risk - on the majority of receipts the baseline already reads
+correctly, while keeping the expensive LLM structuring call to exactly
+one per receipt no matter how many variants are tried.
+"""
+import argparse
+import os
+
+from ocr import ocr as run_ocr
+from plausibility import best_residual, is_plausible
+from preprocess import VARIANT_COUNT, preprocess
+
+
+def preprocess_and_ocr_adaptive(image_path: str, out_dir: str) -> dict:
+    """Returns {"ocr_text", "final_image_path", "variant", "residual",
+    "variants_tried"} - "variant" is whichever one was kept (the first
+    plausible one, or else the one with the lowest residual), not
+    necessarily the last one tried."""
+    best = None  # (residual, ocr_text, final_path, variant)
+    variants_tried = 0
+
+    for variant in range(VARIANT_COUNT):
+        variant_dir = os.path.join(out_dir, f"variant{variant}")
+        final_path = preprocess(image_path, variant_dir, variant=variant)
+        ocr_txt_path = run_ocr(final_path, variant_dir)
+        with open(ocr_txt_path, "r", encoding="utf-8") as f:
+            ocr_text = f.read()
+        variants_tried += 1
+
+        residual = best_residual(ocr_text)
+        if best is None or residual < best[0]:
+            best = (residual, ocr_text, final_path, variant)
+        if is_plausible(residual):
+            break
+
+    residual, ocr_text, final_path, variant = best
+    return {
+        "ocr_text": ocr_text,
+        "final_image_path": final_path,
+        "variant": variant,
+        "residual": residual,
+        "variants_tried": variants_tried,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-in", dest="in_path", required=True)
+    parser.add_argument("-out", dest="out_dir", required=True)
+    args = parser.parse_args()
+
+    result = preprocess_and_ocr_adaptive(args.in_path, args.out_dir)
+
+    ocr_path = os.path.join(args.out_dir, "ocr.txt")
+    with open(ocr_path, "w", encoding="utf-8") as f:
+        f.write(result["ocr_text"])
+
+    print(
+        f"variant={result['variant']} "
+        f"variants_tried={result['variants_tried']} "
+        f"residual={result['residual']:.3f} "
+        f"final_image={result['final_image_path']}"
+    )
+    print(ocr_path)
+
+
+if __name__ == "__main__":
+    main()

tooling-container/src/plausibility.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Cheap, LLM-free plausibility scoring of raw OCR text from a German fuel
+receipt. Used by adaptive.py to rank several preprocessing variants' OCR
+output against each other - and decide whether to bother trying a more
+elaborate variant at all - without paying for an LLM call per variant.
+
+Mirrors the same arithmetic identity and EUR tolerance
+android/.../ReceiptPlausibility.kt and compare_ground_truth.py use
+(total_price == price_per_litre * litres - discount, within 0.05 EUR),
+applied here to candidate numbers found directly in raw OCR text rather
+than already-structured fields, since this runs before (and to help pick
+input for) the single structuring LLM call.
+"""
+import re
+
+TOLERANCE_EUR = 0.05
+
+_NUMBER_RE = re.compile(r"(\d{1,3}),(\d{2,3})")
+
+# Plausible-value ranges for German fuel receipts, used to keep the
+# combination search small and to reject obviously-wrong candidates
+# (transaction IDs, card digits, etc. pulled in by the same regex).
+_PRICE_RANGE = (0.5, 3.5)
+_LITRES_RANGE = (1.0, 150.0)
+_TOTAL_RANGE = (1.0, 500.0)
+_DISCOUNT_RANGE = (0.0, 50.0)
+
+
+def _candidates(ocr_text: str) -> list[float]:
+    out = []
+    for m in _NUMBER_RE.finditer(ocr_text):
+        try:
+            out.append(float(f"{m.group(1)}.{m.group(2)}"))
+        except ValueError:
+            continue
+    return out
+
+
+def best_residual(ocr_text: str) -> float:
+    """Smallest |price_per_litre * litres - discount - total_price| over
+    plausible-range combinations of numbers found in ocr_text. Lower is
+    more plausible; see is_plausible() for the pass/fail threshold. inf
+    if the text doesn't contain enough candidate numbers in the right
+    ranges to try any combination at all."""
+    candidates = _candidates(ocr_text)
+    prices = [c for c in candidates if _PRICE_RANGE[0] <= c <= _PRICE_RANGE[1]]
+    litres = [c for c in candidates if _LITRES_RANGE[0] <= c <= _LITRES_RANGE[1]]
+    totals = [c for c in candidates if _TOTAL_RANGE[0] <= c <= _TOTAL_RANGE[1]]
+    discounts = [0.0] + [
+        c for c in candidates if _DISCOUNT_RANGE[0] < c <= _DISCOUNT_RANGE[1]
+    ]
+
+    best = float("inf")
+    for p in prices:
+        for litre in litres:
+            if litre == p:
+                continue
+            price_times_litres = p * litre
+            for t in totals:
+                if t == p or t == litre:
+                    continue
+                for d in discounts:
+                    residual = abs(price_times_litres - d - t)
+                    if residual < best:
+                        best = residual
+    return best
+
+
+def is_plausible(residual: float) -> bool:
+    return residual <= TOLERANCE_EUR

tooling-container/src/preprocess.py

@@ -8,12 +8,22 @@ the output directory (01_gray, 02_bilateral, 03_final) so the effect of
 each step can be inspected on its own before trusting the final image to
 OCR.
 
-Heavier processing (strong denoising, local/CLAHE contrast, unsharp
+Heavier processing (strong denoising, local/CLAHE contrast, naive unsharp
 masking) was tried and tested worse: each one amplifies the security
 background pattern into OCR noise rather than suppressing it. This
-lighter two-step version was chosen because, across a set of receipts
-that the plain-grayscale baseline OCR'd incorrectly, it recovered data
-the baseline missed and never made a previously-correct read worse.
+lighter two-step version (variant 0 below, and the default) was chosen
+because, across a set of receipts that the plain-grayscale baseline
+OCR'd incorrectly, it recovered data the baseline missed and never made
+a previously-correct read worse.
+
+Variants 1-3 add a high-pass-style detail boost and/or edge enhancement
+before this same bilateral+autocontrast stage. Applied unconditionally
+to every receipt, this combination also tested worse on the full batch
+for the same reason (see
+testing/2026_10_04_gimp_highpass_nlfilter_experiment.md) - but it
+noticeably helps the specific hard receipts it was designed for, so
+adaptive.py tries these variants only when the cheap variant 0 already
+looks implausible, rather than applying them globally.
 """
 import argparse
 import os
@@ -22,6 +32,13 @@ import cv2
 import numpy as np
 from PIL import Image, ImageOps
 
+# Width the variant 1-3 parameters below were calibrated at; their blur
+# radii scale relative to this so they hold up across this project's test
+# photos, which range from ~768px to ~4096px wide.
+REFERENCE_WIDTH = 768
+
+VARIANT_COUNT = 4  # 0=baseline, 1=+highpass, 2=+edge-enhance, 3=+both
+
 
 def load_grayscale(path: str) -> np.ndarray:
     img = Image.open(path)
@@ -29,23 +46,66 @@ def load_grayscale(path: str) -> np.ndarray:
     return np.array(img)
 
 
+def highpass_substitute(
+    gray: np.ndarray, blur_sigma=4.15, low_in=58, high_in=135, gamma=0.96
+) -> np.ndarray:
+    scale = gray.shape[1] / REFERENCE_WIDTH
+    blurred = cv2.GaussianBlur(gray, (0, 0), sigmaX=blur_sigma * scale)
+    grain = np.clip(gray.astype(np.float32) - blurred.astype(np.float32) + 128.0, 0, 255)
+    normalized = np.clip((grain - low_in) / (high_in - low_in), 0, 1)
+    gamma_applied = np.power(normalized, 1.0 / gamma)
+    return np.clip(gamma_applied * 255.0, 0, 255).astype(np.uint8)
+
+
+def edge_enhance(gray: np.ndarray, amount=0.69, sigma=0.52) -> np.ndarray:
+    scale = gray.shape[1] / REFERENCE_WIDTH
+    blurred = cv2.GaussianBlur(gray, (0, 0), sigmaX=sigma * scale)
+    sharpened = gray.astype(np.float32) + amount * (
+        gray.astype(np.float32) - blurred.astype(np.float32)
+    )
+    return np.clip(sharpened, 0, 255).astype(np.uint8)
+
+
 def autocontrast(gray: np.ndarray, low_pct=2, high_pct=98) -> np.ndarray:
     lo, hi = np.percentile(gray, (low_pct, high_pct))
     stretched = (gray.astype(np.float32) - lo) * 255.0 / (hi - lo)
     return np.clip(stretched, 0, 255).astype(np.uint8)
 
 
-def preprocess(path: str, out_dir: str) -> str:
+def preprocess(path: str, out_dir: str, variant: int = 0) -> str:
+    """variant: 0=baseline (bilateral+autocontrast only, unchanged default
+    behavior), 1=+highpass_substitute, 2=+edge_enhance, 3=+both. See
+    adaptive.py for why this is a parameter rather than always-on."""
+    if not 0 <= variant < VARIANT_COUNT:
+        raise ValueError(f"variant must be 0-{VARIANT_COUNT - 1}, got {variant}")
+
     os.makedirs(out_dir, exist_ok=True)
 
-    gray = load_grayscale(path)
-    cv2.imwrite(os.path.join(out_dir, "01_gray.png"), gray)
+    stage = load_grayscale(path)
+    cv2.imwrite(os.path.join(out_dir, "01_gray.png"), stage)
 
-    bilateral = cv2.bilateralFilter(gray, d=5, sigmaColor=50, sigmaSpace=50)
-    cv2.imwrite(os.path.join(out_dir, "02_bilateral.png"), bilateral)
+    # Stage filenames only extend beyond the original 01/02/03 numbering
+    # for variants 1-3, which adaptive.py always runs in their own
+    # per-variant subdirectory - so this never changes what
+    # 1_preprocess_image.sh's default (variant 0) manual single-stage
+    # inspection produces, or any doc/script that hardcodes 03_final.png.
+    next_n = 2
+    if variant in (1, 3):
+        stage = highpass_substitute(stage)
+        cv2.imwrite(os.path.join(out_dir, f"0{next_n}_highpass.png"), stage)
+        next_n += 1
+
+    if variant in (2, 3):
+        stage = edge_enhance(stage)
+        cv2.imwrite(os.path.join(out_dir, f"0{next_n}_edgeenhance.png"), stage)
+        next_n += 1
+
+    bilateral = cv2.bilateralFilter(stage, d=5, sigmaColor=50, sigmaSpace=50)
+    cv2.imwrite(os.path.join(out_dir, f"0{next_n}_bilateral.png"), bilateral)
+    next_n += 1
 
     final = autocontrast(bilateral)
-    final_path = os.path.join(out_dir, "03_final.png")
+    final_path = os.path.join(out_dir, f"0{next_n}_final.png")
     cv2.imwrite(final_path, final)
 
     return final_path
@@ -55,9 +115,10 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("-in", dest="in_path", required=True)
     parser.add_argument("-out", dest="out_dir", required=True)
+    parser.add_argument("-variant", dest="variant", type=int, default=0)
     args = parser.parse_args()
 
-    final_path = preprocess(args.in_path, args.out_dir)
+    final_path = preprocess(args.in_path, args.out_dir, args.variant)
     print(final_path)
 
 

tooling-container/src/server.py

@@ -24,9 +24,8 @@ import requests
 from fastapi import FastAPI, HTTPException, UploadFile
 from fastapi.responses import JSONResponse
 
+from adaptive import preprocess_and_ocr_adaptive
 from extract import structure
-from ocr import ocr as run_ocr
-from preprocess import preprocess
 
 OLLAMA_BASE_URL = os.environ.get("OLLAMA_BASE_URL", "http://127.0.0.1:11434")
 OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "qwen2.5:3b")
@@ -85,15 +84,10 @@ def health() -> dict:
 def _process_job(job_id: str, image_path: str, scratch_dir: str) -> None:
     try:
         _set_job(job_id, status="preprocessing")
-        final_path = preprocess(image_path, scratch_dir)
-
-        _set_job(job_id, status="ocr")
-        ocr_txt_path = run_ocr(final_path, scratch_dir)
-        with open(ocr_txt_path, "r", encoding="utf-8") as f:
-            ocr_text = f.read()
+        adaptive_result = preprocess_and_ocr_adaptive(image_path, scratch_dir)
 
         _set_job(job_id, status="extracting")
-        result = structure(ocr_text)
+        result = structure(adaptive_result["ocr_text"])
 
         _set_job(job_id, status="done", result=result)
     except Exception as exc:  # noqa: BLE001 - reported to the client, not swallowed