From 6e93b7f67254607d5a01fb77ae013d53d0a6e4ac Mon Sep 17 00:00:00 2001
From: kamaji <kamaji@sdanywhere.com>
Date: Sat, 31 Jan 2026 23:35:16 -0600
Subject: [PATCH] Add receipt extraction, manual crop, and UX improvements

- Add Claude Haiku vision integration to extract amount and date from
  receipt photos, re-reading on photo replacement
- Add manual crop overlay with draggable handles for receipt photos
- Open camera directly when tapping + to add new receipt
- Make add/edit modal scrollable on small screens
- Show "Tap to change photo" hint on uploaded photos
- Include api-key in Docker image for Anthropic API access

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 Dockerfile    |   2 +-
 receipts.html | 314 +++++++++++++++++++++++++++++++++++++++++++++++++-
 server.py     |  79 ++++++++++++-
 3 files changed, 388 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f248129..0d3abd6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,7 +5,7 @@ WORKDIR /app
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY server.py receipts.html ./
+COPY server.py receipts.html api-key ./
 
 EXPOSE 8080
 
diff --git a/receipts.html b/receipts.html
index 4f92b65..8831080 100644
--- a/receipts.html
+++ b/receipts.html
@@ -82,6 +82,7 @@
     z-index: 100;
     justify-content: center;
     align-items: flex-end;
+    overflow-y: auto;
   }
   .modal-overlay.open { display: flex; }
 
@@ -89,6 +90,8 @@
     background: #fff;
     width: 100%;
     max-width: 600px;
+    max-height: 90vh;
+    overflow-y: auto;
     border-radius: 16px 16px 0 0;
     padding: 24px 20px 32px;
     animation: slideUp 0.25s ease;
@@ -161,6 +164,13 @@
     border-radius: 6px;
   }
   .photo-area svg { opacity: 0.4; }
+  .extract-status {
+    font-size: 0.8rem;
+    color: #888;
+    text-align: center;
+    padding: 6px 0 0;
+    min-height: 1.4em;
+  }
 
   .btn-row {
     display: flex;
@@ -249,6 +259,51 @@
   .empty-state svg { margin-bottom: 12px; opacity: 0.3; }
   .empty-state p { font-size: 0.95rem; }
 
+  /* Manual crop overlay */
+  .crop-overlay {
+    display: none;
+    position: fixed;
+    inset: 0;
+    background: rgba(0,0,0,0.92);
+    z-index: 150;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+  }
+  .crop-overlay.open { display: flex; }
+  .crop-canvas-wrap {
+    flex: 1;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    width: 100%;
+    overflow: hidden;
+    padding: 12px;
+  }
+  .crop-canvas-wrap canvas {
+    max-width: 100%;
+    max-height: 100%;
+    touch-action: none;
+  }
+  .crop-toolbar {
+    display: flex;
+    gap: 12px;
+    padding: 12px 20px 24px;
+    width: 100%;
+    max-width: 500px;
+  }
+  .crop-toolbar button {
+    flex: 1;
+    padding: 12px;
+    border: none;
+    border-radius: 8px;
+    font-size: 1rem;
+    font-weight: 600;
+    cursor: pointer;
+  }
+  .crop-btn-full { background: rgba(255,255,255,0.15); color: #fff; }
+  .crop-btn-crop { background: #e94560; color: #fff; }
+
   /* Login overlay */
   .login-overlay {
     position: fixed;
@@ -440,6 +495,7 @@
         <span>Tap to take photo</span>
       </div>
       <input type="file" id="photo-input" accept="image/*" capture="environment" hidden>
+      <div class="extract-status" id="extract-status"></div>
     </div>
 
     <div class="field">
@@ -480,6 +536,17 @@
   <img id="photo-preview-img" src="" alt="Receipt photo">
 </div>
 
+<!-- Manual crop overlay -->
+<div class="crop-overlay" id="crop-overlay">
+  <div class="crop-canvas-wrap">
+    <canvas id="crop-canvas"></canvas>
+  </div>
+  <div class="crop-toolbar">
+    <button class="crop-btn-full" id="crop-full">Use Full Image</button>
+    <button class="crop-btn-crop" id="crop-use">Crop &amp; Use</button>
+  </div>
+</div>
+
 <!-- Login overlay -->
 <div class="login-overlay" id="login-overlay">
   <div class="login-box">
@@ -638,6 +705,7 @@
       currentPhotoUrl = null;
     }
     updatePhotoArea();
+    extractStatus.textContent = "";
     overlay.classList.add("open");
     amountInput.focus();
   }
@@ -650,7 +718,7 @@
   function updatePhotoArea() {
     if (currentPhotoUrl) {
       photoArea.classList.add("has-photo");
-      photoArea.innerHTML = `<img src="${currentPhotoUrl}" alt="Receipt photo">`;
+      photoArea.innerHTML = `<img src="${currentPhotoUrl}" alt="Receipt photo"><span style="position:absolute;bottom:0;left:0;right:0;background:rgba(0,0,0,0.5);color:#fff;font-size:0.75rem;text-align:center;padding:4px 0;">Tap to change photo</span>`;
     } else {
       photoArea.classList.remove("has-photo");
       photoArea.innerHTML = `
@@ -823,7 +891,10 @@
   }
 
   // Events
-  document.getElementById("add-btn").addEventListener("click", () => openModal(null));
+  document.getElementById("add-btn").addEventListener("click", () => {
+    openModal(null);
+    photoInput.click();
+  });
   document.getElementById("btn-cancel").addEventListener("click", closeModal);
 
   overlay.addEventListener("click", e => {
@@ -832,12 +903,245 @@
 
   photoArea.addEventListener("click", () => photoInput.click());
 
+  // --- Receipt extraction via Claude ---
+  const extractStatus = document.getElementById("extract-status");
+
+  async function extractReceiptInfo(blob) {
+    extractStatus.textContent = "Reading receipt...";
+    try {
+      const r = await fetch("/api/extract-receipt", {
+        method: "POST",
+        headers: { "Content-Type": "image/jpeg" },
+        body: blob
+      });
+      if (r.ok) {
+        const data = await r.json();
+        if (data.amount != null) {
+          amountInput.value = data.amount;
+        }
+        if (data.date != null) {
+          dateInput.value = data.date;
+        }
+      }
+    } catch (e) {
+      console.error("Receipt extraction failed:", e);
+    } finally {
+      extractStatus.textContent = "";
+    }
+  }
+
+  // --- Manual crop tool ---
+  const cropOverlay = document.getElementById("crop-overlay");
+  const cropCanvas = document.getElementById("crop-canvas");
+  const cropCtx = cropCanvas.getContext("2d");
+
+  let cropImg = null;        // Image element for the selected photo
+  let cropOriginalBlob = null;
+  let cropScale = 1;         // scale factor: canvas pixels → display pixels
+  let cropImgX = 0, cropImgY = 0; // image offset on canvas
+  let cropImgW = 0, cropImgH = 0; // drawn image size on canvas
+
+  // Crop rectangle in canvas coordinates
+  let cropRect = { x: 0, y: 0, w: 0, h: 0 };
+  const HANDLE_SIZE = 18; // touch-friendly handle radius
+
+  let cropDrag = null; // { type: 'move'|'tl'|'tr'|'bl'|'br'|'t'|'b'|'l'|'r', sx, sy, origRect }
+
+  function drawCrop() {
+    const cw = cropCanvas.width, ch = cropCanvas.height;
+    cropCtx.clearRect(0, 0, cw, ch);
+    // Draw image
+    cropCtx.drawImage(cropImg, cropImgX, cropImgY, cropImgW, cropImgH);
+    // Dark mask outside crop rect
+    cropCtx.fillStyle = "rgba(0,0,0,0.55)";
+    // Top
+    cropCtx.fillRect(0, 0, cw, cropRect.y);
+    // Bottom
+    cropCtx.fillRect(0, cropRect.y + cropRect.h, cw, ch - cropRect.y - cropRect.h);
+    // Left
+    cropCtx.fillRect(0, cropRect.y, cropRect.x, cropRect.h);
+    // Right
+    cropCtx.fillRect(cropRect.x + cropRect.w, cropRect.y, cw - cropRect.x - cropRect.w, cropRect.h);
+    // Crop border
+    cropCtx.strokeStyle = "#e94560";
+    cropCtx.lineWidth = 2;
+    cropCtx.strokeRect(cropRect.x, cropRect.y, cropRect.w, cropRect.h);
+    // Corner handles
+    const hs = HANDLE_SIZE / 2;
+    cropCtx.fillStyle = "#e94560";
+    const corners = [
+      [cropRect.x, cropRect.y],
+      [cropRect.x + cropRect.w, cropRect.y],
+      [cropRect.x, cropRect.y + cropRect.h],
+      [cropRect.x + cropRect.w, cropRect.y + cropRect.h],
+    ];
+    corners.forEach(([cx, cy]) => {
+      cropCtx.beginPath();
+      cropCtx.arc(cx, cy, hs, 0, Math.PI * 2);
+      cropCtx.fill();
+    });
+  }
+
+  function getCropPos(e) {
+    const rect = cropCanvas.getBoundingClientRect();
+    const scX = cropCanvas.width / rect.width;
+    const scY = cropCanvas.height / rect.height;
+    const touch = e.touches ? e.touches[0] : e;
+    return { x: (touch.clientX - rect.left) * scX, y: (touch.clientY - rect.top) * scY };
+  }
+
+  function hitTest(pos) {
+    const r = cropRect;
+    const hs = HANDLE_SIZE;
+    // Check corners first
+    if (Math.hypot(pos.x - r.x, pos.y - r.y) < hs) return "tl";
+    if (Math.hypot(pos.x - (r.x + r.w), pos.y - r.y) < hs) return "tr";
+    if (Math.hypot(pos.x - r.x, pos.y - (r.y + r.h)) < hs) return "bl";
+    if (Math.hypot(pos.x - (r.x + r.w), pos.y - (r.y + r.h)) < hs) return "br";
+    // Check edges (within hs distance)
+    if (pos.x > r.x + hs && pos.x < r.x + r.w - hs) {
+      if (Math.abs(pos.y - r.y) < hs) return "t";
+      if (Math.abs(pos.y - (r.y + r.h)) < hs) return "b";
+    }
+    if (pos.y > r.y + hs && pos.y < r.y + r.h - hs) {
+      if (Math.abs(pos.x - r.x) < hs) return "l";
+      if (Math.abs(pos.x - (r.x + r.w)) < hs) return "r";
+    }
+    // Inside rect = move
+    if (pos.x >= r.x && pos.x <= r.x + r.w && pos.y >= r.y && pos.y <= r.y + r.h) return "move";
+    return null;
+  }
+
+  function clampRect(r) {
+    const minS = 30;
+    if (r.w < minS) r.w = minS;
+    if (r.h < minS) r.h = minS;
+    if (r.x < cropImgX) r.x = cropImgX;
+    if (r.y < cropImgY) r.y = cropImgY;
+    if (r.x + r.w > cropImgX + cropImgW) r.x = cropImgX + cropImgW - r.w;
+    if (r.y + r.h > cropImgY + cropImgH) r.y = cropImgY + cropImgH - r.h;
+    // Ensure still within image after clamping position
+    if (r.x < cropImgX) { r.x = cropImgX; r.w = cropImgW; }
+    if (r.y < cropImgY) { r.y = cropImgY; r.h = cropImgH; }
+  }
+
+  function onCropPointerDown(e) {
+    e.preventDefault();
+    const pos = getCropPos(e);
+    const type = hitTest(pos);
+    if (!type) return;
+    cropDrag = { type, sx: pos.x, sy: pos.y, origRect: { ...cropRect } };
+  }
+
+  function onCropPointerMove(e) {
+    if (!cropDrag) return;
+    e.preventDefault();
+    const pos = getCropPos(e);
+    const dx = pos.x - cropDrag.sx;
+    const dy = pos.y - cropDrag.sy;
+    const o = cropDrag.origRect;
+    const minS = 30;
+
+    if (cropDrag.type === "move") {
+      cropRect.x = o.x + dx;
+      cropRect.y = o.y + dy;
+      cropRect.w = o.w;
+      cropRect.h = o.h;
+    } else {
+      let nx = o.x, ny = o.y, nw = o.w, nh = o.h;
+      if (cropDrag.type.includes("l")) { nx = o.x + dx; nw = o.w - dx; }
+      if (cropDrag.type.includes("r")) { nw = o.w + dx; }
+      if (cropDrag.type.includes("t")) { ny = o.y + dy; nh = o.h - dy; }
+      if (cropDrag.type.includes("b")) { nh = o.h + dy; }
+      // Prevent inversion
+      if (nw < minS) { if (cropDrag.type.includes("l")) { nx = o.x + o.w - minS; } nw = minS; }
+      if (nh < minS) { if (cropDrag.type.includes("t")) { ny = o.y + o.h - minS; } nh = minS; }
+      cropRect.x = nx; cropRect.y = ny; cropRect.w = nw; cropRect.h = nh;
+    }
+    clampRect(cropRect);
+    drawCrop();
+  }
+
+  function onCropPointerUp(e) {
+    cropDrag = null;
+  }
+
+  // Attach mouse events
+  cropCanvas.addEventListener("mousedown", onCropPointerDown);
+  window.addEventListener("mousemove", onCropPointerMove);
+  window.addEventListener("mouseup", onCropPointerUp);
+  // Attach touch events
+  cropCanvas.addEventListener("touchstart", onCropPointerDown, { passive: false });
+  window.addEventListener("touchmove", onCropPointerMove, { passive: false });
+  window.addEventListener("touchend", onCropPointerUp);
+
+  function openCropOverlay(blob) {
+    cropOriginalBlob = blob;
+    cropImg = new Image();
+    cropImg.onload = () => {
+      // Show overlay first so the wrapper has layout dimensions
+      cropOverlay.classList.add("open");
+      requestAnimationFrame(() => {
+        const wrap = cropCanvas.parentElement;
+        const maxW = wrap.clientWidth - 24;
+        const maxH = wrap.clientHeight - 24;
+        let dw = cropImg.width, dh = cropImg.height;
+        const ratio = Math.min(maxW / dw, maxH / dh, 1);
+        dw = Math.round(dw * ratio);
+        dh = Math.round(dh * ratio);
+        cropCanvas.width = dw;
+        cropCanvas.height = dh;
+        cropImgX = 0; cropImgY = 0;
+        cropImgW = dw; cropImgH = dh;
+        // Default crop rect: inset 10% from each edge
+        const inset = 0.10;
+        cropRect = {
+          x: Math.round(dw * inset),
+          y: Math.round(dh * inset),
+          w: Math.round(dw * (1 - 2 * inset)),
+          h: Math.round(dh * (1 - 2 * inset))
+        };
+        drawCrop();
+      });
+    };
+    cropImg.src = URL.createObjectURL(blob);
+  }
+
+  document.getElementById("crop-full").addEventListener("click", () => {
+    currentPhotoBlob = cropOriginalBlob;
+    currentPhotoUrl = URL.createObjectURL(currentPhotoBlob);
+    updatePhotoArea();
+    cropOverlay.classList.remove("open");
+    extractReceiptInfo(currentPhotoBlob);
+  });
+
+  document.getElementById("crop-use").addEventListener("click", () => {
+    // Extract crop region from the original image at full resolution
+    const scaleX = cropImg.width / cropImgW;
+    const scaleY = cropImg.height / cropImgH;
+    const sx = Math.round((cropRect.x - cropImgX) * scaleX);
+    const sy = Math.round((cropRect.y - cropImgY) * scaleY);
+    const sw = Math.round(cropRect.w * scaleX);
+    const sh = Math.round(cropRect.h * scaleY);
+
+    const outCanvas = document.createElement("canvas");
+    outCanvas.width = sw;
+    outCanvas.height = sh;
+    outCanvas.getContext("2d").drawImage(cropImg, sx, sy, sw, sh, 0, 0, sw, sh);
+    outCanvas.toBlob(blob => {
+      currentPhotoBlob = blob;
+      currentPhotoUrl = URL.createObjectURL(blob);
+      updatePhotoArea();
+      cropOverlay.classList.remove("open");
+      extractReceiptInfo(currentPhotoBlob);
+    }, "image/jpeg", 0.85);
+  });
+
   photoInput.addEventListener("change", async e => {
     const file = e.target.files[0];
     if (!file) return;
-    currentPhotoBlob = await resizeImage(file, 1024);
-    currentPhotoUrl = URL.createObjectURL(currentPhotoBlob);
-    updatePhotoArea();
+    const resized = await resizeImage(file, 1024);
+    openCropOverlay(resized);
   });
 
   btnSave.addEventListener("click", async () => {
diff --git a/server.py b/server.py
index 7f7d4c8..8f4c720 100644
--- a/server.py
+++ b/server.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 """Receipt Manager backend — proxies receipt data and photos to Nextcloud via WebDAV."""
 
+import base64
 import getpass
 import hashlib
 import io
@@ -30,6 +31,8 @@ PHOTOS_DIR = "photos"
 # Server-side session store: {token: True}
 SESSIONS = {}
 
+ANTHROPIC_API_KEY = ""
+
 
 def hash_password(password):
     """Hash a password with SHA-256."""
@@ -159,6 +162,59 @@ def build_excel(receipts):
     return buf.getvalue()
 
 
+# --- Receipt extraction via Claude Haiku ---------------------------------------
+
+def extract_receipt_info(jpeg_bytes):
+    """Use Claude Haiku vision to extract total amount and date from a receipt image."""
+    try:
+        img_b64 = base64.standard_b64encode(jpeg_bytes).decode("ascii")
+        resp = requests.post(
+            "https://api.anthropic.com/v1/messages",
+            headers={
+                "x-api-key": ANTHROPIC_API_KEY,
+                "anthropic-version": "2023-06-01",
+                "Content-Type": "application/json",
+            },
+            json={
+                "model": "claude-3-haiku-20240307",
+                "max_tokens": 200,
+                "messages": [{
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image",
+                            "source": {
+                                "type": "base64",
+                                "media_type": "image/jpeg",
+                                "data": img_b64,
+                            },
+                        },
+                        {
+                            "type": "text",
+                            "text": 'Extract the total amount and transaction date from this receipt. Reply with JSON only: {"amount": number_or_null, "date": "YYYY-MM-DD"_or_null}',
+                        },
+                    ],
+                }],
+            },
+            timeout=30,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        text = data["content"][0]["text"].strip()
+        # Extract JSON from response (may be wrapped in markdown code block)
+        m = re.search(r"\{.*\}", text, re.DOTALL)
+        if m:
+            parsed = json.loads(m.group())
+            return {
+                "amount": parsed.get("amount"),
+                "date": parsed.get("date"),
+            }
+    except Exception as e:
+        print(f"[extract_receipt_info] Error: {e}")
+    return {"amount": None, "date": None}
+
+
+
 # --- HTTP handler -------------------------------------------------------------
 
 class ReuseTCPServer(HTTPServer):
@@ -347,6 +403,18 @@ class Handler(BaseHTTPRequestHandler):
                 self._send_error(500, str(e))
             return
 
+        # POST /api/extract-receipt — extract total + date from receipt image
+        if self.path == "/api/extract-receipt":
+            if not self._check_session():
+                return
+            try:
+                body = self._read_body()
+                result = extract_receipt_info(body)
+                self._send_json(result)
+            except Exception as e:
+                self._send_json({"amount": None, "date": None})
+            return
+
         # POST /api/photos/<id> — upload photo
         m = re.fullmatch(r"/api/photos/([A-Za-z0-9_-]+)", self.path)
         if m:
@@ -420,9 +488,18 @@ def ensure_folder_path():
 
 
 def main():
-    global NC_USERNAME, NC_PASSWORD, NC_DAV_ROOT, NC_AUTH
+    global NC_USERNAME, NC_PASSWORD, NC_DAV_ROOT, NC_AUTH, ANTHROPIC_API_KEY
 
     print("=== Receipt Manager — Nextcloud Backend ===\n")
+
+    # Load Anthropic API key
+    try:
+        with open("api-key", "r") as f:
+            ANTHROPIC_API_KEY = f.read().strip()
+        print("Anthropic API key loaded.")
+    except FileNotFoundError:
+        print("WARNING: api-key file not found — receipt extraction disabled.")
+
     if not NC_USERNAME:
         NC_USERNAME = input("Nextcloud username: ").strip()
     if not NC_PASSWORD: