From 3da4dec452cddcdf5b8a3d65f23a1a11cc8cbb59 Mon Sep 17 00:00:00 2001 From: Claudio Schaad Date: Thu, 27 Aug 2026 18:43:47 +0200 Subject: [PATCH] Let the user crop the scanned photo before OCR Whole-page photos carry far more text than the one sentence someone wants, hurting OCR accuracy and letting stray text leak into notes. Adds a crop overlay (drag-to-select via Pointer Events, works for touch and mouse) between capture and OCR, with a "use full photo" escape hatch for already-tight shots. recognizeText now takes a Blob so it accepts either the cropped canvas output or the original file. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01C3DGg1pvmvf81zXs4pk1J7 --- client/src/components/ImageCropper.tsx | 150 +++++++++++++++++++++++++ client/src/lib/ocr.ts | 4 +- client/src/pages/NoteFormPage.tsx | 21 +++- client/src/styles.css | 52 +++++++++ 4 files changed, 222 insertions(+), 5 deletions(-) create mode 100644 client/src/components/ImageCropper.tsx diff --git a/client/src/components/ImageCropper.tsx b/client/src/components/ImageCropper.tsx new file mode 100644 index 0000000..c357101 --- /dev/null +++ b/client/src/components/ImageCropper.tsx @@ -0,0 +1,150 @@ +import { useEffect, useRef, useState, type PointerEvent as ReactPointerEvent } from "react"; + +type Rect = { x: number; y: number; w: number; h: number }; + +const MIN_RECT_SIZE = 12; + +function clamp(value: number, min: number, max: number) { + return Math.min(Math.max(value, min), max); +} + +async function cropToBlob(img: HTMLImageElement, rect: Rect): Promise { + const scaleX = img.naturalWidth / img.clientWidth; + const scaleY = img.naturalHeight / img.clientHeight; + + const canvas = document.createElement("canvas"); + canvas.width = Math.round(rect.w * scaleX); + canvas.height = Math.round(rect.h * scaleY); + + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Could not create canvas context"); + + ctx.drawImage( + img, + rect.x * scaleX, + rect.y * scaleY, + rect.w * scaleX, + rect.h * scaleY, + 0, + 0, + canvas.width, + canvas.height + ); + + return new Promise((resolve, reject) => { + canvas.toBlob( + (blob) => (blob ? resolve(blob) : reject(new Error("Could not crop image"))), + "image/jpeg", + 0.92 + ); + }); +} + +export function ImageCropper({ + imageFile, + onCancel, + onCropped +}: { + imageFile: File; + onCancel: () => void; + onCropped: (image: Blob) => void; +}) { + const [objectUrl, setObjectUrl] = useState(null); + const [rect, setRect] = useState(null); + const [isCropping, setIsCropping] = useState(false); + const imgRef = useRef(null); + const wrapRef = useRef(null); + const dragStart = useRef<{ x: number; y: number } | null>(null); + + useEffect(() => { + const url = URL.createObjectURL(imageFile); + setObjectUrl(url); + return () => URL.revokeObjectURL(url); + }, [imageFile]); + + function pointFromEvent(e: ReactPointerEvent) { + const bounds = wrapRef.current!.getBoundingClientRect(); + return { + x: clamp(e.clientX - bounds.left, 0, bounds.width), + y: clamp(e.clientY - bounds.top, 0, bounds.height) + }; + } + + function handlePointerDown(e: ReactPointerEvent) { + e.currentTarget.setPointerCapture(e.pointerId); + dragStart.current = pointFromEvent(e); + setRect({ x: dragStart.current.x, y: dragStart.current.y, w: 0, h: 0 }); + } + + function handlePointerMove(e: ReactPointerEvent) { + if (!dragStart.current) return; + const current = pointFromEvent(e); + const start = dragStart.current; + setRect({ + x: Math.min(start.x, current.x), + y: Math.min(start.y, current.y), + w: Math.abs(current.x - start.x), + h: Math.abs(current.y - start.y) + }); + } + + function handlePointerUp() { + dragStart.current = null; + } + + async function handleConfirmCrop() { + if (!imgRef.current || !rect) return; + setIsCropping(true); + try { + const blob = await cropToBlob(imgRef.current, rect); + onCropped(blob); + } finally { + setIsCropping(false); + } + } + + const hasValidRect = !!rect && rect.w >= MIN_RECT_SIZE && rect.h >= MIN_RECT_SIZE; + + return ( +
+
+

Drag over the photo to select just the text you want.

+ + {objectUrl && ( +
+ Captured page + {rect && ( +
+ )} +
+ )} + +
+ + + +
+
+
+ ); +} diff --git a/client/src/lib/ocr.ts b/client/src/lib/ocr.ts index 3bf5701..5f13f9f 100644 --- a/client/src/lib/ocr.ts +++ b/client/src/lib/ocr.ts @@ -14,10 +14,10 @@ function cleanUpText(raw: string): string { .trim(); } -export async function recognizeText(file: File): Promise { +export async function recognizeText(image: Blob): Promise { const worker = await getWorker(); const { data: { text } - } = await worker.recognize(file); + } = await worker.recognize(image); return cleanUpText(text); } diff --git a/client/src/pages/NoteFormPage.tsx b/client/src/pages/NoteFormPage.tsx index 470bef7..dba4715 100644 --- a/client/src/pages/NoteFormPage.tsx +++ b/client/src/pages/NoteFormPage.tsx @@ -4,6 +4,7 @@ import { fetchBooks } from "../api/books"; import { saveNote } from "../api/notes"; import { logout } from "../api/auth"; import { recognizeText } from "../lib/ocr"; +import { ImageCropper } from "../components/ImageCropper"; const NEW_BOOK_OPTION = "__new__"; @@ -18,6 +19,7 @@ export function NoteFormPage({ onLoggedOut }: { onLoggedOut: () => void }) { const [text, setText] = useState(""); const [isScanning, setIsScanning] = useState(false); const [scanError, setScanError] = useState(null); + const [pendingImage, setPendingImage] = useState(null); const fileInputRef = useRef(null); const saveNoteMutation = useMutation({ @@ -45,20 +47,25 @@ export function NoteFormPage({ onLoggedOut }: { onLoggedOut: () => void }) { } } - async function handleImageSelected(e: ChangeEvent) { + function handleImageSelected(e: ChangeEvent) { const file = e.target.files?.[0]; + e.target.value = ""; if (!file) return; + setScanError(null); + setPendingImage(file); + } + async function handleCropped(image: Blob) { + setPendingImage(null); setIsScanning(true); setScanError(null); try { - const recognized = await recognizeText(file); + const recognized = await recognizeText(image); setText((prev) => (prev.trim() ? `${prev}\n${recognized}` : recognized)); } catch { setScanError("Couldn't recognize text in that photo. Try again with better lighting/focus."); } finally { setIsScanning(false); - e.target.value = ""; } } @@ -77,6 +84,14 @@ export function NoteFormPage({ onLoggedOut }: { onLoggedOut: () => void }) { return (
+ {pendingImage && ( + setPendingImage(null)} + onCropped={handleCropped} + /> + )} +

New note

diff --git a/client/src/styles.css b/client/src/styles.css index e486d09..71dd2a0 100644 --- a/client/src/styles.css +++ b/client/src/styles.css @@ -129,3 +129,55 @@ button.link { color: var(--muted); font-size: 0.85rem; } + +.crop-overlay { + position: fixed; + inset: 0; + background: rgba(0, 0, 0, 0.6); + display: flex; + align-items: center; + justify-content: center; + padding: 1.5rem; + z-index: 100; +} + +.crop-dialog { + width: 100%; + max-width: 480px; + background: var(--card-bg); + border: 1px solid var(--border); + border-radius: 12px; + padding: 1.25rem; + display: flex; + flex-direction: column; + gap: 0.75rem; +} + +.crop-image-wrap { + position: relative; + touch-action: none; + user-select: none; + max-height: 60vh; + overflow: hidden; + border-radius: 8px; +} + +.crop-image-wrap img { + display: block; + width: 100%; + height: auto; +} + +.crop-rect { + position: absolute; + border: 2px solid var(--accent); + background: rgba(31, 111, 235, 0.25); + pointer-events: none; +} + +.crop-actions { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.75rem; +}