Add camera scan-to-text for notes via on-device OCR

Lets a photo of a book page fill in the note text instead of typing it out. OCR runs entirely client-side (tesseract.js, dynamically imported so it doesn't bloat the main bundle) - no server changes, no API keys, and the photo never leaves the device. Recognized text is appended into the textarea for the user to review/edit before saving. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01C3DGg1pvmvf81zXs4pk1J7
2026-08-27 18:19:25 +02:00 · 2026-08-27 18:19:25 +02:00 · 88d965a364
commit 88d965a364
parent 39f7eb8804
6 changed files with 177 additions and 5 deletions
--- a/README.md
+++ b/README.md
@ -1,6 +1,6 @@
 # Notes

-A small PWA for jotting down short notes while reading — e.g. an interesting sentence from a book. Each note can optionally be tagged with a book/topic (existing or new) and a page number. Notes are saved as markdown directly into pages of a Nextcloud **Collective**, one page per book/topic, nested under a parent page (default: "Reading Notes"). Untagged notes are appended to that parent page itself.
+A small PWA for jotting down short notes while reading — e.g. an interesting sentence from a book. Each note is tagged with a book/topic (existing or new) and can optionally include a page number. Notes are saved as markdown directly into pages of a Nextcloud **Collective**, one page per book/topic, nested under a parent page (default: "Reading Notes"). A note's text can be typed by hand or filled in by scanning a photo of the page (on-device OCR via `tesseract.js` — no image or text ever leaves the device for this step).

 ## Stack

--- a/client/package.json
+++ b/client/package.json
@ -12,7 +12,8 @@
    "@notes/shared": "*",
    "@tanstack/react-query": "^5.51.23",
    "react": "^18.3.1",
-    "react-dom": "^18.3.1"
+    "react-dom": "^18.3.1",
+    "tesseract.js": "^7.0.0"
  },
  "devDependencies": {
    "@types/react": "^18.3.3",
--- a/client/src/lib/ocr.ts
+++ b/client/src/lib/ocr.ts
@ -0,0 +1,23 @@
+let workerPromise: Promise<import("tesseract.js").Worker> | null = null;
+
+function getWorker() {
+  if (!workerPromise) {
+    workerPromise = import("tesseract.js").then(({ createWorker }) => createWorker("eng"));
+  }
+  return workerPromise;
+}
+
+function cleanUpText(raw: string): string {
+  return raw
+    .replace(/[ \t]+\n/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+export async function recognizeText(file: File): Promise<string> {
+  const worker = await getWorker();
+  const {
+    data: { text }
+  } = await worker.recognize(file);
+  return cleanUpText(text);
+}
--- a/client/src/pages/NoteFormPage.tsx
+++ b/client/src/pages/NoteFormPage.tsx
@ -1,8 +1,9 @@
-import { useState, type FormEvent } from "react";
+import { useRef, useState, type ChangeEvent, type FormEvent } from "react";
 import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
 import { fetchBooks } from "../api/books";
 import { saveNote } from "../api/notes";
 import { logout } from "../api/auth";
+import { recognizeText } from "../lib/ocr";

 const NEW_BOOK_OPTION = "__new__";

@ -15,6 +16,9 @@ export function NoteFormPage({ onLoggedOut }: { onLoggedOut: () => void }) {
  const [newBookTitle, setNewBookTitle] = useState("");
  const [page, setPage] = useState("");
  const [text, setText] = useState("");
+  const [isScanning, setIsScanning] = useState(false);
+  const [scanError, setScanError] = useState<string | null>(null);
+  const fileInputRef = useRef<HTMLInputElement>(null);

  const saveNoteMutation = useMutation({
    mutationFn: saveNote,
@ -41,6 +45,23 @@ export function NoteFormPage({ onLoggedOut }: { onLoggedOut: () => void }) {
    }
  }

+  async function handleImageSelected(e: ChangeEvent<HTMLInputElement>) {
+    const file = e.target.files?.[0];
+    if (!file) return;
+
+    setIsScanning(true);
+    setScanError(null);
+    try {
+      const recognized = await recognizeText(file);
+      setText((prev) => (prev.trim() ? `${prev}\n${recognized}` : recognized));
+    } catch {
+      setScanError("Couldn't recognize text in that photo. Try again with better lighting/focus.");
+    } finally {
+      setIsScanning(false);
+      e.target.value = "";
+    }
+  }
+
  function handleSubmit(e: FormEvent) {
    e.preventDefault();
    const trimmedBook = (isAddingNewBook ? newBookTitle : book).trim();
@ -65,7 +86,25 @@ export function NoteFormPage({ onLoggedOut }: { onLoggedOut: () => void }) {
        </div>

        <label>
+          <div className="header-row">
            Note
+            <button
+              type="button"
+              className="link"
+              onClick={() => fileInputRef.current?.click()}
+              disabled={isScanning || saveNoteMutation.isPending}
+            >
+              📷 Scan
+            </button>
+          </div>
+          <input
+            ref={fileInputRef}
+            type="file"
+            accept="image/*"
+            capture="environment"
+            onChange={handleImageSelected}
+            style={{ display: "none" }}
+          />
          <textarea
            value={text}
            onChange={(e) => setText(e.target.value)}
@ -73,6 +112,8 @@ export function NoteFormPage({ onLoggedOut }: { onLoggedOut: () => void }) {
            placeholder="An interesting sentence…"
            required
          />
+          {isScanning && <span className="hint">Recognizing text…</span>}
+          {scanError && <span className="error">{scanError}</span>}
        </label>

        <label>
--- a/client/src/styles.css
+++ b/client/src/styles.css
@ -124,3 +124,8 @@ button.link {
  font-size: 0.9rem;
  margin: 0;
 }
+
+.hint {
+  color: var(--muted);
+  font-size: 0.85rem;
+}
--- a/package-lock.json
+++ b/package-lock.json
@ -29,7 +29,8 @@
        "@notes/shared": "*",
        "@tanstack/react-query": "^5.51.23",
        "react": "^18.3.1",
-        "react-dom": "^18.3.1"
+        "react-dom": "^18.3.1",
+        "tesseract.js": "^7.0.0"
      },
      "devDependencies": {
        "@types/react": "^18.3.3",
@ -3492,6 +3493,11 @@
        "node": ">=6.0.0"
      }
    },
+    "node_modules/bmp-js": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
+      "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw=="
+    },
    "node_modules/body-parser": {
      "version": "1.20.6",
      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.6.tgz",
@ -5317,6 +5323,11 @@
      "integrity": "sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ==",
      "dev": true
    },
+    "node_modules/idb-keyval": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.3.0.tgz",
+      "integrity": "sha512-um+2dgAWmYsu615EXpWVwSmapJhON0G43t3Ka/EVaohzPQXSMqKEqeDK/oIW3Ow+BXaF2PvSc+oBTFp793A5Ow=="
+    },
    "node_modules/ignore": {
      "version": "5.3.2",
      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
@ -5782,6 +5793,11 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/is-url": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
+      "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww=="
+    },
    "node_modules/is-weakmap": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz",
@ -6202,6 +6218,25 @@
        "node": "^18 || ^20 || >= 21"
      }
    },
+    "node_modules/node-fetch": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
+      "engines": {
+        "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/node-gyp-build": {
      "version": "4.8.4",
      "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz",
@ -6289,6 +6324,14 @@
        "wrappy": "1"
      }
    },
+    "node_modules/opencollective-postinstall": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz",
+      "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==",
+      "bin": {
+        "opencollective-postinstall": "index.js"
+      }
+    },
    "node_modules/optionator": {
      "version": "0.9.4",
      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@ -6692,6 +6735,11 @@
        "node": ">=4"
      }
    },
+    "node_modules/regenerator-runtime": {
+      "version": "0.13.11",
+      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
+      "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg=="
+    },
    "node_modules/regexp.prototype.flags": {
      "version": "1.5.4",
      "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz",
@ -7591,6 +7639,28 @@
        "node": ">=10"
      }
    },
+    "node_modules/tesseract.js": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz",
+      "integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==",
+      "hasInstallScript": true,
+      "dependencies": {
+        "bmp-js": "^0.1.0",
+        "idb-keyval": "^6.2.0",
+        "is-url": "^1.2.4",
+        "node-fetch": "^2.6.9",
+        "opencollective-postinstall": "^2.0.3",
+        "regenerator-runtime": "^0.13.3",
+        "tesseract.js-core": "^7.0.0",
+        "wasm-feature-detect": "^1.8.0",
+        "zlibjs": "^0.3.1"
+      }
+    },
+    "node_modules/tesseract.js-core": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz",
+      "integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw=="
+    },
    "node_modules/text-table": {
      "version": "0.2.0",
      "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@ -7662,6 +7732,11 @@
        "node": ">=0.6"
      }
    },
+    "node_modules/tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
+    },
    "node_modules/tree-kill": {
      "version": "1.2.2",
      "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
@ -8589,6 +8664,25 @@
        "url": "https://github.com/sponsors/jonschlinkert"
      }
    },
+    "node_modules/wasm-feature-detect": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.9.0.tgz",
+      "integrity": "sha512-zonE+xlIIYtxPy++L24ow0hAD8CICb4+FgPyROd3buyXIqsJvUEDkBgfCCoXOd1Hu3DUr0GOfnPIdcGV+YpNaA=="
+    },
+    "node_modules/webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
+    },
+    "node_modules/whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+      "dependencies": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
    "node_modules/which": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@ -9222,6 +9316,14 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/zlibjs": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz",
+      "integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==",
+      "engines": {
+        "node": "*"
+      }
+    },
    "node_modules/zod": {
      "version": "3.25.76",
      "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",