Initial commit

Photo-based book cataloger with AI identification.
Room → Cabinet → Shelf → Book hierarchy; FastAPI + SQLite backend;
vanilla JS SPA; OpenAI-compatible plugin system for boundary
detection, text recognition, and archive search.
This commit is contained in:
2026-03-09 14:17:13 +03:00
commit 084d1aebd5
64 changed files with 8605 additions and 0 deletions

407
src/api.py Normal file
View File

@@ -0,0 +1,407 @@
"""
API routes for the bookshelf cataloger.
Each handler: parse payload → validate existence → call logic → return response.
No SQL here; no business logic here.
"""
import asyncio
import dataclasses
import json
from typing import Any, TypeVar
from fastapi import APIRouter, File, HTTPException, Request, UploadFile
from mashumaro.codecs import BasicDecoder
import db
import logic
import plugins as plugin_registry
from config import get_config
from files import del_photo, save_photo
from logic.boundaries import book_spine_source, shelf_source
from logic.images import crop_save, serve_crop
from models import (
BoundariesPayload,
CropPayload,
DismissFieldPayload,
ReorderPayload,
UpdateBookPayload,
UpdateNamePayload,
)
router = APIRouter()
# ── Payload decoders ──────────────────────────────────────────────────────────
_name_dec: BasicDecoder[UpdateNamePayload] = BasicDecoder(UpdateNamePayload)
_book_dec: BasicDecoder[UpdateBookPayload] = BasicDecoder(UpdateBookPayload)
_boundaries_dec: BasicDecoder[BoundariesPayload] = BasicDecoder(BoundariesPayload)
_crop_dec: BasicDecoder[CropPayload] = BasicDecoder(CropPayload)
_dismiss_dec: BasicDecoder[DismissFieldPayload] = BasicDecoder(DismissFieldPayload)
_reorder_dec: BasicDecoder[ReorderPayload] = BasicDecoder(ReorderPayload)
_T = TypeVar("_T")
async def _parse(decoder: BasicDecoder[_T], request: Request) -> _T:
try:
return decoder.decode(await request.json())
except Exception as e:
raise HTTPException(422, str(e))
# ── Config ────────────────────────────────────────────────────────────────────
@router.get("/api/config")
def api_config() -> dict[str, Any]:
return {
"boundary_grab_px": get_config().ui.boundary_grab_px,
"plugins": plugin_registry.get_manifest(),
}
# ── Tree ──────────────────────────────────────────────────────────────────────
@router.get("/api/tree")
def get_tree() -> list[dict[str, Any]]:
with db.connection() as c:
return db.get_tree(c)
# ── Rooms ─────────────────────────────────────────────────────────────────────
@router.post("/api/rooms")
async def create_room() -> dict[str, Any]:
with db.transaction() as c:
room = db.create_room(c)
return {**dataclasses.asdict(room), "cabinets": []}
@router.put("/api/rooms/{room_id}")
async def update_room(room_id: str, request: Request) -> dict[str, Any]:
payload = await _parse(_name_dec, request)
with db.connection() as c:
if not db.get_room(c, room_id):
raise HTTPException(404, "Room not found")
with db.transaction() as c:
db.rename_room(c, room_id, payload.name.strip())
return {"ok": True}
@router.delete("/api/rooms/{room_id}")
async def delete_room(room_id: str) -> dict[str, Any]:
with db.connection() as c:
if not db.get_room(c, room_id):
raise HTTPException(404, "Room not found")
photos = db.collect_room_photos(c, room_id)
with db.transaction() as c:
db.delete_room(c, room_id)
for fn in photos:
del_photo(fn)
return {"ok": True}
# ── Cabinets ──────────────────────────────────────────────────────────────────
@router.post("/api/rooms/{room_id}/cabinets")
async def create_cabinet(room_id: str) -> dict[str, Any]:
with db.connection() as c:
if not db.get_room(c, room_id):
raise HTTPException(404, "Room not found")
with db.transaction() as c:
cabinet = db.create_cabinet(c, room_id)
return {**dataclasses.asdict(cabinet), "shelves": []}
@router.put("/api/cabinets/{cabinet_id}")
async def update_cabinet(cabinet_id: str, request: Request) -> dict[str, Any]:
payload = await _parse(_name_dec, request)
with db.connection() as c:
if not db.get_cabinet(c, cabinet_id):
raise HTTPException(404, "Cabinet not found")
with db.transaction() as c:
db.rename_cabinet(c, cabinet_id, payload.name.strip())
return {"ok": True}
@router.delete("/api/cabinets/{cabinet_id}")
async def delete_cabinet(cabinet_id: str) -> dict[str, Any]:
with db.connection() as c:
if not db.get_cabinet(c, cabinet_id):
raise HTTPException(404, "Cabinet not found")
photos = db.collect_cabinet_photos(c, cabinet_id)
with db.transaction() as c:
db.delete_cabinet(c, cabinet_id)
for fn in photos:
del_photo(fn)
return {"ok": True}
@router.post("/api/cabinets/{cabinet_id}/photo")
async def cabinet_photo(cabinet_id: str, image: UploadFile = File(...)) -> dict[str, Any]:
with db.connection() as c:
if not db.get_cabinet(c, cabinet_id):
raise HTTPException(404, "Cabinet not found")
old = db.get_cabinet_photo(c, cabinet_id)
del_photo(old)
fn = await save_photo(image)
with db.transaction() as c:
db.set_cabinet_photo(c, cabinet_id, fn)
return {"photo_filename": fn}
@router.patch("/api/cabinets/{cabinet_id}/boundaries")
async def update_cabinet_boundaries(cabinet_id: str, request: Request) -> dict[str, Any]:
payload = await _parse(_boundaries_dec, request)
with db.connection() as c:
if not db.get_cabinet(c, cabinet_id):
raise HTTPException(404, "Cabinet not found")
with db.transaction() as c:
db.set_cabinet_boundaries(c, cabinet_id, json.dumps(payload.boundaries))
return {"ok": True}
@router.post("/api/cabinets/{cabinet_id}/crop")
async def crop_cabinet_photo(cabinet_id: str, request: Request) -> dict[str, Any]:
payload = await _parse(_crop_dec, request)
with db.connection() as c:
fn = db.get_cabinet_photo(c, cabinet_id)
if not fn:
raise HTTPException(400, "No photo to crop")
from files import IMAGES_DIR
crop_save(IMAGES_DIR / fn, payload.x, payload.y, payload.w, payload.h)
return {"ok": True}
# ── Shelves ───────────────────────────────────────────────────────────────────
@router.post("/api/cabinets/{cabinet_id}/shelves")
async def create_shelf(cabinet_id: str) -> dict[str, Any]:
with db.connection() as c:
if not db.get_cabinet(c, cabinet_id):
raise HTTPException(404, "Cabinet not found")
with db.transaction() as c:
shelf = db.create_shelf(c, cabinet_id)
return {**dataclasses.asdict(shelf), "books": []}
@router.put("/api/shelves/{shelf_id}")
async def update_shelf(shelf_id: str, request: Request) -> dict[str, Any]:
payload = await _parse(_name_dec, request)
with db.connection() as c:
if not db.get_shelf(c, shelf_id):
raise HTTPException(404, "Shelf not found")
with db.transaction() as c:
db.rename_shelf(c, shelf_id, payload.name.strip())
return {"ok": True}
@router.delete("/api/shelves/{shelf_id}")
async def delete_shelf(shelf_id: str) -> dict[str, Any]:
with db.connection() as c:
if not db.get_shelf(c, shelf_id):
raise HTTPException(404, "Shelf not found")
photos = db.collect_shelf_photos(c, shelf_id)
with db.transaction() as c:
db.delete_shelf(c, shelf_id)
for fn in photos:
del_photo(fn)
return {"ok": True}
@router.post("/api/shelves/{shelf_id}/photo")
async def shelf_photo(shelf_id: str, image: UploadFile = File(...)) -> dict[str, Any]:
with db.connection() as c:
if not db.get_shelf(c, shelf_id):
raise HTTPException(404, "Shelf not found")
old = db.get_shelf_photo(c, shelf_id)
del_photo(old)
fn = await save_photo(image)
with db.transaction() as c:
db.set_shelf_photo(c, shelf_id, fn)
return {"photo_filename": fn}
@router.patch("/api/shelves/{shelf_id}/boundaries")
async def update_shelf_boundaries(shelf_id: str, request: Request) -> dict[str, Any]:
payload = await _parse(_boundaries_dec, request)
with db.connection() as c:
if not db.get_shelf(c, shelf_id):
raise HTTPException(404, "Shelf not found")
with db.transaction() as c:
db.set_shelf_boundaries(c, shelf_id, json.dumps(payload.boundaries))
return {"ok": True}
@router.post("/api/shelves/{shelf_id}/crop")
async def crop_shelf_photo(shelf_id: str, request: Request) -> dict[str, Any]:
payload = await _parse(_crop_dec, request)
with db.connection() as c:
fn = db.get_shelf_photo(c, shelf_id)
if not fn:
raise HTTPException(400, "No override photo to crop")
from files import IMAGES_DIR
crop_save(IMAGES_DIR / fn, payload.x, payload.y, payload.w, payload.h)
return {"ok": True}
@router.get("/api/shelves/{shelf_id}/image")
def shelf_image(shelf_id: str) -> Any:
with db.connection() as c:
path, crop = shelf_source(c, shelf_id)
return serve_crop(path, crop)
# ── Books ─────────────────────────────────────────────────────────────────────
@router.post("/api/shelves/{shelf_id}/books")
async def create_book(shelf_id: str) -> dict[str, Any]:
with db.connection() as c:
if not db.get_shelf(c, shelf_id):
raise HTTPException(404, "Shelf not found")
with db.transaction() as c:
book = db.create_book(c, shelf_id)
return dataclasses.asdict(book)
@router.put("/api/books/{book_id}")
async def update_book(book_id: str, request: Request) -> dict[str, Any]:
payload = await _parse(_book_dec, request)
with db.connection() as c:
if not db.get_book(c, book_id):
raise HTTPException(404, "Book not found")
status = logic.save_user_fields(
book_id,
payload.title.strip(),
payload.author.strip(),
payload.year.strip(),
payload.isbn.strip(),
payload.publisher.strip(),
payload.notes.strip(),
)
return {"ok": True, "identification_status": status}
@router.delete("/api/books/{book_id}")
async def delete_book(book_id: str) -> dict[str, Any]:
with db.connection() as c:
if not db.get_book(c, book_id):
raise HTTPException(404, "Book not found")
fn = db.get_book_photo(c, book_id)
with db.transaction() as c:
db.delete_book(c, book_id)
del_photo(fn)
return {"ok": True}
@router.post("/api/books/{book_id}/photo")
async def book_photo(book_id: str, image: UploadFile = File(...)) -> dict[str, Any]:
with db.connection() as c:
if not db.get_book(c, book_id):
raise HTTPException(404, "Book not found")
old = db.get_book_photo(c, book_id)
del_photo(old)
fn = await save_photo(image)
with db.transaction() as c:
db.set_book_photo(c, book_id, fn)
return {"image_filename": fn}
@router.get("/api/books/{book_id}/spine")
def book_spine(book_id: str) -> Any:
with db.connection() as c:
path, crop = book_spine_source(c, book_id)
return serve_crop(path, crop)
@router.post("/api/books/{book_id}/dismiss-field")
async def dismiss_book_field(book_id: str, request: Request) -> dict[str, Any]:
payload = await _parse(_dismiss_dec, request)
if payload.field not in logic.AI_FIELDS:
raise HTTPException(400, f"field must be one of {logic.AI_FIELDS}")
with db.connection() as c:
if not db.get_book(c, book_id):
raise HTTPException(404, "Book not found")
status, candidates = logic.dismiss_field(book_id, payload.field, payload.value.strip())
return {"ok": True, "identification_status": status, "candidates": candidates}
@router.post("/api/books/{book_id}/process")
async def process_book(book_id: str) -> dict[str, Any]:
"""Run full auto-queue pipeline for a single book."""
with db.connection() as c:
if not db.get_book(c, book_id):
raise HTTPException(404, "Book not found")
loop = asyncio.get_event_loop()
await loop.run_in_executor(logic.batch_executor, logic.process_book_sync, book_id)
with db.connection() as c:
book = db.get_book(c, book_id)
if not book:
raise HTTPException(404, "Book not found")
return dataclasses.asdict(book)
# ── Universal plugin endpoint ─────────────────────────────────────────────────
@router.post("/api/{entity_type}/{entity_id}/plugin/{plugin_id}")
async def run_plugin(entity_type: str, entity_id: str, plugin_id: str) -> dict[str, Any]:
"""Run any registered plugin on an entity. Returns updated entity."""
with db.connection() as c:
if entity_type == "cabinets":
if not db.get_cabinet(c, entity_id):
raise HTTPException(404, "Cabinet not found")
elif entity_type == "shelves":
if not db.get_shelf(c, entity_id):
raise HTTPException(404, "Shelf not found")
elif entity_type == "books":
if not db.get_book(c, entity_id):
raise HTTPException(404, "Book not found")
else:
raise HTTPException(400, f"Unknown entity type: {entity_type}")
loop = asyncio.get_event_loop()
return await logic.dispatch_plugin(plugin_id, plugin_registry.get_plugin(plugin_id), entity_type, entity_id, loop)
# ── Batch ─────────────────────────────────────────────────────────────────────
@router.post("/api/batch")
async def start_batch() -> dict[str, Any]:
if logic.batch_state["running"]:
return {"already_running": True}
with db.connection() as c:
ids = db.get_unidentified_book_ids(c)
if not ids:
return {"started": False, "reason": "no_unidentified_books"}
asyncio.create_task(logic.run_batch(ids))
return {"started": True, "total": len(ids)}
@router.get("/api/batch/status")
def batch_status() -> dict[str, Any]:
return dict(logic.batch_state)
# ── Reorder ───────────────────────────────────────────────────────────────────
_REORDER_TABLES = {"rooms", "cabinets", "shelves", "books"}
@router.patch("/api/{kind}/reorder")
async def reorder(kind: str, request: Request) -> dict[str, Any]:
if kind not in _REORDER_TABLES:
raise HTTPException(400, "Invalid kind")
payload = await _parse(_reorder_dec, request)
with db.transaction() as c:
db.reorder_entities(c, kind, payload.ids)
return {"ok": True}

76
src/app.py Normal file
View File

@@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""
Bookshelf cataloger — FastAPI entry point.
Usage:
cp config/credentials.default.yaml config/credentials.user.yaml # fill in your API key
poetry install
poetry run serve
"""
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
import plugins as plugin_registry
from api import router
from config import get_config, load_config
from db import init_db
from files import IMAGES_DIR, init_dirs
from errors import BadRequestError, ConfigError, ImageReadError, NotFoundError
@asynccontextmanager
async def lifespan(app: FastAPI):
load_config()
init_dirs()
init_db()
plugin_registry.load_plugins(get_config())
yield
app = FastAPI(lifespan=lifespan)
app.include_router(router)
@app.exception_handler(NotFoundError)
async def handle_not_found(_: Request, exc: NotFoundError) -> JSONResponse:
return JSONResponse(status_code=404, content={"detail": str(exc)})
@app.exception_handler(BadRequestError)
async def handle_bad_request(_: Request, exc: BadRequestError) -> JSONResponse:
return JSONResponse(status_code=400, content={"detail": str(exc)})
@app.exception_handler(ConfigError)
async def handle_config_error(_: Request, exc: ConfigError) -> JSONResponse:
return JSONResponse(status_code=500, content={"detail": str(exc)})
@app.exception_handler(ImageReadError)
async def handle_image_read_error(_: Request, exc: ImageReadError) -> JSONResponse:
return JSONResponse(status_code=500, content={"detail": str(exc)})
app.mount("/images", StaticFiles(directory=str(IMAGES_DIR)), name="images")
@app.get("/")
def index() -> FileResponse:
return FileResponse("static/index.html")
app.mount("/", StaticFiles(directory="static"), name="static")
def main() -> None:
import uvicorn
uvicorn.run("app:app", host="0.0.0.0", port=8000, reload=True)
if __name__ == "__main__":
main()

176
src/config.py Normal file
View File

@@ -0,0 +1,176 @@
"""
Configuration loading and typed dataclasses for all config categories.
Reads config/*.default.yaml merged with config/*.user.yaml overrides.
Provides typed access via mashumaro dataclasses.
Three-layer config chain:
credentials → models → functions
credentials: API keys + base_url per provider endpoint
models: AI model string + openrouter routing + prompt; references a credential
functions: per-function-type settings (auto_queue, rate_limit, etc.); AI functions
reference a model; archive functions specify type + config dict
Raises:
ConfigFileError: If a config file cannot be read or parsed as YAML.
ConfigValidationError: If merged config data does not match the AppConfig schema.
ConfigNotLoadedError: If get_config() is called before load_config().
"""
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, TypeGuard
import yaml
from mashumaro.codecs import BasicDecoder
from errors import ConfigFileError, ConfigNotLoadedError, ConfigValidationError
_CONFIG_DIR = Path("config")
_CONFIG_CATEGORIES = ["credentials", "models", "functions", "ui"]
@dataclass
class CredentialConfig:
base_url: str = ""
api_key: str = ""
@dataclass
class ModelConfig:
credentials: str = ""
model: str = ""
extra_body: dict[str, Any] = field(default_factory=lambda: {})
prompt: str = ""
@dataclass
class AIFunctionConfig:
model: str = ""
auto_queue: bool = False
rate_limit_seconds: float = 0.0
timeout: int = 30
max_image_px: int = 1600
confidence_threshold: float = 0.8
name: str = ""
@dataclass
class ArchiveSearcherFunctionConfig:
type: str = ""
auto_queue: bool = False
rate_limit_seconds: float = 0.0
timeout: int = 8
name: str = ""
config: dict[str, Any] = field(default_factory=lambda: {})
@dataclass
class FunctionsConfig:
boundary_detectors: dict[str, AIFunctionConfig] = field(default_factory=lambda: {})
text_recognizers: dict[str, AIFunctionConfig] = field(default_factory=lambda: {})
book_identifiers: dict[str, AIFunctionConfig] = field(default_factory=lambda: {})
archive_searchers: dict[str, ArchiveSearcherFunctionConfig] = field(default_factory=lambda: {})
@dataclass
class UIConfig:
boundary_grab_px: int = 14
@dataclass
class AppConfig:
credentials: dict[str, CredentialConfig] = field(default_factory=lambda: {})
models: dict[str, ModelConfig] = field(default_factory=lambda: {})
functions: FunctionsConfig = field(default_factory=FunctionsConfig)
ui: UIConfig = field(default_factory=UIConfig)
_decoder: BasicDecoder[AppConfig] = BasicDecoder(AppConfig)
# ── Merge helpers ─────────────────────────────────────────────────────────────
def _is_str_dict(v: object) -> TypeGuard[dict[str, Any]]:
"""TypeGuard that narrows Any/object to dict[str, Any] after isinstance check."""
return isinstance(v, dict)
def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
"""Recursively merge override into base. Lists in override replace lists in base.
Args:
base: Base dictionary to merge into.
override: Override dictionary whose values take precedence.
Returns:
New merged dictionary; base and override are not modified.
"""
result: dict[str, Any] = dict(base)
for key, val in override.items():
if key in result:
existing = result[key]
if _is_str_dict(existing) and _is_str_dict(val):
result[key] = deep_merge(existing, val)
continue
result[key] = val
return result
# ── Config loading ────────────────────────────────────────────────────────────
config_holder: list[AppConfig] = []
def load_config() -> AppConfig:
"""Load and parse config from config/*.default.yaml merged with config/*.user.yaml.
Each config category (credentials, models, functions, ui) is read from its default
file, then deep-merged with the corresponding user override file if it exists.
Returns:
Parsed and validated AppConfig.
Raises:
ConfigFileError: If the config directory is missing, a file cannot be opened,
or a file contains invalid YAML.
ConfigValidationError: If the merged config does not match the AppConfig schema.
"""
if not _CONFIG_DIR.exists():
raise ConfigFileError(_CONFIG_DIR, "directory not found — see config/*.default.yaml")
merged: dict[str, Any] = {}
for cat in _CONFIG_CATEGORIES:
data: dict[str, Any] = {}
for f_path in [_CONFIG_DIR / f"{cat}.default.yaml", _CONFIG_DIR / f"{cat}.user.yaml"]:
if not f_path.exists():
continue
try:
with open(f_path) as fh:
loaded = yaml.safe_load(fh)
except (OSError, yaml.YAMLError) as exc:
raise ConfigFileError(f_path, str(exc)) from exc
if _is_str_dict(loaded):
data = deep_merge(data, loaded)
merged = deep_merge(merged, data)
try:
cfg = _decoder.decode(merged)
except Exception as exc:
raise ConfigValidationError(str(exc)) from exc
config_holder.clear()
config_holder.append(cfg)
return cfg
def get_config() -> AppConfig:
"""Return the currently loaded config.
Returns:
The AppConfig loaded by the most recent load_config() call.
Raises:
ConfigNotLoadedError: If load_config() has not yet been called.
"""
if not config_holder:
raise ConfigNotLoadedError()
return config_holder[0]

515
src/db.py Normal file
View File

@@ -0,0 +1,515 @@
"""
Database layer: schema, connection/transaction lifecycle, and all query functions.
No file I/O, no config, no business logic. All SQL lives here.
"""
import json
import sqlite3
import uuid
from collections.abc import Iterator
from contextlib import contextmanager
from datetime import datetime
from pathlib import Path
from mashumaro.codecs import BasicDecoder
from models import BookRow, CabinetRow, RoomRow, ShelfRow
DB_PATH = Path("data") / "books.db"
# ── Schema ─────────────────────────────────────────────────────────────────────
SCHEMA = """
CREATE TABLE IF NOT EXISTS rooms (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
position INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS cabinets (
id TEXT PRIMARY KEY,
room_id TEXT NOT NULL REFERENCES rooms(id) ON DELETE CASCADE,
name TEXT NOT NULL,
photo_filename TEXT,
shelf_boundaries TEXT DEFAULT NULL,
ai_shelf_boundaries TEXT DEFAULT NULL,
position INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS shelves (
id TEXT PRIMARY KEY,
cabinet_id TEXT NOT NULL REFERENCES cabinets(id) ON DELETE CASCADE,
name TEXT NOT NULL,
photo_filename TEXT,
book_boundaries TEXT DEFAULT NULL,
ai_book_boundaries TEXT DEFAULT NULL,
position INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS books (
id TEXT PRIMARY KEY,
shelf_id TEXT NOT NULL REFERENCES shelves(id) ON DELETE CASCADE,
position INTEGER NOT NULL DEFAULT 0,
image_filename TEXT,
title TEXT DEFAULT '',
author TEXT DEFAULT '',
year TEXT DEFAULT '',
isbn TEXT DEFAULT '',
publisher TEXT DEFAULT '',
notes TEXT DEFAULT '',
raw_text TEXT DEFAULT '',
ai_title TEXT DEFAULT '',
ai_author TEXT DEFAULT '',
ai_year TEXT DEFAULT '',
ai_isbn TEXT DEFAULT '',
ai_publisher TEXT DEFAULT '',
identification_status TEXT DEFAULT 'unidentified',
title_confidence REAL DEFAULT 0,
analyzed_at TEXT,
created_at TEXT NOT NULL,
candidates TEXT DEFAULT NULL
);
"""
# ── Mashumaro decoders for entity rows ────────────────────────────────────────
_room_dec: BasicDecoder[RoomRow] = BasicDecoder(RoomRow)
_cabinet_dec: BasicDecoder[CabinetRow] = BasicDecoder(CabinetRow)
_shelf_dec: BasicDecoder[ShelfRow] = BasicDecoder(ShelfRow)
_book_dec: BasicDecoder[BookRow] = BasicDecoder(BookRow)
def _room(row: sqlite3.Row) -> RoomRow:
return _room_dec.decode(dict(row))
def _cabinet(row: sqlite3.Row) -> CabinetRow:
return _cabinet_dec.decode(dict(row))
def _shelf(row: sqlite3.Row) -> ShelfRow:
return _shelf_dec.decode(dict(row))
def _book(row: sqlite3.Row) -> BookRow:
return _book_dec.decode(dict(row))
# ── DB init + connection ────────────────────────────────────────────────────────
def init_db() -> None:
DB_PATH.parent.mkdir(exist_ok=True)
c = conn()
c.executescript(SCHEMA)
c.commit()
c.close()
def conn() -> sqlite3.Connection:
c = sqlite3.connect(DB_PATH)
c.row_factory = sqlite3.Row
c.execute("PRAGMA foreign_keys = ON")
return c
# ── Context managers ──────────────────────────────────────────────────────────
@contextmanager
def connection() -> Iterator[sqlite3.Connection]:
"""Read-only context: opens a connection, closes on exit."""
c = conn()
try:
yield c
finally:
c.close()
@contextmanager
def transaction() -> Iterator[sqlite3.Connection]:
"""Write context: opens, commits on success, rolls back on exception."""
c = conn()
try:
yield c
c.commit()
except Exception:
c.rollback()
raise
finally:
c.close()
# ── Helpers ───────────────────────────────────────────────────────────────────
COUNTERS: dict[str, int] = {}
def uid() -> str:
return str(uuid.uuid4())
def now() -> str:
return datetime.now().isoformat()
def next_pos(db: sqlite3.Connection, table: str, parent_col: str, parent_id: str) -> int:
row = db.execute(f"SELECT COALESCE(MAX(position),0)+1 FROM {table} WHERE {parent_col}=?", [parent_id]).fetchone()
return int(row[0])
def next_root_pos(db: sqlite3.Connection, table: str) -> int:
row = db.execute(f"SELECT COALESCE(MAX(position),0)+1 FROM {table}").fetchone()
return int(row[0])
def next_name(prefix: str) -> str:
COUNTERS[prefix] = COUNTERS.get(prefix, 0) + 1
return f"{prefix} {COUNTERS[prefix]}"
# ── Tree ──────────────────────────────────────────────────────────────────────
def get_tree(db: sqlite3.Connection) -> list[dict[str, object]]:
"""Build and return the full nested Room→Cabinet→Shelf→Book tree."""
rooms: list[dict[str, object]] = [dict(r) for r in db.execute("SELECT * FROM rooms ORDER BY position")]
for room in rooms:
cabs: list[dict[str, object]] = [
dict(c) for c in db.execute("SELECT * FROM cabinets WHERE room_id=? ORDER BY position", [room["id"]])
]
for cab in cabs:
shelves: list[dict[str, object]] = [
dict(s) for s in db.execute("SELECT * FROM shelves WHERE cabinet_id=? ORDER BY position", [cab["id"]])
]
for shelf in shelves:
shelf["books"] = [
dict(b) for b in db.execute("SELECT * FROM books WHERE shelf_id=? ORDER BY position", [shelf["id"]])
]
cab["shelves"] = shelves
room["cabinets"] = cabs
return rooms
# ── Rooms ─────────────────────────────────────────────────────────────────────
def get_room(db: sqlite3.Connection, room_id: str) -> RoomRow | None:
row = db.execute("SELECT * FROM rooms WHERE id=?", [room_id]).fetchone()
return _room(row) if row else None
def create_room(db: sqlite3.Connection) -> RoomRow:
data = {"id": uid(), "name": next_name("Room"), "position": next_root_pos(db, "rooms"), "created_at": now()}
db.execute("INSERT INTO rooms VALUES(:id,:name,:position,:created_at)", data)
return _room_dec.decode(data)
def rename_room(db: sqlite3.Connection, room_id: str, name: str) -> None:
db.execute("UPDATE rooms SET name=? WHERE id=?", [name, room_id])
def collect_room_photos(db: sqlite3.Connection, room_id: str) -> list[str]:
"""Return all photo filenames for cabinets/shelves/books under this room."""
photos: list[str] = []
for r in db.execute(
"SELECT image_filename FROM books WHERE shelf_id IN "
"(SELECT id FROM shelves WHERE cabinet_id IN (SELECT id FROM cabinets WHERE room_id=?))",
[room_id],
):
if r[0]:
photos.append(str(r[0]))
for r in db.execute(
"SELECT photo_filename FROM shelves WHERE cabinet_id IN (SELECT id FROM cabinets WHERE room_id=?)", [room_id]
):
if r[0]:
photos.append(str(r[0]))
for r in db.execute("SELECT photo_filename FROM cabinets WHERE room_id=?", [room_id]):
if r[0]:
photos.append(str(r[0]))
return photos
def delete_room(db: sqlite3.Connection, room_id: str) -> None:
"""Delete room; SQLite ON DELETE CASCADE removes all children."""
db.execute("DELETE FROM rooms WHERE id=?", [room_id])
# ── Cabinets ──────────────────────────────────────────────────────────────────
def get_cabinet(db: sqlite3.Connection, cabinet_id: str) -> CabinetRow | None:
row = db.execute("SELECT * FROM cabinets WHERE id=?", [cabinet_id]).fetchone()
return _cabinet(row) if row else None
def create_cabinet(db: sqlite3.Connection, room_id: str) -> CabinetRow:
data: dict[str, object] = {
"id": uid(),
"room_id": room_id,
"name": next_name("Cabinet"),
"photo_filename": None,
"shelf_boundaries": None,
"ai_shelf_boundaries": None,
"position": next_pos(db, "cabinets", "room_id", room_id),
"created_at": now(),
}
db.execute(
"INSERT INTO cabinets VALUES("
":id,:room_id,:name,:photo_filename,:shelf_boundaries,"
":ai_shelf_boundaries,:position,:created_at)",
data,
)
return _cabinet_dec.decode(data)
def rename_cabinet(db: sqlite3.Connection, cabinet_id: str, name: str) -> None:
db.execute("UPDATE cabinets SET name=? WHERE id=?", [name, cabinet_id])
def collect_cabinet_photos(db: sqlite3.Connection, cabinet_id: str) -> list[str]:
photos: list[str] = []
for r in db.execute(
"SELECT image_filename FROM books WHERE shelf_id IN (SELECT id FROM shelves WHERE cabinet_id=?)", [cabinet_id]
):
if r[0]:
photos.append(str(r[0]))
for r in db.execute("SELECT photo_filename FROM shelves WHERE cabinet_id=?", [cabinet_id]):
if r[0]:
photos.append(str(r[0]))
row = db.execute("SELECT photo_filename FROM cabinets WHERE id=?", [cabinet_id]).fetchone()
if row and row[0]:
photos.append(str(row[0]))
return photos
def delete_cabinet(db: sqlite3.Connection, cabinet_id: str) -> None:
db.execute("DELETE FROM cabinets WHERE id=?", [cabinet_id])
def get_cabinet_photo(db: sqlite3.Connection, cabinet_id: str) -> str | None:
row = db.execute("SELECT photo_filename FROM cabinets WHERE id=?", [cabinet_id]).fetchone()
return str(row[0]) if row and row[0] else None
def set_cabinet_photo(db: sqlite3.Connection, cabinet_id: str, filename: str) -> None:
db.execute("UPDATE cabinets SET photo_filename=? WHERE id=?", [filename, cabinet_id])
def set_cabinet_boundaries(db: sqlite3.Connection, cabinet_id: str, boundaries_json: str) -> None:
db.execute("UPDATE cabinets SET shelf_boundaries=? WHERE id=?", [boundaries_json, cabinet_id])
def set_ai_shelf_boundaries(db: sqlite3.Connection, cabinet_id: str, plugin_id: str, boundaries: list[float]) -> None:
row = db.execute("SELECT ai_shelf_boundaries FROM cabinets WHERE id=?", [cabinet_id]).fetchone()
current: dict[str, object] = json.loads(row[0]) if row and row[0] else {}
current[plugin_id] = boundaries
db.execute("UPDATE cabinets SET ai_shelf_boundaries=? WHERE id=?", [json.dumps(current), cabinet_id])
# ── Shelves ───────────────────────────────────────────────────────────────────
def get_shelf(db: sqlite3.Connection, shelf_id: str) -> ShelfRow | None:
row = db.execute("SELECT * FROM shelves WHERE id=?", [shelf_id]).fetchone()
return _shelf(row) if row else None
def create_shelf(db: sqlite3.Connection, cabinet_id: str) -> ShelfRow:
data: dict[str, object] = {
"id": uid(),
"cabinet_id": cabinet_id,
"name": next_name("Shelf"),
"photo_filename": None,
"book_boundaries": None,
"ai_book_boundaries": None,
"position": next_pos(db, "shelves", "cabinet_id", cabinet_id),
"created_at": now(),
}
db.execute(
"INSERT INTO shelves VALUES("
":id,:cabinet_id,:name,:photo_filename,:book_boundaries,:ai_book_boundaries,:position,:created_at)",
data,
)
return _shelf_dec.decode(data)
def rename_shelf(db: sqlite3.Connection, shelf_id: str, name: str) -> None:
db.execute("UPDATE shelves SET name=? WHERE id=?", [name, shelf_id])
def collect_shelf_photos(db: sqlite3.Connection, shelf_id: str) -> list[str]:
photos: list[str] = []
row = db.execute("SELECT photo_filename FROM shelves WHERE id=?", [shelf_id]).fetchone()
if row and row[0]:
photos.append(str(row[0]))
for r in db.execute("SELECT image_filename FROM books WHERE shelf_id=?", [shelf_id]):
if r[0]:
photos.append(str(r[0]))
return photos
def delete_shelf(db: sqlite3.Connection, shelf_id: str) -> None:
db.execute("DELETE FROM shelves WHERE id=?", [shelf_id])
def get_shelf_photo(db: sqlite3.Connection, shelf_id: str) -> str | None:
row = db.execute("SELECT photo_filename FROM shelves WHERE id=?", [shelf_id]).fetchone()
return str(row[0]) if row and row[0] else None
def set_shelf_photo(db: sqlite3.Connection, shelf_id: str, filename: str) -> None:
db.execute("UPDATE shelves SET photo_filename=? WHERE id=?", [filename, shelf_id])
def set_shelf_boundaries(db: sqlite3.Connection, shelf_id: str, boundaries_json: str) -> None:
db.execute("UPDATE shelves SET book_boundaries=? WHERE id=?", [boundaries_json, shelf_id])
def set_ai_book_boundaries(db: sqlite3.Connection, shelf_id: str, plugin_id: str, boundaries: list[float]) -> None:
row = db.execute("SELECT ai_book_boundaries FROM shelves WHERE id=?", [shelf_id]).fetchone()
current: dict[str, object] = json.loads(row[0]) if row and row[0] else {}
current[plugin_id] = boundaries
db.execute("UPDATE shelves SET ai_book_boundaries=? WHERE id=?", [json.dumps(current), shelf_id])
def get_shelf_rank(db: sqlite3.Connection, shelf_id: str) -> int:
"""0-based rank of shelf among its siblings sorted by position."""
row = db.execute("SELECT cabinet_id FROM shelves WHERE id=?", [shelf_id]).fetchone()
if not row:
return 0
siblings = [r[0] for r in db.execute("SELECT id FROM shelves WHERE cabinet_id=? ORDER BY position", [row[0]])]
return siblings.index(shelf_id) if shelf_id in siblings else 0
# ── Books ─────────────────────────────────────────────────────────────────────
def get_book(db: sqlite3.Connection, book_id: str) -> BookRow | None:
row = db.execute("SELECT * FROM books WHERE id=?", [book_id]).fetchone()
return _book(row) if row else None
def create_book(db: sqlite3.Connection, shelf_id: str) -> BookRow:
data: dict[str, object] = {
"id": uid(),
"shelf_id": shelf_id,
"position": next_pos(db, "books", "shelf_id", shelf_id),
"image_filename": None,
"title": "",
"author": "",
"year": "",
"isbn": "",
"publisher": "",
"notes": "",
"raw_text": "",
"ai_title": "",
"ai_author": "",
"ai_year": "",
"ai_isbn": "",
"ai_publisher": "",
"identification_status": "unidentified",
"title_confidence": 0,
"analyzed_at": None,
"created_at": now(),
"candidates": None,
}
db.execute(
"INSERT INTO books VALUES(:id,:shelf_id,:position,:image_filename,:title,:author,:year,:isbn,:publisher,"
":notes,:raw_text,:ai_title,:ai_author,:ai_year,:ai_isbn,:ai_publisher,:identification_status,"
":title_confidence,:analyzed_at,:created_at,:candidates)",
data,
)
return _book_dec.decode(data)
def delete_book(db: sqlite3.Connection, book_id: str) -> None:
db.execute("DELETE FROM books WHERE id=?", [book_id])
def get_book_photo(db: sqlite3.Connection, book_id: str) -> str | None:
row = db.execute("SELECT image_filename FROM books WHERE id=?", [book_id]).fetchone()
return str(row[0]) if row and row[0] else None
def set_book_photo(db: sqlite3.Connection, book_id: str, filename: str) -> None:
db.execute("UPDATE books SET image_filename=? WHERE id=?", [filename, book_id])
def set_user_book_fields(
db: sqlite3.Connection,
book_id: str,
title: str,
author: str,
year: str,
isbn: str,
publisher: str,
notes: str,
) -> None:
"""Set both user fields and ai_* fields (user edit is the authoritative identification)."""
db.execute(
"UPDATE books SET title=?,author=?,year=?,isbn=?,publisher=?,notes=?,"
"ai_title=?,ai_author=?,ai_year=?,ai_isbn=?,ai_publisher=? WHERE id=?",
[title, author, year, isbn, publisher, notes, title, author, year, isbn, publisher, book_id],
)
def set_book_status(db: sqlite3.Connection, book_id: str, status: str) -> None:
db.execute("UPDATE books SET identification_status=? WHERE id=?", [status, book_id])
def set_book_confidence(db: sqlite3.Connection, book_id: str, confidence: float, analyzed_at: str) -> None:
db.execute(
"UPDATE books SET title_confidence=?, analyzed_at=? WHERE id=?",
[confidence, analyzed_at, book_id],
)
def set_book_ai_fields(
db: sqlite3.Connection,
book_id: str,
ai_title: str,
ai_author: str,
ai_year: str,
ai_isbn: str,
ai_publisher: str,
) -> None:
db.execute(
"UPDATE books SET ai_title=?,ai_author=?,ai_year=?,ai_isbn=?,ai_publisher=? WHERE id=?",
[ai_title, ai_author, ai_year, ai_isbn, ai_publisher, book_id],
)
def set_book_ai_field(db: sqlite3.Connection, book_id: str, field: str, value: str) -> None:
"""Set a single ai_* field by name (used in dismiss_field logic)."""
# field is validated by caller to be in AI_FIELDS
db.execute(f"UPDATE books SET ai_{field}=? WHERE id=?", [value, book_id])
def set_book_raw_text(db: sqlite3.Connection, book_id: str, raw_text: str) -> None:
db.execute("UPDATE books SET raw_text=? WHERE id=?", [raw_text, book_id])
def set_book_candidates(db: sqlite3.Connection, book_id: str, candidates_json: str) -> None:
db.execute("UPDATE books SET candidates=? WHERE id=?", [candidates_json, book_id])
def get_book_rank(db: sqlite3.Connection, book_id: str) -> int:
"""0-based rank of book among its siblings sorted by position."""
row = db.execute("SELECT shelf_id FROM books WHERE id=?", [book_id]).fetchone()
if not row:
return 0
siblings = [r[0] for r in db.execute("SELECT id FROM books WHERE shelf_id=? ORDER BY position", [row[0]])]
return siblings.index(book_id) if book_id in siblings else 0
def get_unidentified_book_ids(db: sqlite3.Connection) -> list[str]:
return [str(r[0]) for r in db.execute("SELECT id FROM books WHERE identification_status='unidentified'")]
# ── Reorder ───────────────────────────────────────────────────────────────────
def reorder_entities(db: sqlite3.Connection, table: str, ids: list[str]) -> None:
for i, entity_id in enumerate(ids, 1):
db.execute(f"UPDATE {table} SET position=? WHERE id=?", [i, entity_id])

280
src/errors.py Normal file
View File

@@ -0,0 +1,280 @@
"""Domain exceptions for the bookshelf application.
All layers may import from this module. Exceptions carry structured attributes
so callers can reason about failures programmatically without string parsing.
Hierarchy:
NotFoundError (→ HTTP 404)
BookNotFoundError
ShelfNotFoundError
CabinetNotFoundError
PluginNotFoundError
NoShelfImageError
ImageFileNotFoundError
BadRequestError (→ HTTP 400)
NoCabinetPhotoError
NoRawTextError
InvalidPluginEntityError
PluginTargetMismatchError
ConfigError (→ HTTP 500)
ConfigNotLoadedError
ConfigFileError
ConfigValidationError
ImageError (→ HTTP 500; ImageFileNotFoundError is also NotFoundError → 404)
ImageFileNotFoundError (also NotFoundError)
ImageReadError
Rules for exception classes:
- Constructor accepts only structured data, no message strings.
- All data is available as public attributes (no string parsing needed).
- __str__ performs all formatting; it is the only place with text.
"""
from pathlib import Path
class NotFoundError(Exception):
"""Base for all 'entity not found' errors. Caught globally and mapped to HTTP 404."""
class BadRequestError(Exception):
"""Base for all 'invalid state/input' errors. Caught globally and mapped to HTTP 400."""
# ── Entity not-found errors ───────────────────────────────────────────────────
class BookNotFoundError(NotFoundError):
"""Raised when a book ID cannot be found in the database.
Attributes:
book_id: The book ID that was looked up.
"""
def __init__(self, book_id: str) -> None:
super().__init__()
self.book_id = book_id
def __str__(self) -> str:
return f"Book not found: {self.book_id!r}"
class ShelfNotFoundError(NotFoundError):
"""Raised when a shelf ID cannot be found in the database.
Attributes:
shelf_id: The shelf ID that was looked up.
"""
def __init__(self, shelf_id: str) -> None:
super().__init__()
self.shelf_id = shelf_id
def __str__(self) -> str:
return f"Shelf not found: {self.shelf_id!r}"
class CabinetNotFoundError(NotFoundError):
"""Raised when a cabinet ID cannot be found in the database.
Attributes:
cabinet_id: The cabinet ID that was looked up.
"""
def __init__(self, cabinet_id: str) -> None:
super().__init__()
self.cabinet_id = cabinet_id
def __str__(self) -> str:
return f"Cabinet not found: {self.cabinet_id!r}"
class PluginNotFoundError(NotFoundError):
"""Raised when a plugin ID is not registered in any category.
Attributes:
plugin_id: The plugin ID that was looked up.
"""
def __init__(self, plugin_id: str) -> None:
super().__init__()
self.plugin_id = plugin_id
def __str__(self) -> str:
return f"Plugin not found: {self.plugin_id!r}"
class NoShelfImageError(NotFoundError):
"""Raised when no image is available for a shelf (no override photo and parent cabinet has no photo).
Attributes:
shelf_id: The shelf that has no usable image.
cabinet_id: The parent cabinet that also lacks a photo.
"""
def __init__(self, shelf_id: str, cabinet_id: str) -> None:
super().__init__()
self.shelf_id = shelf_id
self.cabinet_id = cabinet_id
def __str__(self) -> str:
return f"No image available for shelf {self.shelf_id!r} (cabinet {self.cabinet_id!r} has no photo)"
# ── Bad-request errors ────────────────────────────────────────────────────────
class NoCabinetPhotoError(BadRequestError):
"""Raised when boundary detection requires a cabinet photo that has not been uploaded.
Attributes:
cabinet_id: The cabinet that is missing a photo.
"""
def __init__(self, cabinet_id: str) -> None:
super().__init__()
self.cabinet_id = cabinet_id
def __str__(self) -> str:
return f"Cabinet {self.cabinet_id!r} has no photo; upload one before running boundary detection"
class NoRawTextError(BadRequestError):
"""Raised when book identification is attempted before text recognition has been run.
Attributes:
book_id: The book that is missing raw text.
"""
def __init__(self, book_id: str) -> None:
super().__init__()
self.book_id = book_id
def __str__(self) -> str:
return f"Book {self.book_id!r} has no raw text; run text recognizer first"
class InvalidPluginEntityError(BadRequestError):
"""Raised when a plugin category does not support the requested entity type.
Attributes:
plugin_category: The plugin category (e.g. 'text_recognizer').
entity_type: The entity type that was requested (e.g. 'cabinets').
"""
def __init__(self, plugin_category: str, entity_type: str) -> None:
super().__init__()
self.plugin_category = plugin_category
self.entity_type = entity_type
def __str__(self) -> str:
return f"Plugin category {self.plugin_category!r} does not support entity type {self.entity_type!r}"
class PluginTargetMismatchError(BadRequestError):
"""Raised when a boundary detector plugin's target conflicts with the entity being processed.
Attributes:
plugin_id: The plugin whose target is wrong.
expected_target: The target required for the given entity type.
actual_target: The target the plugin actually declares.
"""
def __init__(self, plugin_id: str, expected_target: str, actual_target: str) -> None:
super().__init__()
self.plugin_id = plugin_id
self.expected_target = expected_target
self.actual_target = actual_target
def __str__(self) -> str:
return (
f"Plugin {self.plugin_id!r} targets {self.actual_target!r}; "
f"expected target {self.expected_target!r} for this entity type"
)
# ── Config errors (→ HTTP 500) ────────────────────────────────────────────────
class ConfigError(Exception):
"""Base for all configuration loading and validation errors. Maps to HTTP 500."""
class ConfigNotLoadedError(ConfigError):
"""Raised when get_config() is called before load_config() has been run."""
def __str__(self) -> str:
return "Config not loaded; call load_config() first"
class ConfigFileError(ConfigError):
"""Raised when a config file cannot be opened or parsed.
Attributes:
path: The config file (or directory) that could not be read.
reason: Human-readable description of the underlying error.
"""
def __init__(self, path: Path, reason: str) -> None:
super().__init__()
self.path = path
self.reason = reason
def __str__(self) -> str:
return f"Config file error ({self.path}): {self.reason}"
class ConfigValidationError(ConfigError):
"""Raised when config data does not match the expected schema.
Attributes:
reason: Human-readable description of the validation failure.
"""
def __init__(self, reason: str) -> None:
super().__init__()
self.reason = reason
def __str__(self) -> str:
return f"Config validation error: {self.reason}"
# ── Image errors ──────────────────────────────────────────────────────────────
class ImageError(Exception):
"""Base for image file operation errors."""
class ImageFileNotFoundError(NotFoundError, ImageError):
"""Raised when an image file referenced by an entity is missing from disk.
Inherits from NotFoundError (→ HTTP 404) and ImageError.
Attributes:
path: The file path that does not exist.
"""
def __init__(self, path: Path) -> None:
super().__init__()
self.path = path
def __str__(self) -> str:
return f"Image file not found: {self.path}"
class ImageReadError(ImageError):
"""Raised when an image file exists but cannot be opened or decoded. Maps to HTTP 500.
Attributes:
path: The file path that could not be read.
reason: Human-readable description of the underlying error.
"""
def __init__(self, path: Path, reason: str) -> None:
super().__init__()
self.path = path
self.reason = reason
def __str__(self) -> str:
return f"Image read error ({self.path}): {self.reason}"

40
src/files.py Normal file
View File

@@ -0,0 +1,40 @@
"""
File system layer: data directories and photo upload/delete helpers.
No DB access, no config parsing, no business logic.
"""
import uuid
from pathlib import Path
import aiofiles
from fastapi import UploadFile
DATA_DIR = Path("data")
IMAGES_DIR = DATA_DIR / "images"
_ALLOWED_EXT = {".jpg", ".jpeg", ".png", ".webp"}
# ── Directory init ─────────────────────────────────────────────────────────────
def init_dirs() -> None:
DATA_DIR.mkdir(exist_ok=True)
IMAGES_DIR.mkdir(exist_ok=True)
# ── Photo helpers ──────────────────────────────────────────────────────────────
async def save_photo(upload: UploadFile) -> str:
ext = Path(upload.filename or "").suffix.lower() or ".jpg"
if ext not in _ALLOWED_EXT:
ext = ".jpg"
fn = f"{uuid.uuid4()}{ext}"
async with aiofiles.open(IMAGES_DIR / fn, "wb") as f:
await f.write(await upload.read())
return fn
def del_photo(fn: str | None) -> None:
if fn:
(IMAGES_DIR / fn).unlink(missing_ok=True)

108
src/logic/__init__.py Normal file
View File

@@ -0,0 +1,108 @@
"""Logic package: plugin dispatch orchestration and public re-exports."""
import asyncio
import dataclasses
from typing import Any
import plugins as plugin_registry
from errors import InvalidPluginEntityError, PluginNotFoundError, PluginTargetMismatchError
from models import PluginLookupResult
from logic.archive import run_archive_searcher, run_archive_searcher_bg
from logic.batch import archive_executor, batch_executor, batch_state, process_book_sync, run_batch
from logic.boundaries import book_spine_source, bounds_for_index, run_boundary_detector, shelf_source
from logic.identification import (
AI_FIELDS,
apply_ai_result,
build_query,
compute_status,
dismiss_field,
run_book_identifier,
run_text_recognizer,
save_user_fields,
)
from logic.images import prep_img_b64, crop_save, serve_crop
__all__ = [
"AI_FIELDS",
"apply_ai_result",
"archive_executor",
"batch_executor",
"batch_state",
"book_spine_source",
"bounds_for_index",
"build_query",
"compute_status",
"crop_save",
"dismiss_field",
"dispatch_plugin",
"process_book_sync",
"run_archive_searcher",
"run_archive_searcher_bg",
"run_batch",
"run_book_identifier",
"run_boundary_detector",
"run_text_recognizer",
"save_user_fields",
"serve_crop",
"shelf_source",
"prep_img_b64",
]
async def dispatch_plugin(
plugin_id: str,
lookup: PluginLookupResult,
entity_type: str,
entity_id: str,
loop: asyncio.AbstractEventLoop,
) -> dict[str, Any]:
"""Validate plugin/entity compatibility, run the plugin, and trigger auto-queue follow-ups.
Args:
plugin_id: The plugin ID string (used in error reporting).
lookup: Discriminated tuple from plugins.get_plugin(); (None, None) if not found.
entity_type: Entity type string (e.g. 'cabinets', 'shelves', 'books').
entity_id: ID of the entity to operate on.
loop: Running event loop for executor dispatch.
Returns:
dataclasses.asdict() of the updated entity row.
Raises:
PluginNotFoundError: If lookup is (None, None).
InvalidPluginEntityError: If the entity_type is not compatible with the plugin category.
PluginTargetMismatchError: If a boundary_detector plugin's target mismatches the entity.
"""
match lookup:
case (None, None):
raise PluginNotFoundError(plugin_id)
case ("boundary_detector", plugin):
if entity_type not in ("cabinets", "shelves"):
raise InvalidPluginEntityError("boundary_detector", entity_type)
if entity_type == "cabinets" and plugin.target != "shelves":
raise PluginTargetMismatchError(plugin.plugin_id, "shelves", plugin.target)
if entity_type == "shelves" and plugin.target != "books":
raise PluginTargetMismatchError(plugin.plugin_id, "books", plugin.target)
result = await loop.run_in_executor(None, run_boundary_detector, plugin, entity_type, entity_id)
return dataclasses.asdict(result)
case ("text_recognizer", plugin):
if entity_type != "books":
raise InvalidPluginEntityError("text_recognizer", entity_type)
result = await loop.run_in_executor(None, run_text_recognizer, plugin, entity_id)
for ap in plugin_registry.get_auto_queue("archive_searchers"):
loop.run_in_executor(archive_executor, run_archive_searcher_bg, ap, entity_id)
return dataclasses.asdict(result)
case ("book_identifier", plugin):
if entity_type != "books":
raise InvalidPluginEntityError("book_identifier", entity_type)
result = await loop.run_in_executor(None, run_book_identifier, plugin, entity_id)
return dataclasses.asdict(result)
case ("archive_searcher", plugin):
if entity_type != "books":
raise InvalidPluginEntityError("archive_searcher", entity_type)
result = await loop.run_in_executor(archive_executor, run_archive_searcher, plugin, entity_id)
return dataclasses.asdict(result)

52
src/logic/archive.py Normal file
View File

@@ -0,0 +1,52 @@
"""Archive search plugin runner."""
import json
import db
from errors import BookNotFoundError
from models import ArchiveSearcherPlugin, BookRow, CandidateRecord
from logic.identification import build_query
def run_archive_searcher(plugin: ArchiveSearcherPlugin, book_id: str) -> BookRow:
"""Run an archive search for a book and merge results into the candidates list.
Args:
plugin: The archive searcher plugin to execute.
book_id: ID of the book to search for.
Returns:
Updated BookRow after merging search results.
Raises:
BookNotFoundError: If book_id does not exist.
"""
with db.transaction() as c:
book = db.get_book(c, book_id)
if not book:
raise BookNotFoundError(book_id)
query = build_query(book)
if not query:
return book
results: list[CandidateRecord] = plugin.search(query)
existing: list[CandidateRecord] = json.loads(book.candidates or "[]")
existing = [cd for cd in existing if cd.get("source") != plugin.plugin_id]
existing.extend(results)
db.set_book_candidates(c, book_id, json.dumps(existing))
updated = db.get_book(c, book_id)
if not updated:
raise BookNotFoundError(book_id)
return updated
def run_archive_searcher_bg(plugin: ArchiveSearcherPlugin, book_id: str) -> None:
"""Run an archive search in fire-and-forget mode; all exceptions are suppressed.
Args:
plugin: The archive searcher plugin to execute.
book_id: ID of the book to search for.
"""
try:
run_archive_searcher(plugin, book_id)
except Exception:
pass

66
src/logic/batch.py Normal file
View File

@@ -0,0 +1,66 @@
"""Batch processing pipeline: auto-queue text recognition and archive search."""
import asyncio
from concurrent.futures import ThreadPoolExecutor
import db
import plugins as plugin_registry
from models import BatchState
from logic.identification import run_text_recognizer
from logic.archive import run_archive_searcher
batch_state: BatchState = {"running": False, "total": 0, "done": 0, "errors": 0, "current": ""}
batch_executor = ThreadPoolExecutor(max_workers=1)
archive_executor = ThreadPoolExecutor(max_workers=8)
def process_book_sync(book_id: str) -> None:
"""Run the full auto-queue pipeline for a single book synchronously.
Runs all auto_queue text_recognizers (if book has no raw_text yet), then all
auto_queue archive_searchers. Exceptions from individual plugins are suppressed.
Args:
book_id: ID of the book to process.
"""
with db.connection() as c:
book = db.get_book(c, book_id)
has_text = bool((book.raw_text if book else "").strip())
if not has_text:
for p in plugin_registry.get_auto_queue("text_recognizers"):
try:
run_text_recognizer(p, book_id)
except Exception:
pass
for p in plugin_registry.get_auto_queue("archive_searchers"):
try:
run_archive_searcher(p, book_id)
except Exception:
pass
async def run_batch(book_ids: list[str]) -> None:
"""Process a list of books through the auto-queue pipeline sequentially.
Updates batch_state throughout execution. Exceptions from individual books
are counted in batch_state['errors'] and do not abort the run.
Args:
book_ids: List of book IDs to process.
"""
loop = asyncio.get_event_loop()
batch_state["running"] = True
batch_state["total"] = len(book_ids)
batch_state["done"] = 0
batch_state["errors"] = 0
for bid in book_ids:
batch_state["current"] = bid
try:
await loop.run_in_executor(batch_executor, process_book_sync, bid)
except Exception:
batch_state["errors"] += 1
batch_state["done"] += 1
batch_state["running"] = False
batch_state["current"] = ""

147
src/logic/boundaries.py Normal file
View File

@@ -0,0 +1,147 @@
"""Boundary calculation and image source resolution for shelves and books."""
import json
import sqlite3
from pathlib import Path
import db
from errors import (
BookNotFoundError,
CabinetNotFoundError,
NoCabinetPhotoError,
NoShelfImageError,
ShelfNotFoundError,
)
from files import IMAGES_DIR
from logic.images import prep_img_b64
from models import BoundaryDetectorPlugin, CabinetRow, ShelfRow
def bounds_for_index(boundaries_json: str | None, idx: int) -> tuple[float, float]:
"""Return (start, end) 0-1 fractions for the segment at a 0-based index.
Args:
boundaries_json: JSON-encoded list of interior boundary fractions, or None.
idx: 0-based segment index. Out-of-range values clamp to the last segment.
Returns:
(start, end) fractions in [0, 1].
"""
bounds: list[float] = json.loads(boundaries_json) if boundaries_json else []
full = [0.0] + bounds + [1.0]
if idx + 1 >= len(full):
return (full[-2] if len(full) >= 2 else 0.0, 1.0)
return (full[idx], full[idx + 1])
def shelf_source(c: sqlite3.Connection, shelf_id: str) -> tuple[Path, tuple[float, float, float, float] | None]:
"""Return the image path and optional crop fractions for a shelf's display image.
Uses the shelf's own override photo if present; otherwise derives a crop from
the parent cabinet's photo using the shelf's positional rank and shelf boundaries.
Args:
c: Open database connection.
shelf_id: ID of the shelf to resolve.
Returns:
(image_path, crop_frac_or_None) — crop is None when using the shelf's own photo.
Raises:
ShelfNotFoundError: If shelf_id does not exist.
NoShelfImageError: If the shelf has no override photo and the cabinet has no photo.
"""
shelf = db.get_shelf(c, shelf_id)
if not shelf:
raise ShelfNotFoundError(shelf_id)
if shelf.photo_filename:
return IMAGES_DIR / shelf.photo_filename, None
cab = db.get_cabinet(c, shelf.cabinet_id)
if not cab or not cab.photo_filename:
raise NoShelfImageError(shelf_id, shelf.cabinet_id)
idx = db.get_shelf_rank(c, shelf_id)
y0, y1 = bounds_for_index(cab.shelf_boundaries, idx)
return IMAGES_DIR / cab.photo_filename, (0.0, y0, 1.0, y1)
def book_spine_source(c: sqlite3.Connection, book_id: str) -> tuple[Path, tuple[float, float, float, float]]:
"""Return the image path and crop fractions for a book's spine image.
Composes the shelf's image source with the book's horizontal position within
the shelf's book boundaries.
Args:
c: Open database connection.
book_id: ID of the book to resolve.
Returns:
(image_path, crop_frac) — always returns a crop (never None).
Raises:
BookNotFoundError: If book_id does not exist.
ShelfNotFoundError: If the book's parent shelf does not exist.
NoShelfImageError: If no image is available for the parent shelf.
"""
book = db.get_book(c, book_id)
if not book:
raise BookNotFoundError(book_id)
shelf = db.get_shelf(c, book.shelf_id)
if not shelf:
raise ShelfNotFoundError(book.shelf_id)
base_path, base_crop = shelf_source(c, book.shelf_id)
idx = db.get_book_rank(c, book_id)
x0, x1 = bounds_for_index(shelf.book_boundaries, idx)
if base_crop is None:
return base_path, (x0, 0.0, x1, 1.0)
else:
_, y0, _, y1 = base_crop
return base_path, (x0, y0, x1, y1)
def run_boundary_detector(plugin: BoundaryDetectorPlugin, entity_type: str, entity_id: str) -> CabinetRow | ShelfRow:
"""Run boundary detection on a cabinet or shelf and persist the result.
Args:
plugin: The boundary detector plugin to execute.
entity_type: Either 'cabinets' or 'shelves'.
entity_id: ID of the entity to process.
Returns:
Updated CabinetRow (if entity_type == 'cabinets') or ShelfRow (if 'shelves').
Raises:
CabinetNotFoundError: If entity_type is 'cabinets' and the cabinet does not exist.
NoCabinetPhotoError: If entity_type is 'cabinets' and the cabinet has no photo.
ShelfNotFoundError: If entity_type is 'shelves' and the shelf does not exist.
NoShelfImageError: If entity_type is 'shelves' and no image is available for the shelf.
"""
with db.transaction() as c:
if entity_type == "cabinets":
entity = db.get_cabinet(c, entity_id)
if not entity:
raise CabinetNotFoundError(entity_id)
if not entity.photo_filename:
raise NoCabinetPhotoError(entity_id)
b64, mt = prep_img_b64(IMAGES_DIR / entity.photo_filename, max_px=plugin.max_image_px)
result = plugin.detect(b64, mt)
boundaries: list[float] = list(result.get("boundaries") or [])
db.set_ai_shelf_boundaries(c, entity_id, plugin.plugin_id, boundaries)
updated_cab = db.get_cabinet(c, entity_id)
if not updated_cab:
raise CabinetNotFoundError(entity_id)
return updated_cab
else: # shelves
entity_s = db.get_shelf(c, entity_id)
if not entity_s:
raise ShelfNotFoundError(entity_id)
path, crop = shelf_source(c, entity_id)
b64, mt = prep_img_b64(path, crop, max_px=plugin.max_image_px)
result = plugin.detect(b64, mt)
boundaries = list(result.get("boundaries") or [])
db.set_ai_book_boundaries(c, entity_id, plugin.plugin_id, boundaries)
updated_shelf = db.get_shelf(c, entity_id)
if not updated_shelf:
raise ShelfNotFoundError(entity_id)
return updated_shelf

245
src/logic/identification.py Normal file
View File

@@ -0,0 +1,245 @@
"""Book identification logic: status computation, AI result application, plugin runners."""
import json
import db
from db import now
from errors import BookNotFoundError, NoRawTextError
from logic.boundaries import book_spine_source
from logic.images import prep_img_b64
from models import (
AIIdentifyResult,
BookIdentifierPlugin,
BookRow,
CandidateRecord,
TextRecognizeResult,
TextRecognizerPlugin,
)
AI_FIELDS = ("title", "author", "year", "isbn", "publisher")
_APPROVED_REQUIRED = ("title", "author", "year")
def compute_status(book: BookRow) -> str:
"""Return the identification_status string derived from current book field values.
Args:
book: The book row to evaluate.
Returns:
One of 'unidentified', 'ai_identified', or 'user_approved'.
"""
if not (book.ai_title or "").strip():
return "unidentified"
filled = all((getattr(book, f) or "").strip() for f in _APPROVED_REQUIRED)
no_diff = all(
not (getattr(book, f"ai_{f}") or "").strip()
or (getattr(book, f) or "").strip() == (getattr(book, f"ai_{f}") or "").strip()
for f in AI_FIELDS
)
return "user_approved" if (filled and no_diff) else "ai_identified"
def build_query(book: BookRow) -> str:
"""Build a search query string from the best available candidate fields.
Prefers the first candidate with a non-empty author+title pair; falls back to
AI fields, then raw OCR text.
Args:
book: The book row to build a query for.
Returns:
Query string, empty if no usable data is available.
"""
candidates: list[dict[str, object]] = json.loads(book.candidates or "[]")
for c in candidates:
q = " ".join(filter(None, [(str(c.get("author") or "")).strip(), (str(c.get("title") or "")).strip()]))
if q:
return q
q = " ".join(filter(None, [(book.ai_author or "").strip(), (book.ai_title or "").strip()]))
if q:
return q
return (book.raw_text or "").strip()
def save_user_fields(book_id: str, title: str, author: str, year: str, isbn: str, publisher: str, notes: str) -> str:
"""Persist user-edited fields and recompute identification status.
Also sets ai_* fields to match user values so they are treated as approved.
Args:
book_id: ID of the book to update.
title: User-provided title.
author: User-provided author.
year: User-provided year.
isbn: User-provided ISBN.
publisher: User-provided publisher.
notes: User-provided notes.
Returns:
Updated identification_status string.
"""
with db.transaction() as c:
db.set_user_book_fields(c, book_id, title, author, year, isbn, publisher, notes)
book = db.get_book(c, book_id)
status = compute_status(book) if book else "unidentified"
db.set_book_status(c, book_id, status)
return status
def dismiss_field(book_id: str, field: str, value: str) -> tuple[str, list[CandidateRecord]]:
"""Dismiss a candidate suggestion for a field.
If value is non-empty: removes matching candidates and reverts ai_field to the
user value if it matched. If value is empty: sets ai_field to the current user value.
Args:
book_id: ID of the book.
field: Field name (one of AI_FIELDS).
value: Candidate value to dismiss, or empty string to dismiss the AI suggestion.
Returns:
(identification_status, updated_candidates).
Raises:
BookNotFoundError: If book_id does not exist.
"""
with db.transaction() as c:
book = db.get_book(c, book_id)
if not book:
raise BookNotFoundError(book_id)
candidates: list[CandidateRecord] = json.loads(book.candidates or "[]")
if value:
candidates = [cand for cand in candidates if (str(cand.get(field) or "")).strip() != value]
db.set_book_candidates(c, book_id, json.dumps(candidates))
if (getattr(book, f"ai_{field}") or "").strip() == value:
db.set_book_ai_field(c, book_id, field, str(getattr(book, field) or ""))
else:
db.set_book_ai_field(c, book_id, field, str(getattr(book, field) or ""))
book = db.get_book(c, book_id)
status = compute_status(book) if book else "unidentified"
db.set_book_status(c, book_id, status)
candidates = json.loads(book.candidates or "[]") if book else []
return status, candidates
def apply_ai_result(book_id: str, result: AIIdentifyResult, confidence_threshold: float = 0.8) -> None:
"""Apply an AI identification result to a book.
Stores confidence unconditionally; sets ai_* fields only when confidence meets the threshold.
Args:
book_id: ID of the book to update.
result: AI identification result dict.
confidence_threshold: Minimum confidence to write ai_* fields (default 0.8).
"""
confidence = float(result.get("confidence") or 0)
with db.transaction() as c:
db.set_book_confidence(c, book_id, confidence, now())
if confidence < confidence_threshold:
return
db.set_book_ai_fields(
c,
book_id,
result.get("title") or "",
result.get("author") or "",
result.get("year") or "",
result.get("isbn") or "",
result.get("publisher") or "",
)
book = db.get_book(c, book_id)
if book:
db.set_book_status(c, book_id, compute_status(book))
def run_text_recognizer(plugin: TextRecognizerPlugin, book_id: str) -> BookRow:
"""Recognize text from a book spine image and store the result.
Calls the plugin with the book's spine image, stores raw_text, and merges
the result into the candidates list.
Args:
plugin: The text recognizer plugin to execute.
book_id: ID of the book to process.
Returns:
Updated BookRow after storing the result.
Raises:
BookNotFoundError: If book_id does not exist.
"""
with db.transaction() as c:
book = db.get_book(c, book_id)
if not book:
raise BookNotFoundError(book_id)
spine_path, spine_crop = book_spine_source(c, book_id)
b64, mt = prep_img_b64(spine_path, spine_crop, max_px=plugin.max_image_px)
result: TextRecognizeResult = plugin.recognize(b64, mt)
raw_text = result.get("raw_text") or ""
cand: CandidateRecord = {
"source": plugin.plugin_id,
"title": (result.get("title") or "").strip(),
"author": (result.get("author") or "").strip(),
"year": (result.get("year") or "").strip(),
"publisher": (result.get("publisher") or "").strip(),
"isbn": "",
}
existing: list[CandidateRecord] = json.loads(book.candidates or "[]")
existing = [cd for cd in existing if cd.get("source") != plugin.plugin_id]
if any([cand["title"], cand["author"], cand["year"], cand["publisher"]]):
existing.append(cand)
db.set_book_raw_text(c, book_id, raw_text)
db.set_book_candidates(c, book_id, json.dumps(existing))
updated = db.get_book(c, book_id)
if not updated:
raise BookNotFoundError(book_id)
return updated
def run_book_identifier(plugin: BookIdentifierPlugin, book_id: str) -> BookRow:
"""Identify a book using AI and update ai_* fields and candidates.
Requires raw_text to have been populated by a text recognizer first.
Args:
plugin: The book identifier plugin to execute.
book_id: ID of the book to process.
Returns:
Updated BookRow after storing the identification result.
Raises:
BookNotFoundError: If book_id does not exist.
NoRawTextError: If the book has no raw_text (text recognizer has not run).
"""
with db.transaction() as c:
book = db.get_book(c, book_id)
if not book:
raise BookNotFoundError(book_id)
raw_text = (book.raw_text or "").strip()
if not raw_text:
raise NoRawTextError(book_id)
result: AIIdentifyResult = plugin.identify(raw_text)
# apply_ai_result manages its own transaction
apply_ai_result(book_id, result, plugin.confidence_threshold)
with db.transaction() as c:
book = db.get_book(c, book_id)
if not book:
raise BookNotFoundError(book_id)
cand: CandidateRecord = {
"source": plugin.plugin_id,
"title": (result.get("title") or "").strip(),
"author": (result.get("author") or "").strip(),
"year": (result.get("year") or "").strip(),
"isbn": (result.get("isbn") or "").strip(),
"publisher": (result.get("publisher") or "").strip(),
}
existing: list[CandidateRecord] = json.loads(book.candidates or "[]")
existing = [cd for cd in existing if cd.get("source") != plugin.plugin_id]
existing.append(cand)
db.set_book_candidates(c, book_id, json.dumps(existing))
updated = db.get_book(c, book_id)
if not updated:
raise BookNotFoundError(book_id)
return updated

107
src/logic/images.py Normal file
View File

@@ -0,0 +1,107 @@
"""Image utilities: in-place crop, base64 encoding, and streaming serve."""
import base64
import io
from pathlib import Path
from fastapi.responses import StreamingResponse
from PIL import Image, UnidentifiedImageError
from errors import ImageFileNotFoundError, ImageReadError
def crop_save(path: Path, x: int, y: int, w: int, h: int) -> None:
"""Crop an image file in-place, replacing it with the cropped version.
Args:
path: Path to the image file.
x: Left pixel coordinate of the crop box.
y: Top pixel coordinate of the crop box.
w: Width of the crop box in pixels.
h: Height of the crop box in pixels.
Raises:
ImageFileNotFoundError: If the file does not exist.
ImageReadError: If the file cannot be opened, decoded, or written back.
"""
try:
with Image.open(path) as img:
cropped = img.crop((x, y, x + w, y + h))
cropped.save(path)
except FileNotFoundError:
raise ImageFileNotFoundError(path) from None
except (OSError, UnidentifiedImageError) as exc:
raise ImageReadError(path, str(exc)) from exc
def prep_img_b64(
path: Path,
crop_frac: tuple[float, float, float, float] | None = None,
max_px: int = 1600,
) -> tuple[str, str]:
"""Load an image, optionally crop it, downscale to max_px on the longest side, and encode as base64.
Args:
path: Path to the source image file.
crop_frac: Optional (x0, y0, x1, y1) fractions in [0, 1] to crop before scaling.
max_px: Maximum pixel count for the longest dimension (default 1600).
Returns:
(base64_string, mime_type) — mime_type is always 'image/png'.
Raises:
ImageFileNotFoundError: If the file does not exist.
ImageReadError: If the file cannot be opened or decoded.
"""
try:
with Image.open(path) as img:
img = img.convert("RGB")
if crop_frac is not None:
x0f, y0f, x1f, y1f = crop_frac
iw, ih = img.size
box = (int(x0f * iw), int(y0f * ih), int(x1f * iw), int(y1f * ih))
img = img.crop(box)
w, h = img.size
if max(w, h) > max_px:
size: tuple[int, int] = (max_px, int(h * max_px / w)) if w >= h else (int(w * max_px / h), max_px)
img = img.resize(size, Image.Resampling.LANCZOS) # pyright: ignore[reportUnknownMemberType]
buf = io.BytesIO()
img.save(buf, format="PNG")
b64 = base64.b64encode(buf.getvalue()).decode()
return b64, "image/png"
except FileNotFoundError:
raise ImageFileNotFoundError(path) from None
except (OSError, UnidentifiedImageError) as exc:
raise ImageReadError(path, str(exc)) from exc
def serve_crop(path: Path, crop_frac: tuple[float, float, float, float] | None) -> StreamingResponse:
"""Serve an image (optionally cropped) as a JPEG streaming HTTP response.
Args:
path: Path to the source image file.
crop_frac: Optional (x0, y0, x1, y1) fractions in [0, 1] to crop before serving.
Returns:
StreamingResponse with media_type 'image/jpeg'.
Raises:
ImageFileNotFoundError: If the file does not exist.
ImageReadError: If the file cannot be opened or decoded.
"""
try:
with Image.open(path) as img:
img = img.convert("RGB")
if crop_frac is not None:
x0f, y0f, x1f, y1f = crop_frac
iw, ih = img.size
box = (int(x0f * iw), int(y0f * ih), int(x1f * iw), int(y1f * ih))
img = img.crop(box)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=90)
buf.seek(0)
return StreamingResponse(io.BytesIO(buf.getvalue()), media_type="image/jpeg")
except FileNotFoundError:
raise ImageFileNotFoundError(path) from None
except (OSError, UnidentifiedImageError) as exc:
raise ImageReadError(path, str(exc)) from exc

241
src/models.py Normal file
View File

@@ -0,0 +1,241 @@
"""Shared types: entity dataclasses, API payload dataclasses, plugin Protocols."""
from dataclasses import dataclass, field
from typing import Any, Literal, Protocol, TypedDict
# ── AI plugin result shapes ───────────────────────────────────────────────────
class BoundaryDetectResult(TypedDict, total=False):
boundaries: list[float]
confidence: float
class TextRecognizeResult(TypedDict, total=False):
raw_text: str
title: str
author: str
year: str
publisher: str
other: str
class AIIdentifyResult(TypedDict, total=False):
title: str
author: str
year: str
isbn: str
publisher: str
confidence: float
# ── Candidate + AI config ─────────────────────────────────────────────────────
class CandidateRecord(TypedDict):
source: str
title: str
author: str
year: str
isbn: str
publisher: str
class AIConfig(TypedDict):
base_url: str
api_key: str
model: str
max_image_px: int
confidence_threshold: float
extra_body: dict[str, Any]
# ── Application state ─────────────────────────────────────────────────────────
class BatchState(TypedDict):
running: bool
total: int
done: int
errors: int
current: str
# ── Plugin manifest ───────────────────────────────────────────────────────────
class _PluginManifestBase(TypedDict):
id: str
name: str
category: str
auto_queue: bool
class PluginManifestEntry(_PluginManifestBase, total=False):
target: str
# ── Plugin Protocols ──────────────────────────────────────────────────────────
class BoundaryDetectorPlugin(Protocol):
plugin_id: str
name: str
auto_queue: bool
target: str
@property
def max_image_px(self) -> int: ...
def detect(self, image_b64: str, image_mime: str) -> BoundaryDetectResult: ...
class TextRecognizerPlugin(Protocol):
plugin_id: str
name: str
auto_queue: bool
@property
def max_image_px(self) -> int: ...
def recognize(self, image_b64: str, image_mime: str) -> TextRecognizeResult: ...
class BookIdentifierPlugin(Protocol):
plugin_id: str
name: str
auto_queue: bool
@property
def confidence_threshold(self) -> float: ...
def identify(self, raw_text: str) -> AIIdentifyResult: ...
class ArchiveSearcherPlugin(Protocol):
plugin_id: str
name: str
auto_queue: bool
def search(self, query: str) -> list[CandidateRecord]: ...
# ── Discriminated union for plugin dispatch ───────────────────────────────────
BDPluginResult = tuple[Literal["boundary_detector"], BoundaryDetectorPlugin]
TRPluginResult = tuple[Literal["text_recognizer"], TextRecognizerPlugin]
BIPluginResult = tuple[Literal["book_identifier"], BookIdentifierPlugin]
ASPluginResult = tuple[Literal["archive_searcher"], ArchiveSearcherPlugin]
NotFoundResult = tuple[None, None]
PluginLookupResult = BDPluginResult | TRPluginResult | BIPluginResult | ASPluginResult | NotFoundResult
# ── Entity dataclasses (typed DB rows) ────────────────────────────────────────
def _list_f() -> list[float]:
return []
def _list_s() -> list[str]:
return []
@dataclass
class RoomRow:
id: str
name: str
position: int
created_at: str
@dataclass
class CabinetRow:
id: str
room_id: str
name: str
photo_filename: str | None
shelf_boundaries: str | None
ai_shelf_boundaries: str | None
position: int
created_at: str
@dataclass
class ShelfRow:
id: str
cabinet_id: str
name: str
photo_filename: str | None
book_boundaries: str | None
ai_book_boundaries: str | None
position: int
created_at: str
@dataclass
class BookRow:
id: str
shelf_id: str
position: int
image_filename: str | None
title: str
author: str
year: str
isbn: str
publisher: str
notes: str
raw_text: str
ai_title: str
ai_author: str
ai_year: str
ai_isbn: str
ai_publisher: str
identification_status: str
title_confidence: float
analyzed_at: str | None
created_at: str
candidates: str | None
# ── API request payload dataclasses ──────────────────────────────────────────
@dataclass
class UpdateNamePayload:
name: str
@dataclass
class UpdateBookPayload:
title: str = ""
author: str = ""
year: str = ""
isbn: str = ""
publisher: str = ""
notes: str = ""
@dataclass
class BoundariesPayload:
boundaries: list[float] = field(default_factory=_list_f)
@dataclass
class CropPayload:
x: int = 0
y: int = 0
w: int = 0
h: int = 0
@dataclass
class DismissFieldPayload:
field: str = ""
value: str = ""
@dataclass
class ReorderPayload:
ids: list[str] = field(default_factory=_list_s)

241
src/plugins/__init__.py Normal file
View File

@@ -0,0 +1,241 @@
"""Plugin registry for bookshelf automations.
Functions are loaded from config at startup via load_plugins().
Four categories: boundary_detectors, text_recognizers, book_identifiers, archive_searchers.
"""
import logging
from typing import Any, Literal, overload
from config import AIFunctionConfig, AppConfig, CredentialConfig, ModelConfig
from models import (
AIConfig,
ASPluginResult,
ArchiveSearcherPlugin,
BDPluginResult,
BIPluginResult,
BookIdentifierPlugin,
BoundaryDetectorPlugin,
NotFoundResult,
PluginLookupResult,
PluginManifestEntry,
TextRecognizerPlugin,
TRPluginResult,
)
from .rate_limiter import RateLimiter
RATE_LIMITER = RateLimiter()
_logger = logging.getLogger(__name__)
# ── Typed per-category registries ─────────────────────────────────────────────
_boundary_detectors: dict[str, BoundaryDetectorPlugin] = {}
_text_recognizers: dict[str, TextRecognizerPlugin] = {}
_book_identifiers: dict[str, BookIdentifierPlugin] = {}
_archive_searchers: dict[str, ArchiveSearcherPlugin] = {}
_type_to_class: dict[str, Any] = {} # populated lazily on first call
def _archive_classes() -> dict[str, Any]:
if not _type_to_class:
from .archives.html_scraper import HtmlScraperPlugin
from .archives.openlibrary import OpenLibraryPlugin
from .archives.rsl import RSLPlugin
from .archives.sru_catalog import SRUCatalogPlugin
_type_to_class.update(
{
"openlibrary": OpenLibraryPlugin,
"rsl": RSLPlugin,
"html_scraper": HtmlScraperPlugin,
"sru_catalog": SRUCatalogPlugin,
}
)
return _type_to_class
def _build_ai_cfg(model_cfg: ModelConfig, cred_cfg: CredentialConfig, func: AIFunctionConfig) -> AIConfig:
"""Assemble runtime AIConfig from the 3-layer config (credentials → models → functions)."""
return AIConfig(
base_url=cred_cfg.base_url,
api_key=cred_cfg.api_key,
model=model_cfg.model,
max_image_px=func.max_image_px,
confidence_threshold=func.confidence_threshold,
extra_body=model_cfg.extra_body,
)
def load_plugins(config: AppConfig) -> None:
"""Populate the plugin registry from a typed AppConfig."""
from .ai_compat import (
BookIdentifierPlugin as BIClass,
BoundaryDetectorBooksPlugin,
BoundaryDetectorShelvesPlugin,
TextRecognizerPlugin as TRClass,
)
_boundary_detectors.clear()
_text_recognizers.clear()
_book_identifiers.clear()
_archive_searchers.clear()
archive_cls = _archive_classes()
for key, func in config.functions.boundary_detectors.items():
if key == "shelves":
bd_cls = BoundaryDetectorShelvesPlugin
elif key == "books":
bd_cls = BoundaryDetectorBooksPlugin
else:
_logger.warning("Unknown boundary_detector key %r — must be 'shelves' or 'books'", key)
continue
m = config.models.get(func.model)
if m is None:
_logger.warning("Skipping boundary_detector %r: model %r not found", key, func.model)
continue
c = config.credentials.get(m.credentials)
if c is None:
_logger.warning("Skipping boundary_detector %r: credential %r not found", key, m.credentials)
continue
_boundary_detectors[key] = bd_cls(
plugin_id=key,
name=func.name or key.replace("_", " ").title(),
ai_config=_build_ai_cfg(m, c, func),
prompt_text=m.prompt,
auto_queue=func.auto_queue,
rate_limit_seconds=func.rate_limit_seconds,
)
for key, func in config.functions.text_recognizers.items():
m = config.models.get(func.model)
if m is None:
_logger.warning("Skipping text_recognizer %r: model %r not found", key, func.model)
continue
c = config.credentials.get(m.credentials)
if c is None:
_logger.warning("Skipping text_recognizer %r: credential %r not found", key, m.credentials)
continue
_text_recognizers[key] = TRClass(
plugin_id=key,
name=func.name or key.replace("_", " ").title(),
ai_config=_build_ai_cfg(m, c, func),
prompt_text=m.prompt,
auto_queue=func.auto_queue,
rate_limit_seconds=func.rate_limit_seconds,
)
for key, func in config.functions.book_identifiers.items():
m = config.models.get(func.model)
if m is None:
_logger.warning("Skipping book_identifier %r: model %r not found", key, func.model)
continue
c = config.credentials.get(m.credentials)
if c is None:
_logger.warning("Skipping book_identifier %r: credential %r not found", key, m.credentials)
continue
_book_identifiers[key] = BIClass(
plugin_id=key,
name=func.name or key.replace("_", " ").title(),
ai_config=_build_ai_cfg(m, c, func),
prompt_text=m.prompt,
auto_queue=func.auto_queue,
rate_limit_seconds=func.rate_limit_seconds,
)
for key, func in config.functions.archive_searchers.items():
cls = archive_cls.get(func.type)
if cls is None:
_logger.warning("Skipping archive_searcher %r: unknown type %r", key, func.type)
continue
_archive_searchers[key] = cls(
plugin_id=key,
name=func.name or key.replace("_", " ").title(),
rate_limiter=RATE_LIMITER,
rate_limit_seconds=func.rate_limit_seconds,
auto_queue=func.auto_queue,
timeout=func.timeout,
config=func.config,
)
def get_manifest() -> list[PluginManifestEntry]:
"""Return list of plugin descriptors for the frontend."""
result: list[PluginManifestEntry] = []
for pid, p in _boundary_detectors.items():
result.append(
PluginManifestEntry(
id=pid, name=p.name, category="boundary_detector", auto_queue=p.auto_queue, target=p.target
)
)
for pid, p in _text_recognizers.items():
result.append(PluginManifestEntry(id=pid, name=p.name, category="text_recognizer", auto_queue=p.auto_queue))
for pid, p in _book_identifiers.items():
result.append(PluginManifestEntry(id=pid, name=p.name, category="book_identifier", auto_queue=p.auto_queue))
for pid, p in _archive_searchers.items():
result.append(PluginManifestEntry(id=pid, name=p.name, category="archive_searcher", auto_queue=p.auto_queue))
return result
@overload
def get_auto_queue(category: Literal["boundary_detectors", "boundary_detector"]) -> list[BoundaryDetectorPlugin]: ...
@overload
def get_auto_queue(category: Literal["text_recognizers", "text_recognizer"]) -> list[TextRecognizerPlugin]: ...
@overload
def get_auto_queue(category: Literal["book_identifiers", "book_identifier"]) -> list[BookIdentifierPlugin]: ...
@overload
def get_auto_queue(category: Literal["archive_searchers", "archive_searcher"]) -> list[ArchiveSearcherPlugin]: ...
@overload
def get_auto_queue(
category: str,
) -> (
list[BoundaryDetectorPlugin] | list[TextRecognizerPlugin] | list[BookIdentifierPlugin] | list[ArchiveSearcherPlugin]
): ...
def get_auto_queue(
category: str,
) -> (
list[BoundaryDetectorPlugin] | list[TextRecognizerPlugin] | list[BookIdentifierPlugin] | list[ArchiveSearcherPlugin]
):
"""Return plugin instances for a category that have auto_queue=True."""
match category:
case "boundary_detectors" | "boundary_detector":
return [p for p in _boundary_detectors.values() if p.auto_queue]
case "text_recognizers" | "text_recognizer":
return [p for p in _text_recognizers.values() if p.auto_queue]
case "book_identifiers" | "book_identifier":
return [p for p in _book_identifiers.values() if p.auto_queue]
case "archive_searchers" | "archive_searcher":
return [p for p in _archive_searchers.values() if p.auto_queue]
case _:
return []
def get_plugin(plugin_id: str) -> PluginLookupResult:
"""Find a plugin by ID across all categories. Returns a discriminated (category, plugin) tuple."""
if plugin_id in _boundary_detectors:
bd: BDPluginResult = ("boundary_detector", _boundary_detectors[plugin_id])
return bd
if plugin_id in _text_recognizers:
tr: TRPluginResult = ("text_recognizer", _text_recognizers[plugin_id])
return tr
if plugin_id in _book_identifiers:
bi: BIPluginResult = ("book_identifier", _book_identifiers[plugin_id])
return bi
if plugin_id in _archive_searchers:
asr: ASPluginResult = ("archive_searcher", _archive_searchers[plugin_id])
return asr
nf: NotFoundResult = (None, None)
return nf

View File

@@ -0,0 +1,21 @@
"""AI plugin classes using OpenAI-compatible APIs.
Submodules:
_client.py — shared _AIClient + HTTP helpers (private)
boundary_detector_shelves.py — BoundaryDetectorShelvesPlugin (cabinet → shelf bounds)
boundary_detector_books.py — BoundaryDetectorBooksPlugin (shelf → book bounds)
text_recognizer.py — TextRecognizerPlugin (spine image → raw text + fields)
book_identifier.py — BookIdentifierPlugin (raw text → bibliographic metadata)
"""
from .boundary_detector_books import BoundaryDetectorBooksPlugin
from .boundary_detector_shelves import BoundaryDetectorShelvesPlugin
from .book_identifier import BookIdentifierPlugin
from .text_recognizer import TextRecognizerPlugin
__all__ = [
"BoundaryDetectorShelvesPlugin",
"BoundaryDetectorBooksPlugin",
"TextRecognizerPlugin",
"BookIdentifierPlugin",
]

View File

@@ -0,0 +1,94 @@
"""Internal OpenAI-compatible HTTP client shared by all AI plugins.
Caches openai.OpenAI instances per (base_url, api_key) to avoid re-creating on each call.
AIClient wraps the raw API call: fills prompt template, encodes images, parses JSON response.
"""
import json
import re
from string import Template
from typing import Any, cast
import openai
from openai.types.chat import ChatCompletionMessageParam
from openai.types.chat.chat_completion_content_part_image_param import (
ChatCompletionContentPartImageParam,
ImageURL,
)
from openai.types.chat.chat_completion_content_part_text_param import ChatCompletionContentPartTextParam
from models import AIConfig
# Module-level cache of openai.OpenAI instances keyed by (base_url, api_key)
_clients: dict[tuple[str, str], openai.OpenAI] = {}
def _get_client(base_url: str, api_key: str) -> openai.OpenAI:
key = (base_url, api_key)
if key not in _clients:
_clients[key] = openai.OpenAI(base_url=base_url, api_key=api_key)
return _clients[key]
def _parse_json(text: str) -> dict[str, Any]:
"""Extract and parse the first JSON object found in text.
Raises ValueError if no JSON object is found or the JSON is malformed.
"""
text = text.strip()
m = re.search(r"\{.*\}", text, re.DOTALL)
if not m:
raise ValueError(f"No JSON object found in AI response: {text[:200]!r}")
try:
result = json.loads(m.group())
except json.JSONDecodeError as exc:
raise ValueError(f"Failed to parse AI response as JSON: {exc}") from exc
if not isinstance(result, dict):
raise ValueError(f"Expected JSON object, got {type(result).__name__}")
return cast(dict[str, Any], result)
ContentPart = ChatCompletionContentPartImageParam | ChatCompletionContentPartTextParam
class AIClient:
"""AI client bound to a specific provider config and output format.
cfg must contain: base_url, api_key, model, max_image_px, confidence_threshold.
output_format is the hardcoded JSON schema string injected as ${OUTPUT_FORMAT}.
"""
def __init__(self, cfg: AIConfig, output_format: str):
self.cfg = cfg
self.output_format = output_format
def call(
self,
prompt_template: str,
images: list[tuple[str, str]],
text_vars: dict[str, str] | None = None,
) -> dict[str, Any]:
"""Substitute template vars, call API with optional images, return parsed JSON.
images: list of (base64_str, mime_type) tuples.
text_vars: extra ${KEY} substitutions beyond ${OUTPUT_FORMAT}.
"""
vars_: dict[str, str] = {"OUTPUT_FORMAT": self.output_format}
if text_vars:
vars_.update(text_vars)
prompt = Template(prompt_template).safe_substitute(vars_)
client = _get_client(self.cfg["base_url"], self.cfg["api_key"])
parts: list[ContentPart] = [
ChatCompletionContentPartImageParam(
type="image_url",
image_url=ImageURL(url=f"data:{mt};base64,{b64}"),
)
for b64, mt in images
]
parts.append(ChatCompletionContentPartTextParam(type="text", text=prompt))
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": parts}]
r = client.chat.completions.create(
model=self.cfg["model"], max_tokens=2048, messages=messages, extra_body=self.cfg["extra_body"]
)
raw = r.choices[0].message.content or ""
return _parse_json(raw)

View File

@@ -0,0 +1,56 @@
"""Book identifier plugin — raw spine text → bibliographic metadata.
Input: raw_text string (from text_recognizer).
Output: {"title": "...", "author": "...", "year": "...", "isbn": "...",
"publisher": "...", "confidence": 0.95}
confidence — float 0-1; results below confidence_threshold are discarded by logic.py.
Result added to books.candidates and books.ai_* fields.
"""
from models import AIConfig, AIIdentifyResult
from ._client import AIClient
class BookIdentifierPlugin:
"""Identifies a book from spine text using a VLM with web-search capability."""
category = "book_identifiers"
OUTPUT_FORMAT = (
'{"title": "...", "author": "...", "year": "...", ' '"isbn": "...", "publisher": "...", "confidence": 0.95}'
)
def __init__(
self,
plugin_id: str,
name: str,
ai_config: AIConfig,
prompt_text: str,
auto_queue: bool,
rate_limit_seconds: float,
):
self.plugin_id = plugin_id
self.name = name
self.auto_queue = auto_queue
self.rate_limit_seconds = rate_limit_seconds
self._client = AIClient(ai_config, self.OUTPUT_FORMAT)
self._prompt_text = prompt_text
def identify(self, raw_text: str) -> AIIdentifyResult:
"""Returns AIIdentifyResult with title/author/year/isbn/publisher/confidence."""
raw = self._client.call(self._prompt_text, [], text_vars={"RAW_TEXT": raw_text})
result = AIIdentifyResult(
title=str(raw.get("title") or ""),
author=str(raw.get("author") or ""),
year=str(raw.get("year") or ""),
isbn=str(raw.get("isbn") or ""),
publisher=str(raw.get("publisher") or ""),
)
conf = raw.get("confidence")
if conf is not None:
result["confidence"] = float(conf)
return result
@property
def confidence_threshold(self) -> float:
return self._client.cfg["confidence_threshold"]

View File

@@ -0,0 +1,46 @@
"""Boundary detector plugin for book spine detection.
Input: shelf image (full or cropped from cabinet photo).
Output: {"boundaries": [x0, x1, ...]}
boundaries — interior x-fractions (0=left, 1=right), excluding 0 and 1.
Results stored in shelves.ai_book_boundaries[plugin_id].
"""
from models import AIConfig, BoundaryDetectResult
from ._client import AIClient
class BoundaryDetectorBooksPlugin:
"""Detects vertical book-spine boundaries in a shelf image using a VLM."""
category = "boundary_detectors"
target = "books" # operates on shelf images; stored in ai_book_boundaries
OUTPUT_FORMAT = '{"boundaries": [0.08, 0.16, 0.24, 0.32]}'
def __init__(
self,
plugin_id: str,
name: str,
ai_config: AIConfig,
prompt_text: str,
auto_queue: bool,
rate_limit_seconds: float,
):
self.plugin_id = plugin_id
self.name = name
self.auto_queue = auto_queue
self.rate_limit_seconds = rate_limit_seconds
self._client = AIClient(ai_config, self.OUTPUT_FORMAT)
self._prompt_text = prompt_text
def detect(self, image_b64: str, image_mime: str) -> BoundaryDetectResult:
"""Returns BoundaryDetectResult with 'boundaries' (list[float])."""
raw = self._client.call(self._prompt_text, [(image_b64, image_mime)])
raw_bounds: list[object] = raw.get("boundaries") or []
boundaries: list[float] = [float(b) for b in raw_bounds if isinstance(b, (int, float))]
return BoundaryDetectResult(boundaries=boundaries)
@property
def max_image_px(self) -> int:
return self._client.cfg["max_image_px"]

View File

@@ -0,0 +1,51 @@
"""Boundary detector plugin for shelf detection.
Input: cabinet photo (full image).
Output: {"boundaries": [y0, y1, ...], "confidence": 0.x}
boundaries — interior y-fractions (0=top, 1=bottom), excluding 0 and 1.
confidence — optional float 0-1.
Results stored in cabinets.ai_shelf_boundaries[plugin_id].
"""
from models import AIConfig, BoundaryDetectResult
from ._client import AIClient
class BoundaryDetectorShelvesPlugin:
"""Detects horizontal shelf boundaries in a cabinet photo using a VLM."""
category = "boundary_detectors"
target = "shelves" # operates on cabinet images; stored in ai_shelf_boundaries
OUTPUT_FORMAT = '{"boundaries": [0.24, 0.48, 0.72], "confidence": 0.92}'
def __init__(
self,
plugin_id: str,
name: str,
ai_config: AIConfig,
prompt_text: str,
auto_queue: bool,
rate_limit_seconds: float,
):
self.plugin_id = plugin_id
self.name = name
self.auto_queue = auto_queue
self.rate_limit_seconds = rate_limit_seconds
self._client = AIClient(ai_config, self.OUTPUT_FORMAT)
self._prompt_text = prompt_text
def detect(self, image_b64: str, image_mime: str) -> BoundaryDetectResult:
"""Returns BoundaryDetectResult with 'boundaries' and optionally 'confidence'."""
raw = self._client.call(self._prompt_text, [(image_b64, image_mime)])
raw_bounds: list[object] = raw.get("boundaries") or []
boundaries: list[float] = [float(b) for b in raw_bounds if isinstance(b, (int, float))]
result = BoundaryDetectResult(boundaries=boundaries)
conf = raw.get("confidence")
if conf is not None:
result["confidence"] = float(conf)
return result
@property
def max_image_px(self) -> int:
return self._client.cfg["max_image_px"]

View File

@@ -0,0 +1,56 @@
"""Text recognizer plugin — spine image → raw text + structured fields.
Input: book spine image.
Output: {"raw_text": "...", "title": "...", "author": "...", "year": "...",
"publisher": "...", "other": "..."}
raw_text — all visible text verbatim, line-break separated.
other fields — VLM interpretation of raw_text.
Result added to books.candidates and books.raw_text.
"""
from models import AIConfig, TextRecognizeResult
from ._client import AIClient
class TextRecognizerPlugin:
"""Reads text from a book spine image using a VLM."""
category = "text_recognizers"
OUTPUT_FORMAT = (
'{"raw_text": "The Great Gatsby\\nF. Scott Fitzgerald\\nScribner", '
'"title": "The Great Gatsby", "author": "F. Scott Fitzgerald", '
'"year": "", "publisher": "Scribner", "other": ""}'
)
def __init__(
self,
plugin_id: str,
name: str,
ai_config: AIConfig,
prompt_text: str,
auto_queue: bool,
rate_limit_seconds: float,
):
self.plugin_id = plugin_id
self.name = name
self.auto_queue = auto_queue
self.rate_limit_seconds = rate_limit_seconds
self._client = AIClient(ai_config, self.OUTPUT_FORMAT)
self._prompt_text = prompt_text
def recognize(self, image_b64: str, image_mime: str) -> TextRecognizeResult:
"""Returns TextRecognizeResult with raw_text, title, author, year, publisher, other."""
raw = self._client.call(self._prompt_text, [(image_b64, image_mime)])
return TextRecognizeResult(
raw_text=str(raw.get("raw_text") or ""),
title=str(raw.get("title") or ""),
author=str(raw.get("author") or ""),
year=str(raw.get("year") or ""),
publisher=str(raw.get("publisher") or ""),
other=str(raw.get("other") or ""),
)
@property
def max_image_px(self) -> int:
return self._client.cfg["max_image_px"]

View File

View File

@@ -0,0 +1,121 @@
"""Config-driven HTML scraper for archive sites (rusneb, alib, shpl, etc.)."""
import re
from typing import Any
from urllib.parse import urlparse
import httpx
from models import CandidateRecord
from ..rate_limiter import RateLimiter
_YEAR_RE = re.compile(r"\b(1[0-9]{3}|20[012][0-9])\b")
def _cls_re(cls_frag: str, min_len: int = 3, max_len: int = 120) -> re.Pattern[str]:
return re.compile(rf'class="[^"]*{re.escape(cls_frag)}[^"]*"[^>]*>([^<]{{{min_len},{max_len}}})<')
class HtmlScraperPlugin:
"""
Config-driven HTML scraper. Supported config keys:
url — search URL
search_param — query param name
extra_params — dict of fixed extra query parameters
title_class — CSS class fragment for title elements (class-based strategy)
author_class — CSS class fragment for author elements
link_href_pattern — href regex to find title <a> links (link strategy, e.g. alib)
brief_class — CSS class for brief record rows (brief strategy, e.g. shpl)
"""
category = "archive_searchers"
def __init__(
self,
plugin_id: str,
name: str,
rate_limiter: RateLimiter,
rate_limit_seconds: float,
auto_queue: bool,
timeout: int,
config: dict[str, Any],
):
self.plugin_id = plugin_id
self.name = name
self._rl = rate_limiter
self.rate_limit_seconds = rate_limit_seconds
self.auto_queue = auto_queue
self.timeout = timeout
self.config = config
self._domain: str = urlparse(str(config.get("url") or "")).netloc or plugin_id
def search(self, query: str) -> list[CandidateRecord]:
cfg = self.config
self._rl.wait_and_record(self._domain, self.rate_limit_seconds)
params: dict[str, Any] = dict(cfg.get("extra_params") or {})
params[cfg["search_param"]] = query
r = httpx.get(
cfg["url"],
params=params,
timeout=self.timeout,
headers={"User-Agent": "Mozilla/5.0"},
)
html = r.text
years = _YEAR_RE.findall(html)
# Strategy: link_href_pattern (alib-style)
if "link_href_pattern" in cfg:
return self._parse_link(html, years, cfg)
# Strategy: brief_class (shpl-style)
if "brief_class" in cfg:
return self._parse_brief(html, years, cfg)
# Strategy: title_class + author_class (rusneb-style)
return self._parse_class(html, years, cfg)
def _parse_class(self, html: str, years: list[str], cfg: dict[str, Any]) -> list[CandidateRecord]:
titles = _cls_re(cfg.get("title_class", "title")).findall(html)[:3]
authors = _cls_re(cfg.get("author_class", "author"), 3, 80).findall(html)[:3]
return [
CandidateRecord(
source=self.plugin_id,
title=title.strip(),
author=authors[i].strip() if i < len(authors) else "",
year=years[i] if i < len(years) else "",
isbn="",
publisher="",
)
for i, title in enumerate(titles)
]
def _parse_link(self, html: str, years: list[str], cfg: dict[str, Any]) -> list[CandidateRecord]:
href_pat = cfg.get("link_href_pattern", r"")
titles = re.findall(rf'<a[^>]+href="[^"]*{href_pat}[^"]*"[^>]*>([^<]{{3,120}})</a>', html)[:3]
authors = _cls_re(cfg.get("author_class", "author"), 3, 80).findall(html)[:3]
return [
CandidateRecord(
source=self.plugin_id,
title=title.strip(),
author=authors[i].strip() if i < len(authors) else "",
year=years[i] if i < len(years) else "",
isbn="",
publisher="",
)
for i, title in enumerate(titles)
]
def _parse_brief(self, html: str, years: list[str], cfg: dict[str, Any]) -> list[CandidateRecord]:
titles = _cls_re(cfg.get("brief_class", "brief"), 3, 120).findall(html)[:3]
return [
CandidateRecord(
source=self.plugin_id,
title=t.strip(),
author="",
year=years[i] if i < len(years) else "",
isbn="",
publisher="",
)
for i, t in enumerate(titles)
]

View File

@@ -0,0 +1,54 @@
"""OpenLibrary JSON search API plugin (openlibrary.org/search.json)."""
from typing import Any
import httpx
from models import CandidateRecord
from ..rate_limiter import RateLimiter
_DOMAIN = "openlibrary.org"
class OpenLibraryPlugin:
category = "archive_searchers"
def __init__(
self,
plugin_id: str,
name: str,
rate_limiter: RateLimiter,
rate_limit_seconds: float,
auto_queue: bool,
timeout: int,
config: dict[str, Any],
):
self.plugin_id = plugin_id
self.name = name
self._rl = rate_limiter
self.rate_limit_seconds = rate_limit_seconds
self.auto_queue = auto_queue
self.timeout = timeout
def search(self, query: str) -> list[CandidateRecord]:
self._rl.wait_and_record(_DOMAIN, self.rate_limit_seconds)
r = httpx.get(
"https://openlibrary.org/search.json",
params={"q": query, "limit": 5, "fields": "title,author_name,first_publish_year,isbn,publisher"},
timeout=self.timeout,
)
docs: list[dict[str, Any]] = r.json().get("docs", [])
out: list[CandidateRecord] = []
for d in docs[:3]:
out.append(
CandidateRecord(
source=self.plugin_id,
title=(str(d.get("title") or "")).strip(),
author=", ".join(d.get("author_name") or []).strip(),
year=str(d.get("first_publish_year") or "").strip(),
isbn=((d.get("isbn") or [""])[0]).strip(),
publisher=((d.get("publisher") or [""])[0]).strip(),
)
)
return out

View File

@@ -0,0 +1,59 @@
"""RSL (Russian State Library) AJAX JSON search API plugin (search.rsl.ru)."""
from typing import Any
import httpx
from models import CandidateRecord
from ..rate_limiter import RateLimiter
_DOMAIN = "search.rsl.ru"
class RSLPlugin:
category = "archive_searchers"
def __init__(
self,
plugin_id: str,
name: str,
rate_limiter: RateLimiter,
rate_limit_seconds: float,
auto_queue: bool,
timeout: int,
config: dict[str, Any],
):
self.plugin_id = plugin_id
self.name = name
self._rl = rate_limiter
self.rate_limit_seconds = rate_limit_seconds
self.auto_queue = auto_queue
self.timeout = timeout
def search(self, query: str) -> list[CandidateRecord]:
self._rl.wait_and_record(_DOMAIN, self.rate_limit_seconds)
r = httpx.get(
"https://search.rsl.ru/site/ajax-search",
params={"language": "ru", "q": query, "page": 1, "perPage": 5},
timeout=self.timeout,
headers={"Accept": "application/json"},
)
data: dict[str, Any] = r.json()
records: list[dict[str, Any]] = data.get("records") or data.get("items") or data.get("data") or []
out: list[CandidateRecord] = []
for rec in records[:3]:
title = (str(rec.get("title") or rec.get("name") or "")).strip()
if not title:
continue
out.append(
CandidateRecord(
source=self.plugin_id,
title=title,
author=(str(rec.get("author") or rec.get("authors") or "")).strip(),
year=str(rec.get("year") or rec.get("pubyear") or "").strip(),
isbn=(str(rec.get("isbn") or "")).strip(),
publisher=(str(rec.get("publisher") or "")).strip(),
)
)
return out

View File

@@ -0,0 +1,71 @@
"""SRU XML catalog plugin (NLR and similar SRU-compliant catalogs)."""
import re
from typing import Any
from urllib.parse import urlparse
import httpx
from models import CandidateRecord
from ..rate_limiter import RateLimiter
class SRUCatalogPlugin:
"""
Config-driven SRU catalog searcher. Config keys:
url — SRU endpoint URL
query_prefix — SRU query prefix prepended to search term (e.g. 'title=')
"""
category = "archive_searchers"
def __init__(
self,
plugin_id: str,
name: str,
rate_limiter: RateLimiter,
rate_limit_seconds: float,
auto_queue: bool,
timeout: int,
config: dict[str, Any],
):
self.plugin_id = plugin_id
self.name = name
self._rl = rate_limiter
self.rate_limit_seconds = rate_limit_seconds
self.auto_queue = auto_queue
self.timeout = timeout
self.config = config
self._domain: str = urlparse(str(config.get("url") or "")).netloc or plugin_id
def search(self, query: str) -> list[CandidateRecord]:
cfg = self.config
self._rl.wait_and_record(self._domain, self.rate_limit_seconds)
sru_query = f'{cfg.get("query_prefix", "")}{query}'
r = httpx.get(
cfg["url"],
params={
"operation": "searchRetrieve",
"version": "1.1",
"query": sru_query,
"maximumRecords": "5",
"recordSchema": "dc",
},
timeout=self.timeout,
headers={"User-Agent": "Mozilla/5.0"},
)
titles = re.findall(r"<dc:title>([^<]+)</dc:title>", r.text)[:3]
authors = re.findall(r"<dc:creator>([^<]+)</dc:creator>", r.text)[:3]
years = re.findall(r"<dc:date>(\d{4})</dc:date>", r.text)[:3]
return [
CandidateRecord(
source=self.plugin_id,
title=title.strip(),
author=authors[i].strip() if i < len(authors) else "",
year=years[i] if i < len(years) else "",
isbn="",
publisher="",
)
for i, title in enumerate(titles)
]

View File

@@ -0,0 +1,23 @@
"""Thread-safe in-memory per-domain rate limiter shared across all archive plugin threads."""
import time
from threading import Lock
class RateLimiter:
"""Thread-safe per-domain rate limiter. Shared across all archive plugin threads."""
def __init__(self):
self._lock = Lock()
self._next: dict[str, float] = {}
def wait_and_record(self, domain: str, rate_s: float):
"""Block until rate limit for domain has passed, then record next allowed time."""
if rate_s <= 0:
return
with self._lock:
now = time.time()
delay = self._next.get(domain, 0) - now
self._next[domain] = max(now, self._next.get(domain, now)) + rate_s
if delay > 0:
time.sleep(delay)