From 35628637fdd04eeb6644b8e9ff6cedbc6cae281f Mon Sep 17 00:00:00 2001 From: psuckrow-routine Date: Wed, 16 Sep 2026 21:23:38 +0200 Subject: [PATCH] feat(zotero): support multiple Zotero libraries via per-source library ID (#93) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(zotero): support multiple libraries via per-source library ID Embed an optional numeric library ID in the source string to allow multiple Zotero libraries to be synced from a single daemon instance: source: "zotero:123456:" # all collections in library 123456 source: "zotero:123456:Research%%ML" # specific collection When a library ID is present the connector resolves credentials from per-library env vars before falling back to the global defaults: ZOTERO_API_KEY_ → ZOTERO_API_KEY ZOTERO_LIBRARY_TYPE_ → ZOTERO_LIBRARY_TYPE Existing source strings ("zotero:" or "zotero:Collection%%Sub") are fully backward-compatible — collection names are not purely numeric so there is no ambiguity when parsing. * refactor(zotero): per-library env var resolution via _zotero_env helper Introduce _zotero_env(key, library_id, default) which checks ZOTERO__ before falling back to ZOTERO_. Apply it uniformly to every configurable setting so that each source entry in .oikb.yaml can be tuned independently when multiple libraries are in use: ZOTERO_CHECKSUM_ overrides ZOTERO_CHECKSUM ZOTERO_INCLUDE_NOTES_ ZOTERO_INCLUDE_NOTES ZOTERO_INCLUDE_ANNOTATIONS_ ZOTERO_INCLUDE_ANNOTATIONS ZOTERO_LIBRARY_TYPE_ ZOTERO_LIBRARY_TYPE ZOTERO_EXCLUDE_ ZOTERO_EXCLUDE ZOTERO_UNFILED_DIR_ ZOTERO_UNFILED_DIR ZOTERO_WEBDAV_{URL,USER,PASSWORD}_ ZOTERO_WEBDAV_* Single-library setups are unaffected — the helper falls through to the global vars unchanged. * fix(sync): skip empty file content before upload Files with no extractable text (e.g. Zotero items with no fulltext and no downloadable PDF) were uploaded as zero-byte payloads, causing Open WebUI to reject them with EMPTY_CONTENT errors on every sync cycle. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Pierre-Louis Suckrow Co-authored-by: Pierre-Louis Suckrow Co-authored-by: Claude Sonnet 4.6 --- src/oikb/cli.py | 2 +- src/oikb/connectors/zotero.py | 75 ++++++++++++++++++++++++++++------- src/oikb/sync.py | 6 +++ 3 files changed, 68 insertions(+), 15 deletions(-) diff --git a/src/oikb/cli.py b/src/oikb/cli.py index d21816d..da35517 100644 --- a/src/oikb/cli.py +++ b/src/oikb/cli.py @@ -312,7 +312,7 @@ def _resolve_connector( if source.startswith("zotero:"): from oikb.connectors.zotero import ZoteroConnector, parse_zotero_source parsed = parse_zotero_source(source) - return ZoteroConnector(hierarchy=parsed.get("hierarchy")) + return ZoteroConnector(hierarchy=parsed.get("hierarchy"), library_id=parsed.get("library_id")) # Default: local filesystem. from oikb.connectors.filesystem import FilesystemConnector diff --git a/src/oikb/connectors/zotero.py b/src/oikb/connectors/zotero.py index 7af45f8..bc21883 100644 --- a/src/oikb/connectors/zotero.py +++ b/src/oikb/connectors/zotero.py @@ -9,6 +9,28 @@ Auth via ZOTERO_LIBRARY_ID, ZOTERO_API_KEY, and optional ZOTERO_LIBRARY_TYPE. Source syntax: zotero:%%. Use bare zotero: for all top-level collections and unfiled library items. + +Multiple libraries in a single daemon +-------------------------------------- +Embed a numeric library ID before the collection path to target a specific +library and resolve its credentials from per-library env vars: + + source: "zotero:123456:" # all collections in library 123456 + source: "zotero:123456:Research%%ML" # specific collection + +Every ZOTERO_* env var supports a per-library override by appending the +library ID (e.g. ZOTERO_API_KEY_123456). The global var is the fallback: + + ZOTERO_API_KEY_123456 overrides ZOTERO_API_KEY + ZOTERO_LIBRARY_TYPE_123456 overrides ZOTERO_LIBRARY_TYPE + ZOTERO_CHECKSUM_123456 overrides ZOTERO_CHECKSUM + ZOTERO_INCLUDE_NOTES_123456 ZOTERO_INCLUDE_NOTES + ZOTERO_INCLUDE_ANNOTATIONS_123456 ZOTERO_INCLUDE_ANNOTATIONS + ZOTERO_EXCLUDE_123456 ZOTERO_EXCLUDE + ZOTERO_UNFILED_DIR_123456 ZOTERO_UNFILED_DIR + ZOTERO_WEBDAV_URL_123456 ZOTERO_WEBDAV_URL + ZOTERO_WEBDAV_USER_123456 ZOTERO_WEBDAV_USER + ZOTERO_WEBDAV_PASSWORD_123456 ZOTERO_WEBDAV_PASSWORD """ from __future__ import annotations @@ -30,6 +52,19 @@ from oikb.connectors import BaseConnector, ManifestEntry, SourceFileUnavailable HIERARCHY_SEP = "%%" +def _zotero_env(key: str, library_id: str | None, default: str = "") -> str: + """Return the per-library env var if set, otherwise the global one. + + Allows each Zotero source entry to be configured independently when + multiple libraries are synced from a single daemon instance. + """ + if library_id: + value = os.environ.get(f"{key}_{library_id}") + if value is not None: + return value + return os.environ.get(key, default) + + @dataclass(frozen=True, slots=True) class _ZoteroFile: item: dict[str, Any] @@ -57,7 +92,7 @@ class ZoteroConnector(BaseConnector): ) from exc library_id = library_id or os.environ.get("ZOTERO_LIBRARY_ID", "") - api_key = api_key or os.environ.get("ZOTERO_API_KEY", "") + api_key = api_key or _zotero_env("ZOTERO_API_KEY", library_id) if not library_id or not api_key: raise ValueError( "Zotero credentials required. Set ZOTERO_LIBRARY_ID and ZOTERO_API_KEY." @@ -66,27 +101,28 @@ class ZoteroConnector(BaseConnector): self.hierarchy = hierarchy self._zot = zotero.Zotero( library_id, - library_type or os.environ.get("ZOTERO_LIBRARY_TYPE", "user"), + library_type or _zotero_env("ZOTERO_LIBRARY_TYPE", library_id, "user"), api_key, ) - self.include_notes = os.environ.get("ZOTERO_INCLUDE_NOTES", "").strip().lower() in { + self.include_notes = _zotero_env("ZOTERO_INCLUDE_NOTES", library_id).strip().lower() in { "1", "true", "yes", "on", } - self.include_annotations = os.environ.get( - "ZOTERO_INCLUDE_ANNOTATIONS", "" - ).strip().lower() in {"1", "true", "yes", "on"} - self.checksum_mode = os.environ.get("ZOTERO_CHECKSUM", "version").strip().lower() + self.include_annotations = ( + _zotero_env("ZOTERO_INCLUDE_ANNOTATIONS", library_id).strip().lower() + in {"1", "true", "yes", "on"} + ) + self.checksum_mode = _zotero_env("ZOTERO_CHECKSUM", library_id, "version").strip().lower() if self.checksum_mode not in {"version", "content"}: raise ValueError("ZOTERO_CHECKSUM must be 'version' or 'content'.") self.exclude = [ part.strip().strip("/") - for part in os.environ.get("ZOTERO_EXCLUDE", "").split(",") + for part in _zotero_env("ZOTERO_EXCLUDE", library_id).split(",") if part.strip().strip("/") ] - self.unfiled_dir = os.environ.get("ZOTERO_UNFILED_DIR", "_unfiled").strip("/") - self.webdav_url = os.environ.get("ZOTERO_WEBDAV_URL", "").rstrip("/") - self.webdav_user = os.environ.get("ZOTERO_WEBDAV_USER", "") - self.webdav_password = os.environ.get("ZOTERO_WEBDAV_PASSWORD", "") + self.unfiled_dir = _zotero_env("ZOTERO_UNFILED_DIR", library_id, "_unfiled").strip("/") + self.webdav_url = _zotero_env("ZOTERO_WEBDAV_URL", library_id).rstrip("/") + self.webdav_user = _zotero_env("ZOTERO_WEBDAV_USER", library_id) + self.webdav_password = _zotero_env("ZOTERO_WEBDAV_PASSWORD", library_id) self._files: dict[tuple[str, str], _ZoteroFile] = {} self._text_cache: dict[str, str] = {} @@ -387,5 +423,16 @@ class ZoteroConnector(BaseConnector): def parse_zotero_source(source: str) -> dict[str, str | None]: - hierarchy = source.removeprefix("zotero:") - return {"hierarchy": None if hierarchy in {"", "*"} else hierarchy} + rest = source.removeprefix("zotero:") + library_id: str | None = None + # Optional library ID prefix: "zotero:123456:Collection%%Sub". A numeric + # segment before the first colon is treated as the library ID, allowing + # multiple Zotero libraries in a single .oikb.yaml. Plain collection paths + # (e.g. "zotero:Research%%ML") are unchanged — collection names are not + # purely numeric, so there is no ambiguity. + if ":" in rest: + head, tail = rest.split(":", 1) + if head.isdigit(): + library_id = head + rest = tail + return {"hierarchy": None if rest in {"", "*"} else rest, "library_id": library_id} diff --git a/src/oikb/sync.py b/src/oikb/sync.py index f7e07af..b8c8db6 100644 --- a/src/oikb/sync.py +++ b/src/oikb/sync.py @@ -350,6 +350,12 @@ def _run_sync_inner( check_stop() try: content = connector.read_file(path, filename) + if not content: + if progress is not None: + progress.update(task_id, advance=1, description=f"[yellow]⚠ {display}[/yellow]") + else: + click.echo(click.style(f" ⚠ {display}: empty content, skipping", fg="yellow"), err=True) + return ("warning", f"{display}: empty content, skipping") check_stop() directory_id = directory_map.get(path) if path else None client.upload_file(