From d6c33151ab70ea23cfcf713703f037b795174591 Mon Sep 17 00:00:00 2001 From: Luis Novo Date: Tue, 14 Jul 2026 12:28:48 -0300 Subject: [PATCH] feat(worker): expose OPEN_NOTEBOOK_WORKER_MAX_TASKS to control worker concurrency (#1141) The worker processes all queued tasks up to a fixed concurrency of 5, which overloads single-GPU / local-LLM setups and triggers rate limits (#893). The surreal-commands worker already accepts --max-tasks; wire it to a new OPEN_NOTEBOOK_WORKER_MAX_TASKS env var (default 5, set 1 for sequential) at every launch point. - Makefile (worker-start, start-all): --max-tasks "$${VAR:-5}" (Make-escaped) - dev-init.sh: POSIX ${VAR:-5} default expansion - supervisord.conf: wrap in sh -c so the shell expands the var (command= does not run through a shell) - docker-compose.yml: commented environment example - .env.example + environment-reference.md: document it, incl. the launch-time sourcing behavior Also fixes a phantom doc entry: SURREAL_COMMANDS_MAX_TASKS was documented but is not read anywhere (the worker's --max-tasks Typer option has no envvar); replaced with the real OPEN_NOTEBOOK_WORKER_MAX_TASKS. Closes #893 --- .env.example | 6 ++++++ Makefile | 4 ++-- dev-init.sh | 2 +- docker-compose.yml | 5 +++++ docs/5-CONFIGURATION/environment-reference.md | 6 ++++-- supervisord.conf | 4 +++- 6 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 325cf78a..784bf6e4 100644 --- a/.env.example +++ b/.env.example @@ -54,6 +54,12 @@ SURREAL_DATABASE=open_notebook # CHUNK_SIZE=1500 # CHUNK_OVERLAP=150 +# Max background tasks the worker runs concurrently (default 5). Set to 1 for +# sequential processing on single-GPU / local-LLM setups to avoid rate limits. +# NOTE: consumed at worker launch, not by the app — for `make`/dev-init export +# it in your shell; for Docker set it under `environment:` in docker-compose.yml. +# OPEN_NOTEBOOK_WORKER_MAX_TASKS=1 + # Maximum upload/request body size in MB (default: 100). Raise this if you # need to upload larger audio/video files. A fronting reverse proxy's own # limit (e.g. nginx client_max_body_size) still applies and should be raised diff --git a/Makefile b/Makefile index 420df3f9..6f1bafe1 100644 --- a/Makefile +++ b/Makefile @@ -162,7 +162,7 @@ worker: worker-start worker-start: @echo "Starting surreal-commands worker..." - uv run --env-file .env surreal-commands-worker --import-modules commands + uv run --env-file .env surreal-commands-worker --import-modules commands --max-tasks "$${OPEN_NOTEBOOK_WORKER_MAX_TASKS:-5}" worker-stop: @echo "Stopping surreal-commands worker..." @@ -182,7 +182,7 @@ start-all: @uv run run_api.py & @sleep 3 @echo "⚙️ Starting background worker..." - @uv run --env-file .env surreal-commands-worker --import-modules commands & + @uv run --env-file .env surreal-commands-worker --import-modules commands --max-tasks "$${OPEN_NOTEBOOK_WORKER_MAX_TASKS:-5}" & @sleep 2 @echo "🌐 Starting Next.js frontend..." @echo "✅ All services started!" diff --git a/dev-init.sh b/dev-init.sh index 4bd5c67d..277fc35a 100755 --- a/dev-init.sh +++ b/dev-init.sh @@ -29,7 +29,7 @@ sleep 3 # Start background worker in background echo "Starting background worker..." -uv run --env-file .env surreal-commands-worker --import-modules commands & +uv run --env-file .env surreal-commands-worker --import-modules commands --max-tasks "${OPEN_NOTEBOOK_WORKER_MAX_TASKS:-5}" & sleep 2 # Start frontend (foreground) diff --git a/docker-compose.yml b/docker-compose.yml index 9ed75e35..4b489cc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,11 @@ services: # - OPEN_NOTEBOOK_ENABLE_DOCLING=true # Docling engine + OCR + image sources (large ML stack) # - OPEN_NOTEBOOK_ENABLE_CRAWL4AI=true # local Crawl4AI (bundles a Chromium browser) # - CRAWL4AI_API_URL=http://crawl4ai:11235 # use a remote Crawl4AI server instead (no local install) + + # OPTIONAL: max concurrent background tasks the worker processes at once + # (default 5). Set to 1 for sequential processing on single-GPU / local + # LLM setups to avoid overloading the model with parallel requests. + # - OPEN_NOTEBOOK_WORKER_MAX_TASKS=1 volumes: - ./notebook_data:/app/data depends_on: diff --git a/docs/5-CONFIGURATION/environment-reference.md b/docs/5-CONFIGURATION/environment-reference.md index 109d17fb..577c7f6a 100644 --- a/docs/5-CONFIGURATION/environment-reference.md +++ b/docs/5-CONFIGURATION/environment-reference.md @@ -45,11 +45,13 @@ Comprehensive list of all environment variables available in Open Notebook. --- -## Database: Concurrency +## Worker: Concurrency | Variable | Required? | Default | Description | |----------|-----------|---------|-------------| -| `SURREAL_COMMANDS_MAX_TASKS` | No | 5 | Maximum concurrent database tasks | +| `OPEN_NOTEBOOK_WORKER_MAX_TASKS` | No | 5 | Maximum number of background tasks (source processing, embeddings, podcasts) the worker runs concurrently. Passed to the worker as `--max-tasks` at launch. Set to `1` for **sequential processing** on single-GPU or local-LLM setups, where parallel requests overload the model and trigger rate limits. | + +> **Read at worker launch, from the process environment.** In Docker this comes from the container environment — set it under `environment:` in `docker-compose.yml` (or your orchestrator). For local `make worker-start` / `dev-init.sh`, export it in your shell (e.g. `export OPEN_NOTEBOOK_WORKER_MAX_TASKS=1`) — it is consumed by the shell before the app loads `.env`, so a value placed only in `.env` will not apply to these local launch paths. --- diff --git a/supervisord.conf b/supervisord.conf index 6862da0a..267eb39a 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -20,7 +20,9 @@ autostart=true startsecs=3 [program:worker] -command=uv run --no-sync surreal-commands-worker --import-modules commands +; sh -c so the shell expands ${OPEN_NOTEBOOK_WORKER_MAX_TASKS:-5} (supervisord's +; own command= does not run through a shell). Falls back to 5 when unset. +command=sh -c "uv run --no-sync surreal-commands-worker --import-modules commands --max-tasks ${OPEN_NOTEBOOK_WORKER_MAX_TASKS:-5}" stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr