feat(cli): onyx-cli serve over SSH (#9726)

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jamison Lahman <jamison@lahman.dev>

.github/workflows/helm-chart-releases.yml

@@ -47,7 +47,8 @@ jobs:
           done
 
       - name: Publish Helm charts to gh-pages
-        uses: stefanprodan/helm-gh-pages@0ad2bb377311d61ac04ad9eb6f252fb68e207260 # ratchet:stefanprodan/helm-gh-pages@v1.7.0
+        # NOTE: HEAD of https://github.com/stefanprodan/helm-gh-pages/pull/43
+        uses: stefanprodan/helm-gh-pages@ad32ad3b8720abfeaac83532fd1e9bdfca5bbe27 # zizmor: ignore[impostor-commit]
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           charts_dir: deployment/helm/charts

.github/workflows/release-cli.yml

@@ -13,15 +13,6 @@ jobs:
     permissions:
       id-token: write
     timeout-minutes: 10
-    strategy:
-      matrix:
-        os-arch:
-          - { goos: "linux", goarch: "amd64" }
-          - { goos: "linux", goarch: "arm64" }
-          - { goos: "windows", goarch: "amd64" }
-          - { goos: "windows", goarch: "arm64" }
-          - { goos: "darwin", goarch: "amd64" }
-          - { goos: "darwin", goarch: "arm64" }
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
         with:
@@ -31,9 +22,11 @@ jobs:
           enable-cache: false
           version: "0.9.9"
       - run: |
-          GOOS="${{ matrix.os-arch.goos }}" \
-          GOARCH="${{ matrix.os-arch.goarch }}" \
-          uv build --wheel
+          for goos in linux windows darwin; do
+            for goarch in amd64 arm64; do
+              GOOS="$goos" GOARCH="$goarch" uv build --wheel
+            done
+          done
         working-directory: cli
       - run: uv publish
         working-directory: cli

.greptile/config.json

@@ -0,0 +1,64 @@
+{
+    "labels": [],
+    "comment": "",
+    "fixWithAI": true,
+    "hideFooter": false,
+    "strictness": 3,
+    "statusCheck": true,
+    "commentTypes": [
+      "logic",
+      "syntax",
+      "style"
+    ],
+    "instructions": "",
+    "disabledLabels": [],
+    "excludeAuthors": [
+      "dependabot[bot]",
+      "renovate[bot]"
+    ],
+    "ignoreKeywords": "",
+    "ignorePatterns": "",
+    "includeAuthors": [],
+    "summarySection": {
+      "included": true,
+      "collapsible": false,
+      "defaultOpen": false
+    },
+    "excludeBranches": [],
+    "fileChangeLimit": 300,
+    "includeBranches": [],
+    "includeKeywords": "",
+    "triggerOnUpdates": true,
+    "updateExistingSummaryComment": true,
+    "updateSummaryOnly": false,
+    "issuesTableSection": {
+      "included": true,
+      "collapsible": false,
+      "defaultOpen": false
+    },
+    "statusCommentsEnabled": true,
+    "confidenceScoreSection": {
+      "included": true,
+      "collapsible": false
+    },
+    "sequenceDiagramSection": {
+      "included": true,
+      "collapsible": false,
+      "defaultOpen": false
+    },
+    "shouldUpdateDescription": false,
+    "rules": [
+      {
+        "scope": ["web/**"],
+        "rule": "In Onyx's Next.js app, the `app/ee/admin/` directory is a filesystem convention for Enterprise Edition route overrides — it does NOT add an `/ee/` prefix to the URL. Both `app/admin/groups/page.tsx` and `app/ee/admin/groups/page.tsx` serve the same URL `/admin/groups`. Hardcoded `/admin/...` paths in router.push() calls are correct and do NOT break EE deployments. Do not flag hardcoded admin paths as bugs."
+      },
+      {
+        "scope": ["web/**"],
+        "rule": "In Onyx, each API key creates a unique user row in the database with a unique `user_id` (UUID). There is a 1:1 mapping between API keys and their backing user records. Multiple API keys do NOT share the same `user_id`. Do not flag potential duplicate row IDs when using `user_id` from API key descriptors."
+      },
+      {
+        "scope": ["backend/**/*.py"],
+        "rule": "Never raise HTTPException directly in business code. Use `raise OnyxError(OnyxErrorCode.XXX, \"message\")` from `onyx.error_handling.exceptions`. A global FastAPI exception handler converts OnyxError into structured JSON responses with {\"error_code\": \"...\", \"detail\": \"...\"}. Error codes are defined in `onyx.error_handling.error_codes.OnyxErrorCode`. For upstream errors with dynamic HTTP status codes, use `status_code_override`: `raise OnyxError(OnyxErrorCode.BAD_GATEWAY, detail, status_code_override=upstream_status)`."
+      }
+    ]
+}

.greptile/files.json

@@ -0,0 +1,57 @@
+[
+  {
+    "scope": [],
+    "path": "contributing_guides/best_practices.md",
+    "description": "Best practices for contributing to the codebase"
+  },
+  {
+    "scope": ["web/**"],
+    "path": "web/AGENTS.md",
+    "description": "Frontend coding standards for the web directory"
+  },
+  {
+    "scope": ["web/**"],
+    "path": "web/tests/README.md",
+    "description": "Frontend testing guide and conventions"
+  },
+  {
+    "scope": ["web/**"],
+    "path": "web/CLAUDE.md",
+    "description": "Single source of truth for frontend coding standards"
+  },
+  {
+    "scope": ["web/**"],
+    "path": "web/lib/opal/README.md",
+    "description": "Opal component library usage guide"
+  },
+  {
+    "scope": ["backend/**"],
+    "path": "backend/tests/README.md",
+    "description": "Backend testing guide covering all 4 test types, fixtures, and conventions"
+  },
+  {
+    "scope": ["backend/onyx/connectors/**"],
+    "path": "backend/onyx/connectors/README.md",
+    "description": "Connector development guide covering design, interfaces, and required changes"
+  },
+  {
+    "scope": [],
+    "path": "CLAUDE.md",
+    "description": "Project instructions and coding standards"
+  },
+  {
+    "scope": [],
+    "path": "backend/alembic/README.md",
+    "description": "Migration guidance, including multi-tenant migration behavior"
+  },
+  {
+    "scope": [],
+    "path": "deployment/helm/charts/onyx/values-lite.yaml",
+    "description": "Lite deployment Helm values and service assumptions"
+  },
+  {
+    "scope": [],
+    "path": "deployment/docker_compose/docker-compose.onyx-lite.yml",
+    "description": "Lite deployment Docker Compose overlay and disabled service behavior"
+  }
+]

.greptile/rules.md

@@ -0,0 +1,39 @@
+# Greptile Review Rules
+
+## Type Annotations
+
+Use explicit type annotations for variables to enhance code clarity, especially when moving type hints around in the code.
+
+## Best Practices
+
+Use `contributing_guides/best_practices.md` as core review context. Prefer consistency with existing patterns, fix issues in code you touch, avoid tacking new features onto muddy interfaces, fail loudly instead of silently swallowing errors, keep code strictly typed, preserve clear state boundaries, remove duplicate or dead logic, break up overly long functions, avoid hidden import-time side effects, respect module boundaries, and favor correctness-by-construction over relying on callers to use an API correctly.
+
+## TODOs
+
+Whenever a TODO is added, there must always be an associated name or ticket with that TODO in the style of `TODO(name): ...` or `TODO(1234): ...`
+
+## Debugging Code
+
+Remove temporary debugging code before merging to production, especially tenant-specific debugging logs.
+
+## Hardcoded Booleans
+
+When hardcoding a boolean variable to a constant value, remove the variable entirely and clean up all places where it's used rather than just setting it to a constant.
+
+## Multi-tenant vs Single-tenant
+
+Code changes must consider both multi-tenant and single-tenant deployments. In multi-tenant mode, preserve tenant isolation, ensure tenant context is propagated correctly, and avoid assumptions that only hold for a single shared schema or globally shared state. In single-tenant mode, avoid introducing unnecessary tenant-specific requirements or cloud-only control-plane dependencies.
+
+## Nginx Routing — New Backend Routes
+
+Whenever a new backend route is added that does NOT start with `/api`, it must also be explicitly added to ALL nginx configs:
+- `deployment/helm/charts/onyx/templates/nginx-conf.yaml` (Helm/k8s)
+- `deployment/data/nginx/app.conf.template` (docker-compose dev)
+- `deployment/data/nginx/app.conf.template.prod` (docker-compose prod)
+- `deployment/data/nginx/app.conf.template.no-letsencrypt` (docker-compose no-letsencrypt)
+
+Routes not starting with `/api` are not caught by the existing `^/(api|openapi\.json)` location block and will fall through to `location /`, which proxies to the Next.js web server and returns an HTML 404. The new location block must be placed before the `/api` block. Examples of routes that need this treatment: `/scim`, `/mcp`.
+
+## Full vs Lite Deployments
+
+Code changes must consider both regular Onyx deployments and Onyx lite deployments. Lite deployments disable the vector DB, Redis, model servers, and background workers by default, use PostgreSQL-backed cache/auth/file storage, and rely on the API server to handle background work. Do not assume those services are available unless the code path is explicitly limited to full deployments.

.pre-commit-config.yaml

@@ -122,7 +122,7 @@ repos:
     rev: 5d1e709b7be35cb2025444e19de266b056b7b7ee # frozen: v2.10.1
     hooks:
       - id: golangci-lint
-        language_version: "1.26.0"
+        language_version: "1.26.1"
         entry: bash -c "find . -name go.mod -not -path './.venv/*' -print0 | xargs -0 -I{} bash -c 'cd \"$(dirname {})\" && golangci-lint run ./...'"
 
   - repo: https://github.com/astral-sh/ruff-pre-commit

README.md

@@ -35,7 +35,7 @@ Onyx comes loaded with advanced features like Agents, Web Search, RAG, MCP, Deep
 > [!TIP]
 > Run Onyx with one command (or see deployment section below):
 > ```
-> curl -fsSL https://raw.githubusercontent.com/onyx-dot-app/onyx/main/deployment/docker_compose/install.sh > install.sh && chmod +x install.sh && ./install.sh
+> curl -fsSL https://onyx.app/install_onyx.sh | bash
 > ```
 
 ****

backend/alembic/versions/1d78c0ca7853_remove_voice_provider_deleted_column.py

@@ -0,0 +1,35 @@
+"""remove voice_provider deleted column
+
+Revision ID: 1d78c0ca7853
+Revises: a3f8b2c1d4e5
+Create Date: 2026-03-26 11:30:53.883127
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "1d78c0ca7853"
+down_revision = "a3f8b2c1d4e5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Hard-delete any soft-deleted rows before dropping the column
+    op.execute("DELETE FROM voice_provider WHERE deleted = true")
+    op.drop_column("voice_provider", "deleted")
+
+
+def downgrade() -> None:
+    op.add_column(
+        "voice_provider",
+        sa.Column(
+            "deleted",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+    )

backend/ee/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -28,6 +28,7 @@ from onyx.access.models import DocExternalAccess
 from onyx.access.models import ElementExternalAccess
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_find_task
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
@@ -187,7 +188,6 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str) -> bool | None
     # (which lives on a different db number)
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_CONNECTOR_DOC_PERMISSIONS_SYNC_BEAT_LOCK,
@@ -227,6 +227,7 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str) -> bool | None
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
             try:
+                r_celery = celery_get_broker_client(self.app)
                 validate_permission_sync_fences(
                     tenant_id, r, r_replica, r_celery, lock_beat
                 )
@@ -473,6 +474,8 @@ def connector_permission_sync_generator_task(
             cc_pair = get_connector_credential_pair_from_id(
                 db_session=db_session,
                 cc_pair_id=cc_pair_id,
+                eager_load_connector=True,
+                eager_load_credential=True,
             )
             if cc_pair is None:
                 raise ValueError(

backend/ee/onyx/background/celery/tasks/external_group_syncing/tasks.py

@@ -29,6 +29,7 @@ from ee.onyx.external_permissions.sync_params import (
 from ee.onyx.external_permissions.sync_params import get_source_perm_sync_config
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_find_task
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
 from onyx.background.celery.tasks.beat_schedule import CLOUD_BEAT_MULTIPLIER_DEFAULT
 from onyx.background.error_logging import emit_background_error
@@ -162,7 +163,6 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str) -> bool | None:
     # (which lives on a different db number)
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_CONNECTOR_EXTERNAL_GROUP_SYNC_BEAT_LOCK,
@@ -221,6 +221,7 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str) -> bool | None:
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
             try:
+                r_celery = celery_get_broker_client(self.app)
                 validate_external_group_sync_fences(
                     tenant_id, self.app, r, r_replica, r_celery, lock_beat
                 )

backend/ee/onyx/external_permissions/slack/doc_sync.py

@@ -8,6 +8,7 @@ from ee.onyx.external_permissions.slack.utils import fetch_user_id_to_email_map
 from onyx.access.models import DocExternalAccess
 from onyx.access.models import ExternalAccess
 from onyx.connectors.credentials_provider import OnyxDBCredentialsProvider
+from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import HierarchyNode
 from onyx.connectors.slack.connector import get_channels
 from onyx.connectors.slack.connector import make_paginated_slack_api_call
@@ -105,9 +106,11 @@ def _get_slack_document_access(
     slack_connector: SlackConnector,
     channel_permissions: dict[str, ExternalAccess],  # noqa: ARG001
     callback: IndexingHeartbeatInterface | None,
+    indexing_start: SecondsSinceUnixEpoch | None = None,
 ) -> Generator[DocExternalAccess, None, None]:
     slim_doc_generator = slack_connector.retrieve_all_slim_docs_perm_sync(
-        callback=callback
+        callback=callback,
+        start=indexing_start,
     )
 
     for doc_metadata_batch in slim_doc_generator:
@@ -180,9 +183,15 @@ def slack_doc_sync(
 
     slack_connector = SlackConnector(**cc_pair.connector.connector_specific_config)
     slack_connector.set_credentials_provider(provider)
+    indexing_start_ts: SecondsSinceUnixEpoch | None = (
+        cc_pair.connector.indexing_start.timestamp()
+        if cc_pair.connector.indexing_start is not None
+        else None
+    )
 
     yield from _get_slack_document_access(
-        slack_connector,
+        slack_connector=slack_connector,
         channel_permissions=channel_permissions,
         callback=callback,
+        indexing_start=indexing_start_ts,
     )

backend/ee/onyx/external_permissions/utils.py

@@ -6,6 +6,7 @@ from onyx.access.models import ElementExternalAccess
 from onyx.access.models import ExternalAccess
 from onyx.access.models import NodeExternalAccess
 from onyx.configs.constants import DocumentSource
+from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnectorWithPermSync
 from onyx.connectors.models import HierarchyNode
 from onyx.db.models import ConnectorCredentialPair
@@ -40,10 +41,19 @@ def generic_doc_sync(
 
     logger.info(f"Starting {doc_source} doc sync for CC Pair ID: {cc_pair.id}")
 
+    indexing_start: SecondsSinceUnixEpoch | None = (
+        cc_pair.connector.indexing_start.timestamp()
+        if cc_pair.connector.indexing_start is not None
+        else None
+    )
+
     newly_fetched_doc_ids: set[str] = set()
 
     logger.info(f"Fetching all slim documents from {doc_source}")
-    for doc_batch in slim_connector.retrieve_all_slim_docs_perm_sync(callback=callback):
+    for doc_batch in slim_connector.retrieve_all_slim_docs_perm_sync(
+        start=indexing_start,
+        callback=callback,
+    ):
         logger.info(f"Got {len(doc_batch)} slim documents from {doc_source}")
 
         if callback:

backend/onyx/background/celery/celery_redis.py

@@ -1,5 +1,6 @@
 # These are helper objects for tracking the keys we need to write in redis
 import json
+import threading
 from typing import Any
 from typing import cast
 
@@ -7,7 +8,59 @@ from celery import Celery
 from redis import Redis
 
 from onyx.background.celery.configs.base import CELERY_SEPARATOR
+from onyx.configs.app_configs import REDIS_HEALTH_CHECK_INTERVAL
 from onyx.configs.constants import OnyxCeleryPriority
+from onyx.configs.constants import REDIS_SOCKET_KEEPALIVE_OPTIONS
+
+
+_broker_client: Redis | None = None
+_broker_url: str | None = None
+_broker_client_lock = threading.Lock()
+
+
+def celery_get_broker_client(app: Celery) -> Redis:
+    """Return a shared Redis client connected to the Celery broker DB.
+
+    Uses a module-level singleton so all tasks on a worker share one
+    connection instead of creating a new one per call. The client
+    connects directly to the broker Redis DB (parsed from the broker URL).
+
+    Thread-safe via lock — safe for use in Celery thread-pool workers.
+
+    Usage:
+        r_celery = celery_get_broker_client(self.app)
+        length = celery_get_queue_length(queue, r_celery)
+    """
+    global _broker_client, _broker_url
+    with _broker_client_lock:
+        url = app.conf.broker_url
+        if _broker_client is not None and _broker_url == url:
+            try:
+                _broker_client.ping()
+                return _broker_client
+            except Exception:
+                try:
+                    _broker_client.close()
+                except Exception:
+                    pass
+                _broker_client = None
+        elif _broker_client is not None:
+            try:
+                _broker_client.close()
+            except Exception:
+                pass
+            _broker_client = None
+
+        _broker_url = url
+        _broker_client = Redis.from_url(
+            url,
+            decode_responses=False,
+            health_check_interval=REDIS_HEALTH_CHECK_INTERVAL,
+            socket_keepalive=True,
+            socket_keepalive_options=REDIS_SOCKET_KEEPALIVE_OPTIONS,
+            retry_on_timeout=True,
+        )
+        return _broker_client
 
 
 def celery_get_unacked_length(r: Redis) -> int:

backend/onyx/background/celery/tasks/connector_deletion/tasks.py

@@ -14,6 +14,7 @@ from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.configs.app_configs import JOB_TIMEOUT
@@ -132,7 +133,6 @@ def revoke_tasks_blocking_deletion(
 def check_for_connector_deletion_task(self: Task, *, tenant_id: str) -> bool | None:
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_CONNECTOR_DELETION_BEAT_LOCK,
@@ -149,6 +149,7 @@ def check_for_connector_deletion_task(self: Task, *, tenant_id: str) -> bool | N
         if not r.exists(OnyxRedisSignals.BLOCK_VALIDATE_CONNECTOR_DELETION_FENCES):
             # clear fences that don't have associated celery tasks in progress
             try:
+                r_celery = celery_get_broker_client(self.app)
                 validate_connector_deletion_fences(
                     tenant_id, r, r_replica, r_celery, lock_beat
                 )

backend/onyx/background/celery/tasks/docprocessing/tasks.py

@@ -22,6 +22,7 @@ from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_find_task
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
 from onyx.background.celery.celery_utils import httpx_init_vespa_pool
 from onyx.background.celery.memory_monitoring import emit_process_memory
@@ -449,7 +450,7 @@ def check_indexing_completion(
             ):
                 # Check if the task exists in the celery queue
                 # This handles the case where Redis dies after task creation but before task execution
-                redis_celery = task.app.broker_connection().channel().client  # type: ignore
+                redis_celery = celery_get_broker_client(task.app)
                 task_exists = celery_find_task(
                     attempt.celery_task_id,
                     OnyxCeleryQueues.CONNECTOR_DOC_FETCHING,

backend/onyx/background/celery/tasks/monitoring/tasks.py

@@ -1,6 +1,5 @@
 import json
 import time
-from collections.abc import Callable
 from datetime import timedelta
 from itertools import islice
 from typing import Any
@@ -19,6 +18,7 @@ from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
 from onyx.background.celery.memory_monitoring import emit_process_memory
@@ -698,31 +698,27 @@ def monitor_background_processes(self: Task, *, tenant_id: str) -> None:
         return None
 
     try:
-        # Get Redis client for Celery broker
-        redis_celery = self.app.broker_connection().channel().client  # type: ignore
         redis_std = get_redis_client()
 
-        # Define metric collection functions and their dependencies
-        metric_functions: list[Callable[[], list[Metric]]] = [
-            lambda: _collect_queue_metrics(redis_celery),
-            lambda: _collect_connector_metrics(db_session, redis_std),
-            lambda: _collect_sync_metrics(db_session, redis_std),
-        ]
+        # Collect queue metrics with broker connection
+        r_celery = celery_get_broker_client(self.app)
+        queue_metrics = _collect_queue_metrics(r_celery)
 
-        # Collect and log each metric
+        # Collect remaining metrics (no broker connection needed)
         with get_session_with_current_tenant() as db_session:
-            for metric_fn in metric_functions:
-                metrics = metric_fn()
-                for metric in metrics:
-                    # double check to make sure we aren't double-emitting metrics
-                    if metric.key is None or not _has_metric_been_emitted(
-                        redis_std, metric.key
-                    ):
-                        metric.log()
-                        metric.emit(tenant_id)
+            all_metrics: list[Metric] = queue_metrics
+            all_metrics.extend(_collect_connector_metrics(db_session, redis_std))
+            all_metrics.extend(_collect_sync_metrics(db_session, redis_std))
 
-                    if metric.key is not None:
-                        _mark_metric_as_emitted(redis_std, metric.key)
+            for metric in all_metrics:
+                if metric.key is None or not _has_metric_been_emitted(
+                    redis_std, metric.key
+                ):
+                    metric.log()
+                    metric.emit(tenant_id)
+
+                if metric.key is not None:
+                    _mark_metric_as_emitted(redis_std, metric.key)
 
         task_logger.info("Successfully collected background metrics")
     except SoftTimeLimitExceeded:
@@ -890,7 +886,7 @@ def monitor_celery_queues_helper(
 ) -> None:
     """A task to monitor all celery queue lengths."""
 
-    r_celery = task.app.broker_connection().channel().client  # type: ignore
+    r_celery = celery_get_broker_client(task.app)
     n_celery = celery_get_queue_length(OnyxCeleryQueues.PRIMARY, r_celery)
     n_docfetching = celery_get_queue_length(
         OnyxCeleryQueues.CONNECTOR_DOC_FETCHING, r_celery
@@ -1080,7 +1076,7 @@ def cloud_monitor_celery_pidbox(
     num_deleted = 0
 
     MAX_PIDBOX_IDLE = 24 * 3600  # 1 day in seconds
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
+    r_celery = celery_get_broker_client(self.app)
     for key in r_celery.scan_iter("*.reply.celery.pidbox"):
         key_bytes = cast(bytes, key)
         key_str = key_bytes.decode("utf-8")

backend/onyx/background/celery/tasks/pruning/tasks.py

@@ -17,6 +17,7 @@ from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_find_task
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
@@ -203,7 +204,6 @@ def _is_pruning_due(cc_pair: ConnectorCredentialPair) -> bool:
 def check_for_pruning(self: Task, *, tenant_id: str) -> bool | None:
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_PRUNE_BEAT_LOCK,
@@ -261,6 +261,7 @@ def check_for_pruning(self: Task, *, tenant_id: str) -> bool | None:
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
             try:
+                r_celery = celery_get_broker_client(self.app)
                 validate_pruning_fences(tenant_id, r, r_replica, r_celery, lock_beat)
             except Exception:
                 task_logger.exception("Exception while validating pruning fences")

backend/onyx/background/celery/tasks/user_file_processing/tasks.py

@@ -16,6 +16,7 @@ from sqlalchemy.orm import Session
 
 from onyx.access.access import build_access_for_user_files
 from onyx.background.celery.apps.app_base import task_logger
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_utils import httpx_init_vespa_pool
 from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
@@ -105,7 +106,7 @@ def _user_file_delete_queued_key(user_file_id: str | UUID) -> str:
 
 
 def get_user_file_project_sync_queue_depth(celery_app: Celery) -> int:
-    redis_celery: Redis = celery_app.broker_connection().channel().client  # type: ignore
+    redis_celery = celery_get_broker_client(celery_app)
     return celery_get_queue_length(
         OnyxCeleryQueues.USER_FILE_PROJECT_SYNC, redis_celery
     )
@@ -238,7 +239,7 @@ def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
     skipped_guard = 0
     try:
         # --- Protection 1: queue depth backpressure ---
-        r_celery = self.app.broker_connection().channel().client  # type: ignore
+        r_celery = celery_get_broker_client(self.app)
         queue_len = celery_get_queue_length(
             OnyxCeleryQueues.USER_FILE_PROCESSING, r_celery
         )
@@ -591,7 +592,7 @@ def check_for_user_file_delete(self: Task, *, tenant_id: str) -> None:
         # --- Protection 1: queue depth backpressure ---
         # NOTE: must use the broker's Redis client (not redis_client) because
         # Celery queues live on a separate Redis DB with CELERY_SEPARATOR keys.
-        r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
+        r_celery = celery_get_broker_client(self.app)
         queue_len = celery_get_queue_length(OnyxCeleryQueues.USER_FILE_DELETE, r_celery)
         if queue_len > USER_FILE_DELETE_MAX_QUEUE_DEPTH:
             task_logger.warning(

backend/onyx/configs/app_configs.py

@@ -44,6 +44,31 @@ SEND_USER_METADATA_TO_LLM_PROVIDER = (
 # User Facing Features Configs
 #####
 BLURB_SIZE = 128  # Number Encoder Tokens included in the chunk blurb
+
+# Hard ceiling for the admin-configurable file upload size (in MB).
+# Self-hosted customers can raise or lower this via the environment variable.
+_raw_max_upload_size_mb = int(os.environ.get("MAX_ALLOWED_UPLOAD_SIZE_MB", "250"))
+if _raw_max_upload_size_mb < 0:
+    logger.warning(
+        "MAX_ALLOWED_UPLOAD_SIZE_MB=%d is negative; falling back to 250",
+        _raw_max_upload_size_mb,
+    )
+    _raw_max_upload_size_mb = 250
+MAX_ALLOWED_UPLOAD_SIZE_MB = _raw_max_upload_size_mb
+
+# Default fallback for the per-user file upload size limit (in MB) when no
+# admin-configured value exists.  Clamped to MAX_ALLOWED_UPLOAD_SIZE_MB at
+# runtime so this never silently exceeds the hard ceiling.
+_raw_default_upload_size_mb = int(
+    os.environ.get("DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB", "100")
+)
+if _raw_default_upload_size_mb < 0:
+    logger.warning(
+        "DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB=%d is negative; falling back to 100",
+        _raw_default_upload_size_mb,
+    )
+    _raw_default_upload_size_mb = 100
+DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB = _raw_default_upload_size_mb
 GENERATIVE_MODEL_ACCESS_CHECK_FREQ = int(
     os.environ.get("GENERATIVE_MODEL_ACCESS_CHECK_FREQ") or 86400
 )  # 1 day
@@ -61,17 +86,6 @@ CACHE_BACKEND = CacheBackendType(
     os.environ.get("CACHE_BACKEND", CacheBackendType.REDIS)
 )
 
-# Maximum token count for a single uploaded file. Files exceeding this are rejected.
-# Defaults to 100k tokens (or 10M when vector DB is disabled).
-_DEFAULT_FILE_TOKEN_LIMIT = 10_000_000 if DISABLE_VECTOR_DB else 100_000
-FILE_TOKEN_COUNT_THRESHOLD = int(
-    os.environ.get("FILE_TOKEN_COUNT_THRESHOLD", str(_DEFAULT_FILE_TOKEN_LIMIT))
-)
-
-# Maximum upload size for a single user file (chat/projects) in MB.
-USER_FILE_MAX_UPLOAD_SIZE_MB = int(os.environ.get("USER_FILE_MAX_UPLOAD_SIZE_MB") or 50)
-USER_FILE_MAX_UPLOAD_SIZE_BYTES = USER_FILE_MAX_UPLOAD_SIZE_MB * 1024 * 1024
-
 # If set to true, will show extra/uncommon connectors in the "Other" category
 SHOW_EXTRA_CONNECTORS = os.environ.get("SHOW_EXTRA_CONNECTORS", "").lower() == "true"
 

backend/onyx/connectors/canvas/client.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
+    rl_requests,
+)
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
+
+logger = logging.getLogger(__name__)
+
+# Requests timeout in seconds.
+_CANVAS_CALL_TIMEOUT: int = 30
+_CANVAS_API_VERSION: str = "/api/v1"
+# Matches the "next" URL in a Canvas Link header, e.g.:
+#   <https://canvas.example.com/api/v1/courses?page=2>; rel="next"
+# Captures the URL inside the angle brackets.
+_NEXT_LINK_PATTERN: re.Pattern[str] = re.compile(r'<([^>]+)>;\s*rel="next"')
+
+
+_STATUS_TO_ERROR_CODE: dict[int, OnyxErrorCode] = {
+    401: OnyxErrorCode.CREDENTIAL_EXPIRED,
+    403: OnyxErrorCode.INSUFFICIENT_PERMISSIONS,
+    404: OnyxErrorCode.BAD_GATEWAY,
+    429: OnyxErrorCode.RATE_LIMITED,
+}
+
+
+def _error_code_for_status(status_code: int) -> OnyxErrorCode:
+    """Map an HTTP status code to the appropriate OnyxErrorCode.
+
+    Expects a >= 400 status code. Known codes (401, 403, 404, 429) are
+    mapped to specific error codes; all other codes (unrecognised 4xx
+    and 5xx) map to BAD_GATEWAY as unexpected upstream errors.
+    """
+    if status_code in _STATUS_TO_ERROR_CODE:
+        return _STATUS_TO_ERROR_CODE[status_code]
+    return OnyxErrorCode.BAD_GATEWAY
+
+
+class CanvasApiClient:
+    def __init__(
+        self,
+        bearer_token: str,
+        canvas_base_url: str,
+    ) -> None:
+        parsed_base = urlparse(canvas_base_url)
+        if not parsed_base.hostname:
+            raise ValueError("canvas_base_url must include a valid host")
+        if parsed_base.scheme != "https":
+            raise ValueError("canvas_base_url must use https")
+
+        self._bearer_token = bearer_token
+        self.base_url = (
+            canvas_base_url.rstrip("/").removesuffix(_CANVAS_API_VERSION)
+            + _CANVAS_API_VERSION
+        )
+        # Hostname is already validated above; reuse parsed_base instead
+        # of re-parsing.  Used by _parse_next_link to validate pagination URLs.
+        self._expected_host: str = parsed_base.hostname
+
+    def get(
+        self,
+        endpoint: str = "",
+        params: dict[str, Any] | None = None,
+        full_url: str | None = None,
+    ) -> tuple[Any, str | None]:
+        """Make a GET request to the Canvas API.
+
+        Returns a tuple of (json_body, next_url).
+        next_url is parsed from the Link header and is None if there are no more pages.
+        If full_url is provided, it is used directly (for following pagination links).
+
+        Security note: full_url must only be set to values returned by
+        ``_parse_next_link``, which validates the host against the configured
+        Canvas base URL.  Passing an arbitrary URL would leak the bearer token.
+        """
+        # full_url is used when following pagination (Canvas returns the
+        # next-page URL in the Link header).  For the first request we build
+        # the URL from the endpoint name instead.
+        url = full_url if full_url else self._build_url(endpoint)
+        headers = self._build_headers()
+
+        response = rl_requests.get(
+            url,
+            headers=headers,
+            params=params if not full_url else None,
+            timeout=_CANVAS_CALL_TIMEOUT,
+        )
+
+        try:
+            response_json = response.json()
+        except ValueError as e:
+            if response.status_code < 300:
+                raise OnyxError(
+                    OnyxErrorCode.BAD_GATEWAY,
+                    detail=f"Invalid JSON in Canvas response: {e}",
+                )
+            logger.warning(
+                "Failed to parse JSON from Canvas error response (status=%d): %s",
+                response.status_code,
+                e,
+            )
+            response_json = {}
+
+        if response.status_code >= 400:
+            # Try to extract the most specific error message from the
+            # Canvas response body.  Canvas uses three different shapes
+            # depending on the endpoint and error type:
+            default_error: str = response.reason or f"HTTP {response.status_code}"
+            error = default_error
+            if isinstance(response_json, dict):
+                # Shape 1: {"error": {"message": "Not authorized"}}
+                error_field = response_json.get("error")
+                if isinstance(error_field, dict):
+                    response_error = error_field.get("message", "")
+                    if response_error:
+                        error = response_error
+                # Shape 2: {"error": "Invalid access token"}
+                elif isinstance(error_field, str):
+                    error = error_field
+                # Shape 3: {"errors": [{"message": "..."}]}
+                # Used for validation errors.  Only use as fallback if
+                # we didn't already find a more specific message above.
+                if error == default_error:
+                    errors_list = response_json.get("errors")
+                    if isinstance(errors_list, list) and errors_list:
+                        first_error = errors_list[0]
+                        if isinstance(first_error, dict):
+                            msg = first_error.get("message", "")
+                            if msg:
+                                error = msg
+            raise OnyxError(
+                _error_code_for_status(response.status_code),
+                detail=error,
+                status_code_override=response.status_code,
+            )
+
+        next_url = self._parse_next_link(response.headers.get("Link", ""))
+        return response_json, next_url
+
+    def _parse_next_link(self, link_header: str) -> str | None:
+        """Extract the 'next' URL from a Canvas Link header.
+
+        Only returns URLs whose host matches the configured Canvas base URL
+        to prevent leaking the bearer token to arbitrary hosts.
+        """
+        expected_host = self._expected_host
+        for match in _NEXT_LINK_PATTERN.finditer(link_header):
+            url = match.group(1)
+            parsed_url = urlparse(url)
+            if parsed_url.hostname != expected_host:
+                raise OnyxError(
+                    OnyxErrorCode.BAD_GATEWAY,
+                    detail=(
+                        "Canvas pagination returned an unexpected host "
+                        f"({parsed_url.hostname}); expected {expected_host}"
+                    ),
+                )
+            if parsed_url.scheme != "https":
+                raise OnyxError(
+                    OnyxErrorCode.BAD_GATEWAY,
+                    detail=(
+                        "Canvas pagination link must use https, "
+                        f"got {parsed_url.scheme!r}"
+                    ),
+                )
+            return url
+        return None
+
+    def _build_headers(self) -> dict[str, str]:
+        """Return the Authorization header with the bearer token."""
+        return {"Authorization": f"Bearer {self._bearer_token}"}
+
+    def _build_url(self, endpoint: str) -> str:
+        """Build a full Canvas API URL from an endpoint path.
+
+        Assumes endpoint is non-empty (e.g. ``"courses"``, ``"announcements"``).
+        Only called on a first request, endpoint must be set for first request.
+        Verify endpoint exists in case of future changes where endpoint might be optional.
+        Leading slashes are stripped to avoid double-slash in the result.
+        self.base_url is already normalized with no trailing slash.
+        """
+        final_url = self.base_url
+        clean_endpoint = endpoint.lstrip("/")
+        if clean_endpoint:
+            final_url += "/" + clean_endpoint
+        return final_url

backend/onyx/connectors/canvas/connector.py

@@ -0,0 +1,74 @@
+from typing import Literal
+from typing import TypeAlias
+
+from pydantic import BaseModel
+
+from onyx.connectors.models import ConnectorCheckpoint
+
+
+class CanvasCourse(BaseModel):
+    id: int
+    name: str
+    course_code: str
+    created_at: str
+    workflow_state: str
+
+
+class CanvasPage(BaseModel):
+    page_id: int
+    url: str
+    title: str
+    body: str | None = None
+    created_at: str
+    updated_at: str
+    course_id: int
+
+
+class CanvasAssignment(BaseModel):
+    id: int
+    name: str
+    description: str | None = None
+    html_url: str
+    course_id: int
+    created_at: str
+    updated_at: str
+    due_at: str | None = None
+
+
+class CanvasAnnouncement(BaseModel):
+    id: int
+    title: str
+    message: str | None = None
+    html_url: str
+    posted_at: str | None = None
+    course_id: int
+
+
+CanvasStage: TypeAlias = Literal["pages", "assignments", "announcements"]
+
+
+class CanvasConnectorCheckpoint(ConnectorCheckpoint):
+    """Checkpoint state for resumable Canvas indexing.
+
+    Fields:
+        course_ids: Materialized list of course IDs to process.
+        current_course_index: Index into course_ids for current course.
+        stage: Which item type we're processing for the current course.
+        next_url: Pagination cursor within the current stage. None means
+            start from the first page; a URL means resume from that page.
+
+    Invariant:
+        If current_course_index is incremented, stage must be reset to
+        "pages" and next_url must be reset to None.
+    """
+
+    course_ids: list[int] = []
+    current_course_index: int = 0
+    stage: CanvasStage = "pages"
+    next_url: str | None = None
+
+    def advance_course(self) -> None:
+        """Move to the next course and reset within-course state."""
+        self.current_course_index += 1
+        self.stage = "pages"
+        self.next_url = None

backend/onyx/connectors/confluence/connector.py

@@ -890,8 +890,8 @@ class ConfluenceConnector(
 
     def _retrieve_all_slim_docs(
         self,
-        start: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
-        end: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
         include_permissions: bool = True,
     ) -> GenerateSlimDocumentOutput:
@@ -915,8 +915,8 @@ class ConfluenceConnector(
                 self.confluence_client, doc_id, restrictions, ancestors
             ) or space_level_access_info.get(page_space_key)
 
-        # Query pages
-        page_query = self.base_cql_page_query + self.cql_label_filter
+        # Query pages (with optional time filtering for indexing_start)
+        page_query = self._construct_page_cql_query(start, end)
         for page in self.confluence_client.cql_paginate_all_expansions(
             cql=page_query,
             expand=restrictions_expand,
@@ -950,7 +950,9 @@ class ConfluenceConnector(
 
             # Query attachments for each page
             page_hierarchy_node_yielded = False
-            attachment_query = self._construct_attachment_query(_get_page_id(page))
+            attachment_query = self._construct_attachment_query(
+                _get_page_id(page), start, end
+            )
             for attachment in self.confluence_client.cql_paginate_all_expansions(
                 cql=attachment_query,
                 expand=restrictions_expand,

backend/onyx/connectors/jira/connector.py

@@ -10,6 +10,7 @@ from datetime import timedelta
 from datetime import timezone
 from typing import Any
 
+import requests
 from jira import JIRA
 from jira.exceptions import JIRAError
 from jira.resources import Issue
@@ -239,29 +240,53 @@ def enhanced_search_ids(
     )
 
 
-def bulk_fetch_issues(
-    jira_client: JIRA, issue_ids: list[str], fields: str | None = None
-) -> list[Issue]:
-    # TODO: move away from this jira library if they continue to not support
-    # the endpoints we need. Using private fields is not ideal, but
-    # is likely fine for now since we pin the library version
+def _bulk_fetch_request(
+    jira_client: JIRA, issue_ids: list[str], fields: str | None
+) -> list[dict[str, Any]]:
+    """Raw POST to the bulkfetch endpoint. Returns the list of raw issue dicts."""
     bulk_fetch_path = jira_client._get_url("issue/bulkfetch")
-
     # Prepare the payload according to Jira API v3 specification
     payload: dict[str, Any] = {"issueIdsOrKeys": issue_ids}
-
     # Only restrict fields if specified, might want to explicitly do this in the future
     # to avoid reading unnecessary data
     payload["fields"] = fields.split(",") if fields else ["*all"]
 
+    resp = jira_client._session.post(bulk_fetch_path, json=payload)
+    return resp.json()["issues"]
+
+
+def bulk_fetch_issues(
+    jira_client: JIRA, issue_ids: list[str], fields: str | None = None
+) -> list[Issue]:
+    # TODO(evan): move away from this jira library if they continue to not support
+    # the endpoints we need. Using private fields is not ideal, but
+    # is likely fine for now since we pin the library version
+
     try:
-        response = jira_client._session.post(bulk_fetch_path, json=payload).json()
+        raw_issues = _bulk_fetch_request(jira_client, issue_ids, fields)
+    except requests.exceptions.JSONDecodeError:
+        if len(issue_ids) <= 1:
+            logger.exception(
+                f"Jira bulk-fetch response for issue(s) {issue_ids} could not "
+                f"be decoded as JSON (response too large or truncated)."
+            )
+            raise
+
+        mid = len(issue_ids) // 2
+        logger.warning(
+            f"Jira bulk-fetch JSON decode failed for batch of {len(issue_ids)} issues. "
+            f"Splitting into sub-batches of {mid} and {len(issue_ids) - mid}."
+        )
+        left = bulk_fetch_issues(jira_client, issue_ids[:mid], fields)
+        right = bulk_fetch_issues(jira_client, issue_ids[mid:], fields)
+        return left + right
     except Exception as e:
         logger.error(f"Error fetching issues: {e}")
-        raise e
+        raise
+
     return [
         Issue(jira_client._options, jira_client._session, raw=issue)
-        for issue in response["issues"]
+        for issue in raw_issues
     ]
 
 

backend/onyx/connectors/sharepoint/connector.py

@@ -1765,7 +1765,11 @@ class SharepointConnector(
         checkpoint.current_drive_delta_next_link = None
         checkpoint.seen_document_ids.clear()
 
-    def _fetch_slim_documents_from_sharepoint(self) -> GenerateSlimDocumentOutput:
+    def _fetch_slim_documents_from_sharepoint(
+        self,
+        start: datetime | None = None,
+        end: datetime | None = None,
+    ) -> GenerateSlimDocumentOutput:
         site_descriptors = self._filter_excluded_sites(
             self.site_descriptors or self.fetch_sites()
         )
@@ -1786,7 +1790,9 @@ class SharepointConnector(
             # Process site documents if flag is True
             if self.include_site_documents:
                 for driveitem, drive_name, drive_web_url in self._fetch_driveitems(
-                    site_descriptor=site_descriptor
+                    site_descriptor=site_descriptor,
+                    start=start,
+                    end=end,
                 ):
                     if self._is_driveitem_excluded(driveitem):
                         logger.debug(f"Excluding by path denylist: {driveitem.web_url}")
@@ -1841,7 +1847,9 @@ class SharepointConnector(
 
             # Process site pages if flag is True
             if self.include_site_pages:
-                site_pages = self._fetch_site_pages(site_descriptor)
+                site_pages = self._fetch_site_pages(
+                    site_descriptor, start=start, end=end
+                )
                 for site_page in site_pages:
                     logger.debug(
                         f"Processing site page: {site_page.get('webUrl', site_page.get('name', 'Unknown'))}"
@@ -2565,12 +2573,22 @@ class SharepointConnector(
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
-        end: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,  # noqa: ARG002
     ) -> GenerateSlimDocumentOutput:
-
-        yield from self._fetch_slim_documents_from_sharepoint()
+        start_dt = (
+            datetime.fromtimestamp(start, tz=timezone.utc)
+            if start is not None
+            else None
+        )
+        end_dt = (
+            datetime.fromtimestamp(end, tz=timezone.utc) if end is not None else None
+        )
+        yield from self._fetch_slim_documents_from_sharepoint(
+            start=start_dt,
+            end=end_dt,
+        )
 
 
 if __name__ == "__main__":

backend/onyx/connectors/slack/connector.py

@@ -516,6 +516,8 @@ def _get_all_doc_ids(
     ] = default_msg_filter,
     callback: IndexingHeartbeatInterface | None = None,
     workspace_url: str | None = None,
+    start: SecondsSinceUnixEpoch | None = None,
+    end: SecondsSinceUnixEpoch | None = None,
 ) -> GenerateSlimDocumentOutput:
     """
     Get all document ids in the workspace, channel by channel
@@ -546,6 +548,8 @@ def _get_all_doc_ids(
             client=client,
             channel=channel,
             callback=callback,
+            oldest=str(start) if start else None,  # 0.0 -> None intentionally
+            latest=str(end) if end is not None else None,
         )
 
         for message_batch in channel_message_batches:
@@ -847,8 +851,8 @@ class SlackConnector(
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
-        end: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         if self.client is None:
@@ -861,6 +865,8 @@ class SlackConnector(
             msg_filter_func=self.msg_filter_func,
             callback=callback,
             workspace_url=self._workspace_url,
+            start=start,
+            end=end,
         )
 
     def _load_from_checkpoint(

backend/onyx/db/models.py

@@ -3135,8 +3135,6 @@ class VoiceProvider(Base):
     is_default_stt: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     is_default_tts: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 
-    deleted: Mapped[bool] = mapped_column(Boolean, default=False)
-
     time_created: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True), server_default=func.now()
     )

backend/onyx/db/voice.py

@@ -17,39 +17,30 @@ MAX_VOICE_PLAYBACK_SPEED = 2.0
 def fetch_voice_providers(db_session: Session) -> list[VoiceProvider]:
     """Fetch all voice providers."""
     return list(
-        db_session.scalars(
-            select(VoiceProvider)
-            .where(VoiceProvider.deleted.is_(False))
-            .order_by(VoiceProvider.name)
-        ).all()
+        db_session.scalars(select(VoiceProvider).order_by(VoiceProvider.name)).all()
     )
 
 
 def fetch_voice_provider_by_id(
-    db_session: Session, provider_id: int, include_deleted: bool = False
+    db_session: Session, provider_id: int
 ) -> VoiceProvider | None:
     """Fetch a voice provider by ID."""
-    stmt = select(VoiceProvider).where(VoiceProvider.id == provider_id)
-    if not include_deleted:
-        stmt = stmt.where(VoiceProvider.deleted.is_(False))
-    return db_session.scalar(stmt)
+    return db_session.scalar(
+        select(VoiceProvider).where(VoiceProvider.id == provider_id)
+    )
 
 
 def fetch_default_stt_provider(db_session: Session) -> VoiceProvider | None:
     """Fetch the default STT provider."""
     return db_session.scalar(
-        select(VoiceProvider)
-        .where(VoiceProvider.is_default_stt.is_(True))
-        .where(VoiceProvider.deleted.is_(False))
+        select(VoiceProvider).where(VoiceProvider.is_default_stt.is_(True))
     )
 
 
 def fetch_default_tts_provider(db_session: Session) -> VoiceProvider | None:
     """Fetch the default TTS provider."""
     return db_session.scalar(
-        select(VoiceProvider)
-        .where(VoiceProvider.is_default_tts.is_(True))
-        .where(VoiceProvider.deleted.is_(False))
+        select(VoiceProvider).where(VoiceProvider.is_default_tts.is_(True))
     )
 
 
@@ -58,9 +49,7 @@ def fetch_voice_provider_by_type(
 ) -> VoiceProvider | None:
     """Fetch a voice provider by type."""
     return db_session.scalar(
-        select(VoiceProvider)
-        .where(VoiceProvider.provider_type == provider_type)
-        .where(VoiceProvider.deleted.is_(False))
+        select(VoiceProvider).where(VoiceProvider.provider_type == provider_type)
     )
 
 
@@ -119,10 +108,10 @@ def upsert_voice_provider(
 
 
 def delete_voice_provider(db_session: Session, provider_id: int) -> None:
-    """Soft-delete a voice provider by ID."""
+    """Delete a voice provider by ID."""
     provider = fetch_voice_provider_by_id(db_session, provider_id)
     if provider:
-        provider.deleted = True
+        db_session.delete(provider)
         db_session.flush()
 
 

backend/onyx/document_index/disabled.py

@@ -5,6 +5,7 @@ accidentally reaches the vector DB layer will fail loudly instead of timing
 out against a nonexistent Vespa/OpenSearch instance.
 """
 
+from collections.abc import Iterable
 from typing import Any
 
 from onyx.context.search.models import IndexFilters
@@ -66,7 +67,7 @@ class DisabledDocumentIndex(DocumentIndex):
     # ------------------------------------------------------------------
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],  # noqa: ARG002
+        chunks: Iterable[DocMetadataAwareIndexChunk],  # noqa: ARG002
         index_batch_params: IndexBatchParams,  # noqa: ARG002
     ) -> set[DocumentInsertionRecord]:
         raise RuntimeError(VECTOR_DB_DISABLED_ERROR)

backend/onyx/document_index/interfaces.py

@@ -1,4 +1,5 @@
 import abc
+from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import datetime
 from typing import Any
@@ -206,7 +207,7 @@ class Indexable(abc.ABC):
     @abc.abstractmethod
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
+        chunks: Iterable[DocMetadataAwareIndexChunk],
         index_batch_params: IndexBatchParams,
     ) -> set[DocumentInsertionRecord]:
         """
@@ -226,8 +227,8 @@ class Indexable(abc.ABC):
         it is done automatically outside of this code.
 
         Parameters:
-        - chunks: Document chunks with all of the information needed for indexing to the document
-                index.
+        - chunks: Document chunks with all of the information needed for
+                indexing to the document index.
         - tenant_id: The tenant id of the user whose chunks are being indexed
         - large_chunks_enabled: Whether large chunks are enabled
 

backend/onyx/document_index/interfaces_new.py

@@ -1,4 +1,5 @@
 import abc
+from collections.abc import Iterable
 from typing import Self
 
 from pydantic import BaseModel
@@ -209,10 +210,10 @@ class Indexable(abc.ABC):
     @abc.abstractmethod
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
+        chunks: Iterable[DocMetadataAwareIndexChunk],
         indexing_metadata: IndexingMetadata,
     ) -> list[DocumentInsertionRecord]:
-        """Indexes a list of document chunks into the document index.
+        """Indexes an iterable of document chunks into the document index.
 
         This is often a batch operation including chunks from multiple
         documents.

backend/onyx/document_index/opensearch/opensearch_document_index.py

@@ -1,5 +1,5 @@
 import json
-from collections import defaultdict
+from collections.abc import Iterable
 from typing import Any
 
 import httpx
@@ -351,7 +351,7 @@ class OpenSearchOldDocumentIndex(OldDocumentIndex):
 
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
+        chunks: Iterable[DocMetadataAwareIndexChunk],
         index_batch_params: IndexBatchParams,
     ) -> set[OldDocumentInsertionRecord]:
         """
@@ -647,10 +647,10 @@ class OpenSearchDocumentIndex(DocumentIndex):
 
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
-        indexing_metadata: IndexingMetadata,  # noqa: ARG002
+        chunks: Iterable[DocMetadataAwareIndexChunk],
+        indexing_metadata: IndexingMetadata,
     ) -> list[DocumentInsertionRecord]:
-        """Indexes a list of document chunks into the document index.
+        """Indexes an iterable of document chunks into the document index.
 
         Groups chunks by document ID and for each document, deletes existing
         chunks and indexes the new chunks in bulk.
@@ -673,29 +673,34 @@ class OpenSearchDocumentIndex(DocumentIndex):
                 document is newly indexed or had already existed and was just
                 updated.
         """
-        # Group chunks by document ID.
-        doc_id_to_chunks: dict[str, list[DocMetadataAwareIndexChunk]] = defaultdict(
-            list
+        total_chunks = sum(
+            cc.new_chunk_cnt
+            for cc in indexing_metadata.doc_id_to_chunk_cnt_diff.values()
         )
-        for chunk in chunks:
-            doc_id_to_chunks[chunk.source_document.id].append(chunk)
         logger.debug(
-            f"[OpenSearchDocumentIndex] Indexing {len(chunks)} chunks from {len(doc_id_to_chunks)} "
+            f"[OpenSearchDocumentIndex] Indexing {total_chunks} chunks from {len(indexing_metadata.doc_id_to_chunk_cnt_diff)} "
             f"documents for index {self._index_name}."
         )
 
         document_indexing_results: list[DocumentInsertionRecord] = []
-        # Try to index per-document.
-        for _, chunks in doc_id_to_chunks.items():
+        deleted_doc_ids: set[str] = set()
+        # Buffer chunks per document as they arrive from the iterable.
+        # When the document ID changes flush the buffered chunks.
+        current_doc_id: str | None = None
+        current_chunks: list[DocMetadataAwareIndexChunk] = []
+
+        def _flush_chunks(doc_chunks: list[DocMetadataAwareIndexChunk]) -> None:
+            assert len(doc_chunks) > 0, "doc_chunks is empty"
+
             # Create a batch of OpenSearch-formatted chunks for bulk insertion.
-            # Do this before deleting existing chunks to reduce the amount of
-            # time the document index has no content for a given document, and
-            # to reduce the chance of entering a state where we delete chunks,
-            # then some error happens, and never successfully index new chunks.
+            # Since we are doing this in batches, an error occurring midway
+            # can result in a state where chunks are deleted and not all the
+            # new chunks have been indexed.
             chunk_batch: list[DocumentChunk] = [
-                _convert_onyx_chunk_to_opensearch_document(chunk) for chunk in chunks
+                _convert_onyx_chunk_to_opensearch_document(chunk)
+                for chunk in doc_chunks
             ]
-            onyx_document: Document = chunks[0].source_document
+            onyx_document: Document = doc_chunks[0].source_document
             # First delete the doc's chunks from the index. This is so that
             # there are no dangling chunks in the index, in the event that the
             # new document's content contains fewer chunks than the previous
@@ -704,22 +709,40 @@ class OpenSearchDocumentIndex(DocumentIndex):
             # if the chunk count has actually decreased. This assumes that
             # overlapping chunks are perfectly overwritten. If we can't
             # guarantee that then we need the code as-is.
-            num_chunks_deleted = self.delete(
-                onyx_document.id, onyx_document.chunk_count
-            )
-            # If we see that chunks were deleted we assume the doc already
-            # existed.
-            document_insertion_record = DocumentInsertionRecord(
-                document_id=onyx_document.id,
-                already_existed=num_chunks_deleted > 0,
-            )
+            if onyx_document.id not in deleted_doc_ids:
+                num_chunks_deleted = self.delete(
+                    onyx_document.id, onyx_document.chunk_count
+                )
+                deleted_doc_ids.add(onyx_document.id)
+                # If we see that chunks were deleted we assume the doc already
+                # existed. We record the result before bulk_index_documents
+                # runs. If indexing raises, this entire result list is discarded
+                # by the caller's retry logic, so early recording is safe.
+                document_indexing_results.append(
+                    DocumentInsertionRecord(
+                        document_id=onyx_document.id,
+                        already_existed=num_chunks_deleted > 0,
+                    )
+                )
             # Now index. This will raise if a chunk of the same ID exists, which
             # we do not expect because we should have deleted all chunks.
             self._client.bulk_index_documents(
                 documents=chunk_batch,
                 tenant_state=self._tenant_state,
             )
-            document_indexing_results.append(document_insertion_record)
+
+        for chunk in chunks:
+            doc_id = chunk.source_document.id
+            if doc_id != current_doc_id:
+                if current_chunks:
+                    _flush_chunks(current_chunks)
+                current_doc_id = doc_id
+                current_chunks = [chunk]
+            else:
+                current_chunks.append(chunk)
+
+        if current_chunks:
+            _flush_chunks(current_chunks)
 
         return document_indexing_results
 

backend/onyx/document_index/vespa/index.py

@@ -6,6 +6,7 @@ import re
 import time
 import urllib
 import zipfile
+from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import datetime
 from datetime import timedelta
@@ -461,7 +462,7 @@ class VespaIndex(DocumentIndex):
 
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
+        chunks: Iterable[DocMetadataAwareIndexChunk],
         index_batch_params: IndexBatchParams,
     ) -> set[OldDocumentInsertionRecord]:
         """

backend/onyx/document_index/vespa/vespa_document_index.py

@@ -1,6 +1,8 @@
 import concurrent.futures
 import logging
 import random
+from collections.abc import Generator
+from collections.abc import Iterable
 from typing import Any
 from uuid import UUID
 
@@ -318,7 +320,7 @@ class VespaDocumentIndex(DocumentIndex):
 
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
+        chunks: Iterable[DocMetadataAwareIndexChunk],
         indexing_metadata: IndexingMetadata,
     ) -> list[DocumentInsertionRecord]:
         doc_id_to_chunk_cnt_diff = indexing_metadata.doc_id_to_chunk_cnt_diff
@@ -338,22 +340,31 @@ class VespaDocumentIndex(DocumentIndex):
 
         # Vespa has restrictions on valid characters, yet document IDs come from
         # external w.r.t. this class. We need to sanitize them.
-        cleaned_chunks: list[DocMetadataAwareIndexChunk] = [
-            clean_chunk_id_copy(chunk) for chunk in chunks
-        ]
-        assert len(cleaned_chunks) == len(
-            chunks
-        ), "Bug: Cleaned chunks and input chunks have different lengths."
+        #
+        # Instead of materializing all cleaned chunks upfront, we stream them
+        # through a generator that cleans IDs and builds the original-ID mapping
+        # incrementally as chunks flow into Vespa.
+        def _clean_and_track(
+            chunks_iter: Iterable[DocMetadataAwareIndexChunk],
+            id_map: dict[str, str],
+            seen_ids: set[str],
+        ) -> Generator[DocMetadataAwareIndexChunk, None, None]:
+            """Cleans chunk IDs and builds the original-ID mapping
+            incrementally as chunks flow through, avoiding a separate
+            materialization pass."""
+            for chunk in chunks_iter:
+                original_id = chunk.source_document.id
+                cleaned = clean_chunk_id_copy(chunk)
+                cleaned_id = cleaned.source_document.id
+                # Needed so the final DocumentInsertionRecord returned can have
+                # the original document ID. cleaned_chunks might not contain IDs
+                # exactly as callers supplied them.
+                id_map[cleaned_id] = original_id
+                seen_ids.add(cleaned_id)
+                yield cleaned
 
-        # Needed so the final DocumentInsertionRecord returned can have the
-        # original document ID. cleaned_chunks might not contain IDs exactly as
-        # callers supplied them.
-        new_document_id_to_original_document_id: dict[str, str] = dict()
-        for i, cleaned_chunk in enumerate(cleaned_chunks):
-            old_chunk = chunks[i]
-            new_document_id_to_original_document_id[
-                cleaned_chunk.source_document.id
-            ] = old_chunk.source_document.id
+        new_document_id_to_original_document_id: dict[str, str] = {}
+        all_cleaned_doc_ids: set[str] = set()
 
         existing_docs: set[str] = set()
 
@@ -409,7 +420,13 @@ class VespaDocumentIndex(DocumentIndex):
                     executor=executor,
                 )
 
-            # Insert new Vespa documents.
+            # Insert new Vespa documents, streaming through the cleaning
+            # pipeline so chunks are never fully materialized.
+            cleaned_chunks = _clean_and_track(
+                chunks,
+                new_document_id_to_original_document_id,
+                all_cleaned_doc_ids,
+            )
             for chunk_batch in batch_generator(cleaned_chunks, BATCH_SIZE):
                 batch_index_vespa_chunks(
                     chunks=chunk_batch,
@@ -419,10 +436,6 @@ class VespaDocumentIndex(DocumentIndex):
                     executor=executor,
                 )
 
-        all_cleaned_doc_ids: set[str] = {
-            chunk.source_document.id for chunk in cleaned_chunks
-        }
-
         return [
             DocumentInsertionRecord(
                 document_id=new_document_id_to_original_document_id[cleaned_doc_id],

backend/onyx/file_processing/extract_file_text.py

@@ -44,6 +44,7 @@ KNOWN_OPENPYXL_BUGS = [
     "Value must be either numerical or a string containing a wildcard",
     "File contains no valid workbook part",
     "Unable to read workbook: could not read stylesheet from None",
+    "Colors must be aRGB hex values",
 ]
 
 

backend/onyx/indexing/adapters/user_file_indexing_adapter.py

@@ -29,6 +29,7 @@ from onyx.indexing.models import DocMetadataAwareIndexChunk
 from onyx.indexing.models import IndexChunk
 from onyx.indexing.models import UpdatableChunkData
 from onyx.llm.factory import get_default_llm
+from onyx.natural_language_processing.utils import count_tokens
 from onyx.natural_language_processing.utils import get_tokenizer
 from onyx.utils.logger import setup_logger
 
@@ -173,8 +174,10 @@ class UserFileIndexingAdapter:
                     [chunk.content for chunk in user_file_chunks]
                 )
                 user_file_id_to_raw_text[str(user_file_id)] = combined_content
-                token_count = (
-                    len(llm_tokenizer.encode(combined_content)) if llm_tokenizer else 0
+                token_count: int = (
+                    count_tokens(combined_content, llm_tokenizer)
+                    if llm_tokenizer
+                    else 0
                 )
                 user_file_id_to_token_count[str(user_file_id)] = token_count
             else:

backend/onyx/llm/constants.py

@@ -25,6 +25,7 @@ class LlmProviderNames(str, Enum):
     LM_STUDIO = "lm_studio"
     MISTRAL = "mistral"
     LITELLM_PROXY = "litellm_proxy"
+    BIFROST = "bifrost"
 
     def __str__(self) -> str:
         """Needed so things like:
@@ -44,6 +45,7 @@ WELL_KNOWN_PROVIDER_NAMES = [
     LlmProviderNames.OLLAMA_CHAT,
     LlmProviderNames.LM_STUDIO,
     LlmProviderNames.LITELLM_PROXY,
+    LlmProviderNames.BIFROST,
 ]
 
 
@@ -61,6 +63,7 @@ PROVIDER_DISPLAY_NAMES: dict[str, str] = {
     LlmProviderNames.OLLAMA_CHAT: "Ollama",
     LlmProviderNames.LM_STUDIO: "LM Studio",
     LlmProviderNames.LITELLM_PROXY: "LiteLLM Proxy",
+    LlmProviderNames.BIFROST: "Bifrost",
     "groq": "Groq",
     "anyscale": "Anyscale",
     "deepseek": "DeepSeek",
@@ -112,6 +115,7 @@ AGGREGATOR_PROVIDERS: set[str] = {
     LlmProviderNames.VERTEX_AI,
     LlmProviderNames.AZURE,
     LlmProviderNames.LITELLM_PROXY,
+    LlmProviderNames.BIFROST,
 }
 
 # Model family name mappings for display name generation

backend/onyx/llm/multi_llm.py

@@ -185,6 +185,21 @@ def _messages_contain_tool_content(messages: list[dict[str, Any]]) -> bool:
     return False
 
 
+def _prompt_contains_tool_call_history(prompt: LanguageModelInput) -> bool:
+    """Check if the prompt contains any assistant messages with tool_calls.
+
+    When Anthropic's extended thinking is enabled, the API requires every
+    assistant message to start with a thinking block before any tool_use
+    blocks.  Since we don't preserve thinking_blocks (they carry
+    cryptographic signatures that can't be reconstructed), we must skip
+    the thinking param whenever history contains prior tool-calling turns.
+    """
+    from onyx.llm.models import AssistantMessage
+
+    msgs = prompt if isinstance(prompt, list) else [prompt]
+    return any(isinstance(msg, AssistantMessage) and msg.tool_calls for msg in msgs)
+
+
 def _is_vertex_model_rejecting_output_config(model_name: str) -> bool:
     normalized_model_name = model_name.lower()
     return any(
@@ -290,6 +305,17 @@ class LitellmLLM(LLM):
         ):
             model_kwargs[VERTEX_LOCATION_KWARG] = "global"
 
+        # Bifrost: OpenAI-compatible proxy that expects model names in
+        # provider/model format (e.g. "anthropic/claude-sonnet-4-6").
+        # We route through LiteLLM's openai provider with the Bifrost base URL,
+        # and ensure /v1 is appended.
+        if model_provider == LlmProviderNames.BIFROST:
+            self._custom_llm_provider = "openai"
+            if self._api_base is not None:
+                base = self._api_base.rstrip("/")
+                self._api_base = base if base.endswith("/v1") else f"{base}/v1"
+                model_kwargs["api_base"] = self._api_base
+
         # This is needed for Ollama to do proper function calling
         if model_provider == LlmProviderNames.OLLAMA_CHAT and api_base is not None:
             model_kwargs["api_base"] = api_base
@@ -401,14 +427,20 @@ class LitellmLLM(LLM):
         optional_kwargs: dict[str, Any] = {}
 
         # Model name
+        is_bifrost = self._model_provider == LlmProviderNames.BIFROST
         model_provider = (
             f"{self.config.model_provider}/responses"
             if is_openai_model  # Uses litellm's completions -> responses bridge
             else self.config.model_provider
         )
-        model = (
-            f"{model_provider}/{self.config.deployment_name or self.config.model_name}"
-        )
+        if is_bifrost:
+            # Bifrost expects model names in provider/model format
+            # (e.g. "anthropic/claude-sonnet-4-6") sent directly to its
+            # OpenAI-compatible endpoint. We use custom_llm_provider="openai"
+            # so LiteLLM doesn't try to route based on the provider prefix.
+            model = self.config.deployment_name or self.config.model_name
+        else:
+            model = f"{model_provider}/{self.config.deployment_name or self.config.model_name}"
 
         # Tool choice
         if is_claude_model and tool_choice == ToolChoiceOptions.REQUIRED:
@@ -449,7 +481,20 @@ class LitellmLLM(LLM):
                     reasoning_effort
                 )
 
-                if budget_tokens is not None:
+                # Anthropic requires every assistant message with tool_use
+                # blocks to start with a thinking block that carries a
+                # cryptographic signature.  We don't preserve those blocks
+                # across turns, so skip thinking when the history already
+                # contains tool-calling assistant messages.  LiteLLM's
+                # modify_params workaround doesn't cover all providers
+                # (notably Bedrock).
+                can_enable_thinking = (
+                    budget_tokens is not None
+                    and not _prompt_contains_tool_call_history(prompt)
+                )
+
+                if can_enable_thinking:
+                    assert budget_tokens is not None  # mypy
                     if max_tokens is not None:
                         # Anthropic has a weird rule where max token has to be at least as much as budget tokens if set
                         # and the minimum budget tokens is 1024
@@ -483,10 +528,11 @@ class LitellmLLM(LLM):
         if structured_response_format:
             optional_kwargs["response_format"] = structured_response_format
 
-        if not (is_claude_model or is_ollama or is_mistral):
+        if not (is_claude_model or is_ollama or is_mistral) or is_bifrost:
             # Litellm bug: tool_choice is dropped silently if not specified here for OpenAI
             # However, this param breaks Anthropic and Mistral models,
-            # so it must be conditionally included.
+            # so it must be conditionally included unless the request is
+            # routed through Bifrost's OpenAI-compatible endpoint.
             # Additionally, tool_choice is not supported by Ollama and causes warnings if included.
             # See also, https://github.com/ollama/ollama/issues/11171
             optional_kwargs["allowed_openai_params"] = ["tool_choice"]

backend/onyx/llm/well_known_providers/constants.py

@@ -13,6 +13,8 @@ LM_STUDIO_API_KEY_CONFIG_KEY = "LM_STUDIO_API_KEY"
 
 LITELLM_PROXY_PROVIDER_NAME = "litellm_proxy"
 
+BIFROST_PROVIDER_NAME = "bifrost"
+
 # Providers that use optional Bearer auth from custom_config
 PROVIDERS_WITH_SPECIAL_API_KEY_HANDLING: dict[str, str] = {
     LlmProviderNames.OLLAMA_CHAT: OLLAMA_API_KEY_CONFIG_KEY,

backend/onyx/llm/well_known_providers/llm_provider_options.py

@@ -15,6 +15,7 @@ from onyx.llm.well_known_providers.auto_update_service import (
 from onyx.llm.well_known_providers.constants import ANTHROPIC_PROVIDER_NAME
 from onyx.llm.well_known_providers.constants import AZURE_PROVIDER_NAME
 from onyx.llm.well_known_providers.constants import BEDROCK_PROVIDER_NAME
+from onyx.llm.well_known_providers.constants import BIFROST_PROVIDER_NAME
 from onyx.llm.well_known_providers.constants import LITELLM_PROXY_PROVIDER_NAME
 from onyx.llm.well_known_providers.constants import LM_STUDIO_PROVIDER_NAME
 from onyx.llm.well_known_providers.constants import OLLAMA_PROVIDER_NAME
@@ -49,6 +50,7 @@ def _get_provider_to_models_map() -> dict[str, list[str]]:
         LM_STUDIO_PROVIDER_NAME: [],  # Dynamic - fetched from LM Studio API
         OPENROUTER_PROVIDER_NAME: [],  # Dynamic - fetched from OpenRouter API
         LITELLM_PROXY_PROVIDER_NAME: [],  # Dynamic - fetched from LiteLLM proxy API
+        BIFROST_PROVIDER_NAME: [],  # Dynamic - fetched from Bifrost API
     }
 
 

backend/onyx/natural_language_processing/utils.py

@@ -175,6 +175,32 @@ def get_tokenizer(
     return _check_tokenizer_cache(provider_type, model_name)
 
 
+# Max characters per encode() call.
+_ENCODE_CHUNK_SIZE = 500_000
+
+
+def count_tokens(
+    text: str,
+    tokenizer: BaseTokenizer,
+    token_limit: int | None = None,
+) -> int:
+    """Count tokens, chunking the input to avoid tiktoken stack overflow.
+
+    If token_limit is provided and the text is large enough to require
+    multiple chunks (> 500k chars), stops early once the count exceeds it.
+    When early-exiting, the returned value exceeds token_limit but may be
+    less than the true full token count.
+    """
+    if len(text) <= _ENCODE_CHUNK_SIZE:
+        return len(tokenizer.encode(text))
+    total = 0
+    for start in range(0, len(text), _ENCODE_CHUNK_SIZE):
+        total += len(tokenizer.encode(text[start : start + _ENCODE_CHUNK_SIZE]))
+        if token_limit is not None and total > token_limit:
+            return total  # Already over — skip remaining chunks
+    return total
+
+
 def tokenizer_trim_content(
     content: str, desired_length: int, tokenizer: BaseTokenizer
 ) -> str:

backend/onyx/server/features/build/sandbox/kubernetes/docker/templates/outputs/web/package-lock.json

@@ -3844,9 +3844,9 @@
       }
     },
     "node_modules/@ts-morph/common/node_modules/brace-expansion": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.3.tgz",
-      "integrity": "sha512-fy6KJm2RawA5RcHkLa1z/ScpBeA762UF9KmZQxwIbDtRJrgLzM10depAiEQ+CXYcoiqW1/m96OAAoke2nE9EeA==",
+      "version": "5.0.5",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz",
+      "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==",
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
@@ -4224,9 +4224,9 @@
       }
     },
     "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
-      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz",
+      "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -5007,9 +5007,9 @@
       }
     },
     "node_modules/brace-expansion": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
-      "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
+      "version": "1.1.13",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz",
+      "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {

backend/onyx/server/features/hooks/api.py

@@ -44,11 +44,12 @@ def _check_ssrf_safety(endpoint_url: str) -> None:
     """Raise OnyxError if endpoint_url could be used for SSRF.
 
     Delegates to validate_outbound_http_url with https_only=True.
+    Uses BAD_GATEWAY so the frontend maps the error to the Endpoint URL field.
     """
     try:
         validate_outbound_http_url(endpoint_url, https_only=True)
     except (SSRFException, ValueError) as e:
-        raise OnyxError(OnyxErrorCode.INVALID_INPUT, str(e))
+        raise OnyxError(OnyxErrorCode.BAD_GATEWAY, str(e))
 
 
 # ---------------------------------------------------------------------------
@@ -122,9 +123,8 @@ def _validate_endpoint(
     (not reachable — indicates the api_key is invalid).
 
     Timeout handling:
-    - ConnectTimeout: TCP handshake never completed → cannot_connect.
-    - ReadTimeout / WriteTimeout: TCP was established, server responded slowly → timeout
-      (operator should consider increasing timeout_seconds).
+    - Any httpx.TimeoutException (ConnectTimeout, ReadTimeout, WriteTimeout, PoolTimeout) →
+      timeout (operator should consider increasing timeout_seconds).
     - All other exceptions → cannot_connect.
     """
     _check_ssrf_safety(endpoint_url)
@@ -141,19 +141,11 @@ def _validate_endpoint(
             )
         return HookValidateResponse(status=HookValidateStatus.passed)
     except httpx.TimeoutException as exc:
-        # ConnectTimeout: TCP handshake never completed → cannot_connect.
-        # ReadTimeout / WriteTimeout: TCP was established, server just responded slowly → timeout.
-        if isinstance(exc, httpx.ConnectTimeout):
-            logger.warning(
-                "Hook endpoint validation: connect timeout for %s",
-                endpoint_url,
-                exc_info=exc,
-            )
-            return HookValidateResponse(
-                status=HookValidateStatus.cannot_connect, error_message=str(exc)
-            )
+        # Any timeout (connect, read, or write) means the configured timeout_seconds
+        # is too low for this endpoint. Report as timeout so the UI directs the user
+        # to increase the timeout setting.
         logger.warning(
-            "Hook endpoint validation: read/write timeout for %s",
+            "Hook endpoint validation: timeout for %s",
             endpoint_url,
             exc_info=exc,
         )

backend/onyx/server/features/projects/projects_file_utils.py

@@ -9,20 +9,15 @@ from pydantic import ConfigDict
 from pydantic import Field
 from sqlalchemy.orm import Session
 
-from onyx.configs.app_configs import FILE_TOKEN_COUNT_THRESHOLD
-from onyx.configs.app_configs import USER_FILE_MAX_UPLOAD_SIZE_BYTES
-from onyx.configs.app_configs import USER_FILE_MAX_UPLOAD_SIZE_MB
 from onyx.db.llm import fetch_default_llm_model
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_processing.extract_file_text import get_file_ext
 from onyx.file_processing.file_types import OnyxFileExtensions
 from onyx.file_processing.password_validation import is_file_password_protected
+from onyx.natural_language_processing.utils import count_tokens
 from onyx.natural_language_processing.utils import get_tokenizer
+from onyx.server.settings.store import load_settings
 from onyx.utils.logger import setup_logger
-from shared_configs.configs import MULTI_TENANT
-from shared_configs.configs import SKIP_USERFILE_THRESHOLD
-from shared_configs.configs import SKIP_USERFILE_THRESHOLD_TENANT_LIST
-from shared_configs.contextvars import get_current_tenant_id
 
 
 logger = setup_logger()
@@ -161,8 +156,8 @@ def categorize_uploaded_files(
       document formats (.pdf, .docx, …) and falls back to a text-detection
       heuristic for unknown extensions (.py, .js, .rs, …).
     - Uses default tokenizer to compute token length.
-    - If token length > threshold, reject file (unless threshold skip is enabled).
-    - If text cannot be extracted, reject file.
+    - If token length exceeds the admin-configured threshold, reject file.
+    - If extension unsupported or text cannot be extracted, reject file.
     - Otherwise marked as acceptable.
     """
 
@@ -173,36 +168,33 @@ def categorize_uploaded_files(
     provider_type = default_model.llm_provider.provider if default_model else None
     tokenizer = get_tokenizer(model_name=model_name, provider_type=provider_type)
 
-    # Check if threshold checks should be skipped
-    skip_threshold = False
-
-    # Check global skip flag (works for both single-tenant and multi-tenant)
-    if SKIP_USERFILE_THRESHOLD:
-        skip_threshold = True
-        logger.info("Skipping userfile threshold check (global setting)")
-    # Check tenant-specific skip list (only applicable in multi-tenant)
-    elif MULTI_TENANT and SKIP_USERFILE_THRESHOLD_TENANT_LIST:
-        try:
-            current_tenant_id = get_current_tenant_id()
-            skip_threshold = current_tenant_id in SKIP_USERFILE_THRESHOLD_TENANT_LIST
-            if skip_threshold:
-                logger.info(
-                    f"Skipping userfile threshold check for tenant: {current_tenant_id}"
-                )
-        except RuntimeError as e:
-            logger.warning(f"Failed to get current tenant ID: {str(e)}")
+    # Derive limits from admin-configurable settings.
+    # For upload size: load_settings() resolves 0/None to a positive default.
+    # For token threshold: 0 means "no limit" (converted to None below).
+    settings = load_settings()
+    max_upload_size_mb = (
+        settings.user_file_max_upload_size_mb
+    )  # always positive after load_settings()
+    max_upload_size_bytes = (
+        max_upload_size_mb * 1024 * 1024 if max_upload_size_mb else None
+    )
+    token_threshold_k = settings.file_token_count_threshold_k
+    token_threshold = (
+        token_threshold_k * 1000 if token_threshold_k else None
+    )  # 0 → None = no limit
 
     for upload in files:
         try:
             filename = get_safe_filename(upload)
 
-            # Size limit is a hard safety cap and is enforced even when token
-            # threshold checks are skipped via SKIP_USERFILE_THRESHOLD settings.
-            if is_upload_too_large(upload, USER_FILE_MAX_UPLOAD_SIZE_BYTES):
+            # Size limit is a hard safety cap.
+            if max_upload_size_bytes is not None and is_upload_too_large(
+                upload, max_upload_size_bytes
+            ):
                 results.rejected.append(
                     RejectedFile(
                         filename=filename,
-                        reason=f"Exceeds {USER_FILE_MAX_UPLOAD_SIZE_MB} MB file size limit",
+                        reason=f"Exceeds {max_upload_size_mb} MB file size limit",
                     )
                 )
                 continue
@@ -224,11 +216,11 @@ def categorize_uploaded_files(
                     )
                     continue
 
-                if not skip_threshold and token_count > FILE_TOKEN_COUNT_THRESHOLD:
+                if token_threshold is not None and token_count > token_threshold:
                     results.rejected.append(
                         RejectedFile(
                             filename=filename,
-                            reason=f"Exceeds {FILE_TOKEN_COUNT_THRESHOLD} token limit",
+                            reason=f"Exceeds {token_threshold_k}K token limit",
                         )
                     )
                 else:
@@ -269,12 +261,14 @@ def categorize_uploaded_files(
                     )
                     continue
 
-                token_count = len(tokenizer.encode(text_content))
-                if not skip_threshold and token_count > FILE_TOKEN_COUNT_THRESHOLD:
+                token_count = count_tokens(
+                    text_content, tokenizer, token_limit=token_threshold
+                )
+                if token_threshold is not None and token_count > token_threshold:
                     results.rejected.append(
                         RejectedFile(
                             filename=filename,
-                            reason=f"Exceeds {FILE_TOKEN_COUNT_THRESHOLD} token limit",
+                            reason=f"Exceeds {token_threshold_k}K token limit",
                         )
                     )
                 else:

backend/onyx/server/manage/llm/api.py

@@ -57,6 +57,8 @@ from onyx.llm.well_known_providers.llm_provider_options import (
 )
 from onyx.server.manage.llm.models import BedrockFinalModelResponse
 from onyx.server.manage.llm.models import BedrockModelsRequest
+from onyx.server.manage.llm.models import BifrostFinalModelResponse
+from onyx.server.manage.llm.models import BifrostModelsRequest
 from onyx.server.manage.llm.models import DefaultModel
 from onyx.server.manage.llm.models import LitellmFinalModelResponse
 from onyx.server.manage.llm.models import LitellmModelDetails
@@ -1422,11 +1424,26 @@ def _get_litellm_models_response(api_key: str, api_base: str) -> dict:
     cleaned_api_base = api_base.strip().rstrip("/")
     url = f"{cleaned_api_base}/v1/models"
 
+    return _get_openai_compatible_models_response(
+        url=url,
+        source_name="LiteLLM proxy",
+        api_key=api_key,
+    )
+
+
+def _get_openai_compatible_models_response(
+    url: str,
+    source_name: str,
+    api_key: str | None = None,
+) -> dict:
+    """Fetch model metadata from an OpenAI-compatible `/models` endpoint."""
     headers = {
         "Authorization": f"Bearer {api_key}",
         "HTTP-Referer": "https://onyx.app",
         "X-Title": "Onyx",
     }
+    if not api_key:
+        headers.pop("Authorization")
 
     try:
         response = httpx.get(url, headers=headers, timeout=10.0)
@@ -1436,20 +1453,125 @@ def _get_litellm_models_response(api_key: str, api_base: str) -> dict:
         if e.response.status_code == 401:
             raise OnyxError(
                 OnyxErrorCode.VALIDATION_ERROR,
-                "Authentication failed: invalid or missing API key for LiteLLM proxy.",
+                f"Authentication failed: invalid or missing API key for {source_name}.",
             )
         elif e.response.status_code == 404:
             raise OnyxError(
                 OnyxErrorCode.VALIDATION_ERROR,
-                f"LiteLLM models endpoint not found at {url}. Please verify the API base URL.",
+                f"{source_name} models endpoint not found at {url}. Please verify the API base URL.",
             )
         else:
             raise OnyxError(
                 OnyxErrorCode.BAD_GATEWAY,
-                f"Failed to fetch LiteLLM models: {e}",
+                f"Failed to fetch {source_name} models: {e}",
             )
-    except Exception as e:
+    except httpx.RequestError as e:
+        logger.warning(
+            "Failed to fetch models from OpenAI-compatible endpoint",
+            extra={"source": source_name, "url": url, "error": str(e)},
+            exc_info=True,
+        )
         raise OnyxError(
             OnyxErrorCode.BAD_GATEWAY,
-            f"Failed to fetch LiteLLM models: {e}",
+            f"Failed to fetch {source_name} models: {e}",
         )
+    except ValueError as e:
+        logger.warning(
+            "Received invalid model response from OpenAI-compatible endpoint",
+            extra={"source": source_name, "url": url, "error": str(e)},
+            exc_info=True,
+        )
+        raise OnyxError(
+            OnyxErrorCode.BAD_GATEWAY,
+            f"Failed to fetch {source_name} models: {e}",
+        )
+
+
+@admin_router.post("/bifrost/available-models")
+def get_bifrost_available_models(
+    request: BifrostModelsRequest,
+    _: User = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
+) -> list[BifrostFinalModelResponse]:
+    """Fetch available models from Bifrost gateway /v1/models endpoint."""
+    response_json = _get_bifrost_models_response(
+        api_base=request.api_base, api_key=request.api_key
+    )
+
+    models = response_json.get("data", [])
+    if not isinstance(models, list) or len(models) == 0:
+        raise OnyxError(
+            OnyxErrorCode.VALIDATION_ERROR,
+            "No models found from your Bifrost endpoint",
+        )
+
+    results: list[BifrostFinalModelResponse] = []
+    for model in models:
+        try:
+            model_id = model.get("id", "")
+            model_name = model.get("name", model_id)
+
+            if not model_id:
+                continue
+
+            # Skip embedding models
+            if is_embedding_model(model_id):
+                continue
+
+            results.append(
+                BifrostFinalModelResponse(
+                    name=model_id,
+                    display_name=model_name,
+                    max_input_tokens=model.get("context_length"),
+                    supports_image_input=infer_vision_support(model_id),
+                    supports_reasoning=is_reasoning_model(model_id, model_name),
+                )
+            )
+        except Exception as e:
+            logger.warning(
+                "Failed to parse Bifrost model entry",
+                extra={"error": str(e), "item": str(model)[:1000]},
+            )
+
+    if not results:
+        raise OnyxError(
+            OnyxErrorCode.VALIDATION_ERROR,
+            "No compatible models found from Bifrost",
+        )
+
+    sorted_results = sorted(results, key=lambda m: m.name.lower())
+
+    # Sync new models to DB if provider_name is specified
+    if request.provider_name:
+        _sync_fetched_models(
+            db_session=db_session,
+            provider_name=request.provider_name,
+            models=[
+                SyncModelEntry(
+                    name=r.name,
+                    display_name=r.display_name,
+                    max_input_tokens=r.max_input_tokens,
+                    supports_image_input=r.supports_image_input,
+                )
+                for r in sorted_results
+            ],
+            source_label="Bifrost",
+        )
+
+    return sorted_results
+
+
+def _get_bifrost_models_response(api_base: str, api_key: str | None = None) -> dict:
+    """Perform GET to Bifrost /v1/models and return parsed JSON."""
+    cleaned_api_base = api_base.strip().rstrip("/")
+    # Ensure we hit /v1/models
+    if cleaned_api_base.endswith("/v1"):
+        url = f"{cleaned_api_base}/models"
+    else:
+        url = f"{cleaned_api_base}/v1/models"
+
+    return _get_openai_compatible_models_response(
+        url=url,
+        source_name="Bifrost",
+        api_key=api_key,
+    )

backend/onyx/server/manage/llm/models.py

@@ -449,3 +449,18 @@ class LitellmModelDetails(BaseModel):
 class LitellmFinalModelResponse(BaseModel):
     provider_name: str  # Provider name (e.g. "openai")
     model_name: str  # Model ID (e.g. "gpt-4o")
+
+
+# Bifrost dynamic models fetch
+class BifrostModelsRequest(BaseModel):
+    api_base: str
+    api_key: str | None = None
+    provider_name: str | None = None  # Optional: to save models to existing provider
+
+
+class BifrostFinalModelResponse(BaseModel):
+    name: str  # Model ID in provider/model format (e.g. "anthropic/claude-sonnet-4-6")
+    display_name: str  # Human-readable name from Bifrost API
+    max_input_tokens: int | None
+    supports_image_input: bool
+    supports_reasoning: bool

backend/onyx/server/manage/llm/utils.py

@@ -25,6 +25,7 @@ DYNAMIC_LLM_PROVIDERS = frozenset(
         LlmProviderNames.BEDROCK,
         LlmProviderNames.OLLAMA_CHAT,
         LlmProviderNames.LM_STUDIO,
+        LlmProviderNames.BIFROST,
     }
 )
 
@@ -50,6 +51,25 @@ BEDROCK_VISION_MODELS = frozenset(
     }
 )
 
+# Known Bifrost/OpenAI-compatible vision-capable model families where the
+# source API does not expose this metadata directly.
+BIFROST_VISION_MODEL_FAMILIES = frozenset(
+    {
+        "anthropic/claude-3",
+        "anthropic/claude-4",
+        "amazon/nova-pro",
+        "amazon/nova-lite",
+        "amazon/nova-premier",
+        "openai/gpt-4o",
+        "openai/gpt-4.1",
+        "google/gemini",
+        "meta-llama/llama-3.2",
+        "mistral/pixtral",
+        "qwen/qwen2.5-vl",
+        "qwen/qwen-vl",
+    }
+)
+
 
 def is_valid_bedrock_model(
     model_id: str,
@@ -76,11 +96,18 @@ def is_valid_bedrock_model(
 def infer_vision_support(model_id: str) -> bool:
     """Infer vision support from model ID when base model metadata unavailable.
 
-    Used for cross-region inference profiles when the base model isn't
-    available in the user's region.
+    Used for providers like Bedrock and Bifrost where vision support may
+    need to be inferred from vendor/model naming conventions.
     """
     model_id_lower = model_id.lower()
-    return any(vision_model in model_id_lower for vision_model in BEDROCK_VISION_MODELS)
+    if any(vision_model in model_id_lower for vision_model in BEDROCK_VISION_MODELS):
+        return True
+
+    normalized_model_id = model_id_lower.replace(".", "/")
+    return any(
+        vision_model in normalized_model_id
+        for vision_model in BIFROST_VISION_MODEL_FAMILIES
+    )
 
 
 def generate_bedrock_display_name(model_id: str) -> str:
@@ -322,7 +349,7 @@ def extract_vendor_from_model_name(model_name: str, provider: str) -> str | None
         - Ollama: "llama3:70b" → "Meta"
         - Ollama: "qwen2.5:7b" → "Alibaba"
     """
-    if provider == LlmProviderNames.OPENROUTER:
+    if provider in (LlmProviderNames.OPENROUTER, LlmProviderNames.BIFROST):
         # Format: "vendor/model-name" e.g., "anthropic/claude-3-5-sonnet"
         if "/" in model_name:
             vendor_key = model_name.split("/")[0].lower()

backend/onyx/server/metrics/indexing_pipeline.py

@@ -12,7 +12,6 @@ stale, which is fine for monitoring dashboards.
 import json
 import threading
 import time
-from collections.abc import Callable
 from datetime import datetime
 from datetime import timezone
 from typing import Any
@@ -104,25 +103,23 @@ class _CachedCollector(Collector):
 
 
 class QueueDepthCollector(_CachedCollector):
-    """Reads Celery queue lengths from the broker Redis on each scrape.
-
-    Uses a Redis client factory (callable) rather than a stored client
-    reference so the connection is always fresh from Celery's pool.
-    """
+    """Reads Celery queue lengths from the broker Redis on each scrape."""
 
     def __init__(self, cache_ttl: float = _DEFAULT_CACHE_TTL) -> None:
         super().__init__(cache_ttl)
-        self._get_redis: Callable[[], Redis] | None = None
+        self._celery_app: Any | None = None
 
-    def set_redis_factory(self, factory: Callable[[], Redis]) -> None:
-        """Set a callable that returns a broker Redis client on demand."""
-        self._get_redis = factory
+    def set_celery_app(self, app: Any) -> None:
+        """Set the Celery app for broker Redis access."""
+        self._celery_app = app
 
     def _collect_fresh(self) -> list[GaugeMetricFamily]:
-        if self._get_redis is None:
+        if self._celery_app is None:
             return []
 
-        redis_client = self._get_redis()
+        from onyx.background.celery.celery_redis import celery_get_broker_client
+
+        redis_client = celery_get_broker_client(self._celery_app)
 
         depth = GaugeMetricFamily(
             "onyx_queue_depth",
@@ -404,17 +401,19 @@ class RedisHealthCollector(_CachedCollector):
 
     def __init__(self, cache_ttl: float = _DEFAULT_CACHE_TTL) -> None:
         super().__init__(cache_ttl)
-        self._get_redis: Callable[[], Redis] | None = None
+        self._celery_app: Any | None = None
 
-    def set_redis_factory(self, factory: Callable[[], Redis]) -> None:
-        """Set a callable that returns a broker Redis client on demand."""
-        self._get_redis = factory
+    def set_celery_app(self, app: Any) -> None:
+        """Set the Celery app for broker Redis access."""
+        self._celery_app = app
 
     def _collect_fresh(self) -> list[GaugeMetricFamily]:
-        if self._get_redis is None:
+        if self._celery_app is None:
             return []
 
-        redis_client = self._get_redis()
+        from onyx.background.celery.celery_redis import celery_get_broker_client
+
+        redis_client = celery_get_broker_client(self._celery_app)
 
         memory_used = GaugeMetricFamily(
             "onyx_redis_memory_used_bytes",
@@ -449,40 +448,128 @@ class RedisHealthCollector(_CachedCollector):
         return [memory_used, memory_peak, memory_frag, connected_clients]
 
 
-class WorkerHealthCollector(_CachedCollector):
-    """Collects Celery worker count and process count via inspect ping.
+class WorkerHeartbeatMonitor:
+    """Monitors Celery worker health via the event stream.
 
-    Uses a longer cache TTL (60s) since inspect.ping() is a broadcast
-    command that takes a couple seconds to complete.
-
-    Maintains a set of known worker short-names so that when a worker
-    stops responding, we emit ``up=0`` instead of silently dropping the
-    metric (which would make ``absent()``-style alerts impossible).
+    Subscribes to ``worker-heartbeat``, ``worker-online``, and
+    ``worker-offline`` events via a single persistent connection.
+    Runs in a daemon thread started once during worker setup.
     """
 
-    # Remove a worker from _known_workers after this many consecutive
-    # missed pings (at 60s TTL ≈ 10 minutes of being unreachable).
-    _MAX_CONSECUTIVE_MISSES = 10
+    # Consider a worker down if no heartbeat received for this long.
+    _HEARTBEAT_TIMEOUT_SECONDS = 120.0
 
-    def __init__(self, cache_ttl: float = 60.0) -> None:
+    def __init__(self, celery_app: Any) -> None:
+        self._app = celery_app
+        self._worker_last_seen: dict[str, float] = {}
+        self._lock = threading.Lock()
+        self._running = False
+        self._thread: threading.Thread | None = None
+
+    def start(self) -> None:
+        """Start the background event listener thread.
+
+        Safe to call multiple times — only starts one thread.
+        """
+        if self._thread is not None and self._thread.is_alive():
+            return
+        self._running = True
+        self._thread = threading.Thread(target=self._listen, daemon=True)
+        self._thread.start()
+        logger.info("WorkerHeartbeatMonitor started")
+
+    def stop(self) -> None:
+        self._running = False
+
+    def _listen(self) -> None:
+        """Background loop: connect to event stream and process heartbeats."""
+        while self._running:
+            try:
+                with self._app.connection() as conn:
+                    recv = self._app.events.Receiver(
+                        conn,
+                        handlers={
+                            "worker-heartbeat": self._on_heartbeat,
+                            "worker-online": self._on_heartbeat,
+                            "worker-offline": self._on_offline,
+                        },
+                    )
+                    recv.capture(
+                        limit=None, timeout=self._HEARTBEAT_TIMEOUT_SECONDS, wakeup=True
+                    )
+            except Exception:
+                if self._running:
+                    logger.debug(
+                        "Heartbeat listener disconnected, reconnecting in 5s",
+                        exc_info=True,
+                    )
+                    time.sleep(5.0)
+            else:
+                # capture() returned normally (timeout with no events); reconnect
+                if self._running:
+                    logger.debug("Heartbeat capture timed out, reconnecting")
+                    time.sleep(5.0)
+
+    def _on_heartbeat(self, event: dict[str, Any]) -> None:
+        hostname = event.get("hostname")
+        if hostname:
+            with self._lock:
+                self._worker_last_seen[hostname] = time.monotonic()
+
+    def _on_offline(self, event: dict[str, Any]) -> None:
+        hostname = event.get("hostname")
+        if hostname:
+            with self._lock:
+                self._worker_last_seen.pop(hostname, None)
+
+    def get_worker_status(self) -> dict[str, bool]:
+        """Return {hostname: is_alive} for all known workers.
+
+        Thread-safe. Called by WorkerHealthCollector on each scrape.
+        Also prunes workers that have been dead longer than 2x the
+        heartbeat timeout to prevent unbounded growth.
+        """
+        now = time.monotonic()
+        prune_threshold = self._HEARTBEAT_TIMEOUT_SECONDS * 2
+        with self._lock:
+            # Prune workers that have been gone for 2x the timeout
+            stale = [
+                h
+                for h, ts in self._worker_last_seen.items()
+                if (now - ts) > prune_threshold
+            ]
+            for h in stale:
+                del self._worker_last_seen[h]
+
+            result: dict[str, bool] = {}
+            for hostname, last_seen in self._worker_last_seen.items():
+                alive = (now - last_seen) < self._HEARTBEAT_TIMEOUT_SECONDS
+                result[hostname] = alive
+            return result
+
+
+class WorkerHealthCollector(_CachedCollector):
+    """Collects Celery worker health from the heartbeat monitor.
+
+    Reads worker status from ``WorkerHeartbeatMonitor`` which listens
+    to the Celery event stream via a single persistent connection.
+    """
+
+    def __init__(self, cache_ttl: float = 30.0) -> None:
         super().__init__(cache_ttl)
-        self._celery_app: Any | None = None
-        # worker short-name → consecutive miss count.
-        # Workers start at 0 and reset to 0 each time they respond.
-        # Removed after _MAX_CONSECUTIVE_MISSES missed collects.
-        self._known_workers: dict[str, int] = {}
+        self._monitor: WorkerHeartbeatMonitor | None = None
 
-    def set_celery_app(self, app: Any) -> None:
-        """Set the Celery app instance for inspect commands."""
-        self._celery_app = app
+    def set_monitor(self, monitor: WorkerHeartbeatMonitor) -> None:
+        """Set the heartbeat monitor instance."""
+        self._monitor = monitor
 
     def _collect_fresh(self) -> list[GaugeMetricFamily]:
-        if self._celery_app is None:
+        if self._monitor is None:
             return []
 
         active_workers = GaugeMetricFamily(
             "onyx_celery_active_worker_count",
-            "Number of active Celery workers responding to ping",
+            "Number of active Celery workers with recent heartbeats",
         )
         worker_up = GaugeMetricFamily(
             "onyx_celery_worker_up",
@@ -491,37 +578,15 @@ class WorkerHealthCollector(_CachedCollector):
         )
 
         try:
-            inspector = self._celery_app.control.inspect(timeout=3.0)
-            ping_result = inspector.ping()
+            status = self._monitor.get_worker_status()
+            alive_count = sum(1 for alive in status.values() if alive)
+            active_workers.add_metric([], alive_count)
 
-            responding: set[str] = set()
-            if ping_result:
-                active_workers.add_metric([], len(ping_result))
-                for worker_name in ping_result:
-                    # Strip hostname suffix for cleaner labels
-                    short_name = worker_name.split("@")[0]
-                    responding.add(short_name)
-            else:
-                active_workers.add_metric([], 0)
-
-            # Register newly-seen workers and reset miss count for
-            # workers that responded.
-            for short_name in responding:
-                self._known_workers[short_name] = 0
-
-            # Increment miss count for non-responding workers and evict
-            # those that have been missing too long.
-            stale = []
-            for short_name in list(self._known_workers):
-                if short_name not in responding:
-                    self._known_workers[short_name] += 1
-                    if self._known_workers[short_name] >= self._MAX_CONSECUTIVE_MISSES:
-                        stale.append(short_name)
-            for short_name in stale:
-                del self._known_workers[short_name]
-
-            for short_name in sorted(self._known_workers):
-                worker_up.add_metric([short_name], 1 if short_name in responding else 0)
+            for hostname in sorted(status):
+                # Use short name (before @) for single-host deployments,
+                # full hostname when multiple hosts share a worker type.
+                label = hostname.split("@")[0]
+                worker_up.add_metric([label], 1 if status[hostname] else 0)
         except Exception:
             logger.debug("Failed to collect worker health metrics", exc_info=True)
 

backend/onyx/server/metrics/indexing_pipeline_setup.py

@@ -3,24 +3,21 @@
 Called once by the monitoring celery worker after Redis and DB are ready.
 """
 
-from collections.abc import Callable
-from typing import Any
-
 from celery import Celery
 from prometheus_client.registry import REGISTRY
-from redis import Redis
 
 from onyx.server.metrics.indexing_pipeline import ConnectorHealthCollector
 from onyx.server.metrics.indexing_pipeline import IndexAttemptCollector
 from onyx.server.metrics.indexing_pipeline import QueueDepthCollector
 from onyx.server.metrics.indexing_pipeline import RedisHealthCollector
 from onyx.server.metrics.indexing_pipeline import WorkerHealthCollector
+from onyx.server.metrics.indexing_pipeline import WorkerHeartbeatMonitor
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
 # Module-level singletons — these are lightweight objects (no connections or DB
-# state) until configure() / set_redis_factory() is called. Keeping them at
+# state) until configure() / set_celery_app() is called. Keeping them at
 # module level ensures they survive the lifetime of the worker process and are
 # only registered with the Prometheus registry once.
 _queue_collector = QueueDepthCollector()
@@ -28,75 +25,28 @@ _attempt_collector = IndexAttemptCollector()
 _connector_collector = ConnectorHealthCollector()
 _redis_health_collector = RedisHealthCollector()
 _worker_health_collector = WorkerHealthCollector()
-
-
-def _make_broker_redis_factory(celery_app: Celery) -> Callable[[], Redis]:
-    """Create a factory that returns a cached broker Redis client.
-
-    Reuses a single connection across scrapes to avoid leaking connections.
-    Reconnects automatically if the cached connection becomes stale.
-    """
-    _cached_client: list[Redis | None] = [None]
-    # Keep a reference to the Kombu Connection so we can close it on
-    # reconnect (the raw Redis client outlives the Kombu wrapper).
-    _cached_kombu_conn: list[Any] = [None]
-
-    def _close_client(client: Redis) -> None:
-        """Best-effort close of a Redis client."""
-        try:
-            client.close()
-        except Exception:
-            logger.debug("Failed to close stale Redis client", exc_info=True)
-
-    def _close_kombu_conn() -> None:
-        """Best-effort close of the cached Kombu Connection."""
-        conn = _cached_kombu_conn[0]
-        if conn is not None:
-            try:
-                conn.close()
-            except Exception:
-                logger.debug("Failed to close Kombu connection", exc_info=True)
-            _cached_kombu_conn[0] = None
-
-    def _get_broker_redis() -> Redis:
-        client = _cached_client[0]
-        if client is not None:
-            try:
-                client.ping()
-                return client
-            except Exception:
-                logger.debug("Cached Redis client stale, reconnecting")
-                _close_client(client)
-                _cached_client[0] = None
-                _close_kombu_conn()
-
-        # Get a fresh Redis client from the broker connection.
-        # We hold this client long-term (cached above) rather than using a
-        # context manager, because we need it to persist across scrapes.
-        # The caching logic above ensures we only ever hold one connection,
-        # and we close it explicitly on reconnect.
-        conn = celery_app.broker_connection()
-        # kombu's Channel exposes .client at runtime (the underlying Redis
-        # client) but the type stubs don't declare it.
-        new_client: Redis = conn.channel().client  # type: ignore[attr-defined]
-        _cached_client[0] = new_client
-        _cached_kombu_conn[0] = conn
-        return new_client
-
-    return _get_broker_redis
+_heartbeat_monitor: WorkerHeartbeatMonitor | None = None
 
 
 def setup_indexing_pipeline_metrics(celery_app: Celery) -> None:
     """Register all indexing pipeline collectors with the default registry.
 
     Args:
-        celery_app: The Celery application instance. Used to obtain a fresh
+        celery_app: The Celery application instance. Used to obtain a
             broker Redis client on each scrape for queue depth metrics.
     """
-    redis_factory = _make_broker_redis_factory(celery_app)
-    _queue_collector.set_redis_factory(redis_factory)
-    _redis_health_collector.set_redis_factory(redis_factory)
-    _worker_health_collector.set_celery_app(celery_app)
+    _queue_collector.set_celery_app(celery_app)
+    _redis_health_collector.set_celery_app(celery_app)
+
+    # Start the heartbeat monitor daemon thread — uses a single persistent
+    # connection to receive worker-heartbeat events.
+    # Module-level singleton prevents duplicate threads on re-entry.
+    global _heartbeat_monitor
+    if _heartbeat_monitor is None:
+        _heartbeat_monitor = WorkerHeartbeatMonitor(celery_app)
+        _heartbeat_monitor.start()
+    _worker_health_collector.set_monitor(_heartbeat_monitor)
+
     _attempt_collector.configure()
     _connector_collector.configure()
 

backend/onyx/server/settings/api.py

@@ -9,7 +9,9 @@ from onyx import __version__ as onyx_version
 from onyx.auth.users import current_admin_user
 from onyx.auth.users import current_user
 from onyx.auth.users import is_user_admin
+from onyx.configs.app_configs import DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB
 from onyx.configs.app_configs import DISABLE_VECTOR_DB
+from onyx.configs.app_configs import MAX_ALLOWED_UPLOAD_SIZE_MB
 from onyx.configs.constants import KV_REINDEX_KEY
 from onyx.configs.constants import NotificationType
 from onyx.db.engine.sql_engine import get_session
@@ -17,10 +19,16 @@ from onyx.db.models import User
 from onyx.db.notification import dismiss_all_notifications
 from onyx.db.notification import get_notifications
 from onyx.db.notification import update_notification_last_shown
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
 from onyx.hooks.utils import HOOKS_AVAILABLE
 from onyx.key_value_store.factory import get_kv_store
 from onyx.key_value_store.interface import KvKeyNotFoundError
 from onyx.server.features.build.utils import is_onyx_craft_enabled
+from onyx.server.settings.models import (
+    DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB,
+)
+from onyx.server.settings.models import DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB
 from onyx.server.settings.models import Notification
 from onyx.server.settings.models import Settings
 from onyx.server.settings.models import UserSettings
@@ -41,6 +49,15 @@ basic_router = APIRouter(prefix="/settings")
 def admin_put_settings(
     settings: Settings, _: User = Depends(current_admin_user)
 ) -> None:
+    if (
+        settings.user_file_max_upload_size_mb is not None
+        and settings.user_file_max_upload_size_mb > 0
+        and settings.user_file_max_upload_size_mb > MAX_ALLOWED_UPLOAD_SIZE_MB
+    ):
+        raise OnyxError(
+            OnyxErrorCode.INVALID_INPUT,
+            f"File upload size limit cannot exceed {MAX_ALLOWED_UPLOAD_SIZE_MB} MB",
+        )
     store_settings(settings)
 
 
@@ -83,6 +100,16 @@ def fetch_settings(
         vector_db_enabled=not DISABLE_VECTOR_DB,
         hooks_enabled=HOOKS_AVAILABLE,
         version=onyx_version,
+        max_allowed_upload_size_mb=MAX_ALLOWED_UPLOAD_SIZE_MB,
+        default_user_file_max_upload_size_mb=min(
+            DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB,
+            MAX_ALLOWED_UPLOAD_SIZE_MB,
+        ),
+        default_file_token_count_threshold_k=(
+            DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB
+            if DISABLE_VECTOR_DB
+            else DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB
+        ),
     )
 
 

backend/onyx/server/settings/models.py

@@ -2,12 +2,19 @@ from datetime import datetime
 from enum import Enum
 
 from pydantic import BaseModel
+from pydantic import Field
 
+from onyx.configs.app_configs import DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB
+from onyx.configs.app_configs import DISABLE_VECTOR_DB
+from onyx.configs.app_configs import MAX_ALLOWED_UPLOAD_SIZE_MB
 from onyx.configs.constants import NotificationType
 from onyx.configs.constants import QueryHistoryType
 from onyx.db.models import Notification as NotificationDBModel
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
+DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB = 200
+DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB = 10000
+
 
 class PageType(str, Enum):
     CHAT = "chat"
@@ -78,7 +85,12 @@ class Settings(BaseModel):
 
     # User Knowledge settings
     user_knowledge_enabled: bool | None = True
-    user_file_max_upload_size_mb: int | None = None
+    user_file_max_upload_size_mb: int | None = Field(
+        default=DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB, ge=0
+    )
+    file_token_count_threshold_k: int | None = Field(
+        default=None, ge=0  # thousands of tokens; None = context-aware default
+    )
 
     # Connector settings
     show_extra_connectors: bool | None = True
@@ -108,3 +120,14 @@ class UserSettings(Settings):
     hooks_enabled: bool = False
     # Application version, read from the ONYX_VERSION env var at startup.
     version: str | None = None
+    # Hard ceiling for user_file_max_upload_size_mb, derived from env var.
+    max_allowed_upload_size_mb: int = MAX_ALLOWED_UPLOAD_SIZE_MB
+    # Factory defaults so the frontend can show a "restore default" button.
+    default_user_file_max_upload_size_mb: int = DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB
+    default_file_token_count_threshold_k: int = Field(
+        default_factory=lambda: (
+            DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB
+            if DISABLE_VECTOR_DB
+            else DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB
+        )
+    )

backend/onyx/server/settings/store.py

@@ -1,13 +1,19 @@
 from onyx.cache.factory import get_cache_backend
+from onyx.configs.app_configs import DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB
 from onyx.configs.app_configs import DISABLE_USER_KNOWLEDGE
+from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.configs.app_configs import ENABLE_OPENSEARCH_INDEXING_FOR_ONYX
+from onyx.configs.app_configs import MAX_ALLOWED_UPLOAD_SIZE_MB
 from onyx.configs.app_configs import ONYX_QUERY_HISTORY_TYPE
 from onyx.configs.app_configs import SHOW_EXTRA_CONNECTORS
-from onyx.configs.app_configs import USER_FILE_MAX_UPLOAD_SIZE_MB
 from onyx.configs.constants import KV_SETTINGS_KEY
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.key_value_store.factory import get_kv_store
 from onyx.key_value_store.interface import KvKeyNotFoundError
+from onyx.server.settings.models import (
+    DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB,
+)
+from onyx.server.settings.models import DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB
 from onyx.server.settings.models import Settings
 from onyx.utils.logger import setup_logger
 
@@ -51,9 +57,36 @@ def load_settings() -> Settings:
     if DISABLE_USER_KNOWLEDGE:
         settings.user_knowledge_enabled = False
 
-    settings.user_file_max_upload_size_mb = USER_FILE_MAX_UPLOAD_SIZE_MB
     settings.show_extra_connectors = SHOW_EXTRA_CONNECTORS
     settings.opensearch_indexing_enabled = ENABLE_OPENSEARCH_INDEXING_FOR_ONYX
+
+    # Resolve context-aware defaults for token threshold.
+    # None = admin hasn't set a value yet → use context-aware default.
+    # 0 = admin explicitly chose "no limit" → preserve as-is.
+    if settings.file_token_count_threshold_k is None:
+        settings.file_token_count_threshold_k = (
+            DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB
+            if DISABLE_VECTOR_DB
+            else DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB
+        )
+
+    # Upload size: 0 and None are treated as "unset" (not "no limit") →
+    # fall back to min(configured default, hard ceiling).
+    if not settings.user_file_max_upload_size_mb:
+        settings.user_file_max_upload_size_mb = min(
+            DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB,
+            MAX_ALLOWED_UPLOAD_SIZE_MB,
+        )
+
+    # Clamp to env ceiling so stale KV values are capped even if the
+    # operator lowered MAX_ALLOWED_UPLOAD_SIZE_MB after a higher value
+    # was already saved (api.py only guards new writes).
+    if (
+        settings.user_file_max_upload_size_mb > 0
+        and settings.user_file_max_upload_size_mb > MAX_ALLOWED_UPLOAD_SIZE_MB
+    ):
+        settings.user_file_max_upload_size_mb = MAX_ALLOWED_UPLOAD_SIZE_MB
+
     return settings
 
 

backend/requirements/default.txt

@@ -187,7 +187,7 @@ coloredlogs==15.0.1
     # via onnxruntime
 courlan==1.3.2
     # via trafilatura
-cryptography==46.0.5
+cryptography==46.0.6
     # via
     #   authlib
     #   google-auth
@@ -449,7 +449,7 @@ kombu==5.5.4
     # via celery
 kubernetes==31.0.0
     # via onyx
-langchain-core==1.2.11
+langchain-core==1.2.22
     # via onyx
 langdetect==1.0.9
     # via unstructured

backend/requirements/dev.txt

@@ -97,7 +97,7 @@ comm==0.2.3
     # via ipykernel
 contourpy==1.3.3
     # via matplotlib
-cryptography==46.0.5
+cryptography==46.0.6
     # via
     #   google-auth
     #   pyjwt
@@ -263,7 +263,7 @@ oauthlib==3.2.2
     # via
     #   kubernetes
     #   requests-oauthlib
-onyx-devtools==0.7.1
+onyx-devtools==0.7.2
     # via onyx
 openai==2.14.0
     # via

backend/requirements/ee.txt

@@ -76,7 +76,7 @@ colorama==0.4.6 ; sys_platform == 'win32'
     # via
     #   click
     #   tqdm
-cryptography==46.0.5
+cryptography==46.0.6
     # via
     #   google-auth
     #   pyjwt

backend/requirements/model_server.txt

@@ -92,7 +92,7 @@ colorama==0.4.6 ; sys_platform == 'win32'
     # via
     #   click
     #   tqdm
-cryptography==46.0.5
+cryptography==46.0.6
     # via
     #   google-auth
     #   pyjwt

backend/shared_configs/configs.py

@@ -191,25 +191,6 @@ IGNORED_SYNCING_TENANT_LIST = (
     else None
 )
 
-# Global flag to skip userfile threshold for all users/tenants
-SKIP_USERFILE_THRESHOLD = (
-    os.environ.get("SKIP_USERFILE_THRESHOLD", "").lower() == "true"
-)
-
-# Comma-separated list of specific tenant IDs to skip threshold (multi-tenant only)
-SKIP_USERFILE_THRESHOLD_TENANT_IDS = os.environ.get(
-    "SKIP_USERFILE_THRESHOLD_TENANT_IDS"
-)
-SKIP_USERFILE_THRESHOLD_TENANT_LIST = (
-    [
-        tenant.strip()
-        for tenant in SKIP_USERFILE_THRESHOLD_TENANT_IDS.split(",")
-        if tenant.strip()
-    ]
-    if SKIP_USERFILE_THRESHOLD_TENANT_IDS
-    else None
-)
-
 ENVIRONMENT = os.environ.get("ENVIRONMENT") or "not_explicitly_set"
 
 

backend/tests/daily/connectors/slack/test_slack_perm_sync.py

@@ -1,4 +1,6 @@
 import time
+from datetime import datetime
+from datetime import timezone
 
 import pytest
 
@@ -17,6 +19,10 @@ PRIVATE_CHANNEL_USERS = [
     "test_user_2@onyx-test.com",
 ]
 
+# Predates any test workspace messages, so the result set should match
+# the "no start time" case while exercising the oldest= parameter.
+OLDEST_TS_2016 = datetime(2016, 1, 1, tzinfo=timezone.utc).timestamp()
+
 pytestmark = pytest.mark.usefixtures("enable_ee")
 
 
@@ -105,15 +111,17 @@ def test_load_from_checkpoint_access__private_channel(
     ],
     indirect=True,
 )
+@pytest.mark.parametrize("start_ts", [None, OLDEST_TS_2016])
 def test_slim_documents_access__public_channel(
     slack_connector: SlackConnector,
+    start_ts: float | None,
 ) -> None:
     """Test that retrieve_all_slim_docs_perm_sync returns correct access information for slim documents."""
     if not slack_connector.client:
         raise RuntimeError("Web client must be defined")
 
     slim_docs_generator = slack_connector.retrieve_all_slim_docs_perm_sync(
-        start=0.0,
+        start=start_ts,
         end=time.time(),
     )
 
@@ -149,7 +157,7 @@ def test_slim_documents_access__private_channel(
         raise RuntimeError("Web client must be defined")
 
     slim_docs_generator = slack_connector.retrieve_all_slim_docs_perm_sync(
-        start=0.0,
+        start=None,
         end=time.time(),
     )
 

backend/tests/external_dependency_unit/celery/test_persona_file_sync.py

@@ -129,6 +129,10 @@ def _patch_task_app(task: Any, mock_app: MagicMock) -> Generator[None, None, Non
             return_value=mock_app,
         ),
         patch(_PATCH_QUEUE_DEPTH, return_value=0),
+        patch(
+            "onyx.background.celery.tasks.user_file_processing.tasks.celery_get_broker_client",
+            return_value=MagicMock(),
+        ),
     ):
         yield
 

backend/tests/external_dependency_unit/celery/test_user_file_delete_queue.py

@@ -88,10 +88,22 @@ def _patch_task_app(task: Any, mock_app: MagicMock) -> Generator[None, None, Non
     the actual task instance.  We patch ``app`` on that instance's class
     (a unique Celery-generated Task subclass) so the mock is scoped to this
     task only.
+
+    Also patches ``celery_get_broker_client`` so the mock app doesn't need
+    a real broker URL.
     """
     task_instance = task.run.__self__
-    with patch.object(
-        type(task_instance), "app", new_callable=PropertyMock, return_value=mock_app
+    with (
+        patch.object(
+            type(task_instance),
+            "app",
+            new_callable=PropertyMock,
+            return_value=mock_app,
+        ),
+        patch(
+            "onyx.background.celery.tasks.user_file_processing.tasks.celery_get_broker_client",
+            return_value=MagicMock(),
+        ),
     ):
         yield
 

backend/tests/external_dependency_unit/celery/test_user_file_processing_queue.py

@@ -90,8 +90,17 @@ def _patch_task_app(task: Any, mock_app: MagicMock) -> Generator[None, None, Non
     task only.
     """
     task_instance = task.run.__self__
-    with patch.object(
-        type(task_instance), "app", new_callable=PropertyMock, return_value=mock_app
+    with (
+        patch.object(
+            type(task_instance),
+            "app",
+            new_callable=PropertyMock,
+            return_value=mock_app,
+        ),
+        patch(
+            "onyx.background.celery.tasks.user_file_processing.tasks.celery_get_broker_client",
+            return_value=MagicMock(),
+        ),
     ):
         yield
 

backend/tests/external_dependency_unit/connectors/confluence/test_confluence_group_sync.py

@@ -103,6 +103,11 @@ _EXPECTED_CONFLUENCE_GROUPS = [
         user_emails={"oauth@onyx.app"},
         gives_anyone_access=False,
     ),
+    ExternalUserGroupSet(
+        id="no yuhong allowed",
+        user_emails={"hagen@danswer.ai", "pablo@onyx.app", "chris@onyx.app"},
+        gives_anyone_access=False,
+    ),
 ]
 
 

backend/tests/external_dependency_unit/document_index/test_document_index.py

@@ -6,6 +6,7 @@ These tests assume Vespa and OpenSearch are running.
 import time
 import uuid
 from collections.abc import Generator
+from collections.abc import Iterator
 
 import httpx
 import pytest
@@ -21,6 +22,7 @@ from onyx.document_index.opensearch.opensearch_document_index import (
 )
 from onyx.document_index.vespa.index import VespaIndex
 from onyx.document_index.vespa.vespa_document_index import VespaDocumentIndex
+from onyx.indexing.models import DocMetadataAwareIndexChunk
 from tests.external_dependency_unit.constants import TEST_TENANT_ID
 from tests.external_dependency_unit.document_index.conftest import EMBEDDING_DIM
 from tests.external_dependency_unit.document_index.conftest import make_chunk
@@ -201,3 +203,25 @@ class TestDocumentIndexNew:
             assert len(result_map) == 2
             assert result_map[existing_doc] is True
             assert result_map[new_doc] is False
+
+    def test_index_accepts_generator(
+        self,
+        document_indices: list[DocumentIndexNew],
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """index() accepts a generator (any iterable), not just a list."""
+        for document_index in document_indices:
+            doc_id = f"test_gen_{uuid.uuid4().hex[:8]}"
+            metadata = make_indexing_metadata([doc_id], old_counts=[0], new_counts=[3])
+
+            def chunk_gen() -> Iterator[DocMetadataAwareIndexChunk]:
+                for i in range(3):
+                    yield make_chunk(doc_id, chunk_id=i)
+
+            results = document_index.index(
+                chunks=chunk_gen(), indexing_metadata=metadata
+            )
+
+            assert len(results) == 1
+            assert results[0].document_id == doc_id
+            assert results[0].already_existed is False

backend/tests/external_dependency_unit/document_index/test_document_index_old.py

@@ -5,6 +5,7 @@ These tests assume Vespa and OpenSearch are running.
 
 import time
 from collections.abc import Generator
+from collections.abc import Iterator
 
 import pytest
 
@@ -166,3 +167,29 @@ class TestDocumentIndexOld:
                 batch_retrieval=True,
             )
             assert len(inference_chunks) == 0
+
+    def test_index_accepts_generator(
+        self,
+        document_indices: list[DocumentIndex],
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """index() accepts a generator (any iterable), not just a list."""
+        for document_index in document_indices:
+
+            def chunk_gen() -> Iterator[DocMetadataAwareIndexChunk]:
+                for i in range(3):
+                    yield make_chunk("test_doc_gen", chunk_id=i)
+
+            index_batch_params = IndexBatchParams(
+                doc_id_to_previous_chunk_cnt={"test_doc_gen": 0},
+                doc_id_to_new_chunk_cnt={"test_doc_gen": 3},
+                tenant_id=get_current_tenant_id(),
+                large_chunks_enabled=False,
+            )
+
+            results = document_index.index(chunk_gen(), index_batch_params)
+
+            assert len(results) == 1
+            record = results.pop()
+            assert record.document_id == "test_doc_gen"
+            assert record.already_existed is False

backend/tests/unit/onyx/background/celery/test_celery_redis.py

@@ -0,0 +1,87 @@
+"""Tests for celery_get_broker_client singleton."""
+
+from collections.abc import Iterator
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import pytest
+
+from onyx.background.celery import celery_redis
+
+
+@pytest.fixture(autouse=True)
+def reset_singleton() -> Iterator[None]:
+    """Reset the module-level singleton between tests."""
+    celery_redis._broker_client = None
+    celery_redis._broker_url = None
+    yield
+    celery_redis._broker_client = None
+    celery_redis._broker_url = None
+
+
+def _make_mock_app(broker_url: str = "redis://localhost:6379/15") -> MagicMock:
+    app = MagicMock()
+    app.conf.broker_url = broker_url
+    return app
+
+
+class TestCeleryGetBrokerClient:
+    @patch("onyx.background.celery.celery_redis.Redis")
+    def test_creates_client_on_first_call(self, mock_redis_cls: MagicMock) -> None:
+        mock_client = MagicMock()
+        mock_redis_cls.from_url.return_value = mock_client
+
+        app = _make_mock_app()
+        result = celery_redis.celery_get_broker_client(app)
+
+        assert result is mock_client
+        call_args = mock_redis_cls.from_url.call_args
+        assert call_args[0][0] == "redis://localhost:6379/15"
+        assert call_args[1]["decode_responses"] is False
+        assert call_args[1]["socket_keepalive"] is True
+        assert call_args[1]["retry_on_timeout"] is True
+
+    @patch("onyx.background.celery.celery_redis.Redis")
+    def test_reuses_cached_client(self, mock_redis_cls: MagicMock) -> None:
+        mock_client = MagicMock()
+        mock_client.ping.return_value = True
+        mock_redis_cls.from_url.return_value = mock_client
+
+        app = _make_mock_app()
+        client1 = celery_redis.celery_get_broker_client(app)
+        client2 = celery_redis.celery_get_broker_client(app)
+
+        assert client1 is client2
+        # from_url called only once
+        assert mock_redis_cls.from_url.call_count == 1
+
+    @patch("onyx.background.celery.celery_redis.Redis")
+    def test_reconnects_on_ping_failure(self, mock_redis_cls: MagicMock) -> None:
+        stale_client = MagicMock()
+        stale_client.ping.side_effect = ConnectionError("disconnected")
+
+        fresh_client = MagicMock()
+        fresh_client.ping.return_value = True
+
+        mock_redis_cls.from_url.side_effect = [stale_client, fresh_client]
+
+        app = _make_mock_app()
+
+        # First call creates stale_client
+        client1 = celery_redis.celery_get_broker_client(app)
+        assert client1 is stale_client
+
+        # Second call: ping fails, creates fresh_client
+        client2 = celery_redis.celery_get_broker_client(app)
+        assert client2 is fresh_client
+        assert mock_redis_cls.from_url.call_count == 2
+
+    @patch("onyx.background.celery.celery_redis.Redis")
+    def test_uses_broker_url_from_app_config(self, mock_redis_cls: MagicMock) -> None:
+        mock_redis_cls.from_url.return_value = MagicMock()
+
+        app = _make_mock_app("redis://custom-host:6380/3")
+        celery_redis.celery_get_broker_client(app)
+
+        call_args = mock_redis_cls.from_url.call_args
+        assert call_args[0][0] == "redis://custom-host:6380/3"

backend/tests/unit/onyx/connectors/canvas/test_canvas_connector.py

@@ -0,0 +1,381 @@
+"""Tests for Canvas connector — client (PR1)."""
+
+from typing import Any
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import pytest
+
+from onyx.connectors.canvas.client import CanvasApiClient
+from onyx.error_handling.exceptions import OnyxError
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+FAKE_BASE_URL = "https://myschool.instructure.com"
+FAKE_TOKEN = "fake-canvas-token"
+
+
+def _mock_response(
+    status_code: int = 200,
+    json_data: Any = None,
+    link_header: str = "",
+) -> MagicMock:
+    """Create a mock HTTP response with status, json, and Link header."""
+    resp = MagicMock()
+    resp.status_code = status_code
+    resp.reason = "OK" if status_code < 300 else "Error"
+    resp.json.return_value = json_data if json_data is not None else []
+    resp.headers = {"Link": link_header}
+    return resp
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient.__init__ tests
+# ---------------------------------------------------------------------------
+
+
+class TestCanvasApiClientInit:
+    def test_success(self) -> None:
+        client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=FAKE_BASE_URL,
+        )
+
+        expected_base_url = f"{FAKE_BASE_URL}/api/v1"
+        expected_host = "myschool.instructure.com"
+
+        assert client.base_url == expected_base_url
+        assert client._expected_host == expected_host
+
+    def test_normalizes_trailing_slash(self) -> None:
+        client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=f"{FAKE_BASE_URL}/",
+        )
+
+        expected_base_url = f"{FAKE_BASE_URL}/api/v1"
+
+        assert client.base_url == expected_base_url
+
+    def test_normalizes_existing_api_v1(self) -> None:
+        client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=f"{FAKE_BASE_URL}/api/v1",
+        )
+
+        expected_base_url = f"{FAKE_BASE_URL}/api/v1"
+
+        assert client.base_url == expected_base_url
+
+    def test_rejects_non_https_scheme(self) -> None:
+        with pytest.raises(ValueError, match="must use https"):
+            CanvasApiClient(
+                bearer_token=FAKE_TOKEN,
+                canvas_base_url="ftp://myschool.instructure.com",
+            )
+
+    def test_rejects_http(self) -> None:
+        with pytest.raises(ValueError, match="must use https"):
+            CanvasApiClient(
+                bearer_token=FAKE_TOKEN,
+                canvas_base_url="http://myschool.instructure.com",
+            )
+
+    def test_rejects_missing_host(self) -> None:
+        with pytest.raises(ValueError, match="must include a valid host"):
+            CanvasApiClient(
+                bearer_token=FAKE_TOKEN,
+                canvas_base_url="https://",
+            )
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient._build_url tests
+# ---------------------------------------------------------------------------
+
+
+class TestBuildUrl:
+    def setup_method(self) -> None:
+        self.client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=FAKE_BASE_URL,
+        )
+
+    def test_appends_endpoint(self) -> None:
+        result = self.client._build_url("courses")
+        expected = f"{FAKE_BASE_URL}/api/v1/courses"
+
+        assert result == expected
+
+    def test_strips_leading_slash_from_endpoint(self) -> None:
+        result = self.client._build_url("/courses")
+        expected = f"{FAKE_BASE_URL}/api/v1/courses"
+
+        assert result == expected
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient._build_headers tests
+# ---------------------------------------------------------------------------
+
+
+class TestBuildHeaders:
+    def setup_method(self) -> None:
+        self.client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=FAKE_BASE_URL,
+        )
+
+    def test_returns_bearer_auth(self) -> None:
+        result = self.client._build_headers()
+        expected = {"Authorization": f"Bearer {FAKE_TOKEN}"}
+
+        assert result == expected
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient.get tests
+# ---------------------------------------------------------------------------
+
+
+class TestGet:
+    def setup_method(self) -> None:
+        self.client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=FAKE_BASE_URL,
+        )
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_success_returns_json_and_next_url(self, mock_requests: MagicMock) -> None:
+        next_link = f"<{FAKE_BASE_URL}/api/v1/courses?page=2>; " 'rel="next"'
+        mock_requests.get.return_value = _mock_response(
+            json_data=[{"id": 1}], link_header=next_link
+        )
+
+        data, next_url = self.client.get("courses")
+
+        expected_data = [{"id": 1}]
+        expected_next = f"{FAKE_BASE_URL}/api/v1/courses?page=2"
+
+        assert data == expected_data
+        assert next_url == expected_next
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_success_no_next_page(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(json_data=[{"id": 1}])
+
+        data, next_url = self.client.get("courses")
+
+        assert data == [{"id": 1}]
+        assert next_url is None
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_raises_on_error_status(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(403, {})
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        assert exc_info.value.status_code == 403
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_raises_on_404(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(404, {})
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        assert exc_info.value.status_code == 404
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_raises_on_429(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(429, {})
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        assert exc_info.value.status_code == 429
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_skips_params_when_using_full_url(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(json_data=[])
+        full = f"{FAKE_BASE_URL}/api/v1/courses?page=2"
+
+        self.client.get(params={"per_page": "100"}, full_url=full)
+
+        _, kwargs = mock_requests.get.call_args
+        assert kwargs["params"] is None
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_extracts_message_from_error_dict(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Shape 1: {"error": {"message": "Not authorized"}}"""
+        mock_requests.get.return_value = _mock_response(
+            403, {"error": {"message": "Not authorized"}}
+        )
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Not authorized"
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_extracts_message_from_error_string(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Shape 2: {"error": "Invalid access token"}"""
+        mock_requests.get.return_value = _mock_response(
+            401, {"error": "Invalid access token"}
+        )
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Invalid access token"
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_extracts_message_from_errors_list(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Shape 3: {"errors": [{"message": "Invalid query"}]}"""
+        mock_requests.get.return_value = _mock_response(
+            400, {"errors": [{"message": "Invalid query"}]}
+        )
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Invalid query"
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_dict_takes_priority_over_errors_list(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """When both error shapes are present, error dict wins."""
+        mock_requests.get.return_value = _mock_response(
+            403, {"error": "Specific error", "errors": [{"message": "Generic"}]}
+        )
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Specific error"
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_falls_back_to_reason_when_no_json_message(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Empty error body falls back to response.reason."""
+        mock_requests.get.return_value = _mock_response(500, {})
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Error"  # from _mock_response's reason for >= 300
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_invalid_json_on_success_raises(self, mock_requests: MagicMock) -> None:
+        """Invalid JSON on a 2xx response raises OnyxError."""
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.json.side_effect = ValueError("No JSON")
+        resp.headers = {"Link": ""}
+        mock_requests.get.return_value = resp
+
+        with pytest.raises(OnyxError, match="Invalid JSON"):
+            self.client.get("courses")
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_invalid_json_on_error_falls_back_to_reason(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Invalid JSON on a 4xx response falls back to response.reason."""
+        resp = MagicMock()
+        resp.status_code = 500
+        resp.reason = "Internal Server Error"
+        resp.json.side_effect = ValueError("No JSON")
+        resp.headers = {"Link": ""}
+        mock_requests.get.return_value = resp
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Internal Server Error"
+
+        assert result == expected
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient._parse_next_link tests
+# ---------------------------------------------------------------------------
+
+
+class TestParseNextLink:
+    def setup_method(self) -> None:
+        self.client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url="https://canvas.example.com",
+        )
+
+    def test_found(self) -> None:
+        header = '<https://canvas.example.com/api/v1/courses?page=2>; rel="next"'
+
+        result = self.client._parse_next_link(header)
+        expected = "https://canvas.example.com/api/v1/courses?page=2"
+
+        assert result == expected
+
+    def test_not_found(self) -> None:
+        header = '<https://canvas.example.com/api/v1/courses?page=1>; rel="current"'
+
+        result = self.client._parse_next_link(header)
+
+        assert result is None
+
+    def test_empty(self) -> None:
+        result = self.client._parse_next_link("")
+
+        assert result is None
+
+    def test_multiple_rels(self) -> None:
+        header = (
+            '<https://canvas.example.com/api/v1/courses?page=1>; rel="current", '
+            '<https://canvas.example.com/api/v1/courses?page=2>; rel="next"'
+        )
+
+        result = self.client._parse_next_link(header)
+        expected = "https://canvas.example.com/api/v1/courses?page=2"
+
+        assert result == expected
+
+    def test_rejects_host_mismatch(self) -> None:
+        header = '<https://evil.example.com/api/v1/courses?page=2>; rel="next"'
+
+        with pytest.raises(OnyxError, match="unexpected host"):
+            self.client._parse_next_link(header)
+
+    def test_rejects_non_https_link(self) -> None:
+        header = '<http://canvas.example.com/api/v1/courses?page=2>; rel="next"'
+
+        with pytest.raises(OnyxError, match="must use https"):
+            self.client._parse_next_link(header)

backend/tests/unit/onyx/connectors/jira/test_jira_bulk_fetch.py

@@ -0,0 +1,147 @@
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+import requests
+from jira import JIRA
+from jira.resources import Issue
+
+from onyx.connectors.jira.connector import bulk_fetch_issues
+
+
+def _make_raw_issue(issue_id: str) -> dict[str, Any]:
+    return {
+        "id": issue_id,
+        "key": f"TEST-{issue_id}",
+        "fields": {"summary": f"Issue {issue_id}"},
+    }
+
+
+def _mock_jira_client() -> MagicMock:
+    mock = MagicMock(spec=JIRA)
+    mock._options = {"server": "https://jira.example.com"}
+    mock._session = MagicMock()
+    mock._get_url = MagicMock(
+        return_value="https://jira.example.com/rest/api/3/issue/bulkfetch"
+    )
+    return mock
+
+
+def test_bulk_fetch_success() -> None:
+    """Happy path: all issues fetched in one request."""
+    client = _mock_jira_client()
+    raw = [_make_raw_issue("1"), _make_raw_issue("2"), _make_raw_issue("3")]
+    resp = MagicMock()
+    resp.json.return_value = {"issues": raw}
+    client._session.post.return_value = resp
+
+    result = bulk_fetch_issues(client, ["1", "2", "3"])
+    assert len(result) == 3
+    assert all(isinstance(r, Issue) for r in result)
+    client._session.post.assert_called_once()
+
+
+def test_bulk_fetch_splits_on_json_error() -> None:
+    """When the full batch fails with JSONDecodeError, sub-batches succeed."""
+    client = _mock_jira_client()
+
+    call_count = 0
+
+    def _post_side_effect(url: str, json: dict[str, Any]) -> MagicMock:  # noqa: ARG001
+        nonlocal call_count
+        call_count += 1
+        ids = json["issueIdsOrKeys"]
+        if len(ids) > 2:
+            resp = MagicMock()
+            resp.json.side_effect = requests.exceptions.JSONDecodeError(
+                "Expecting ',' delimiter", "doc", 2294125
+            )
+            return resp
+
+        resp = MagicMock()
+        resp.json.return_value = {"issues": [_make_raw_issue(i) for i in ids]}
+        return resp
+
+    client._session.post.side_effect = _post_side_effect
+
+    result = bulk_fetch_issues(client, ["1", "2", "3", "4"])
+    assert len(result) == 4
+    returned_ids = {r.raw["id"] for r in result}
+    assert returned_ids == {"1", "2", "3", "4"}
+    assert call_count > 1
+
+
+def test_bulk_fetch_raises_on_single_unfetchable_issue() -> None:
+    """A single issue that always fails JSON decode raises after splitting."""
+    client = _mock_jira_client()
+
+    def _post_side_effect(url: str, json: dict[str, Any]) -> MagicMock:  # noqa: ARG001
+        ids = json["issueIdsOrKeys"]
+        if "bad" in ids:
+            resp = MagicMock()
+            resp.json.side_effect = requests.exceptions.JSONDecodeError(
+                "Expecting ',' delimiter", "doc", 100
+            )
+            return resp
+
+        resp = MagicMock()
+        resp.json.return_value = {"issues": [_make_raw_issue(i) for i in ids]}
+        return resp
+
+    client._session.post.side_effect = _post_side_effect
+
+    with pytest.raises(requests.exceptions.JSONDecodeError):
+        bulk_fetch_issues(client, ["1", "bad", "2"])
+
+
+def test_bulk_fetch_non_json_error_propagates() -> None:
+    """Non-JSONDecodeError exceptions still propagate."""
+    client = _mock_jira_client()
+
+    resp = MagicMock()
+    resp.json.side_effect = ValueError("something else broke")
+    client._session.post.return_value = resp
+
+    try:
+        bulk_fetch_issues(client, ["1"])
+        assert False, "Expected ValueError to propagate"
+    except ValueError:
+        pass
+
+
+def test_bulk_fetch_with_fields() -> None:
+    """Fields parameter is forwarded correctly."""
+    client = _mock_jira_client()
+    raw = [_make_raw_issue("1")]
+    resp = MagicMock()
+    resp.json.return_value = {"issues": raw}
+    client._session.post.return_value = resp
+
+    bulk_fetch_issues(client, ["1"], fields="summary,description")
+
+    call_payload = client._session.post.call_args[1]["json"]
+    assert call_payload["fields"] == ["summary", "description"]
+
+
+def test_bulk_fetch_recursive_splitting_raises_on_bad_issue() -> None:
+    """With a 6-issue batch where one is bad, recursion isolates it and raises."""
+    client = _mock_jira_client()
+    bad_id = "BAD"
+
+    def _post_side_effect(url: str, json: dict[str, Any]) -> MagicMock:  # noqa: ARG001
+        ids = json["issueIdsOrKeys"]
+        if bad_id in ids:
+            resp = MagicMock()
+            resp.json.side_effect = requests.exceptions.JSONDecodeError(
+                "truncated", "doc", 999
+            )
+            return resp
+
+        resp = MagicMock()
+        resp.json.return_value = {"issues": [_make_raw_issue(i) for i in ids]}
+        return resp
+
+    client._session.post.side_effect = _post_side_effect
+
+    with pytest.raises(requests.exceptions.JSONDecodeError):
+        bulk_fetch_issues(client, ["1", "2", bad_id, "3", "4", "5"])

backend/tests/unit/onyx/connectors/jira/test_jira_permission_sync.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+from datetime import timezone
 from unittest.mock import MagicMock
 from unittest.mock import patch
 
@@ -31,6 +33,7 @@ def mock_jira_cc_pair(
         "jira_base_url": jira_base_url,
         "project_key": project_key,
     }
+    mock_cc_pair.connector.indexing_start = None
 
     return mock_cc_pair
 
@@ -65,3 +68,75 @@ def test_jira_permission_sync(
             fetch_all_existing_docs_ids_fn=mock_fetch_all_existing_docs_ids_fn,
         ):
             print(doc)
+
+
+def test_jira_doc_sync_passes_indexing_start(
+    jira_connector: JiraConnector,
+    mock_jira_cc_pair: MagicMock,
+    mock_fetch_all_existing_docs_fn: MagicMock,
+    mock_fetch_all_existing_docs_ids_fn: MagicMock,
+) -> None:
+    """Verify that generic_doc_sync derives indexing_start from cc_pair
+    and forwards it to retrieve_all_slim_docs_perm_sync."""
+    indexing_start_dt = datetime(2025, 6, 1, tzinfo=timezone.utc)
+    mock_jira_cc_pair.connector.indexing_start = indexing_start_dt
+
+    with patch("onyx.connectors.jira.connector.build_jira_client") as mock_build_client:
+        mock_build_client.return_value = jira_connector._jira_client
+        assert jira_connector._jira_client is not None
+        jira_connector._jira_client._options = MagicMock()
+        jira_connector._jira_client._options.return_value = {
+            "rest_api_version": JIRA_SERVER_API_VERSION
+        }
+
+        with patch.object(
+            type(jira_connector),
+            "retrieve_all_slim_docs_perm_sync",
+            return_value=iter([]),
+        ) as mock_retrieve:
+            list(
+                jira_doc_sync(
+                    cc_pair=mock_jira_cc_pair,
+                    fetch_all_existing_docs_fn=mock_fetch_all_existing_docs_fn,
+                    fetch_all_existing_docs_ids_fn=mock_fetch_all_existing_docs_ids_fn,
+                )
+            )
+
+            mock_retrieve.assert_called_once()
+            call_kwargs = mock_retrieve.call_args
+            assert call_kwargs.kwargs["start"] == indexing_start_dt.timestamp()
+
+
+def test_jira_doc_sync_passes_none_when_no_indexing_start(
+    jira_connector: JiraConnector,
+    mock_jira_cc_pair: MagicMock,
+    mock_fetch_all_existing_docs_fn: MagicMock,
+    mock_fetch_all_existing_docs_ids_fn: MagicMock,
+) -> None:
+    """Verify that indexing_start is None when the connector has no indexing_start set."""
+    mock_jira_cc_pair.connector.indexing_start = None
+
+    with patch("onyx.connectors.jira.connector.build_jira_client") as mock_build_client:
+        mock_build_client.return_value = jira_connector._jira_client
+        assert jira_connector._jira_client is not None
+        jira_connector._jira_client._options = MagicMock()
+        jira_connector._jira_client._options.return_value = {
+            "rest_api_version": JIRA_SERVER_API_VERSION
+        }
+
+        with patch.object(
+            type(jira_connector),
+            "retrieve_all_slim_docs_perm_sync",
+            return_value=iter([]),
+        ) as mock_retrieve:
+            list(
+                jira_doc_sync(
+                    cc_pair=mock_jira_cc_pair,
+                    fetch_all_existing_docs_fn=mock_fetch_all_existing_docs_fn,
+                    fetch_all_existing_docs_ids_fn=mock_fetch_all_existing_docs_ids_fn,
+                )
+            )
+
+            mock_retrieve.assert_called_once()
+            call_kwargs = mock_retrieve.call_args
+            assert call_kwargs.kwargs["start"] is None

backend/tests/unit/onyx/db/test_voice.py

@@ -272,13 +272,13 @@ class TestUpsertVoiceProvider:
 class TestDeleteVoiceProvider:
     """Tests for delete_voice_provider."""
 
-    def test_soft_deletes_provider_when_found(self, mock_db_session: MagicMock) -> None:
+    def test_hard_deletes_provider_when_found(self, mock_db_session: MagicMock) -> None:
         provider = _make_voice_provider(id=1)
         mock_db_session.scalar.return_value = provider
 
         delete_voice_provider(mock_db_session, 1)
 
-        assert provider.deleted is True
+        mock_db_session.delete.assert_called_once_with(provider)
         mock_db_session.flush.assert_called_once()
 
     def test_does_nothing_when_provider_not_found(

backend/tests/unit/onyx/llm/test_multi_llm.py

@@ -11,6 +11,7 @@ from litellm.types.utils import ChatCompletionDeltaToolCall
 from litellm.types.utils import Delta
 from litellm.types.utils import Function as LiteLLMFunction
 
+import onyx.llm.models
 from onyx.configs.app_configs import MOCK_LLM_RESPONSE
 from onyx.llm.constants import LlmProviderNames
 from onyx.llm.interfaces import LLMUserIdentity
@@ -1462,3 +1463,210 @@ def test_no_tool_choice_sent_when_no_tools(default_multi_llm: LitellmLLM) -> Non
         assert (
             "tool_choice" not in kwargs
         ), "tool_choice must not be sent to providers when no tools are provided"
+
+
+def test_bifrost_normalizes_api_base_in_model_kwargs() -> None:
+    llm = LitellmLLM(
+        api_key="test_key",
+        api_base="https://bifrost.example.com/",
+        timeout=30,
+        model_provider=LlmProviderNames.BIFROST,
+        model_name="anthropic/claude-sonnet-4-6",
+        max_input_tokens=32000,
+    )
+
+    assert llm._custom_llm_provider == "openai"
+    assert llm._api_base == "https://bifrost.example.com/v1"
+    assert llm._model_kwargs["api_base"] == "https://bifrost.example.com/v1"
+
+
+def test_prompt_contains_tool_call_history_true() -> None:
+    from onyx.llm.multi_llm import _prompt_contains_tool_call_history
+
+    messages: LanguageModelInput = [
+        UserMessage(content="What's the weather?"),
+        AssistantMessage(
+            content=None,
+            tool_calls=[
+                ToolCall(
+                    id="tc_1",
+                    function=FunctionCall(name="get_weather", arguments="{}"),
+                )
+            ],
+        ),
+    ]
+    assert _prompt_contains_tool_call_history(messages) is True
+
+
+def test_prompt_contains_tool_call_history_false_no_tools() -> None:
+    from onyx.llm.multi_llm import _prompt_contains_tool_call_history
+
+    messages: LanguageModelInput = [
+        UserMessage(content="Hello"),
+        AssistantMessage(content="Hi there!"),
+    ]
+    assert _prompt_contains_tool_call_history(messages) is False
+
+
+def test_prompt_contains_tool_call_history_false_user_only() -> None:
+    from onyx.llm.multi_llm import _prompt_contains_tool_call_history
+
+    messages: LanguageModelInput = [UserMessage(content="Hello")]
+    assert _prompt_contains_tool_call_history(messages) is False
+
+
+def test_bedrock_claude_drops_thinking_when_thinking_blocks_missing() -> None:
+    """When thinking is enabled but assistant messages with tool_calls lack
+    thinking_blocks, the thinking param must be dropped to avoid the Bedrock
+    BadRequestError about missing thinking blocks."""
+    llm = LitellmLLM(
+        api_key=None,
+        timeout=30,
+        model_provider=LlmProviderNames.BEDROCK,
+        model_name="anthropic.claude-sonnet-4-20250514-v1:0",
+        max_input_tokens=200000,
+    )
+
+    messages: LanguageModelInput = [
+        UserMessage(content="What's the weather?"),
+        AssistantMessage(
+            content=None,
+            tool_calls=[
+                ToolCall(
+                    id="tc_1",
+                    function=FunctionCall(
+                        name="get_weather",
+                        arguments='{"city": "Paris"}',
+                    ),
+                )
+            ],
+        ),
+        onyx.llm.models.ToolMessage(
+            content="22°C sunny",
+            tool_call_id="tc_1",
+        ),
+    ]
+
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get the weather",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"city": {"type": "string"}},
+                },
+            },
+        }
+    ]
+
+    with (
+        patch("litellm.completion") as mock_completion,
+        patch("onyx.llm.multi_llm.model_is_reasoning_model", return_value=True),
+    ):
+        mock_completion.return_value = []
+
+        list(llm.stream(messages, tools=tools, reasoning_effort=ReasoningEffort.HIGH))
+
+        kwargs = mock_completion.call_args.kwargs
+        assert "thinking" not in kwargs, (
+            "thinking param should be dropped when thinking_blocks are missing "
+            "from assistant messages with tool_calls"
+        )
+
+
+def test_bedrock_claude_keeps_thinking_when_no_tool_history() -> None:
+    """When thinking is enabled and there are no historical assistant messages
+    with tool_calls, the thinking param should be preserved."""
+    llm = LitellmLLM(
+        api_key=None,
+        timeout=30,
+        model_provider=LlmProviderNames.BEDROCK,
+        model_name="anthropic.claude-sonnet-4-20250514-v1:0",
+        max_input_tokens=200000,
+    )
+
+    messages: LanguageModelInput = [
+        UserMessage(content="What's the weather?"),
+    ]
+
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get the weather",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"city": {"type": "string"}},
+                },
+            },
+        }
+    ]
+
+    with (
+        patch("litellm.completion") as mock_completion,
+        patch("onyx.llm.multi_llm.model_is_reasoning_model", return_value=True),
+    ):
+        mock_completion.return_value = []
+
+        list(llm.stream(messages, tools=tools, reasoning_effort=ReasoningEffort.HIGH))
+
+        kwargs = mock_completion.call_args.kwargs
+        assert "thinking" in kwargs, (
+            "thinking param should be preserved when no assistant messages "
+            "with tool_calls exist in history"
+        )
+        assert kwargs["thinking"]["type"] == "enabled"
+
+
+def test_bifrost_claude_includes_allowed_openai_params() -> None:
+    llm = LitellmLLM(
+        api_key="test_key",
+        api_base="https://bifrost.example.com",
+        timeout=30,
+        model_provider=LlmProviderNames.BIFROST,
+        model_name="anthropic/claude-sonnet-4-6",
+        max_input_tokens=32000,
+    )
+
+    messages: LanguageModelInput = [UserMessage(content="Use a tool if needed")]
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "lookup",
+                "description": "Look up data",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"query": {"type": "string"}},
+                    "required": ["query"],
+                },
+            },
+        }
+    ]
+    mock_stream_chunks = [
+        litellm.ModelResponse(
+            id="chatcmpl-123",
+            choices=[
+                litellm.Choices(
+                    delta=_create_delta(role="assistant", content="Done"),
+                    finish_reason="stop",
+                    index=0,
+                )
+            ],
+            model="anthropic/claude-sonnet-4-6",
+        ),
+    ]
+
+    with patch("litellm.completion") as mock_completion:
+        mock_completion.return_value = mock_stream_chunks
+
+        llm.invoke(messages, tools=tools)
+
+        kwargs = mock_completion.call_args.kwargs
+        assert kwargs["model"] == "anthropic/claude-sonnet-4-6"
+        assert kwargs["base_url"] == "https://bifrost.example.com/v1"
+        assert kwargs["custom_llm_provider"] == "openai"
+        assert kwargs["allowed_openai_params"] == ["tool_choice"]

backend/tests/unit/onyx/server/features/hooks/test_api.py

@@ -3,7 +3,7 @@
 Covers:
 - _check_ssrf_safety: scheme enforcement and private-IP blocklist
 - _validate_endpoint: httpx exception → HookValidateStatus mapping
-  ConnectTimeout     → cannot_connect  (TCP handshake never completed)
+  ConnectTimeout     → timeout         (any timeout directs user to increase timeout_seconds)
   ConnectError       → cannot_connect  (DNS / TLS failure)
   ReadTimeout et al. → timeout         (TCP connected, server slow)
   Any other exc      → cannot_connect
@@ -61,7 +61,7 @@ class TestCheckSsrfSafety:
     def test_non_https_scheme_rejected(self, url: str) -> None:
         with pytest.raises(OnyxError) as exc_info:
             self._call(url)
-        assert exc_info.value.error_code == OnyxErrorCode.INVALID_INPUT
+        assert exc_info.value.error_code == OnyxErrorCode.BAD_GATEWAY
         assert "https" in (exc_info.value.detail or "").lower()
 
     # --- private IP blocklist ---
@@ -87,7 +87,7 @@ class TestCheckSsrfSafety:
         ):
             mock_dns.return_value = [(None, None, None, None, (ip, 0))]
             self._call("https://internal.example.com/hook")
-        assert exc_info.value.error_code == OnyxErrorCode.INVALID_INPUT
+        assert exc_info.value.error_code == OnyxErrorCode.BAD_GATEWAY
         assert ip in (exc_info.value.detail or "")
 
     def test_public_ip_is_allowed(self) -> None:
@@ -106,7 +106,7 @@ class TestCheckSsrfSafety:
             pytest.raises(OnyxError) as exc_info,
         ):
             self._call("https://no-such-host.example.com/hook")
-        assert exc_info.value.error_code == OnyxErrorCode.INVALID_INPUT
+        assert exc_info.value.error_code == OnyxErrorCode.BAD_GATEWAY
 
 
 # ---------------------------------------------------------------------------
@@ -158,13 +158,11 @@ class TestValidateEndpoint:
         assert self._call().status == HookValidateStatus.passed
 
     @patch("onyx.server.features.hooks.api.httpx.Client")
-    def test_connect_timeout_returns_cannot_connect(
-        self, mock_client_cls: MagicMock
-    ) -> None:
+    def test_connect_timeout_returns_timeout(self, mock_client_cls: MagicMock) -> None:
         mock_client_cls.return_value.__enter__.return_value.post.side_effect = (
             httpx.ConnectTimeout("timed out")
         )
-        assert self._call().status == HookValidateStatus.cannot_connect
+        assert self._call().status == HookValidateStatus.timeout
 
     @patch("onyx.server.features.hooks.api.httpx.Client")
     @pytest.mark.parametrize(

backend/tests/unit/onyx/server/manage/llm/test_fetch_models_api.py

@@ -12,6 +12,8 @@ import httpx
 import pytest
 
 from onyx.error_handling.exceptions import OnyxError
+from onyx.server.manage.llm.models import BifrostFinalModelResponse
+from onyx.server.manage.llm.models import BifrostModelsRequest
 from onyx.server.manage.llm.models import LitellmFinalModelResponse
 from onyx.server.manage.llm.models import LitellmModelsRequest
 from onyx.server.manage.llm.models import LMStudioFinalModelResponse
@@ -850,13 +852,15 @@ class TestGetLitellmAvailableModels:
         mock_session = MagicMock()
 
         with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
-            mock_get.side_effect = Exception("Connection refused")
+            mock_get.side_effect = httpx.ConnectError(
+                "Connection refused", request=MagicMock()
+            )
 
             request = LitellmModelsRequest(
                 api_base="http://localhost:4000",
                 api_key="test-key",
             )
-            with pytest.raises(OnyxError, match="Failed to fetch LiteLLM models"):
+            with pytest.raises(OnyxError, match="Failed to fetch LiteLLM proxy models"):
                 get_litellm_available_models(request, MagicMock(), mock_session)
 
     def test_401_raises_authentication_error(self) -> None:
@@ -898,3 +902,113 @@ class TestGetLitellmAvailableModels:
             )
             with pytest.raises(OnyxError, match="endpoint not found"):
                 get_litellm_available_models(request, MagicMock(), mock_session)
+
+
+class TestGetBifrostAvailableModels:
+    """Tests for the Bifrost model fetch endpoint."""
+
+    @pytest.fixture
+    def mock_bifrost_response(self) -> dict:
+        """Mock response from Bifrost /v1/models endpoint."""
+        return {
+            "data": [
+                {
+                    "id": "anthropic/claude-3-5-sonnet",
+                    "name": "Claude 3.5 Sonnet",
+                    "context_length": 200000,
+                },
+                {
+                    "id": "openai/gpt-4o",
+                    "name": "GPT-4o",
+                    "context_length": 128000,
+                },
+                {
+                    "id": "deepseek/deepseek-r1",
+                    "name": "DeepSeek R1",
+                    "context_length": 64000,
+                },
+            ]
+        }
+
+    def test_returns_model_list(self, mock_bifrost_response: dict) -> None:
+        """Test that endpoint returns properly formatted non-embedding models."""
+        from onyx.server.manage.llm.api import get_bifrost_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_bifrost_response
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = BifrostModelsRequest(api_base="https://bifrost.example.com")
+            results = get_bifrost_available_models(request, MagicMock(), mock_session)
+
+            assert len(results) == 3
+            assert all(isinstance(r, BifrostFinalModelResponse) for r in results)
+            assert [r.name for r in results] == sorted(
+                [r.name for r in results], key=str.lower
+            )
+
+    def test_infers_vision_support(self, mock_bifrost_response: dict) -> None:
+        """Test that vision support is inferred from provider/model IDs."""
+        from onyx.server.manage.llm.api import get_bifrost_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_bifrost_response
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = BifrostModelsRequest(api_base="https://bifrost.example.com")
+            results = get_bifrost_available_models(request, MagicMock(), mock_session)
+
+            claude = next(r for r in results if r.name == "anthropic/claude-3-5-sonnet")
+            gpt4o = next(r for r in results if r.name == "openai/gpt-4o")
+            deepseek = next(r for r in results if r.name == "deepseek/deepseek-r1")
+
+            assert claude.supports_image_input is True
+            assert gpt4o.supports_image_input is True
+            assert deepseek.supports_image_input is False
+
+    def test_existing_v1_suffix_is_not_duplicated(self) -> None:
+        """Test that an existing /v1 suffix still hits a single /v1/models endpoint."""
+        from onyx.server.manage.llm.api import get_bifrost_available_models
+
+        mock_session = MagicMock()
+        response = {"data": [{"id": "openai/gpt-4o", "name": "GPT-4o"}]}
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = response
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = BifrostModelsRequest(api_base="https://bifrost.example.com/v1")
+            get_bifrost_available_models(request, MagicMock(), mock_session)
+
+            called_url = mock_get.call_args[0][0]
+            assert called_url == "https://bifrost.example.com/v1/models"
+
+    def test_request_failure_is_logged_and_wrapped(self) -> None:
+        """Test that request-layer failures are logged before raising OnyxError."""
+        from onyx.server.manage.llm.api import get_bifrost_available_models
+
+        mock_session = MagicMock()
+
+        with (
+            patch("onyx.server.manage.llm.api.httpx.get") as mock_get,
+            patch("onyx.server.manage.llm.api.logger.warning") as mock_warning,
+        ):
+            mock_get.side_effect = httpx.ConnectError(
+                "Connection refused", request=MagicMock()
+            )
+
+            request = BifrostModelsRequest(api_base="https://bifrost.example.com")
+            with pytest.raises(OnyxError, match="Failed to fetch Bifrost models"):
+                get_bifrost_available_models(request, MagicMock(), mock_session)
+
+            mock_warning.assert_called_once()

backend/tests/unit/onyx/server/manage/llm/test_llm_provider_utils.py

@@ -176,6 +176,14 @@ class TestInferVisionSupport:
         """Test Nova Pro has vision."""
         assert infer_vision_support("amazon.nova-pro-v1") is True
 
+    def test_bifrost_claude_has_vision(self) -> None:
+        """Test Bifrost Claude models are recognized as vision-capable."""
+        assert infer_vision_support("anthropic/claude-3-5-sonnet") is True
+
+    def test_bifrost_gpt4o_has_vision(self) -> None:
+        """Test Bifrost GPT-4o models are recognized as vision-capable."""
+        assert infer_vision_support("openai/gpt-4o") is True
+
     def test_mistral_no_vision(self) -> None:
         """Test Mistral doesn't have vision (not in known list)."""
         assert infer_vision_support("mistral.mistral-large") is False

backend/tests/unit/onyx/server/test_projects_file_utils.py

@@ -4,13 +4,23 @@ from unittest.mock import MagicMock
 import pytest
 from fastapi import UploadFile
 
+from onyx.natural_language_processing import utils as nlp_utils
+from onyx.natural_language_processing.utils import BaseTokenizer
+from onyx.natural_language_processing.utils import count_tokens
 from onyx.server.features.projects import projects_file_utils as utils
+from onyx.server.settings.models import Settings
 
 
-class _Tokenizer:
+class _Tokenizer(BaseTokenizer):
     def encode(self, text: str) -> list[int]:
         return [1] * len(text)
 
+    def tokenize(self, text: str) -> list[str]:
+        return list(text)
+
+    def decode(self, _tokens: list[int]) -> str:
+        return ""
+
 
 class _NonSeekableFile(BytesIO):
     def tell(self) -> int:
@@ -29,10 +39,26 @@ def _make_upload_no_size(filename: str, content: bytes) -> UploadFile:
     return UploadFile(filename=filename, file=BytesIO(content), size=None)
 
 
-def _patch_common_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:
+def _make_settings(upload_size_mb: int = 1, token_threshold_k: int = 100) -> Settings:
+    return Settings(
+        user_file_max_upload_size_mb=upload_size_mb,
+        file_token_count_threshold_k=token_threshold_k,
+    )
+
+
+def _patch_common_dependencies(
+    monkeypatch: pytest.MonkeyPatch,
+    upload_size_mb: int = 1,
+    token_threshold_k: int = 100,
+) -> None:
     monkeypatch.setattr(utils, "fetch_default_llm_model", lambda _db: None)
     monkeypatch.setattr(utils, "get_tokenizer", lambda **_kwargs: _Tokenizer())
     monkeypatch.setattr(utils, "is_file_password_protected", lambda **_kwargs: False)
+    monkeypatch.setattr(
+        utils,
+        "load_settings",
+        lambda: _make_settings(upload_size_mb, token_threshold_k),
+    )
 
 
 def test_get_upload_size_bytes_falls_back_to_stream_size() -> None:
@@ -76,9 +102,8 @@ def test_is_upload_too_large_logs_warning_when_size_unknown(
 def test_categorize_uploaded_files_accepts_size_under_limit(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    _patch_common_dependencies(monkeypatch)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_BYTES", 100)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_MB", 1)
+    # upload_size_mb=1 → max_bytes = 1*1024*1024; file size 99 is well under
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1)
     monkeypatch.setattr(utils, "estimate_image_tokens_for_upload", lambda _upload: 10)
 
     upload = _make_upload("small.png", size=99)
@@ -91,9 +116,7 @@ def test_categorize_uploaded_files_accepts_size_under_limit(
 def test_categorize_uploaded_files_uses_seek_fallback_when_upload_size_missing(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    _patch_common_dependencies(monkeypatch)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_BYTES", 100)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_MB", 1)
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1)
     monkeypatch.setattr(utils, "estimate_image_tokens_for_upload", lambda _upload: 10)
 
     upload = _make_upload_no_size("small.png", content=b"x" * 99)
@@ -106,12 +129,11 @@ def test_categorize_uploaded_files_uses_seek_fallback_when_upload_size_missing(
 def test_categorize_uploaded_files_accepts_size_at_limit(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    _patch_common_dependencies(monkeypatch)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_BYTES", 100)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_MB", 1)
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1)
     monkeypatch.setattr(utils, "estimate_image_tokens_for_upload", lambda _upload: 10)
 
-    upload = _make_upload("edge.png", size=100)
+    # 1 MB = 1048576 bytes; file at exactly that boundary should be accepted
+    upload = _make_upload("edge.png", size=1048576)
     result = utils.categorize_uploaded_files([upload], MagicMock())
 
     assert len(result.acceptable) == 1
@@ -121,12 +143,10 @@ def test_categorize_uploaded_files_accepts_size_at_limit(
 def test_categorize_uploaded_files_rejects_size_over_limit_with_reason(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    _patch_common_dependencies(monkeypatch)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_BYTES", 100)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_MB", 1)
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1)
     monkeypatch.setattr(utils, "estimate_image_tokens_for_upload", lambda _upload: 10)
 
-    upload = _make_upload("large.png", size=101)
+    upload = _make_upload("large.png", size=1048577)  # 1 byte over 1 MB
     result = utils.categorize_uploaded_files([upload], MagicMock())
 
     assert len(result.acceptable) == 0
@@ -137,13 +157,11 @@ def test_categorize_uploaded_files_rejects_size_over_limit_with_reason(
 def test_categorize_uploaded_files_mixed_batch_keeps_valid_and_rejects_oversized(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    _patch_common_dependencies(monkeypatch)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_BYTES", 100)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_MB", 1)
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1)
     monkeypatch.setattr(utils, "estimate_image_tokens_for_upload", lambda _upload: 10)
 
     small = _make_upload("small.png", size=50)
-    large = _make_upload("large.png", size=101)
+    large = _make_upload("large.png", size=1048577)
 
     result = utils.categorize_uploaded_files([small, large], MagicMock())
 
@@ -153,15 +171,12 @@ def test_categorize_uploaded_files_mixed_batch_keeps_valid_and_rejects_oversized
     assert result.rejected[0].reason == "Exceeds 1 MB file size limit"
 
 
-def test_categorize_uploaded_files_enforces_size_limit_even_when_threshold_is_skipped(
+def test_categorize_uploaded_files_enforces_size_limit_always(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    _patch_common_dependencies(monkeypatch)
-    monkeypatch.setattr(utils, "SKIP_USERFILE_THRESHOLD", True)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_BYTES", 100)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_MB", 1)
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1)
 
-    upload = _make_upload("oversized.pdf", size=101)
+    upload = _make_upload("oversized.pdf", size=1048577)
     result = utils.categorize_uploaded_files([upload], MagicMock())
 
     assert len(result.acceptable) == 0
@@ -172,14 +187,12 @@ def test_categorize_uploaded_files_enforces_size_limit_even_when_threshold_is_sk
 def test_categorize_uploaded_files_checks_size_before_text_extraction(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    _patch_common_dependencies(monkeypatch)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_BYTES", 100)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_MB", 1)
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1)
 
     extract_mock = MagicMock(return_value="this should not run")
     monkeypatch.setattr(utils, "extract_file_text", extract_mock)
 
-    oversized_doc = _make_upload("oversized.pdf", size=101)
+    oversized_doc = _make_upload("oversized.pdf", size=1048577)
     result = utils.categorize_uploaded_files([oversized_doc], MagicMock())
 
     extract_mock.assert_not_called()
@@ -188,40 +201,219 @@ def test_categorize_uploaded_files_checks_size_before_text_extraction(
     assert result.rejected[0].reason == "Exceeds 1 MB file size limit"
 
 
-def test_categorize_uploaded_files_accepts_python_file(
+def test_categorize_enforces_size_limit_when_upload_size_mb_is_positive(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    _patch_common_dependencies(monkeypatch)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_BYTES", 10_000)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_MB", 1)
+    """A positive upload_size_mb is always enforced."""
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1)
+    monkeypatch.setattr(utils, "estimate_image_tokens_for_upload", lambda _upload: 10)
 
-    py_source = b'def hello():\n    print("world")\n'
-    monkeypatch.setattr(
-        utils, "extract_file_text", lambda **_kwargs: py_source.decode()
-    )
-
-    upload = _make_upload("script.py", size=len(py_source), content=py_source)
-    result = utils.categorize_uploaded_files([upload], MagicMock())
-
-    assert len(result.acceptable) == 1
-    assert result.acceptable[0].filename == "script.py"
-    assert len(result.rejected) == 0
-
-
-def test_categorize_uploaded_files_rejects_binary_file(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    _patch_common_dependencies(monkeypatch)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_BYTES", 10_000)
-    monkeypatch.setattr(utils, "USER_FILE_MAX_UPLOAD_SIZE_MB", 1)
-
-    monkeypatch.setattr(utils, "extract_file_text", lambda **_kwargs: "")
-
-    binary_content = bytes(range(256)) * 4
-    upload = _make_upload("data.bin", size=len(binary_content), content=binary_content)
+    upload = _make_upload("huge.png", size=1048577, content=b"x")
     result = utils.categorize_uploaded_files([upload], MagicMock())
 
     assert len(result.acceptable) == 0
     assert len(result.rejected) == 1
-    assert result.rejected[0].filename == "data.bin"
-    assert "Unsupported file type" in result.rejected[0].reason
+
+
+def test_categorize_enforces_token_limit_when_threshold_k_is_positive(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A positive token_threshold_k is always enforced."""
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1000, token_threshold_k=5)
+    monkeypatch.setattr(utils, "estimate_image_tokens_for_upload", lambda _upload: 6000)
+
+    upload = _make_upload("big_image.png", size=100)
+    result = utils.categorize_uploaded_files([upload], MagicMock())
+
+    assert len(result.acceptable) == 0
+    assert len(result.rejected) == 1
+
+
+def test_categorize_no_token_limit_when_threshold_k_is_zero(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """token_threshold_k=0 means no token limit; high-token files are accepted."""
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1000, token_threshold_k=0)
+    monkeypatch.setattr(
+        utils, "estimate_image_tokens_for_upload", lambda _upload: 999_999
+    )
+
+    upload = _make_upload("huge_image.png", size=100)
+    result = utils.categorize_uploaded_files([upload], MagicMock())
+
+    assert len(result.rejected) == 0
+    assert len(result.acceptable) == 1
+
+
+def test_categorize_both_limits_enforced(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Both positive limits are enforced; file exceeding token limit is rejected."""
+    _patch_common_dependencies(monkeypatch, upload_size_mb=10, token_threshold_k=5)
+    monkeypatch.setattr(utils, "estimate_image_tokens_for_upload", lambda _upload: 6000)
+
+    upload = _make_upload("over_tokens.png", size=100)
+    result = utils.categorize_uploaded_files([upload], MagicMock())
+
+    assert len(result.acceptable) == 0
+    assert len(result.rejected) == 1
+    assert result.rejected[0].reason == "Exceeds 5K token limit"
+
+
+def test_categorize_rejection_reason_contains_dynamic_values(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Rejection reasons reflect the admin-configured limits, not hardcoded values."""
+    _patch_common_dependencies(monkeypatch, upload_size_mb=42, token_threshold_k=7)
+    monkeypatch.setattr(utils, "estimate_image_tokens_for_upload", lambda _upload: 8000)
+
+    # File within size limit but over token limit
+    upload = _make_upload("tokens.png", size=100)
+    result = utils.categorize_uploaded_files([upload], MagicMock())
+
+    assert result.rejected[0].reason == "Exceeds 7K token limit"
+
+    # File over size limit
+    _patch_common_dependencies(monkeypatch, upload_size_mb=42, token_threshold_k=7)
+    oversized = _make_upload("big.png", size=42 * 1024 * 1024 + 1)
+    result2 = utils.categorize_uploaded_files([oversized], MagicMock())
+
+    assert result2.rejected[0].reason == "Exceeds 42 MB file size limit"
+
+
+# --- count_tokens tests ---
+
+
+def test_count_tokens_small_text() -> None:
+    """Small text should be encoded in a single call and return correct count."""
+    tokenizer = _Tokenizer()
+    text = "hello world"
+    assert count_tokens(text, tokenizer) == len(tokenizer.encode(text))
+
+
+def test_count_tokens_chunked_matches_single_call() -> None:
+    """Chunked encoding should produce the same result as single-call for small text."""
+    tokenizer = _Tokenizer()
+    text = "a" * 1000
+    assert count_tokens(text, tokenizer) == len(tokenizer.encode(text))
+
+
+def test_count_tokens_large_text_is_chunked(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Text exceeding _ENCODE_CHUNK_SIZE should be split into multiple encode calls."""
+    monkeypatch.setattr(nlp_utils, "_ENCODE_CHUNK_SIZE", 100)
+    tokenizer = _Tokenizer()
+    text = "a" * 250
+    # _Tokenizer returns 1 token per char, so total should be 250
+    assert count_tokens(text, tokenizer) == 250
+
+
+def test_count_tokens_with_token_limit_exits_early(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """When token_limit is set and exceeded, count_tokens should stop early."""
+    monkeypatch.setattr(nlp_utils, "_ENCODE_CHUNK_SIZE", 100)
+
+    encode_call_count = 0
+    original_tokenizer = _Tokenizer()
+
+    class _CountingTokenizer(BaseTokenizer):
+        def encode(self, text: str) -> list[int]:
+            nonlocal encode_call_count
+            encode_call_count += 1
+            return original_tokenizer.encode(text)
+
+        def tokenize(self, text: str) -> list[str]:
+            return list(text)
+
+        def decode(self, _tokens: list[int]) -> str:
+            return ""
+
+    tokenizer = _CountingTokenizer()
+    # 500 chars → 5 chunks of 100; limit=150 → should stop after 2 chunks
+    text = "a" * 500
+    result = count_tokens(text, tokenizer, token_limit=150)
+
+    assert result == 200  # 2 chunks × 100 tokens each
+    assert encode_call_count == 2, "Should have stopped after 2 chunks"
+
+
+def test_count_tokens_with_token_limit_not_exceeded(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """When token_limit is set but not exceeded, all chunks are encoded."""
+    monkeypatch.setattr(nlp_utils, "_ENCODE_CHUNK_SIZE", 100)
+    tokenizer = _Tokenizer()
+    text = "a" * 250
+    result = count_tokens(text, tokenizer, token_limit=1000)
+    assert result == 250
+
+
+def test_count_tokens_no_limit_encodes_all_chunks(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Without token_limit, all chunks are encoded regardless of count."""
+    monkeypatch.setattr(nlp_utils, "_ENCODE_CHUNK_SIZE", 100)
+    tokenizer = _Tokenizer()
+    text = "a" * 500
+    result = count_tokens(text, tokenizer)
+    assert result == 500
+
+
+# --- early exit via token_limit in categorize tests ---
+
+
+def test_categorize_early_exits_tokenization_for_large_text(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Large text files should be rejected via early-exit tokenization
+    without encoding all chunks."""
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1000, token_threshold_k=1)
+    # token_threshold = 1000; _ENCODE_CHUNK_SIZE = 100 → text of 500 chars = 5 chunks
+    # Should stop after 2nd chunk (200 tokens > 1000? No... need 1 token per char)
+    # With _Tokenizer: 1 token per char. threshold=1000, chunk=100 → need 11 chunks
+    # Let's use a bigger text
+    monkeypatch.setattr(nlp_utils, "_ENCODE_CHUNK_SIZE", 100)
+    large_text = "x" * 5000  # 5000 tokens, threshold 1000
+    monkeypatch.setattr(utils, "extract_file_text", lambda **_kwargs: large_text)
+
+    encode_call_count = 0
+    original_tokenizer = _Tokenizer()
+
+    class _CountingTokenizer(BaseTokenizer):
+        def encode(self, text: str) -> list[int]:
+            nonlocal encode_call_count
+            encode_call_count += 1
+            return original_tokenizer.encode(text)
+
+        def tokenize(self, text: str) -> list[str]:
+            return list(text)
+
+        def decode(self, _tokens: list[int]) -> str:
+            return ""
+
+    monkeypatch.setattr(utils, "get_tokenizer", lambda **_kwargs: _CountingTokenizer())
+
+    upload = _make_upload("big.txt", size=5000, content=large_text.encode())
+    result = utils.categorize_uploaded_files([upload], MagicMock())
+
+    assert len(result.rejected) == 1
+    assert "token limit" in result.rejected[0].reason
+    # 5000 chars / 100 chunk_size = 50 chunks total; should stop well before all 50
+    assert (
+        encode_call_count < 50
+    ), f"Expected early exit but encoded {encode_call_count} chunks out of 50"
+
+
+def test_categorize_text_under_token_limit_accepted(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Text files under the token threshold should be accepted with exact count."""
+    _patch_common_dependencies(monkeypatch, upload_size_mb=1000, token_threshold_k=1)
+    small_text = "x" * 500  # 500 tokens < 1000 threshold
+    monkeypatch.setattr(utils, "extract_file_text", lambda **_kwargs: small_text)
+
+    upload = _make_upload("ok.txt", size=500, content=small_text.encode())
+    result = utils.categorize_uploaded_files([upload], MagicMock())
+
+    assert len(result.acceptable) == 1
+    assert result.acceptable_file_to_token_count["ok.txt"] == 500

backend/tests/unit/onyx/server/test_settings_store.py

@@ -1,12 +1,23 @@
 import pytest
 
+from onyx.configs.app_configs import DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB
 from onyx.key_value_store.interface import KvKeyNotFoundError
 from onyx.server.settings import store as settings_store
+from onyx.server.settings.models import (
+    DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB,
+)
+from onyx.server.settings.models import DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB
+from onyx.server.settings.models import Settings
 
 
 class _FakeKvStore:
+    def __init__(self, data: dict | None = None) -> None:
+        self._data = data
+
     def load(self, _key: str) -> dict:
-        raise KvKeyNotFoundError()
+        if self._data is None:
+            raise KvKeyNotFoundError()
+        return self._data
 
 
 class _FakeCache:
@@ -20,13 +31,140 @@ class _FakeCache:
         self._vals[key] = value.encode("utf-8")
 
 
-def test_load_settings_includes_user_file_max_upload_size_mb(
+def test_load_settings_uses_model_defaults_when_no_stored_value(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
+    """When no settings are stored (vector DB enabled), load_settings() should
+    resolve the default token threshold to 200."""
     monkeypatch.setattr(settings_store, "get_kv_store", lambda: _FakeKvStore())
     monkeypatch.setattr(settings_store, "get_cache_backend", lambda: _FakeCache())
-    monkeypatch.setattr(settings_store, "USER_FILE_MAX_UPLOAD_SIZE_MB", 77)
+    monkeypatch.setattr(settings_store, "DISABLE_VECTOR_DB", False)
 
     settings = settings_store.load_settings()
 
-    assert settings.user_file_max_upload_size_mb == 77
+    assert settings.user_file_max_upload_size_mb == DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB
+    assert (
+        settings.file_token_count_threshold_k
+        == DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB
+    )
+
+
+def test_load_settings_uses_high_token_default_when_vector_db_disabled(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """When vector DB is disabled and no settings are stored, the token
+    threshold should default to 10000 (10M tokens)."""
+    monkeypatch.setattr(settings_store, "get_kv_store", lambda: _FakeKvStore())
+    monkeypatch.setattr(settings_store, "get_cache_backend", lambda: _FakeCache())
+    monkeypatch.setattr(settings_store, "DISABLE_VECTOR_DB", True)
+
+    settings = settings_store.load_settings()
+
+    assert settings.user_file_max_upload_size_mb == DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB
+    assert (
+        settings.file_token_count_threshold_k
+        == DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB
+    )
+
+
+def test_load_settings_preserves_explicit_value_when_vector_db_disabled(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """When vector DB is disabled but admin explicitly set a token threshold,
+    that value should be preserved (not overridden by the 10000 default)."""
+    stored = Settings(file_token_count_threshold_k=500).model_dump()
+    monkeypatch.setattr(settings_store, "get_kv_store", lambda: _FakeKvStore(stored))
+    monkeypatch.setattr(settings_store, "get_cache_backend", lambda: _FakeCache())
+    monkeypatch.setattr(settings_store, "DISABLE_VECTOR_DB", True)
+
+    settings = settings_store.load_settings()
+
+    assert settings.file_token_count_threshold_k == 500
+
+
+def test_load_settings_preserves_zero_token_threshold(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A value of 0 means 'no limit' and should be preserved."""
+    stored = Settings(file_token_count_threshold_k=0).model_dump()
+    monkeypatch.setattr(settings_store, "get_kv_store", lambda: _FakeKvStore(stored))
+    monkeypatch.setattr(settings_store, "get_cache_backend", lambda: _FakeCache())
+    monkeypatch.setattr(settings_store, "DISABLE_VECTOR_DB", True)
+
+    settings = settings_store.load_settings()
+
+    assert settings.file_token_count_threshold_k == 0
+
+
+def test_load_settings_resolves_zero_upload_size_to_default(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A value of 0 should be treated as unset and resolved to the default."""
+    stored = Settings(user_file_max_upload_size_mb=0).model_dump()
+    monkeypatch.setattr(settings_store, "get_kv_store", lambda: _FakeKvStore(stored))
+    monkeypatch.setattr(settings_store, "get_cache_backend", lambda: _FakeCache())
+
+    settings = settings_store.load_settings()
+
+    assert settings.user_file_max_upload_size_mb == DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB
+
+
+def test_load_settings_clamps_upload_size_to_env_max(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """When the stored upload size exceeds MAX_ALLOWED_UPLOAD_SIZE_MB, it should
+    be clamped to the env-configured maximum."""
+    stored = Settings(user_file_max_upload_size_mb=500).model_dump()
+    monkeypatch.setattr(settings_store, "get_kv_store", lambda: _FakeKvStore(stored))
+    monkeypatch.setattr(settings_store, "get_cache_backend", lambda: _FakeCache())
+    monkeypatch.setattr(settings_store, "MAX_ALLOWED_UPLOAD_SIZE_MB", 250)
+
+    settings = settings_store.load_settings()
+
+    assert settings.user_file_max_upload_size_mb == 250
+
+
+def test_load_settings_preserves_upload_size_within_max(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """When the stored upload size is within MAX_ALLOWED_UPLOAD_SIZE_MB, it should
+    be preserved unchanged."""
+    stored = Settings(user_file_max_upload_size_mb=150).model_dump()
+    monkeypatch.setattr(settings_store, "get_kv_store", lambda: _FakeKvStore(stored))
+    monkeypatch.setattr(settings_store, "get_cache_backend", lambda: _FakeCache())
+    monkeypatch.setattr(settings_store, "MAX_ALLOWED_UPLOAD_SIZE_MB", 250)
+
+    settings = settings_store.load_settings()
+
+    assert settings.user_file_max_upload_size_mb == 150
+
+
+def test_load_settings_zero_upload_size_resolves_to_default(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A value of 0 should be treated as unset and resolved to the default,
+    clamped to MAX_ALLOWED_UPLOAD_SIZE_MB."""
+    stored = Settings(user_file_max_upload_size_mb=0).model_dump()
+    monkeypatch.setattr(settings_store, "get_kv_store", lambda: _FakeKvStore(stored))
+    monkeypatch.setattr(settings_store, "get_cache_backend", lambda: _FakeCache())
+    monkeypatch.setattr(settings_store, "MAX_ALLOWED_UPLOAD_SIZE_MB", 100)
+    monkeypatch.setattr(settings_store, "DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB", 100)
+
+    settings = settings_store.load_settings()
+
+    assert settings.user_file_max_upload_size_mb == 100
+
+
+def test_load_settings_default_clamped_to_max(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """When DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB exceeds MAX_ALLOWED_UPLOAD_SIZE_MB,
+    the effective default should be min(DEFAULT, MAX)."""
+    monkeypatch.setattr(settings_store, "get_kv_store", lambda: _FakeKvStore())
+    monkeypatch.setattr(settings_store, "get_cache_backend", lambda: _FakeCache())
+    monkeypatch.setattr(settings_store, "DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB", 100)
+    monkeypatch.setattr(settings_store, "MAX_ALLOWED_UPLOAD_SIZE_MB", 50)
+
+    settings = settings_store.load_settings()
+
+    assert settings.user_file_max_upload_size_mb == 50

backend/tests/unit/server/metrics/test_indexing_pipeline_collectors.py

@@ -1,5 +1,6 @@
 """Tests for indexing pipeline Prometheus collectors."""
 
+from collections.abc import Iterator
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
@@ -13,6 +14,16 @@ from onyx.server.metrics.indexing_pipeline import IndexAttemptCollector
 from onyx.server.metrics.indexing_pipeline import QueueDepthCollector
 
 
+@pytest.fixture(autouse=True)
+def _mock_broker_client() -> Iterator[None]:
+    """Patch celery_get_broker_client for all collector tests."""
+    with patch(
+        "onyx.background.celery.celery_redis.celery_get_broker_client",
+        return_value=MagicMock(),
+    ):
+        yield
+
+
 class TestQueueDepthCollector:
     def test_returns_empty_when_factory_not_set(self) -> None:
         collector = QueueDepthCollector()
@@ -24,8 +35,7 @@ class TestQueueDepthCollector:
 
     def test_collects_queue_depths(self) -> None:
         collector = QueueDepthCollector(cache_ttl=0)
-        mock_redis = MagicMock()
-        collector.set_redis_factory(lambda: mock_redis)
+        collector.set_celery_app(MagicMock())
 
         with (
             patch(
@@ -60,8 +70,8 @@ class TestQueueDepthCollector:
 
     def test_handles_redis_error_gracefully(self) -> None:
         collector = QueueDepthCollector(cache_ttl=0)
-        mock_redis = MagicMock()
-        collector.set_redis_factory(lambda: mock_redis)
+        MagicMock()
+        collector.set_celery_app(MagicMock())
 
         with patch(
             "onyx.server.metrics.indexing_pipeline.celery_get_queue_length",
@@ -74,8 +84,8 @@ class TestQueueDepthCollector:
 
     def test_caching_returns_stale_within_ttl(self) -> None:
         collector = QueueDepthCollector(cache_ttl=60)
-        mock_redis = MagicMock()
-        collector.set_redis_factory(lambda: mock_redis)
+        MagicMock()
+        collector.set_celery_app(MagicMock())
 
         with (
             patch(
@@ -98,31 +108,10 @@ class TestQueueDepthCollector:
 
         assert first is second  # Same object, from cache
 
-    def test_factory_called_each_scrape(self) -> None:
-        """Verify the Redis factory is called on each fresh collect, not cached."""
-        collector = QueueDepthCollector(cache_ttl=0)
-        factory = MagicMock(return_value=MagicMock())
-        collector.set_redis_factory(factory)
-
-        with (
-            patch(
-                "onyx.server.metrics.indexing_pipeline.celery_get_queue_length",
-                return_value=0,
-            ),
-            patch(
-                "onyx.server.metrics.indexing_pipeline.celery_get_unacked_task_ids",
-                return_value=set(),
-            ),
-        ):
-            collector.collect()
-            collector.collect()
-
-        assert factory.call_count == 2
-
     def test_error_returns_stale_cache(self) -> None:
         collector = QueueDepthCollector(cache_ttl=0)
-        mock_redis = MagicMock()
-        collector.set_redis_factory(lambda: mock_redis)
+        MagicMock()
+        collector.set_celery_app(MagicMock())
 
         # First call succeeds
         with (

backend/tests/unit/server/metrics/test_indexing_pipeline_setup.py

@@ -1,96 +1,22 @@
-"""Tests for indexing pipeline setup (Redis factory caching)."""
+"""Tests for indexing pipeline setup."""
 
 from unittest.mock import MagicMock
 
-from onyx.server.metrics.indexing_pipeline_setup import _make_broker_redis_factory
+from onyx.server.metrics.indexing_pipeline import QueueDepthCollector
+from onyx.server.metrics.indexing_pipeline import RedisHealthCollector
 
 
-def _make_mock_app(client: MagicMock) -> MagicMock:
-    """Create a mock Celery app whose broker_connection().channel().client
-    returns the given client."""
-    mock_app = MagicMock()
-    mock_conn = MagicMock()
-    mock_conn.channel.return_value.client = client
+class TestCollectorCeleryAppSetup:
+    def test_queue_depth_collector_uses_celery_app(self) -> None:
+        """QueueDepthCollector.set_celery_app stores the app for broker access."""
+        collector = QueueDepthCollector()
+        mock_app = MagicMock()
+        collector.set_celery_app(mock_app)
+        assert collector._celery_app is mock_app
 
-    mock_app.broker_connection.return_value = mock_conn
-
-    return mock_app
-
-
-class TestMakeBrokerRedisFactory:
-    def test_caches_redis_client_across_calls(self) -> None:
-        """Factory should reuse the same client on subsequent calls."""
-        mock_client = MagicMock()
-        mock_client.ping.return_value = True
-        mock_app = _make_mock_app(mock_client)
-
-        factory = _make_broker_redis_factory(mock_app)
-
-        client1 = factory()
-        client2 = factory()
-
-        assert client1 is client2
-        # broker_connection should only be called once
-        assert mock_app.broker_connection.call_count == 1
-
-    def test_reconnects_when_ping_fails(self) -> None:
-        """Factory should create a new client if ping fails (stale connection)."""
-        mock_client_stale = MagicMock()
-        mock_client_stale.ping.side_effect = ConnectionError("disconnected")
-
-        mock_client_fresh = MagicMock()
-        mock_client_fresh.ping.return_value = True
-
-        mock_app = _make_mock_app(mock_client_stale)
-
-        factory = _make_broker_redis_factory(mock_app)
-
-        # First call — creates and caches
-        client1 = factory()
-        assert client1 is mock_client_stale
-        assert mock_app.broker_connection.call_count == 1
-
-        # Switch to fresh client for next connection
-        mock_conn_fresh = MagicMock()
-        mock_conn_fresh.channel.return_value.client = mock_client_fresh
-        mock_app.broker_connection.return_value = mock_conn_fresh
-
-        # Second call — ping fails on stale, reconnects
-        client2 = factory()
-        assert client2 is mock_client_fresh
-        assert mock_app.broker_connection.call_count == 2
-
-    def test_reconnect_closes_stale_client(self) -> None:
-        """When ping fails, the old client should be closed before reconnecting."""
-        mock_client_stale = MagicMock()
-        mock_client_stale.ping.side_effect = ConnectionError("disconnected")
-
-        mock_client_fresh = MagicMock()
-        mock_client_fresh.ping.return_value = True
-
-        mock_app = _make_mock_app(mock_client_stale)
-
-        factory = _make_broker_redis_factory(mock_app)
-
-        # First call — creates and caches
-        factory()
-
-        # Switch to fresh client
-        mock_conn_fresh = MagicMock()
-        mock_conn_fresh.channel.return_value.client = mock_client_fresh
-        mock_app.broker_connection.return_value = mock_conn_fresh
-
-        # Second call — ping fails, should close stale client
-        factory()
-        mock_client_stale.close.assert_called_once()
-
-    def test_first_call_creates_connection(self) -> None:
-        """First call should always create a new connection."""
-        mock_client = MagicMock()
-        mock_app = _make_mock_app(mock_client)
-
-        factory = _make_broker_redis_factory(mock_app)
-        client = factory()
-
-        assert client is mock_client
-        mock_app.broker_connection.assert_called_once()
+    def test_redis_health_collector_uses_celery_app(self) -> None:
+        """RedisHealthCollector.set_celery_app stores the app for broker access."""
+        collector = RedisHealthCollector()
+        mock_app = MagicMock()
+        collector.set_celery_app(mock_app)
+        assert collector._celery_app is mock_app

backend/tests/unit/server/metrics/test_worker_health.py

@@ -0,0 +1,170 @@
+"""Tests for WorkerHeartbeatMonitor and WorkerHealthCollector."""
+
+import time
+from unittest.mock import MagicMock
+
+from onyx.server.metrics.indexing_pipeline import WorkerHealthCollector
+from onyx.server.metrics.indexing_pipeline import WorkerHeartbeatMonitor
+
+
+class TestWorkerHeartbeatMonitor:
+    def test_heartbeat_registers_worker(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        monitor._on_heartbeat({"hostname": "primary@host1"})
+
+        status = monitor.get_worker_status()
+        assert "primary@host1" in status
+        assert status["primary@host1"] is True
+
+    def test_multiple_workers(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        monitor._on_heartbeat({"hostname": "primary@host1"})
+        monitor._on_heartbeat({"hostname": "docfetching@host1"})
+        monitor._on_heartbeat({"hostname": "monitoring@host1"})
+
+        status = monitor.get_worker_status()
+        assert len(status) == 3
+        assert all(alive for alive in status.values())
+
+    def test_offline_removes_worker(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        monitor._on_heartbeat({"hostname": "primary@host1"})
+        monitor._on_offline({"hostname": "primary@host1"})
+
+        status = monitor.get_worker_status()
+        assert "primary@host1" not in status
+
+    def test_stale_heartbeat_marks_worker_down(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        with monitor._lock:
+            monitor._worker_last_seen["primary@host1"] = (
+                time.monotonic() - monitor._HEARTBEAT_TIMEOUT_SECONDS - 10
+            )
+
+        status = monitor.get_worker_status()
+        assert status["primary@host1"] is False
+
+    def test_very_stale_worker_is_pruned(self) -> None:
+        """Workers dead for 2x the timeout are pruned from the dict."""
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        with monitor._lock:
+            monitor._worker_last_seen["gone@host1"] = (
+                time.monotonic() - monitor._HEARTBEAT_TIMEOUT_SECONDS * 2 - 10
+            )
+
+        status = monitor.get_worker_status()
+        assert "gone@host1" not in status
+        assert monitor.get_worker_status() == {}
+
+    def test_heartbeat_refreshes_stale_worker(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        with monitor._lock:
+            monitor._worker_last_seen["primary@host1"] = (
+                time.monotonic() - monitor._HEARTBEAT_TIMEOUT_SECONDS - 10
+            )
+        assert monitor.get_worker_status()["primary@host1"] is False
+
+        monitor._on_heartbeat({"hostname": "primary@host1"})
+        assert monitor.get_worker_status()["primary@host1"] is True
+
+    def test_ignores_empty_hostname(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        monitor._on_heartbeat({})
+        monitor._on_heartbeat({"hostname": ""})
+        monitor._on_offline({})
+
+        assert monitor.get_worker_status() == {}
+
+    def test_returns_full_hostname_as_key(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        monitor._on_heartbeat({"hostname": "docprocessing@my-long-host.local"})
+
+        status = monitor.get_worker_status()
+        assert "docprocessing@my-long-host.local" in status
+
+    def test_start_is_idempotent(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        # Mock the thread so we don't actually start one
+        mock_thread = MagicMock()
+        mock_thread.is_alive.return_value = True
+        monitor._thread = mock_thread
+        monitor._running = True
+
+        # Second start should be a no-op
+        monitor.start()
+        # Thread constructor should not have been called again
+        assert monitor._thread is mock_thread
+
+    def test_thread_safety(self) -> None:
+        """get_worker_status should not raise even if heartbeats arrive concurrently."""
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        monitor._on_heartbeat({"hostname": "primary@host1"})
+        status = monitor.get_worker_status()
+        monitor._on_heartbeat({"hostname": "primary@host1"})
+        status2 = monitor.get_worker_status()
+        assert status == status2
+
+
+class TestWorkerHealthCollector:
+    def test_returns_empty_when_no_monitor(self) -> None:
+        collector = WorkerHealthCollector(cache_ttl=0)
+        assert collector.collect() == []
+
+    def test_collects_active_workers(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        monitor._on_heartbeat({"hostname": "primary@host1"})
+        monitor._on_heartbeat({"hostname": "docfetching@host1"})
+        monitor._on_heartbeat({"hostname": "monitoring@host1"})
+
+        collector = WorkerHealthCollector(cache_ttl=0)
+        collector.set_monitor(monitor)
+
+        families = collector.collect()
+        assert len(families) == 2
+
+        active = families[0]
+        assert active.name == "onyx_celery_active_worker_count"
+        assert active.samples[0].value == 3
+
+        up = families[1]
+        assert up.name == "onyx_celery_worker_up"
+        assert len(up.samples) == 3
+        # Labels use short names (before @)
+        labels = {s.labels["worker"] for s in up.samples}
+        assert labels == {"primary", "docfetching", "monitoring"}
+        for sample in up.samples:
+            assert sample.value == 1
+
+    def test_reports_dead_worker(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+        monitor._on_heartbeat({"hostname": "primary@host1"})
+        with monitor._lock:
+            monitor._worker_last_seen["monitoring@host1"] = (
+                time.monotonic() - monitor._HEARTBEAT_TIMEOUT_SECONDS - 10
+            )
+
+        collector = WorkerHealthCollector(cache_ttl=0)
+        collector.set_monitor(monitor)
+
+        families = collector.collect()
+        active = families[0]
+        assert active.samples[0].value == 1
+
+        up = families[1]
+        samples_by_name = {s.labels["worker"]: s.value for s in up.samples}
+        assert samples_by_name["primary"] == 1
+        assert samples_by_name["monitoring"] == 0
+
+    def test_empty_monitor_returns_zero(self) -> None:
+        monitor = WorkerHeartbeatMonitor(MagicMock())
+
+        collector = WorkerHealthCollector(cache_ttl=0)
+        collector.set_monitor(monitor)
+
+        families = collector.collect()
+        assert len(families) == 2
+        active = families[0]
+        assert active.samples[0].value == 0
+        up = families[1]
+        assert up.name == "onyx_celery_worker_up"
+        assert len(up.samples) == 0

cli/.gitignore

@@ -1,3 +1,4 @@
 onyx-cli
 cli
 onyx.cli
+__pycache__

cli/README.md

@@ -63,6 +63,31 @@ onyx-cli agents
 onyx-cli agents --json
 ```
 
+### Serve over SSH
+
+```shell
+# Start a public SSH endpoint for the CLI TUI
+onyx-cli serve --host 0.0.0.0 --port 2222
+
+# Connect as a client
+ssh your-host -p 2222
+```
+
+Clients can either:
+- paste an API key at the login prompt, or
+- skip the prompt by sending `ONYX_API_KEY` over SSH:
+
+```shell
+export ONYX_API_KEY=your-key
+ssh -o SendEnv=ONYX_API_KEY your-host -p 2222
+```
+
+Useful hardening flags:
+- `--idle-timeout` (default `15m`)
+- `--max-session-timeout` (default `8h`)
+- `--rate-limit-per-minute` (default `20`)
+- `--rate-limit-burst` (default `40`)
+
 ## Commands
 
 | Command | Description |
@@ -70,6 +95,7 @@ onyx-cli agents --json
 | `chat` | Launch the interactive chat TUI (default) |
 | `ask` | Ask a one-shot question (non-interactive) |
 | `agents` | List available agents |
+| `serve` | Serve the interactive chat TUI over SSH |
 | `configure` | Configure server URL and API key |
 | `validate-config` | Validate configuration and test connection |
 

cli/cmd/root.go

@@ -1,7 +1,17 @@
 // Package cmd implements Cobra CLI commands for the Onyx CLI.
 package cmd
 
-import "github.com/spf13/cobra"
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/onyx-dot-app/onyx/cli/internal/api"
+	"github.com/onyx-dot-app/onyx/cli/internal/config"
+	"github.com/onyx-dot-app/onyx/cli/internal/version"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
 
 // Version and Commit are set via ldflags at build time.
 var (
@@ -16,15 +26,69 @@ func fullVersion() string {
 	return Version
 }
 
+func printVersion(cmd *cobra.Command) {
+	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Client version: %s\n", fullVersion())
+
+	cfg := config.Load()
+	if !cfg.IsConfigured() {
+		_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Server version: unknown (not configured)\n")
+		return
+	}
+
+	client := api.NewClient(cfg)
+	ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second)
+	defer cancel()
+
+	log.Debug("fetching backend version from /api/version")
+	backendVersion, err := client.GetBackendVersion(ctx)
+	if err != nil {
+		log.WithError(err).Debug("could not fetch backend version")
+		_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Server version: unknown (could not reach server)\n")
+		return
+	}
+
+	if backendVersion == "" {
+		_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Server version: unknown (empty response)\n")
+		return
+	}
+
+	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Server version: %s\n", backendVersion)
+
+	min := version.MinServer()
+	if sv, ok := version.Parse(backendVersion); ok && sv.LessThan(min) {
+		log.Warnf("Server version %s is below minimum required %d.%d, please upgrade",
+			backendVersion, min.Major, min.Minor)
+	}
+}
+
 // Execute creates and runs the root command.
 func Execute() error {
+	opts := struct {
+		Debug bool
+	}{}
+
 	rootCmd := &cobra.Command{
-		Use:     "onyx-cli",
-		Short:   "Terminal UI for chatting with Onyx",
-		Long:    "Onyx CLI — a terminal interface for chatting with your Onyx agent.",
-		Version: fullVersion(),
+		Use:   "onyx-cli",
+		Short: "Terminal UI for chatting with Onyx",
+		Long:  "Onyx CLI — a terminal interface for chatting with your Onyx agent.",
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			if opts.Debug {
+				log.SetLevel(log.DebugLevel)
+			} else {
+				log.SetLevel(log.InfoLevel)
+			}
+			log.SetFormatter(&log.TextFormatter{
+				DisableTimestamp: true,
+			})
+		},
 	}
 
+	rootCmd.PersistentFlags().BoolVar(&opts.Debug, "debug", false, "run in debug mode")
+
+	// Custom --version flag instead of Cobra's built-in (which only shows one version string)
+	var showVersion bool
+	rootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "Print client and server version information")
+
 	// Register subcommands
 	chatCmd := newChatCmd()
 	rootCmd.AddCommand(chatCmd)
@@ -32,9 +96,16 @@ func Execute() error {
 	rootCmd.AddCommand(newAgentsCmd())
 	rootCmd.AddCommand(newConfigureCmd())
 	rootCmd.AddCommand(newValidateConfigCmd())
+	rootCmd.AddCommand(newServeCmd())
 
-	// Default command is chat
-	rootCmd.RunE = chatCmd.RunE
+	// Default command is chat, but intercept --version first
+	rootCmd.RunE = func(cmd *cobra.Command, args []string) error {
+		if showVersion {
+			printVersion(cmd)
+			return nil
+		}
+		return chatCmd.RunE(cmd, args)
+	}
 
 	return rootCmd.Execute()
 }

cli/cmd/serve.go

@@ -0,0 +1,450 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/charmbracelet/bubbles/textinput"
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/charmbracelet/log"
+	"github.com/charmbracelet/ssh"
+	"github.com/charmbracelet/wish"
+	"github.com/charmbracelet/wish/activeterm"
+	"github.com/charmbracelet/wish/bubbletea"
+	"github.com/charmbracelet/wish/logging"
+	"github.com/charmbracelet/wish/ratelimiter"
+	"github.com/onyx-dot-app/onyx/cli/internal/api"
+	"github.com/onyx-dot-app/onyx/cli/internal/config"
+	"github.com/onyx-dot-app/onyx/cli/internal/tui"
+	"github.com/spf13/cobra"
+	"golang.org/x/time/rate"
+)
+
+const (
+	defaultServeIdleTimeout        = 15 * time.Minute
+	defaultServeMaxSessionTimeout  = 8 * time.Hour
+	defaultServeRateLimitPerMinute = 20
+	defaultServeRateLimitBurst     = 40
+	defaultServeRateLimitCacheSize = 4096
+	maxAPIKeyLength                = 512
+	apiKeyValidationTimeout        = 15 * time.Second
+	maxAPIKeyRetries               = 5
+)
+
+func sessionEnv(s ssh.Session, key string) string {
+	prefix := key + "="
+	for _, env := range s.Environ() {
+		if strings.HasPrefix(env, prefix) {
+			return env[len(prefix):]
+		}
+	}
+	return ""
+}
+
+func validateAPIKey(serverURL string, apiKey string) error {
+	trimmedKey := strings.TrimSpace(apiKey)
+	if len(trimmedKey) > maxAPIKeyLength {
+		return fmt.Errorf("API key is too long (max %d characters)", maxAPIKeyLength)
+	}
+
+	cfg := config.OnyxCliConfig{
+		ServerURL: serverURL,
+		APIKey:    trimmedKey,
+	}
+	client := api.NewClient(cfg)
+	ctx, cancel := context.WithTimeout(context.Background(), apiKeyValidationTimeout)
+	defer cancel()
+	return client.TestConnection(ctx)
+}
+
+// --- auth prompt (bubbletea model) ---
+
+type authState int
+
+const (
+	authInput authState = iota
+	authValidating
+	authDone
+)
+
+type authValidatedMsg struct {
+	key string
+	err error
+}
+
+type authModel struct {
+	input     textinput.Model
+	serverURL string
+	state     authState
+	apiKey    string // set on successful validation
+	errMsg    string
+	retries   int
+	aborted   bool
+}
+
+func newAuthModel(serverURL, initialErr string) authModel {
+	ti := textinput.New()
+	ti.Prompt = "  API Key: "
+	ti.EchoMode = textinput.EchoPassword
+	ti.EchoCharacter = '•'
+	ti.CharLimit = maxAPIKeyLength
+	ti.Width = 80
+	ti.Focus()
+
+	return authModel{
+		input:     ti,
+		serverURL: serverURL,
+		errMsg:    initialErr,
+	}
+}
+
+func (m authModel) Update(msg tea.Msg) (authModel, tea.Cmd) {
+	switch msg := msg.(type) {
+	case tea.WindowSizeMsg:
+		m.input.Width = max(msg.Width-14, 20) // account for prompt width
+		return m, nil
+	case tea.KeyMsg:
+		switch msg.Type {
+		case tea.KeyCtrlC, tea.KeyCtrlD:
+			m.aborted = true
+			return m, nil
+		default:
+			if m.state == authValidating {
+				return m, nil
+			}
+		}
+		switch msg.Type {
+		case tea.KeyEnter:
+			key := strings.TrimSpace(m.input.Value())
+			if key == "" {
+				m.errMsg = "No key entered."
+				m.retries++
+				if m.retries >= maxAPIKeyRetries {
+					m.errMsg = "Too many failed attempts. Disconnecting."
+					m.aborted = true
+					return m, nil
+				}
+				m.input.SetValue("")
+				return m, nil
+			}
+			m.state = authValidating
+			m.errMsg = ""
+			serverURL := m.serverURL
+			return m, func() tea.Msg {
+				return authValidatedMsg{key: key, err: validateAPIKey(serverURL, key)}
+			}
+		}
+
+	case authValidatedMsg:
+		if msg.err != nil {
+			m.state = authInput
+			m.errMsg = msg.err.Error()
+			m.retries++
+			if m.retries >= maxAPIKeyRetries {
+				m.errMsg = "Too many failed attempts. Disconnecting."
+				m.aborted = true
+				return m, nil
+			}
+			m.input.SetValue("")
+			return m, m.input.Focus()
+		}
+		m.apiKey = msg.key
+		m.state = authDone
+		return m, nil
+	}
+
+	if m.state == authInput {
+		var cmd tea.Cmd
+		m.input, cmd = m.input.Update(msg)
+		return m, cmd
+	}
+	return m, nil
+}
+
+func (m authModel) View() string {
+	settingsURL := strings.TrimRight(m.serverURL, "/") + "/app/settings/accounts-access"
+
+	var b strings.Builder
+	b.WriteString("\n")
+	b.WriteString("  \x1b[1;35mOnyx CLI\x1b[0m\n")
+	b.WriteString("  \x1b[90m" + m.serverURL + "\x1b[0m\n")
+	b.WriteString("\n")
+	b.WriteString("  Generate an API key at:\n")
+	b.WriteString("  \x1b[4;34m" + settingsURL + "\x1b[0m\n")
+	b.WriteString("\n")
+	b.WriteString("  \x1b[90mTip: skip this prompt by passing your key via SSH:\x1b[0m\n")
+	b.WriteString("  \x1b[90m  export ONYX_API_KEY=<key>\x1b[0m\n")
+	b.WriteString("  \x1b[90m  ssh -o SendEnv=ONYX_API_KEY <host> -p <port>\x1b[0m\n")
+	b.WriteString("\n")
+
+	if m.errMsg != "" {
+		b.WriteString("  \x1b[1;31m" + m.errMsg + "\x1b[0m\n\n")
+	}
+
+	switch m.state {
+	case authDone:
+		b.WriteString("  \x1b[32mAuthenticated.\x1b[0m\n")
+	case authValidating:
+		b.WriteString("  \x1b[90mValidating…\x1b[0m\n")
+	default:
+		b.WriteString(m.input.View() + "\n")
+	}
+
+	return b.String()
+}
+
+// --- serve model (wraps auth → TUI in a single bubbletea program) ---
+
+type serveModel struct {
+	auth      authModel
+	tui       tea.Model
+	authed    bool
+	serverCfg config.OnyxCliConfig
+	width     int
+	height    int
+}
+
+func newServeModel(serverCfg config.OnyxCliConfig, initialErr string) serveModel {
+	return serveModel{
+		auth:      newAuthModel(serverCfg.ServerURL, initialErr),
+		serverCfg: serverCfg,
+	}
+}
+
+func (m serveModel) Init() tea.Cmd {
+	return textinput.Blink
+}
+
+func (m serveModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+	if !m.authed {
+		if ws, ok := msg.(tea.WindowSizeMsg); ok {
+			m.width = ws.Width
+			m.height = ws.Height
+		}
+
+		var cmd tea.Cmd
+		m.auth, cmd = m.auth.Update(msg)
+
+		if m.auth.aborted {
+			return m, tea.Quit
+		}
+		if m.auth.apiKey != "" {
+			cfg := config.OnyxCliConfig{
+				ServerURL:      m.serverCfg.ServerURL,
+				APIKey:         m.auth.apiKey,
+				DefaultAgentID: m.serverCfg.DefaultAgentID,
+			}
+			m.tui = tui.NewModel(cfg)
+			m.authed = true
+			w, h := m.width, m.height
+			return m, tea.Batch(
+				tea.EnterAltScreen,
+				tea.EnableMouseCellMotion,
+				m.tui.Init(),
+				func() tea.Msg { return tea.WindowSizeMsg{Width: w, Height: h} },
+			)
+		}
+		return m, cmd
+	}
+
+	var cmd tea.Cmd
+	m.tui, cmd = m.tui.Update(msg)
+	return m, cmd
+}
+
+func (m serveModel) View() string {
+	if !m.authed {
+		return m.auth.View()
+	}
+	return m.tui.View()
+}
+
+// --- serve command ---
+
+func newServeCmd() *cobra.Command {
+	var (
+		host              string
+		port              int
+		keyPath           string
+		idleTimeout       time.Duration
+		maxSessionTimeout time.Duration
+		rateLimitPerMin   int
+		rateLimitBurst    int
+		rateLimitCache    int
+	)
+
+	cmd := &cobra.Command{
+		Use:   "serve",
+		Short: "Serve the Onyx TUI over SSH",
+		Long: `Start an SSH server that presents the interactive Onyx chat TUI to
+connecting clients. Each SSH session gets its own independent TUI instance.
+
+Clients are prompted for their Onyx API key on connect. The key can also be
+provided via the ONYX_API_KEY environment variable to skip the prompt:
+
+  ssh -o SendEnv=ONYX_API_KEY host -p port
+
+The server URL is taken from the server operator's config. The server
+auto-generates an Ed25519 host key on first run if the key file does not
+already exist. The host key path can also be set via the ONYX_SSH_HOST_KEY
+environment variable (the --host-key flag takes precedence).
+
+Example:
+  onyx-cli serve --port 2222
+  ssh localhost -p 2222`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			serverCfg := config.Load()
+			if serverCfg.ServerURL == "" {
+				return fmt.Errorf("server URL is not configured; run 'onyx-cli configure' first")
+			}
+			if !cmd.Flags().Changed("host-key") {
+				if v := os.Getenv(config.EnvSSHHostKey); v != "" {
+					keyPath = v
+				}
+			}
+			if rateLimitPerMin <= 0 {
+				return fmt.Errorf("--rate-limit-per-minute must be > 0")
+			}
+			if rateLimitBurst <= 0 {
+				return fmt.Errorf("--rate-limit-burst must be > 0")
+			}
+			if rateLimitCache <= 0 {
+				return fmt.Errorf("--rate-limit-cache must be > 0")
+			}
+
+			addr := net.JoinHostPort(host, fmt.Sprintf("%d", port))
+			connectionLimiter := ratelimiter.NewRateLimiter(
+				rate.Limit(float64(rateLimitPerMin)/60.0),
+				rateLimitBurst,
+				rateLimitCache,
+			)
+
+			handler := func(s ssh.Session) (tea.Model, []tea.ProgramOption) {
+				apiKey := strings.TrimSpace(sessionEnv(s, config.EnvAPIKey))
+				var envErr string
+
+				if apiKey != "" {
+					if err := validateAPIKey(serverCfg.ServerURL, apiKey); err != nil {
+						envErr = fmt.Sprintf("ONYX_API_KEY from SSH environment is invalid: %s", err.Error())
+						apiKey = ""
+					}
+				}
+
+				if apiKey != "" {
+					// Env key is valid — go straight to the TUI.
+					cfg := config.OnyxCliConfig{
+						ServerURL:      serverCfg.ServerURL,
+						APIKey:         apiKey,
+						DefaultAgentID: serverCfg.DefaultAgentID,
+					}
+					return tui.NewModel(cfg), []tea.ProgramOption{
+						tea.WithAltScreen(),
+						tea.WithMouseCellMotion(),
+					}
+				}
+
+				// No valid env key — show auth prompt, then transition
+				// to the TUI within the same bubbletea program.
+				return newServeModel(serverCfg, envErr), []tea.ProgramOption{
+					tea.WithMouseCellMotion(),
+				}
+			}
+
+			serverOptions := []ssh.Option{
+				wish.WithAddress(addr),
+				wish.WithHostKeyPath(keyPath),
+				wish.WithMiddleware(
+					bubbletea.Middleware(handler),
+					activeterm.Middleware(),
+					ratelimiter.Middleware(connectionLimiter),
+					logging.Middleware(),
+				),
+			}
+			if idleTimeout > 0 {
+				serverOptions = append(serverOptions, wish.WithIdleTimeout(idleTimeout))
+			}
+			if maxSessionTimeout > 0 {
+				serverOptions = append(serverOptions, wish.WithMaxTimeout(maxSessionTimeout))
+			}
+
+			s, err := wish.NewServer(serverOptions...)
+			if err != nil {
+				return fmt.Errorf("could not create SSH server: %w", err)
+			}
+
+			done := make(chan os.Signal, 1)
+			signal.Notify(done, os.Interrupt, syscall.SIGTERM)
+
+			log.Info("Starting Onyx SSH server", "addr", addr)
+			log.Info("Connect with", "cmd", fmt.Sprintf("ssh %s -p %d", host, port))
+
+			errCh := make(chan error, 1)
+			go func() {
+				if err := s.ListenAndServe(); err != nil && !errors.Is(err, ssh.ErrServerClosed) {
+					log.Error("SSH server failed", "error", err)
+					errCh <- err
+				}
+			}()
+
+			var serverErr error
+			select {
+			case <-done:
+			case serverErr = <-errCh:
+			}
+
+			signal.Stop(done)
+			log.Info("Shutting down SSH server")
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			if shutdownErr := s.Shutdown(ctx); shutdownErr != nil {
+				return errors.Join(serverErr, shutdownErr)
+			}
+			return serverErr
+		},
+	}
+
+	cmd.Flags().StringVar(&host, "host", "localhost", "Host address to bind to")
+	cmd.Flags().IntVarP(&port, "port", "p", 2222, "Port to listen on")
+	cmd.Flags().StringVar(&keyPath, "host-key", filepath.Join(config.ConfigDir(), "host_ed25519"),
+		"Path to SSH host key (auto-generated if missing)")
+	cmd.Flags().DurationVar(
+		&idleTimeout,
+		"idle-timeout",
+		defaultServeIdleTimeout,
+		"Disconnect idle clients after this duration (set 0 to disable)",
+	)
+	cmd.Flags().DurationVar(
+		&maxSessionTimeout,
+		"max-session-timeout",
+		defaultServeMaxSessionTimeout,
+		"Maximum lifetime of a client session (set 0 to disable)",
+	)
+	cmd.Flags().IntVar(
+		&rateLimitPerMin,
+		"rate-limit-per-minute",
+		defaultServeRateLimitPerMinute,
+		"Per-IP connection rate limit (new sessions per minute)",
+	)
+	cmd.Flags().IntVar(
+		&rateLimitBurst,
+		"rate-limit-burst",
+		defaultServeRateLimitBurst,
+		"Per-IP burst limit for connection attempts",
+	)
+	cmd.Flags().IntVar(
+		&rateLimitCache,
+		"rate-limit-cache",
+		defaultServeRateLimitCacheSize,
+		"Maximum number of IP limiter entries tracked in memory",
+	)
+
+	return cmd
+}

cli/cmd/validate.go

@@ -1,10 +1,14 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
+	"time"
 
 	"github.com/onyx-dot-app/onyx/cli/internal/api"
 	"github.com/onyx-dot-app/onyx/cli/internal/config"
+	"github.com/onyx-dot-app/onyx/cli/internal/version"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -35,6 +39,25 @@ func newValidateConfigCmd() *cobra.Command {
 			}
 
 			_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Status:  connected and authenticated")
+
+			// Check backend version compatibility
+			vCtx, vCancel := context.WithTimeout(cmd.Context(), 5*time.Second)
+			defer vCancel()
+
+			backendVersion, err := client.GetBackendVersion(vCtx)
+			if err != nil {
+				log.WithError(err).Debug("could not fetch backend version")
+			} else if backendVersion == "" {
+				log.Debug("server returned empty version string")
+			} else {
+				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Version: %s\n", backendVersion)
+				min := version.MinServer()
+				if sv, ok := version.Parse(backendVersion); ok && sv.LessThan(min) {
+					log.Warnf("Server version %s is below minimum required %d.%d, please upgrade",
+						backendVersion, min.Major, min.Minor)
+				}
+			}
+
 			return nil
 		},
 	}

cli/go.mod

@@ -1,45 +1,63 @@
 module github.com/onyx-dot-app/onyx/cli
 
-go 1.26.0
+go 1.26.1
 
 require (
-	github.com/charmbracelet/bubbles v0.20.0
-	github.com/charmbracelet/bubbletea v1.3.4
-	github.com/charmbracelet/glamour v0.8.0
-	github.com/charmbracelet/lipgloss v1.1.0
-	github.com/spf13/cobra v1.9.1
-	golang.org/x/term v0.30.0
-	golang.org/x/text v0.34.0
+	github.com/charmbracelet/bubbles v1.0.0
+	github.com/charmbracelet/bubbletea v1.3.10
+	github.com/charmbracelet/glamour v1.0.0
+	github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834
+	github.com/charmbracelet/log v1.0.0
+	github.com/charmbracelet/ssh v0.0.0-20250826160808-ebfa259c7309
+	github.com/charmbracelet/wish v1.4.7
+	github.com/sirupsen/logrus v1.9.4
+	github.com/spf13/cobra v1.10.2
+	golang.org/x/term v0.41.0
+	golang.org/x/text v0.35.0
+	golang.org/x/time v0.15.0
 )
 
 require (
-	github.com/alecthomas/chroma/v2 v2.14.0 // indirect
+	github.com/alecthomas/chroma/v2 v2.23.1 // indirect
+	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
-	github.com/charmbracelet/x/ansi v0.8.0 // indirect
-	github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
-	github.com/charmbracelet/x/term v0.2.1 // indirect
-	github.com/dlclark/regexp2 v1.11.0 // indirect
+	github.com/charmbracelet/colorprofile v0.4.3 // indirect
+	github.com/charmbracelet/keygen v0.5.4 // indirect
+	github.com/charmbracelet/x/ansi v0.11.6 // indirect
+	github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
+	github.com/charmbracelet/x/conpty v0.2.0 // indirect
+	github.com/charmbracelet/x/exp/slice v0.0.0-20260323091123-df7b1bcffcca // indirect
+	github.com/charmbracelet/x/input v0.3.7 // indirect
+	github.com/charmbracelet/x/term v0.2.2 // indirect
+	github.com/charmbracelet/x/termios v0.1.1 // indirect
+	github.com/charmbracelet/x/windows v0.2.2 // indirect
+	github.com/clipperhouse/displaywidth v0.11.0 // indirect
+	github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
+	github.com/creack/pty v1.1.24 // indirect
+	github.com/dlclark/regexp2 v1.11.5 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
+	github.com/go-logfmt/logfmt v0.6.1 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
-	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/mattn/go-runewidth v0.0.21 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.27 // indirect
 	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
-	github.com/yuin/goldmark v1.7.4 // indirect
-	github.com/yuin/goldmark-emoji v1.0.3 // indirect
-	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.31.0 // indirect
+	github.com/yuin/goldmark v1.8.2 // indirect
+	github.com/yuin/goldmark-emoji v1.0.6 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
+	golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect
+	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
 )

cli/go.sum

@@ -1,55 +1,89 @@
-github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE=
-github.com/alecthomas/assert/v2 v2.7.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
-github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46aU4V9E=
-github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I=
-github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
-github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
+github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/chroma/v2 v2.23.1 h1:nv2AVZdTyClGbVQkIzlDm/rnhk1E9bU9nXwmZ/Vk/iY=
+github.com/alecthomas/chroma/v2 v2.23.1/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o=
+github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
+github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
-github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
-github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
+github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY=
+github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
-github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE=
-github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU=
-github.com/charmbracelet/bubbletea v1.3.4 h1:kCg7B+jSCFPLYRA52SDZjr51kG/fMUEoPoZrkaDHyoI=
-github.com/charmbracelet/bubbletea v1.3.4/go.mod h1:dtcUCyCGEX3g9tosuYiut3MXgY/Jsv9nKVdibKKRRXo=
-github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs=
-github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk=
-github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs=
-github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw=
-github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
-github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
-github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE=
-github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q=
-github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8=
-github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs=
-github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b h1:MnAMdlwSltxJyULnrYbkZpp4k58Co7Tah3ciKhSNo0Q=
-github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
-github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
-github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
+github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc=
+github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E=
+github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
+github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
+github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
+github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
+github.com/charmbracelet/glamour v1.0.0 h1:AWMLOVFHTsysl4WV8T8QgkQ0s/ZNZo7CiE4WKhk8l08=
+github.com/charmbracelet/glamour v1.0.0/go.mod h1:DSdohgOBkMr2ZQNhw4LZxSGpx3SvpeujNoXrQyH2hxo=
+github.com/charmbracelet/keygen v0.5.4 h1:XQYgf6UEaTGgQSSmiPpIQ78WfseNQp4Pz8N/c1OsrdA=
+github.com/charmbracelet/keygen v0.5.4/go.mod h1:t4oBRr41bvK7FaJsAaAQhhkUuHslzFXVjOBwA55CZNM=
+github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE=
+github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA=
+github.com/charmbracelet/log v1.0.0 h1:HVVVMmfOorfj3BA9i8X8UL69Hoz9lI0PYwXfJvOdRc4=
+github.com/charmbracelet/log v1.0.0/go.mod h1:uYgY3SmLpwJWxmlrPwXvzVYujxis1vAKRV/0VQB7yWA=
+github.com/charmbracelet/ssh v0.0.0-20250826160808-ebfa259c7309 h1:dCVbCRRtg9+tsfiTXTp0WupDlHruAXyp+YoxGVofHHc=
+github.com/charmbracelet/ssh v0.0.0-20250826160808-ebfa259c7309/go.mod h1:R9cISUs5kAH4Cq/rguNbSwcR+slE5Dfm8FEs//uoIGE=
+github.com/charmbracelet/wish v1.4.7 h1:O+jdLac3s6GaqkOHHSwezejNK04vl6VjO1A+hl8J8Yc=
+github.com/charmbracelet/wish v1.4.7/go.mod h1:OBZ8vC62JC5cvbxJLh+bIWtG7Ctmct+ewziuUWK+G14=
+github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8=
+github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ=
+github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
+github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
+github.com/charmbracelet/x/conpty v0.2.0 h1:eKtA2hm34qNfgJCDp/M6Dc0gLy7e07YEK4qAdNGOvVY=
+github.com/charmbracelet/x/conpty v0.2.0/go.mod h1:fexgUnVrZgw8scD49f6VSi0Ggj9GWYIrpedRthAwW/8=
+github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ=
+github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
+github.com/charmbracelet/x/exp/slice v0.0.0-20260323091123-df7b1bcffcca h1:QQoyQLgUzojMNWHVHToN6d9qTvT0KWtxUKIRPx/Ox5o=
+github.com/charmbracelet/x/exp/slice v0.0.0-20260323091123-df7b1bcffcca/go.mod h1:vqEfX6xzqW1pKKZUUiFOKg0OQ7bCh54Q2vR/tserrRA=
+github.com/charmbracelet/x/input v0.3.7 h1:UzVbkt1vgM9dBQ+K+uRolBlN6IF2oLchmPKKo/aucXo=
+github.com/charmbracelet/x/input v0.3.7/go.mod h1:ZSS9Cia6Cycf2T6ToKIOxeTBTDwl25AGwArJuGaOBH8=
+github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
+github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
+github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY=
+github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo=
+github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM=
+github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k=
+github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8=
+github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
+github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
+github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
-github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
-github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
+github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
+github.com/go-logfmt/logfmt v0.6.1 h1:4hvbpePJKnIzH1B+8OR/JPbTx37NktoI9LE2QZBBkvE=
+github.com/go-logfmt/logfmt v0.6.1/go.mod h1:EV2pOAQoZaT1ZXZbqDl5hrymndi4SY9ED9/z6CO0XAk=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
 github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
-github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
+github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
 github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
-github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w=
+github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
 github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
 github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
 github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
@@ -60,35 +94,47 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
 github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
-github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
-github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
-github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
-github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
-github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
-github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
-github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4=
-github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U=
-golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
-golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
-golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
-golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
+github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
+github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs=
+github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
+golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 h1:jiDhWWeC7jfWqR9c/uplMOqJ0sbNlNWv0UkzE0vX1MA=
+golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/uscMRjUBKtIxworgEcEi+/n9NQDQ=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
-golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
-golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

cli/hatch_build.py

@@ -34,8 +34,7 @@ class CustomBuildHook(BuildHookInterface):
         # Build the Go binary (always rebuild to ensure correct version injection)
         if not os.path.exists(binary_name):
             print(f"Building Go binary '{binary_name}'...")
-            pkg = "github.com/onyx-dot-app/onyx/cli/cmd"
-            ldflags = f"-X {pkg}.version={tag}" f" -X {pkg}.commit={commit}" " -s -w"
+            ldflags = f"-X main.version={tag} -X main.commit={commit} -s -w"
             subprocess.check_call(  # noqa: S603
                 ["go", "build", f"-ldflags={ldflags}", "-o", binary_name],
             )

cli/internal/api/client.go

@@ -270,6 +270,17 @@ func (c *Client) UploadFile(ctx context.Context, filePath string) (*models.FileD
 	}, nil
 }
 
+// GetBackendVersion fetches the backend version string from /api/version.
+func (c *Client) GetBackendVersion(ctx context.Context) (string, error) {
+	var resp struct {
+		BackendVersion string `json:"backend_version"`
+	}
+	if err := c.doJSON(ctx, "GET", "/api/version", nil, &resp); err != nil {
+		return "", err
+	}
+	return resp.BackendVersion, nil
+}
+
 // StopChatSession sends a stop signal for a streaming session (best-effort).
 func (c *Client) StopChatSession(ctx context.Context, sessionID string) {
 	req, err := c.newRequest(ctx, "POST", "/api/chat/stop-chat-session/"+sessionID, nil)

cli/internal/config/config.go

@@ -9,9 +9,10 @@ import (
 )
 
 const (
-	EnvServerURL    = "ONYX_SERVER_URL"
-	EnvAPIKey = "ONYX_API_KEY"
+	EnvServerURL  = "ONYX_SERVER_URL"
+	EnvAPIKey     = "ONYX_API_KEY"
 	EnvAgentID    = "ONYX_PERSONA_ID"
+	EnvSSHHostKey = "ONYX_SSH_HOST_KEY"
 )
 
 // OnyxCliConfig holds the CLI configuration.
@@ -35,8 +36,8 @@ func (c OnyxCliConfig) IsConfigured() bool {
 	return c.APIKey != ""
 }
 
-// configDir returns ~/.config/onyx-cli
-func configDir() string {
+// ConfigDir returns ~/.config/onyx-cli
+func ConfigDir() string {
 	if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
 		return filepath.Join(xdg, "onyx-cli")
 	}
@@ -49,7 +50,7 @@ func configDir() string {
 
 // ConfigFilePath returns the full path to the config file.
 func ConfigFilePath() string {
-	return filepath.Join(configDir(), "config.json")
+	return filepath.Join(ConfigDir(), "config.json")
 }
 
 // ConfigExists checks if the config file exists on disk.
@@ -87,7 +88,7 @@ func Load() OnyxCliConfig {
 
 // Save writes the config to disk, creating parent directories if needed.
 func Save(cfg OnyxCliConfig) error {
-	dir := configDir()
+	dir := ConfigDir()
 	if err := os.MkdirAll(dir, 0o755); err != nil {
 		return err
 	}

cli/internal/version/version.go

@@ -0,0 +1,58 @@
+// Package version provides semver parsing and compatibility checks.
+package version
+
+import (
+	"strconv"
+	"strings"
+)
+
+// Semver holds parsed semantic version components.
+type Semver struct {
+	Major int
+	Minor int
+	Patch int
+}
+
+// minServer is the minimum backend version required by this CLI.
+var minServer = Semver{Major: 3, Minor: 0, Patch: 0}
+
+// MinServer returns the minimum backend version required by this CLI.
+func MinServer() Semver { return minServer }
+
+// Parse extracts major, minor, patch from a version string like "3.1.2" or "v3.1.2".
+// Returns ok=false if the string is not valid semver.
+func Parse(v string) (Semver, bool) {
+	v = strings.TrimPrefix(v, "v")
+	// Strip any pre-release suffix (e.g. "-beta.1") and build metadata (e.g. "+build.1")
+	if idx := strings.IndexAny(v, "-+"); idx != -1 {
+		v = v[:idx]
+	}
+	parts := strings.SplitN(v, ".", 3)
+	if len(parts) != 3 {
+		return Semver{}, false
+	}
+	major, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return Semver{}, false
+	}
+	minor, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return Semver{}, false
+	}
+	patch, err := strconv.Atoi(parts[2])
+	if err != nil {
+		return Semver{}, false
+	}
+	return Semver{Major: major, Minor: minor, Patch: patch}, true
+}
+
+// LessThan reports whether s is strictly less than other.
+func (s Semver) LessThan(other Semver) bool {
+	if s.Major != other.Major {
+		return s.Major < other.Major
+	}
+	if s.Minor != other.Minor {
+		return s.Minor < other.Minor
+	}
+	return s.Patch < other.Patch
+}

cli/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling", "go-bin~=1.24.11", "manygo"]
+requires = ["hatchling==1.29.0", "go-bin~=1.26.1", "manygo==0.2.0"]
 build-backend = "hatchling.build"
 
 [project]

deployment/data/nginx/app.conf.template

@@ -39,6 +39,22 @@ server {
     # Conditionally include MCP location configuration
     include /etc/nginx/conf.d/mcp.conf.inc;
 
+    location ~ ^/scim(/.*)?$ {
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_set_header X-Forwarded-Port $server_port;
+        proxy_set_header Host $host;
+        proxy_http_version 1.1;
+        proxy_buffering off;
+        proxy_redirect off;
+        proxy_connect_timeout ${NGINX_PROXY_CONNECT_TIMEOUT}s;
+        proxy_send_timeout ${NGINX_PROXY_SEND_TIMEOUT}s;
+        proxy_read_timeout ${NGINX_PROXY_READ_TIMEOUT}s;
+        proxy_pass http://api_server;
+    }
+
     # Match both /api/* and /openapi.json in a single rule
     location ~ ^/(api|openapi.json)(/.*)?$ {
         # Rewrite /api prefixed matched paths

deployment/data/nginx/app.conf.template.no-letsencrypt

@@ -39,6 +39,20 @@ server {
     # Conditionally include MCP location configuration
     include /etc/nginx/conf.d/mcp.conf.inc;
 
+    location ~ ^/scim(/.*)?$ {
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        # don't trust client-supplied X-Forwarded-* headers — use nginx's own values
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_set_header X-Forwarded-Port $server_port;
+        proxy_set_header Host $host;
+        proxy_http_version 1.1;
+        proxy_buffering off;
+        proxy_redirect off;
+        proxy_pass http://api_server;
+    }
+
     # Match both /api/* and /openapi.json in a single rule
     location ~ ^/(api|openapi.json)(/.*)?$ {
         # Rewrite /api prefixed matched paths

deployment/data/nginx/app.conf.template.prod

@@ -39,6 +39,23 @@ server {
     # Conditionally include MCP location configuration 
     include /etc/nginx/conf.d/mcp.conf.inc;
 
+    location ~ ^/scim(/.*)?$ {
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        # don't trust client-supplied X-Forwarded-* headers — use nginx's own values
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_set_header X-Forwarded-Port $server_port;
+        proxy_set_header Host $host;
+        proxy_http_version 1.1;
+        proxy_buffering off;
+        proxy_redirect off;
+        proxy_connect_timeout ${NGINX_PROXY_CONNECT_TIMEOUT}s;
+        proxy_send_timeout ${NGINX_PROXY_SEND_TIMEOUT}s;
+        proxy_read_timeout ${NGINX_PROXY_READ_TIMEOUT}s;
+        proxy_pass http://api_server;
+    }
+
     # Match both /api/* and /openapi.json in a single rule
     location ~ ^/(api|openapi.json)(/.*)?$ {
         # Rewrite /api prefixed matched paths

deployment/docker_compose/env.prod.template

@@ -66,10 +66,3 @@ DB_READONLY_PASSWORD=password
 # Show extra/uncommon connectors
 # See https://docs.onyx.app/admins/connectors/overview for a full list of connectors
 SHOW_EXTRA_CONNECTORS=False
-
-# User File Upload Configuration
-# Skip the token count threshold check (100,000 tokens) for uploaded files
-# For self-hosted: set to true to skip for all users
-#SKIP_USERFILE_THRESHOLD=false
-# For multi-tenant: comma-separated list of tenant IDs to skip threshold
-#SKIP_USERFILE_THRESHOLD_TENANT_IDS=

deployment/docker_compose/env.template

@@ -35,6 +35,10 @@ USER_AUTH_SECRET=""
 
 ## Chat Configuration
 # HARD_DELETE_CHATS=
+# MAX_ALLOWED_UPLOAD_SIZE_MB=250
+# Default per-user upload size limit (MB) when no admin value is set.
+# Automatically clamped to MAX_ALLOWED_UPLOAD_SIZE_MB at runtime.
+# DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB=100
 
 ## Base URL for redirects
 # WEB_DOMAIN=
@@ -42,13 +46,6 @@ USER_AUTH_SECRET=""
 ## Enterprise Features, requires a paid plan and licenses
 ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=false
 
-## User File Upload Configuration
-# Skip the token count threshold check (100,000 tokens) for uploaded files
-# For self-hosted: set to true to skip for all users
-# SKIP_USERFILE_THRESHOLD=false
-# For multi-tenant: comma-separated list of tenant IDs to skip threshold
-# SKIP_USERFILE_THRESHOLD_TENANT_IDS=
-
 
 ################################################################################
 ## SERVICES CONFIGURATIONS

deployment/helm/charts/onyx/Chart.yaml

@@ -5,7 +5,7 @@ home: https://www.onyx.app/
 sources:
   - "https://github.com/onyx-dot-app/onyx"
 type: application
-version: 0.4.36
+version: 0.4.38
 appVersion: latest
 annotations:
   category: Productivity

deployment/helm/charts/onyx/templates/celery-worker-docfetching-metrics-service.yaml

@@ -0,0 +1,26 @@
+{{- /* Metrics port must match the default in metrics_server.py (_DEFAULT_PORTS).
+       Do NOT use PROMETHEUS_METRICS_PORT env var in Helm — each worker needs its own port. */ -}}
+{{- if and .Values.vectorDB.enabled (gt (int .Values.celery_worker_docfetching.replicaCount) 0) }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "onyx.fullname" . }}-celery-worker-docfetching-metrics
+  labels:
+    {{- include "onyx.labels" . | nindent 4 }}
+    {{- if .Values.celery_worker_docfetching.deploymentLabels }}
+    {{- toYaml .Values.celery_worker_docfetching.deploymentLabels | nindent 4 }}
+    {{- end }}
+    metrics: "true"
+spec:
+  type: ClusterIP
+  ports:
+    - port: 9092
+      targetPort: metrics
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "onyx.selectorLabels" . | nindent 4 }}
+    {{- if .Values.celery_worker_docfetching.deploymentLabels }}
+    {{- toYaml .Values.celery_worker_docfetching.deploymentLabels | nindent 4 }}
+    {{- end }}
+{{- end }}

deployment/helm/charts/onyx/templates/celery-worker-docfetching.yaml

@@ -73,6 +73,10 @@ spec:
               "-Q",
               "connector_doc_fetching",
             ]
+          ports:
+            - name: metrics
+              containerPort: 9092
+              protocol: TCP
           resources:
             {{- toYaml .Values.celery_worker_docfetching.resources | nindent 12 }}
           envFrom:

deployment/helm/charts/onyx/templates/celery-worker-docprocessing-metrics-service.yaml

@@ -0,0 +1,26 @@
+{{- /* Metrics port must match the default in metrics_server.py (_DEFAULT_PORTS).
+       Do NOT use PROMETHEUS_METRICS_PORT env var in Helm — each worker needs its own port. */ -}}
+{{- if and .Values.vectorDB.enabled (gt (int .Values.celery_worker_docprocessing.replicaCount) 0) }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "onyx.fullname" . }}-celery-worker-docprocessing-metrics
+  labels:
+    {{- include "onyx.labels" . | nindent 4 }}
+    {{- if .Values.celery_worker_docprocessing.deploymentLabels }}
+    {{- toYaml .Values.celery_worker_docprocessing.deploymentLabels | nindent 4 }}
+    {{- end }}
+    metrics: "true"
+spec:
+  type: ClusterIP
+  ports:
+    - port: 9093
+      targetPort: metrics
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "onyx.selectorLabels" . | nindent 4 }}
+    {{- if .Values.celery_worker_docprocessing.deploymentLabels }}
+    {{- toYaml .Values.celery_worker_docprocessing.deploymentLabels | nindent 4 }}
+    {{- end }}
+{{- end }}

deployment/helm/charts/onyx/templates/celery-worker-docprocessing.yaml

@@ -73,6 +73,10 @@ spec:
               "-Q",
               "docprocessing",
             ]
+          ports:
+            - name: metrics
+              containerPort: 9093
+              protocol: TCP
           resources:
             {{- toYaml .Values.celery_worker_docprocessing.resources | nindent 12 }}
           envFrom: