.

Co-authored-by: Nik <nikolas.garza5@gmail.com>

.github/actions/build-integration-image/action.yml

@@ -54,6 +54,7 @@ runs:
       shell: bash
       env:
         RUNS_ON_ECR_CACHE: ${{ inputs.runs-on-ecr-cache }}
+        INTEGRATION_REPOSITORY: ${{ inputs.runs-on-ecr-cache }}
         TAG: nightly-llm-it-${{ inputs.run-id }}
         CACHE_SUFFIX: ${{ steps.format-branch.outputs.cache-suffix }}
         HEAD_SHA: ${{ inputs.github-sha }}

.github/actions/run-nightly-provider-chat-test/action.yml

@@ -9,7 +9,8 @@ inputs:
     required: true
   provider-api-key:
     description: "API key for NIGHTLY_LLM_API_KEY"
-    required: true
+    required: false
+    default: ""
   strict:
     description: "String true/false for NIGHTLY_LLM_STRICT"
     required: true
@@ -17,6 +18,14 @@ inputs:
     description: "Optional NIGHTLY_LLM_API_BASE"
     required: false
     default: ""
+  api-version:
+    description: "Optional NIGHTLY_LLM_API_VERSION"
+    required: false
+    default: ""
+  deployment-name:
+    description: "Optional NIGHTLY_LLM_DEPLOYMENT_NAME"
+    required: false
+    default: ""
   custom-config-json:
     description: "Optional NIGHTLY_LLM_CUSTOM_CONFIG_JSON"
     required: false
@@ -59,6 +68,7 @@ runs:
         DISABLE_TELEMETRY=true
         INTEGRATION_TESTS_MODE=true
         AUTO_LLM_UPDATE_INTERVAL_SECONDS=10
+        AWS_REGION_NAME=us-west-2
         ONYX_BACKEND_IMAGE=${ECR_CACHE}:nightly-llm-it-backend-${RUN_ID}
         ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:nightly-llm-it-model-server-${RUN_ID}
         EOF2
@@ -82,6 +92,8 @@ runs:
         NIGHTLY_LLM_PROVIDER: ${{ inputs.provider }}
         NIGHTLY_LLM_API_KEY: ${{ inputs.provider-api-key }}
         NIGHTLY_LLM_API_BASE: ${{ inputs.api-base }}
+        NIGHTLY_LLM_API_VERSION: ${{ inputs.api-version }}
+        NIGHTLY_LLM_DEPLOYMENT_NAME: ${{ inputs.deployment-name }}
         NIGHTLY_LLM_CUSTOM_CONFIG_JSON: ${{ inputs.custom-config-json }}
         NIGHTLY_LLM_STRICT: ${{ inputs.strict }}
         RUNS_ON_ECR_CACHE: ${{ inputs.runs-on-ecr-cache }}
@@ -91,11 +103,6 @@ runs:
         max_attempts: 2
         retry_wait_seconds: 10
         command: |
-          if [ -z "${MODELS}" ]; then
-            echo "Input 'models' must be non-empty for provider '${NIGHTLY_LLM_PROVIDER}'."
-            exit 1
-          fi
-
           docker run --rm --network onyx_default \
             --name test-runner \
             -e POSTGRES_HOST=relational_db \
@@ -110,10 +117,13 @@ runs:
             -e REDIS_HOST=cache \
             -e API_SERVER_HOST=api_server \
             -e TEST_WEB_HOSTNAME=test-runner \
+            -e AWS_REGION_NAME=us-west-2 \
             -e NIGHTLY_LLM_PROVIDER="${NIGHTLY_LLM_PROVIDER}" \
             -e NIGHTLY_LLM_MODELS="${MODELS}" \
             -e NIGHTLY_LLM_API_KEY="${NIGHTLY_LLM_API_KEY}" \
             -e NIGHTLY_LLM_API_BASE="${NIGHTLY_LLM_API_BASE}" \
+            -e NIGHTLY_LLM_API_VERSION="${NIGHTLY_LLM_API_VERSION}" \
+            -e NIGHTLY_LLM_DEPLOYMENT_NAME="${NIGHTLY_LLM_DEPLOYMENT_NAME}" \
             -e NIGHTLY_LLM_CUSTOM_CONFIG_JSON="${NIGHTLY_LLM_CUSTOM_CONFIG_JSON}" \
             -e NIGHTLY_LLM_STRICT="${NIGHTLY_LLM_STRICT}" \
             ${RUNS_ON_ECR_CACHE}:nightly-llm-it-${RUN_ID} \

.github/workflows/deployment.yml

@@ -426,8 +426,9 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-amd64,mode=max
@@ -499,8 +500,9 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-arm64,mode=max
@@ -646,8 +648,8 @@ jobs:
             NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-amd64,mode=max
@@ -728,8 +730,8 @@ jobs:
             NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-arm64,mode=max
@@ -862,8 +864,9 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-amd64,mode=max
@@ -934,8 +937,9 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-arm64,mode=max
@@ -1072,8 +1076,8 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             ENABLE_CRAFT=true
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-amd64,mode=max
@@ -1145,8 +1149,8 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             ENABLE_CRAFT=true
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-arm64,mode=max
@@ -1287,8 +1291,9 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-amd64,mode=max
@@ -1366,8 +1371,9 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-arm64,mode=max

.github/workflows/nightly-llm-provider-chat-openai.yml

@@ -1,44 +0,0 @@
-name: Nightly LLM Provider Chat Tests (OpenAI)
-concurrency:
-  group: Nightly-LLM-Provider-Chat-OpenAI-${{ github.workflow }}-${{ github.ref_name }}
-  cancel-in-progress: true
-
-on:
-  schedule:
-    # Runs daily at 10:30 UTC (2:30 AM PST / 3:30 AM PDT)
-    - cron: "30 10 * * *"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  openai-provider-chat-test:
-    uses: ./.github/workflows/reusable-nightly-llm-provider-chat.yml
-    with:
-      provider: openai
-      models: ${{ vars.NIGHTLY_LLM_OPENAI_MODELS }}
-      strict: true
-    secrets:
-      provider_api_key: ${{ secrets.OPENAI_API_KEY }}
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
-
-  notify-slack-on-failure:
-    needs: [openai-provider-chat-test]
-    if: failure() && github.event_name == 'schedule'
-    runs-on: ubuntu-slim
-    timeout-minutes: 5
-    steps:
-      - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Send Slack notification
-        uses: ./.github/actions/slack-notify
-        with:
-          webhook-url: ${{ secrets.SLACK_WEBHOOK }}
-          failed-jobs: openai-provider-chat-test
-          title: "🚨 Scheduled OpenAI Provider Chat Tests failed!"
-          ref-name: ${{ github.ref_name }}

.github/workflows/nightly-llm-provider-chat.yml

@@ -0,0 +1,50 @@
+name: Nightly LLM Provider Chat Tests
+concurrency:
+  group: Nightly-LLM-Provider-Chat-${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+on:
+  schedule:
+    # Runs daily at 10:30 UTC (2:30 AM PST / 3:30 AM PDT)
+    - cron: "30 10 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  provider-chat-test:
+    uses: ./.github/workflows/reusable-nightly-llm-provider-chat.yml
+    secrets: inherit
+    permissions:
+      contents: read
+      id-token: write
+    with:
+      openai_models: ${{ vars.NIGHTLY_LLM_OPENAI_MODELS }}
+      anthropic_models: ${{ vars.NIGHTLY_LLM_ANTHROPIC_MODELS }}
+      bedrock_models: ${{ vars.NIGHTLY_LLM_BEDROCK_MODELS }}
+      vertex_ai_models: ${{ vars.NIGHTLY_LLM_VERTEX_AI_MODELS }}
+      azure_models: ${{ vars.NIGHTLY_LLM_AZURE_MODELS }}
+      azure_api_base: ${{ vars.NIGHTLY_LLM_AZURE_API_BASE }}
+      ollama_models: ${{ vars.NIGHTLY_LLM_OLLAMA_MODELS }}
+      openrouter_models: ${{ vars.NIGHTLY_LLM_OPENROUTER_MODELS }}
+      strict: true
+
+  notify-slack-on-failure:
+    needs: [provider-chat-test]
+    if: failure() && github.event_name == 'schedule'
+    runs-on: ubuntu-slim
+    timeout-minutes: 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Send Slack notification
+        uses: ./.github/actions/slack-notify
+        with:
+          webhook-url: ${{ secrets.SLACK_WEBHOOK }}
+          failed-jobs: provider-chat-test
+          title: "🚨 Scheduled LLM Provider Chat Tests failed!"
+          ref-name: ${{ github.ref_name }}

.github/workflows/pr-external-dependency-unit-tests.yml

@@ -160,7 +160,7 @@ jobs:
           cd deployment/docker_compose
 
           # Get list of running containers
-          containers=$(docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.opensearch.yml ps -q)
+          containers=$(docker compose -f docker-compose.yml -f docker-compose.dev.yml ps -q)
 
           # Collect logs from each container
           for container in $containers; do

.github/workflows/pr-integration-tests.yml

@@ -335,7 +335,6 @@ jobs:
           # TODO(Nik): https://linear.app/onyx-app/issue/ENG-1/update-test-infra-to-use-test-license
           LICENSE_ENFORCEMENT_ENABLED=false
           CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS=0.001
-          USE_LIGHTWEIGHT_BACKGROUND_WORKER=false
           EOF
           fi
 
@@ -471,13 +470,13 @@ jobs:
           path: ${{ github.workspace }}/docker-compose.log
       # ------------------------------------------------------------
 
-  no-vectordb-tests:
+  onyx-lite-tests:
     needs: [build-backend-image, build-integration-image]
     runs-on:
       [
         runs-on,
         runner=4cpu-linux-arm64,
-        "run-id=${{ github.run_id }}-no-vectordb-tests",
+        "run-id=${{ github.run_id }}-onyx-lite-tests",
         "extras=ecr-cache",
       ]
     timeout-minutes: 45
@@ -495,13 +494,12 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
-      - name: Create .env file for no-vectordb Docker Compose
+      - name: Create .env file for Onyx Lite Docker Compose
         env:
           ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
           RUN_ID: ${{ github.run_id }}
         run: |
           cat <<EOF > deployment/docker_compose/.env
-          COMPOSE_PROFILES=s3-filestore
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
           LICENSE_ENFORCEMENT_ENABLED=false
           AUTH_TYPE=basic
@@ -509,28 +507,23 @@ jobs:
           POSTGRES_USE_NULL_POOL=true
           REQUIRE_EMAIL_VERIFICATION=false
           DISABLE_TELEMETRY=true
-          DISABLE_VECTOR_DB=true
           ONYX_BACKEND_IMAGE=${ECR_CACHE}:integration-test-backend-test-${RUN_ID}
           INTEGRATION_TESTS_MODE=true
-          USE_LIGHTWEIGHT_BACKGROUND_WORKER=true
           EOF
 
-      # Start only the services needed for no-vectordb mode (no Vespa, no model servers)
-      - name: Start Docker containers (no-vectordb)
+      # Start only the services needed for Onyx Lite (Postgres + API server)
+      - name: Start Docker containers (onyx-lite)
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.no-vectordb.yml -f docker-compose.dev.yml up \
+          docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml -f docker-compose.dev.yml up \
             relational_db \
-            cache \
-            minio \
             api_server \
-            background \
             -d
-        id: start_docker_no_vectordb
+        id: start_docker_onyx_lite
 
       - name: Wait for services to be ready
         run: |
-          echo "Starting wait-for-service script (no-vectordb)..."
+          echo "Starting wait-for-service script (onyx-lite)..."
           start_time=$(date +%s)
           timeout=300
           while true; do
@@ -552,14 +545,14 @@ jobs:
             sleep 5
           done
 
-      - name: Run No-VectorDB Integration Tests
+      - name: Run Onyx Lite Integration Tests
         uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
         with:
           timeout_minutes: 20
           max_attempts: 3
           retry_wait_seconds: 10
           command: |
-            echo "Running no-vectordb integration tests..."
+            echo "Running onyx-lite integration tests..."
             docker run --rm --network onyx_default \
               --name test-runner \
               -e POSTGRES_HOST=relational_db \
@@ -570,39 +563,38 @@ jobs:
               -e DB_READONLY_PASSWORD=password \
               -e POSTGRES_POOL_PRE_PING=true \
               -e POSTGRES_USE_NULL_POOL=true \
-              -e REDIS_HOST=cache \
               -e API_SERVER_HOST=api_server \
               -e OPENAI_API_KEY=${OPENAI_API_KEY} \
               -e TEST_WEB_HOSTNAME=test-runner \
               ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-${{ github.run_id }} \
               /app/tests/integration/tests/no_vectordb
 
-      - name: Dump API server logs (no-vectordb)
+      - name: Dump API server logs (onyx-lite)
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.no-vectordb.yml -f docker-compose.dev.yml \
-            logs --no-color api_server > $GITHUB_WORKSPACE/api_server_no_vectordb.log || true
+          docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml -f docker-compose.dev.yml \
+            logs --no-color api_server > $GITHUB_WORKSPACE/api_server_onyx_lite.log || true
 
-      - name: Dump all-container logs (no-vectordb)
+      - name: Dump all-container logs (onyx-lite)
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.no-vectordb.yml -f docker-compose.dev.yml \
-            logs --no-color > $GITHUB_WORKSPACE/docker-compose-no-vectordb.log || true
+          docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml -f docker-compose.dev.yml \
+            logs --no-color > $GITHUB_WORKSPACE/docker-compose-onyx-lite.log || true
 
-      - name: Upload logs (no-vectordb)
+      - name: Upload logs (onyx-lite)
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
-          name: docker-all-logs-no-vectordb
-          path: ${{ github.workspace }}/docker-compose-no-vectordb.log
+          name: docker-all-logs-onyx-lite
+          path: ${{ github.workspace }}/docker-compose-onyx-lite.log
 
-      - name: Stop Docker containers (no-vectordb)
+      - name: Stop Docker containers (onyx-lite)
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.no-vectordb.yml -f docker-compose.dev.yml down -v
+          docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml -f docker-compose.dev.yml down -v
 
   multitenant-tests:
     needs:
@@ -744,7 +736,7 @@ jobs:
     # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
     runs-on: ubuntu-slim
     timeout-minutes: 45
-    needs: [integration-tests, no-vectordb-tests, multitenant-tests]
+    needs: [integration-tests, onyx-lite-tests, multitenant-tests]
     if: ${{ always() }}
     steps:
       - name: Check job status

.github/workflows/pr-playwright-tests.yml

@@ -268,10 +268,11 @@ jobs:
           persist-credentials: false
 
       - name: Setup node
+        # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
         uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v4
         with:
           node-version: 22
-          cache: "npm"
+          cache: "npm" # zizmor: ignore[cache-poisoning]
           cache-dependency-path: ./web/package-lock.json
 
       - name: Install node dependencies
@@ -279,6 +280,7 @@ jobs:
         run: npm ci
 
       - name: Cache playwright cache
+        # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
         uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
         with:
           path: ~/.cache/ms-playwright
@@ -590,6 +592,108 @@ jobs:
           name: docker-logs-${{ matrix.project }}-${{ github.run_id }}
           path: ${{ github.workspace }}/docker-compose.log
 
+  playwright-tests-lite:
+    needs: [build-web-image, build-backend-image]
+    name: Playwright Tests (lite)
+    runs-on:
+      - runs-on
+      - runner=4cpu-linux-arm64
+      - "run-id=${{ github.run_id }}-playwright-tests-lite"
+      - "extras=ecr-cache"
+    timeout-minutes: 30
+    steps:
+      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
+
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Setup node
+        # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "npm" # zizmor: ignore[cache-poisoning]
+          cache-dependency-path: ./web/package-lock.json
+
+      - name: Install node dependencies
+        working-directory: ./web
+        run: npm ci
+
+      - name: Cache playwright cache
+        # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
+        uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-npm-${{ hashFiles('web/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-playwright-npm-
+
+      - name: Install playwright browsers
+        working-directory: ./web
+        run: npx playwright install --with-deps
+
+      - name: Create .env file for Docker Compose
+        env:
+          OPENAI_API_KEY_VALUE: ${{ env.OPENAI_API_KEY }}
+          ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          cat <<EOF > deployment/docker_compose/.env
+          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
+          LICENSE_ENFORCEMENT_ENABLED=false
+          AUTH_TYPE=basic
+          INTEGRATION_TESTS_MODE=true
+          GEN_AI_API_KEY=${OPENAI_API_KEY_VALUE}
+          MOCK_LLM_RESPONSE=true
+          REQUIRE_EMAIL_VERIFICATION=false
+          DISABLE_TELEMETRY=true
+          ONYX_BACKEND_IMAGE=${ECR_CACHE}:playwright-test-backend-${RUN_ID}
+          ONYX_WEB_SERVER_IMAGE=${ECR_CACHE}:playwright-test-web-${RUN_ID}
+          EOF
+
+      # needed for pulling external images otherwise, we hit the "Unauthenticated users" limit
+      # https://docs.docker.com/docker-hub/usage/
+      - name: Login to Docker Hub
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Start Docker containers (lite)
+        run: |
+          cd deployment/docker_compose
+          docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml -f docker-compose.dev.yml up -d
+        id: start_docker
+
+      - name: Run Playwright tests (lite)
+        working-directory: ./web
+        run: npx playwright test --project lite
+
+      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        if: always()
+        with:
+          name: playwright-test-results-lite-${{ github.run_id }}
+          path: ./web/output/playwright/
+          retention-days: 30
+
+      - name: Save Docker logs
+        if: success() || failure()
+        env:
+          WORKSPACE: ${{ github.workspace }}
+        run: |
+          cd deployment/docker_compose
+          docker compose logs > docker-compose.log
+          mv docker-compose.log ${WORKSPACE}/docker-compose.log
+
+      - name: Upload logs
+        if: success() || failure()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        with:
+          name: docker-logs-lite-${{ github.run_id }}
+          path: ${{ github.workspace }}/docker-compose.log
+
   # Post a single combined visual regression comment after all matrix jobs finish
   visual-regression-comment:
     needs: [playwright-tests]
@@ -603,7 +707,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Download visual diff summaries
-        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # ratchet:actions/download-artifact@v4
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
         with:
           pattern: screenshot-diff-summary-*
           path: summaries/
@@ -686,7 +790,7 @@ jobs:
     # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
     runs-on: ubuntu-slim
     timeout-minutes: 45
-    needs: [playwright-tests]
+    needs: [playwright-tests, playwright-tests-lite]
     if: ${{ always() }}
     steps:
       - name: Check job status

.github/workflows/pr-python-checks.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
   push:
     tags:
       - "v*.*.*"
@@ -21,7 +21,13 @@ jobs:
     # See https://runs-on.com/runners/linux/
     # Note: Mypy seems quite optimized for x64 compared to arm64.
     # Similarly, mypy is single-threaded and incremental, so 2cpu is sufficient.
-    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}-mypy-check", "extras=s3-cache"]
+    runs-on:
+      [
+        runs-on,
+        runner=2cpu-linux-x64,
+        "run-id=${{ github.run_id }}-mypy-check",
+        "extras=s3-cache",
+      ]
     timeout-minutes: 45
 
     steps:
@@ -52,21 +58,14 @@ jobs:
         if: ${{ vars.DISABLE_MYPY_CACHE != 'true' }}
         uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
         with:
-          path: backend/.mypy_cache
-          key: mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-${{ hashFiles('**/*.py', '**/*.pyi', 'backend/pyproject.toml') }}
+          path: .mypy_cache
+          key: mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-${{ hashFiles('**/*.py', '**/*.pyi', 'pyproject.toml') }}
           restore-keys: |
             mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-
             mypy-${{ runner.os }}-
 
       - name: Run MyPy
-        working-directory: ./backend
         env:
           MYPY_FORCE_COLOR: 1
           TERM: xterm-256color
         run: mypy .
-
-      - name: Run MyPy (tools/)
-        env:
-          MYPY_FORCE_COLOR: 1
-          TERM: xterm-256color
-        run: mypy tools/

.github/workflows/pr-python-connector-tests.yml

@@ -89,6 +89,10 @@ env:
   SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
   SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ vars.SHAREPOINT_CLIENT_DIRECTORY_ID }}
   SHAREPOINT_SITE: ${{ vars.SHAREPOINT_SITE }}
+  PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
+  PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
+  PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
+  PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
 
   # Github
   ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}

.github/workflows/reusable-nightly-llm-provider-chat.yml

@@ -3,64 +3,58 @@ name: Reusable Nightly LLM Provider Chat Tests
 on:
   workflow_call:
     inputs:
-      provider:
-        description: "Provider slug passed to NIGHTLY_LLM_PROVIDER (e.g. openai, anthropic)"
-        required: true
+      openai_models:
+        description: "Comma-separated models for openai"
+        required: false
+        default: ""
         type: string
-      models:
-        description: "Comma-separated model list passed to NIGHTLY_LLM_MODELS"
-        required: true
+      anthropic_models:
+        description: "Comma-separated models for anthropic"
+        required: false
+        default: ""
+        type: string
+      bedrock_models:
+        description: "Comma-separated models for bedrock"
+        required: false
+        default: ""
+        type: string
+      vertex_ai_models:
+        description: "Comma-separated models for vertex_ai"
+        required: false
+        default: ""
+        type: string
+      azure_models:
+        description: "Comma-separated models for azure"
+        required: false
+        default: ""
+        type: string
+      ollama_models:
+        description: "Comma-separated models for ollama_chat"
+        required: false
+        default: ""
+        type: string
+      openrouter_models:
+        description: "Comma-separated models for openrouter"
+        required: false
+        default: ""
+        type: string
+      azure_api_base:
+        description: "API base for azure provider"
+        required: false
+        default: ""
         type: string
       strict:
-        description: "Pass-through value for NIGHTLY_LLM_STRICT"
+        description: "Default NIGHTLY_LLM_STRICT passed to tests"
         required: false
         default: true
         type: boolean
-      api_base:
-        description: "Optional NIGHTLY_LLM_API_BASE override"
-        required: false
-        default: ""
-        type: string
-      custom_config_json:
-        description: "Optional NIGHTLY_LLM_CUSTOM_CONFIG_JSON override"
-        required: false
-        default: ""
-        type: string
-    secrets:
-      provider_api_key:
-        description: "Provider API key passed to NIGHTLY_LLM_API_KEY"
-        required: true
-      DOCKER_USERNAME:
-        required: true
-      DOCKER_TOKEN:
-        required: true
 
 permissions:
   contents: read
-
-env:
-  NIGHTLY_LLM_PROVIDER: ${{ inputs.provider }}
-  NIGHTLY_LLM_MODELS: ${{ inputs.models }}
-  NIGHTLY_LLM_API_KEY: ${{ secrets.provider_api_key }}
-  NIGHTLY_LLM_API_BASE: ${{ inputs.api_base }}
-  NIGHTLY_LLM_CUSTOM_CONFIG_JSON: ${{ inputs.custom_config_json }}
-  NIGHTLY_LLM_STRICT: ${{ inputs.strict && 'true' || 'false' }}
+  id-token: write
 
 jobs:
-  validate-inputs:
-    # NOTE: Keep this cheap and fail before image builds if required inputs are missing.
-    runs-on: ubuntu-slim
-    timeout-minutes: 5
-    steps:
-      - name: Validate required nightly provider inputs
-        run: |
-          if [ -z "${NIGHTLY_LLM_MODELS}" ]; then
-            echo "Input 'models' must be non-empty for provider '${NIGHTLY_LLM_PROVIDER}'."
-            exit 1
-          fi
-
   build-backend-image:
-    needs: [validate-inputs]
     runs-on:
       [
         runs-on,
@@ -69,6 +63,7 @@ jobs:
         "extras=ecr-cache",
       ]
     timeout-minutes: 45
+    environment: ci-protected
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
@@ -77,6 +72,19 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+
+      - name: Get AWS Secrets
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
+        with:
+          secret-ids: |
+            DOCKER_USERNAME, test/docker-username
+            DOCKER_TOKEN, test/docker-token
+
       - name: Build backend image
         uses: ./.github/actions/build-backend-image
         with:
@@ -85,12 +93,11 @@ jobs:
           pr-number: ${{ github.event.pull_request.number }}
           github-sha: ${{ github.sha }}
           run-id: ${{ github.run_id }}
-          docker-username: ${{ secrets.DOCKER_USERNAME }}
-          docker-token: ${{ secrets.DOCKER_TOKEN }}
+          docker-username: ${{ env.DOCKER_USERNAME }}
+          docker-token: ${{ env.DOCKER_TOKEN }}
           docker-no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' && 'true' || 'false' }}
 
   build-model-server-image:
-    needs: [validate-inputs]
     runs-on:
       [
         runs-on,
@@ -99,6 +106,7 @@ jobs:
         "extras=ecr-cache",
       ]
     timeout-minutes: 45
+    environment: ci-protected
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
@@ -107,6 +115,19 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+
+      - name: Get AWS Secrets
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
+        with:
+          secret-ids: |
+            DOCKER_USERNAME, test/docker-username
+            DOCKER_TOKEN, test/docker-token
+
       - name: Build model server image
         uses: ./.github/actions/build-model-server-image
         with:
@@ -115,11 +136,10 @@ jobs:
           pr-number: ${{ github.event.pull_request.number }}
           github-sha: ${{ github.sha }}
           run-id: ${{ github.run_id }}
-          docker-username: ${{ secrets.DOCKER_USERNAME }}
-          docker-token: ${{ secrets.DOCKER_TOKEN }}
+          docker-username: ${{ env.DOCKER_USERNAME }}
+          docker-token: ${{ env.DOCKER_TOKEN }}
 
   build-integration-image:
-    needs: [validate-inputs]
     runs-on:
       [
         runs-on,
@@ -128,6 +148,7 @@ jobs:
         "extras=ecr-cache",
       ]
     timeout-minutes: 45
+    environment: ci-protected
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
@@ -136,6 +157,19 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+
+      - name: Get AWS Secrets
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
+        with:
+          secret-ids: |
+            DOCKER_USERNAME, test/docker-username
+            DOCKER_TOKEN, test/docker-token
+
       - name: Build integration image
         uses: ./.github/actions/build-integration-image
         with:
@@ -144,18 +178,83 @@ jobs:
           pr-number: ${{ github.event.pull_request.number }}
           github-sha: ${{ github.sha }}
           run-id: ${{ github.run_id }}
-          docker-username: ${{ secrets.DOCKER_USERNAME }}
-          docker-token: ${{ secrets.DOCKER_TOKEN }}
+          docker-username: ${{ env.DOCKER_USERNAME }}
+          docker-token: ${{ env.DOCKER_TOKEN }}
 
   provider-chat-test:
     needs:
-      [build-backend-image, build-model-server-image, build-integration-image]
+      [
+        build-backend-image,
+        build-model-server-image,
+        build-integration-image,
+      ]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - provider: openai
+            models: ${{ inputs.openai_models }}
+            api_key_env: OPENAI_API_KEY
+            custom_config_env: ""
+            api_base: ""
+            api_version: ""
+            deployment_name: ""
+            required: true
+          - provider: anthropic
+            models: ${{ inputs.anthropic_models }}
+            api_key_env: ANTHROPIC_API_KEY
+            custom_config_env: ""
+            api_base: ""
+            api_version: ""
+            deployment_name: ""
+            required: true
+          - provider: bedrock
+            models: ${{ inputs.bedrock_models }}
+            api_key_env: BEDROCK_API_KEY
+            custom_config_env: ""
+            api_base: ""
+            api_version: ""
+            deployment_name: ""
+            required: false
+          - provider: vertex_ai
+            models: ${{ inputs.vertex_ai_models }}
+            api_key_env: ""
+            custom_config_env: NIGHTLY_LLM_VERTEX_AI_CUSTOM_CONFIG_JSON
+            api_base: ""
+            api_version: ""
+            deployment_name: ""
+            required: false
+          - provider: azure
+            models: ${{ inputs.azure_models }}
+            api_key_env: AZURE_API_KEY
+            custom_config_env: ""
+            api_base: ${{ inputs.azure_api_base }}
+            api_version: "2025-04-01-preview"
+            deployment_name: ""
+            required: false
+          - provider: ollama_chat
+            models: ${{ inputs.ollama_models }}
+            api_key_env: OLLAMA_API_KEY
+            custom_config_env: ""
+            api_base: "https://ollama.com"
+            api_version: ""
+            deployment_name: ""
+            required: false
+          - provider: openrouter
+            models: ${{ inputs.openrouter_models }}
+            api_key_env: OPENROUTER_API_KEY
+            custom_config_env: ""
+            api_base: "https://openrouter.ai/api/v1"
+            api_version: ""
+            deployment_name: ""
+            required: false
     runs-on:
       - runs-on
       - runner=4cpu-linux-arm64
-      - "run-id=${{ github.run_id }}-nightly-${{ inputs.provider }}-provider-chat-test"
+      - "run-id=${{ github.run_id }}-nightly-${{ matrix.provider }}-provider-chat-test"
       - extras=ecr-cache
     timeout-minutes: 45
+    environment: ci-protected
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
@@ -164,19 +263,43 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+
+      - name: Get AWS Secrets
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
+        with:
+          # Keep JSON values unparsed so vertex custom config is passed as raw JSON.
+          parse-json-secrets: false
+          secret-ids: |
+            DOCKER_USERNAME, test/docker-username
+            DOCKER_TOKEN, test/docker-token
+            OPENAI_API_KEY, test/openai-api-key
+            ANTHROPIC_API_KEY, test/anthropic-api-key
+            BEDROCK_API_KEY, test/bedrock-api-key
+            NIGHTLY_LLM_VERTEX_AI_CUSTOM_CONFIG_JSON, test/nightly-llm-vertex-ai-custom-config-json
+            AZURE_API_KEY, test/azure-api-key
+            OLLAMA_API_KEY, test/ollama-api-key
+            OPENROUTER_API_KEY, test/openrouter-api-key
+
       - name: Run nightly provider chat test
         uses: ./.github/actions/run-nightly-provider-chat-test
         with:
-          provider: ${{ env.NIGHTLY_LLM_PROVIDER }}
-          models: ${{ env.NIGHTLY_LLM_MODELS }}
-          provider-api-key: ${{ secrets.provider_api_key }}
-          strict: ${{ env.NIGHTLY_LLM_STRICT }}
-          api-base: ${{ env.NIGHTLY_LLM_API_BASE }}
-          custom-config-json: ${{ env.NIGHTLY_LLM_CUSTOM_CONFIG_JSON }}
+          provider: ${{ matrix.provider }}
+          models: ${{ matrix.models }}
+          provider-api-key: ${{ matrix.api_key_env && env[matrix.api_key_env] || '' }}
+          strict: ${{ inputs.strict && 'true' || 'false' }}
+          api-base: ${{ matrix.api_base }}
+          api-version: ${{ matrix.api_version }}
+          deployment-name: ${{ matrix.deployment_name }}
+          custom-config-json: ${{ matrix.custom_config_env && env[matrix.custom_config_env] || '' }}
           runs-on-ecr-cache: ${{ env.RUNS_ON_ECR_CACHE }}
           run-id: ${{ github.run_id }}
-          docker-username: ${{ secrets.DOCKER_USERNAME }}
-          docker-token: ${{ secrets.DOCKER_TOKEN }}
+          docker-username: ${{ env.DOCKER_USERNAME }}
+          docker-token: ${{ env.DOCKER_TOKEN }}
 
       - name: Dump API server logs
         if: always()
@@ -194,7 +317,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
-          name: docker-all-logs-nightly-${{ inputs.provider }}-llm-provider
+          name: docker-all-logs-nightly-${{ matrix.provider }}-llm-provider
           path: |
             ${{ github.workspace }}/api_server.log
             ${{ github.workspace }}/docker-compose.log

.vscode/launch.json

@@ -40,19 +40,7 @@
       }
     },
     {
-      "name": "Celery (lightweight mode)",
-      "configurations": [
-        "Celery primary",
-        "Celery background",
-        "Celery beat"
-      ],
-      "presentation": {
-        "group": "1"
-      },
-      "stopAll": true
-    },
-    {
-      "name": "Celery (standard mode)",
+      "name": "Celery",
       "configurations": [
         "Celery primary",
         "Celery light",
@@ -253,35 +241,6 @@
       },
       "consoleTitle": "Celery light Console"
     },
-    {
-      "name": "Celery background",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "celery",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "INFO",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
-      },
-      "args": [
-        "-A",
-        "onyx.background.celery.versioned_apps.background",
-        "worker",
-        "--pool=threads",
-        "--concurrency=20",
-        "--prefetch-multiplier=4",
-        "--loglevel=INFO",
-        "--hostname=background@%n",
-        "-Q",
-        "vespa_metadata_sync,connector_deletion,doc_permissions_upsert,checkpoint_cleanup,index_attempt_cleanup,docprocessing,connector_doc_fetching,connector_pruning,connector_doc_permissions_sync,connector_external_group_sync,csv_generation,kg_processing,monitoring,user_file_processing,user_file_project_sync,user_file_delete,opensearch_migration"
-      ],
-      "presentation": {
-        "group": "2"
-      },
-      "consoleTitle": "Celery background Console"
-    },
     {
       "name": "Celery heavy",
       "type": "debugpy",

AGENTS.md

@@ -86,37 +86,6 @@ Onyx uses Celery for asynchronous task processing with multiple specialized work
      - Monitoring tasks (every 5 minutes)
      - Cleanup tasks (hourly)
 
-#### Worker Deployment Modes
-
-Onyx supports two deployment modes for background workers, controlled by the `USE_LIGHTWEIGHT_BACKGROUND_WORKER` environment variable:
-
-**Lightweight Mode** (default, `USE_LIGHTWEIGHT_BACKGROUND_WORKER=true`):
-
-- Runs a single consolidated `background` worker that handles all background tasks:
-  - Light worker tasks (Vespa operations, permissions sync, deletion)
-  - Document processing (indexing pipeline)
-  - Document fetching (connector data retrieval)
-  - Pruning operations (from `heavy` worker)
-  - Knowledge graph processing (from `kg_processing` worker)
-  - Monitoring tasks (from `monitoring` worker)
-  - User file processing (from `user_file_processing` worker)
-- Lower resource footprint (fewer worker processes)
-- Suitable for smaller deployments or development environments
-- Default concurrency: 20 threads (increased to handle combined workload)
-
-**Standard Mode** (`USE_LIGHTWEIGHT_BACKGROUND_WORKER=false`):
-
-- Runs separate specialized workers as documented above (light, docprocessing, docfetching, heavy, kg_processing, monitoring, user_file_processing)
-- Better isolation and scalability
-- Can scale individual workers independently based on workload
-- Suitable for production deployments with higher load
-
-The deployment mode affects:
-
-- **Backend**: Worker processes spawned by supervisord or dev scripts
-- **Helm**: Which Kubernetes deployments are created
-- **Dev Environment**: Which workers `dev_run_background_jobs.py` spawns
-
 #### Key Features
 
 - **Thread-based Workers**: All workers use thread pools (not processes) for stability
@@ -617,6 +586,45 @@ Keep it high level. You can reference certain files or functions though.
 
 Before writing your plan, make sure to do research. Explore the relevant sections in the codebase.
 
+## Error Handling
+
+**Always raise `OnyxError` from `onyx.error_handling.exceptions` instead of `HTTPException`.
+Never hardcode status codes or use `starlette.status` / `fastapi.status` constants directly.**
+
+A global FastAPI exception handler converts `OnyxError` into a JSON response with the standard
+`{"error_code": "...", "message": "..."}` shape. This eliminates boilerplate and keeps error
+handling consistent across the entire backend.
+
+```python
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
+
+# ✅ Good
+raise OnyxError(OnyxErrorCode.NOT_FOUND, "Session not found")
+
+# ✅ Good — no extra message needed
+raise OnyxError(OnyxErrorCode.UNAUTHENTICATED)
+
+# ✅ Good — upstream service with dynamic status code
+raise OnyxError(OnyxErrorCode.BAD_GATEWAY, detail, status_code_override=upstream_status)
+
+# ❌ Bad — using HTTPException directly
+raise HTTPException(status_code=404, detail="Session not found")
+
+# ❌ Bad — starlette constant
+raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Access denied")
+```
+
+Available error codes are defined in `backend/onyx/error_handling/error_codes.py`. If a new error
+category is needed, add it there first — do not invent ad-hoc codes.
+
+**Upstream service errors:** When forwarding errors from an upstream service where the HTTP
+status code is dynamic (comes from the upstream response), use `status_code_override`:
+
+```python
+raise OnyxError(OnyxErrorCode.BAD_GATEWAY, detail, status_code_override=e.response.status_code)
+```
+
 ## Best Practices
 
 In addition to the other content in this file, best practices for contributing

backend/alembic/versions/2664261bfaab_add_cache_store_table.py

@@ -0,0 +1,37 @@
+"""add cache_store table
+
+Revision ID: 2664261bfaab
+Revises: 4a1e4b1c89d2
+Create Date: 2026-02-27 00:00:00.000000
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "2664261bfaab"
+down_revision = "4a1e4b1c89d2"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "cache_store",
+        sa.Column("key", sa.String(), nullable=False),
+        sa.Column("value", sa.LargeBinary(), nullable=True),
+        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.PrimaryKeyConstraint("key"),
+    )
+    op.create_index(
+        "ix_cache_store_expires",
+        "cache_store",
+        ["expires_at"],
+        postgresql_where=sa.text("expires_at IS NOT NULL"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_cache_store_expires", table_name="cache_store")
+    op.drop_table("cache_store")

backend/alembic/versions/4a1e4b1c89d2_add_indexing_to_userfilestatus.py

@@ -0,0 +1,51 @@
+"""Add INDEXING to UserFileStatus
+
+Revision ID: 4a1e4b1c89d2
+Revises: 6b3b4083c5aa
+Create Date: 2026-02-28 00:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "4a1e4b1c89d2"
+down_revision = "6b3b4083c5aa"
+branch_labels = None
+depends_on = None
+
+TABLE = "user_file"
+COLUMN = "status"
+CONSTRAINT_NAME = "ck_user_file_status"
+
+OLD_VALUES = ("PROCESSING", "COMPLETED", "FAILED", "CANCELED", "DELETING")
+NEW_VALUES = ("PROCESSING", "INDEXING", "COMPLETED", "FAILED", "CANCELED", "DELETING")
+
+
+def _drop_status_check_constraint() -> None:
+    """Drop the existing CHECK constraint on user_file.status.
+
+    The constraint name is auto-generated by SQLAlchemy and unknown,
+    so we look it up via the inspector.
+    """
+    inspector = sa.inspect(op.get_bind())
+    for constraint in inspector.get_check_constraints(TABLE):
+        if COLUMN in constraint.get("sqltext", ""):
+            constraint_name = constraint["name"]
+            if constraint_name is not None:
+                op.drop_constraint(constraint_name, TABLE, type_="check")
+
+
+def upgrade() -> None:
+    _drop_status_check_constraint()
+    in_clause = ", ".join(f"'{v}'" for v in NEW_VALUES)
+    op.create_check_constraint(CONSTRAINT_NAME, TABLE, f"{COLUMN} IN ({in_clause})")
+
+
+def downgrade() -> None:
+    op.execute(
+        f"UPDATE {TABLE} SET {COLUMN} = 'PROCESSING' WHERE {COLUMN} = 'INDEXING'"
+    )
+    op.drop_constraint(CONSTRAINT_NAME, TABLE, type_="check")
+    in_clause = ", ".join(f"'{v}'" for v in OLD_VALUES)
+    op.create_check_constraint(CONSTRAINT_NAME, TABLE, f"{COLUMN} IN ({in_clause})")

backend/alembic/versions/57122d037335_add_python_tool_on_default.py

@@ -0,0 +1,69 @@
+"""add python tool on default
+
+Revision ID: 57122d037335
+Revises: c0c937d5c9e5
+Create Date: 2026-02-27 10:10:40.124925
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "57122d037335"
+down_revision = "c0c937d5c9e5"
+branch_labels = None
+depends_on = None
+
+
+PYTHON_TOOL_NAME = "python"
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+
+    # Look up the PythonTool id
+    result = conn.execute(
+        sa.text("SELECT id FROM tool WHERE name = :name"),
+        {"name": PYTHON_TOOL_NAME},
+    ).fetchone()
+
+    if not result:
+        return
+
+    tool_id = result[0]
+
+    # Attach to the default persona (id=0) if not already attached
+    conn.execute(
+        sa.text(
+            """
+            INSERT INTO persona__tool (persona_id, tool_id)
+            VALUES (0, :tool_id)
+            ON CONFLICT DO NOTHING
+            """
+        ),
+        {"tool_id": tool_id},
+    )
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+
+    result = conn.execute(
+        sa.text("SELECT id FROM tool WHERE name = :name"),
+        {"name": PYTHON_TOOL_NAME},
+    ).fetchone()
+
+    if not result:
+        return
+
+    conn.execute(
+        sa.text(
+            """
+            DELETE FROM persona__tool
+            WHERE persona_id = 0 AND tool_id = :tool_id
+            """
+        ),
+        {"tool_id": result[0]},
+    )

backend/alembic/versions/6b3b4083c5aa_persona_cleanup_and_featured.py

@@ -0,0 +1,112 @@
+"""persona cleanup and featured
+
+Revision ID: 6b3b4083c5aa
+Revises: 57122d037335
+Create Date: 2026-02-26 12:00:00.000000
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "6b3b4083c5aa"
+down_revision = "57122d037335"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Add featured column with nullable=True first
+    op.add_column("persona", sa.Column("featured", sa.Boolean(), nullable=True))
+
+    # Migrate data from is_default_persona to featured
+    op.execute("UPDATE persona SET featured = is_default_persona")
+
+    # Make featured non-nullable with default=False
+    op.alter_column(
+        "persona",
+        "featured",
+        existing_type=sa.Boolean(),
+        nullable=False,
+        server_default=sa.false(),
+    )
+
+    # Drop is_default_persona column
+    op.drop_column("persona", "is_default_persona")
+
+    # Drop unused columns
+    op.drop_column("persona", "num_chunks")
+    op.drop_column("persona", "chunks_above")
+    op.drop_column("persona", "chunks_below")
+    op.drop_column("persona", "llm_relevance_filter")
+    op.drop_column("persona", "llm_filter_extraction")
+    op.drop_column("persona", "recency_bias")
+
+
+def downgrade() -> None:
+    # Add back recency_bias column
+    op.add_column(
+        "persona",
+        sa.Column(
+            "recency_bias",
+            sa.VARCHAR(),
+            nullable=False,
+            server_default="base_decay",
+        ),
+    )
+
+    # Add back llm_filter_extraction column
+    op.add_column(
+        "persona",
+        sa.Column(
+            "llm_filter_extraction",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+
+    # Add back llm_relevance_filter column
+    op.add_column(
+        "persona",
+        sa.Column(
+            "llm_relevance_filter",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+
+    # Add back chunks_below column
+    op.add_column(
+        "persona",
+        sa.Column("chunks_below", sa.Integer(), nullable=False, server_default="0"),
+    )
+
+    # Add back chunks_above column
+    op.add_column(
+        "persona",
+        sa.Column("chunks_above", sa.Integer(), nullable=False, server_default="0"),
+    )
+
+    # Add back num_chunks column
+    op.add_column("persona", sa.Column("num_chunks", sa.Float(), nullable=True))
+
+    # Add back is_default_persona column
+    op.add_column(
+        "persona",
+        sa.Column(
+            "is_default_persona",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+
+    # Migrate data from featured to is_default_persona
+    op.execute("UPDATE persona SET is_default_persona = featured")
+
+    # Drop featured column
+    op.drop_column("persona", "featured")

backend/alembic/versions/a3b8d9e2f1c4_make_scim_external_id_nullable.py

@@ -0,0 +1,34 @@
+"""make scim_user_mapping.external_id nullable
+
+Revision ID: a3b8d9e2f1c4
+Revises: 2664261bfaab
+Create Date: 2026-03-02
+
+"""
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "a3b8d9e2f1c4"
+down_revision = "2664261bfaab"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "scim_user_mapping",
+        "external_id",
+        nullable=True,
+    )
+
+
+def downgrade() -> None:
+    # Delete any rows where external_id is NULL before re-applying NOT NULL
+    op.execute("DELETE FROM scim_user_mapping WHERE external_id IS NULL")
+    op.alter_column(
+        "scim_user_mapping",
+        "external_id",
+        nullable=False,
+    )

backend/alembic/versions/c0c937d5c9e5_llm_provider_deprecate_fields.py

@@ -1,8 +1,8 @@
-"""LLMProvider deprecated fields are nullable
+"""llm provider deprecate fields
 
-Revision ID: 001984c88745
-Revises: 7616121f6e97
-Create Date: 2026-02-01 22:24:34.171100
+Revision ID: c0c937d5c9e5
+Revises: 8ffcc2bcfc11
+Create Date: 2026-02-25 17:35:46.125102
 
 """
 
@@ -11,8 +11,8 @@ import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = "001984c88745"
-down_revision = "7616121f6e97"
+revision = "c0c937d5c9e5"
+down_revision = "8ffcc2bcfc11"
 branch_labels = None
 depends_on = None
 
@@ -26,6 +26,13 @@ def upgrade() -> None:
         nullable=True,
     )
 
+    # Drop unique constraint on is_default_provider (defaults now tracked via LLMModelFlow)
+    op.drop_constraint(
+        "llm_provider_is_default_provider_key",
+        "llm_provider",
+        type_="unique",
+    )
+
     # Remove server_default from is_default_vision_provider (was server_default=false())
     op.alter_column(
         "llm_provider",
@@ -34,8 +41,6 @@ def upgrade() -> None:
         server_default=None,
     )
 
-    # is_default_provider and default_vision_model are already nullable with no server_default
-
 
 def downgrade() -> None:
     # Restore default_model_name to NOT NULL (set empty string for any NULLs first)
@@ -49,6 +54,13 @@ def downgrade() -> None:
         nullable=False,
     )
 
+    # Restore unique constraint on is_default_provider
+    op.create_unique_constraint(
+        "llm_provider_is_default_provider_key",
+        "llm_provider",
+        ["is_default_provider"],
+    )
+
     # Restore server_default for is_default_vision_provider
     op.alter_column(
         "llm_provider",

backend/ee/onyx/background/celery/apps/background.py

@@ -1,15 +0,0 @@
-from onyx.background.celery.apps import app_base
-from onyx.background.celery.apps.background import celery_app
-
-
-celery_app.autodiscover_tasks(
-    app_base.filter_task_modules(
-        [
-            "ee.onyx.background.celery.tasks.doc_permission_syncing",
-            "ee.onyx.background.celery.tasks.external_group_syncing",
-            "ee.onyx.background.celery.tasks.cleanup",
-            "ee.onyx.background.celery.tasks.tenant_provisioning",
-            "ee.onyx.background.celery.tasks.query_history",
-        ]
-    )
-)

backend/ee/onyx/db/license.py

@@ -11,11 +11,10 @@ from ee.onyx.server.license.models import LicenseMetadata
 from ee.onyx.server.license.models import LicensePayload
 from ee.onyx.server.license.models import LicenseSource
 from onyx.auth.schemas import UserRole
+from onyx.cache.factory import get_cache_backend
 from onyx.configs.constants import ANONYMOUS_USER_EMAIL
 from onyx.db.models import License
 from onyx.db.models import User
-from onyx.redis.redis_pool import get_redis_client
-from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.utils.logger import setup_logger
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.contextvars import get_current_tenant_id
@@ -142,7 +141,7 @@ def get_used_seats(tenant_id: str | None = None) -> int:
 
 def get_cached_license_metadata(tenant_id: str | None = None) -> LicenseMetadata | None:
     """
-    Get license metadata from Redis cache.
+    Get license metadata from cache.
 
     Args:
         tenant_id: Tenant ID (for multi-tenant deployments)
@@ -150,38 +149,34 @@ def get_cached_license_metadata(tenant_id: str | None = None) -> LicenseMetadata
     Returns:
         LicenseMetadata if cached, None otherwise
     """
-    tenant = tenant_id or get_current_tenant_id()
-    redis_client = get_redis_replica_client(tenant_id=tenant)
+    cache = get_cache_backend(tenant_id=tenant_id)
+    cached = cache.get(LICENSE_METADATA_KEY)
+    if not cached:
+        return None
 
-    cached = redis_client.get(LICENSE_METADATA_KEY)
-    if cached:
-        try:
-            cached_str: str
-            if isinstance(cached, bytes):
-                cached_str = cached.decode("utf-8")
-            else:
-                cached_str = str(cached)
-            return LicenseMetadata.model_validate_json(cached_str)
-        except Exception as e:
-            logger.warning(f"Failed to parse cached license metadata: {e}")
-            return None
-    return None
+    try:
+        cached_str = (
+            cached.decode("utf-8") if isinstance(cached, bytes) else str(cached)
+        )
+        return LicenseMetadata.model_validate_json(cached_str)
+    except Exception as e:
+        logger.warning(f"Failed to parse cached license metadata: {e}")
+        return None
 
 
 def invalidate_license_cache(tenant_id: str | None = None) -> None:
     """
     Invalidate the license metadata cache (not the license itself).
 
-    This deletes the cached LicenseMetadata from Redis. The actual license
-    in the database is not affected. Redis delete is idempotent - if the
-    key doesn't exist, this is a no-op.
+    Deletes the cached LicenseMetadata. The actual license in the database
+    is not affected. Delete is idempotent — if the key doesn't exist, this
+    is a no-op.
 
     Args:
         tenant_id: Tenant ID (for multi-tenant deployments)
     """
-    tenant = tenant_id or get_current_tenant_id()
-    redis_client = get_redis_client(tenant_id=tenant)
-    redis_client.delete(LICENSE_METADATA_KEY)
+    cache = get_cache_backend(tenant_id=tenant_id)
+    cache.delete(LICENSE_METADATA_KEY)
     logger.info("License cache invalidated")
 
 
@@ -192,7 +187,7 @@ def update_license_cache(
     tenant_id: str | None = None,
 ) -> LicenseMetadata:
     """
-    Update the Redis cache with license metadata.
+    Update the cache with license metadata.
 
     We cache all license statuses (ACTIVE, GRACE_PERIOD, GATED_ACCESS) because:
     1. Frontend needs status to show appropriate UI/banners
@@ -211,7 +206,7 @@ def update_license_cache(
     from ee.onyx.utils.license import get_license_status
 
     tenant = tenant_id or get_current_tenant_id()
-    redis_client = get_redis_client(tenant_id=tenant)
+    cache = get_cache_backend(tenant_id=tenant_id)
 
     used_seats = get_used_seats(tenant)
     status = get_license_status(payload, grace_period_end)
@@ -230,7 +225,7 @@ def update_license_cache(
         stripe_subscription_id=payload.stripe_subscription_id,
     )
 
-    redis_client.set(
+    cache.set(
         LICENSE_METADATA_KEY,
         metadata.model_dump_json(),
         ex=LICENSE_CACHE_TTL_SECONDS,

backend/ee/onyx/db/scim.py

@@ -34,6 +34,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
 
 from ee.onyx.server.scim.filtering import ScimFilter
 from ee.onyx.server.scim.filtering import ScimFilterOperator
+from ee.onyx.server.scim.models import ScimMappingFields
 from onyx.db.dal import DAL
 from onyx.db.models import ScimGroupMapping
 from onyx.db.models import ScimToken
@@ -125,15 +126,26 @@ class ScimDAL(DAL):
 
     def create_user_mapping(
         self,
-        external_id: str,
+        external_id: str | None,
         user_id: UUID,
         scim_username: str | None = None,
+        fields: ScimMappingFields | None = None,
     ) -> ScimUserMapping:
-        """Create a mapping between a SCIM externalId and an Onyx user."""
+        """Create a SCIM mapping for a user.
+
+        ``external_id`` may be ``None`` when the IdP omits it (RFC 7643
+        allows this). The mapping still marks the user as SCIM-managed.
+        """
+        f = fields or ScimMappingFields()
         mapping = ScimUserMapping(
             external_id=external_id,
             user_id=user_id,
             scim_username=scim_username,
+            department=f.department,
+            manager=f.manager,
+            given_name=f.given_name,
+            family_name=f.family_name,
+            scim_emails_json=f.scim_emails_json,
         )
         self._session.add(mapping)
         self._session.flush()
@@ -262,8 +274,13 @@ class ScimDAL(DAL):
         Raises:
             ValueError: If the filter uses an unsupported attribute.
         """
-        query = select(User).where(
-            User.role.notin_([UserRole.SLACK_USER, UserRole.EXT_PERM_USER])
+        # Inner-join with ScimUserMapping so only SCIM-managed users appear.
+        # Pre-existing system accounts (anonymous, admin, etc.) are excluded
+        # unless they were explicitly linked via SCIM provisioning.
+        query = (
+            select(User)
+            .join(ScimUserMapping, ScimUserMapping.user_id == User.id)
+            .where(User.role.notin_([UserRole.SLACK_USER, UserRole.EXT_PERM_USER]))
         )
 
         if scim_filter:
@@ -311,23 +328,39 @@ class ScimDAL(DAL):
         user_id: UUID,
         new_external_id: str | None,
         scim_username: str | None = None,
+        fields: ScimMappingFields | None = None,
     ) -> None:
-        """Create, update, or delete the external ID mapping for a user."""
+        """Sync the SCIM mapping for a user.
+
+        If a mapping already exists, its fields are updated (including
+        setting ``external_id`` to ``None`` when the IdP omits it).
+        If no mapping exists and ``new_external_id`` is provided, a new
+        mapping is created.  A mapping is never deleted here — SCIM-managed
+        users must retain their mapping to remain visible in ``GET /Users``.
+
+        When *fields* is provided, all mapping fields are written
+        unconditionally — including ``None`` values — so that a caller can
+        clear a previously-set field (e.g. removing a department).
+        """
         mapping = self.get_user_mapping_by_user_id(user_id)
-        if new_external_id:
-            if mapping:
-                if mapping.external_id != new_external_id:
-                    mapping.external_id = new_external_id
-                if scim_username is not None:
-                    mapping.scim_username = scim_username
-            else:
-                self.create_user_mapping(
-                    external_id=new_external_id,
-                    user_id=user_id,
-                    scim_username=scim_username,
-                )
-        elif mapping:
-            self.delete_user_mapping(mapping.id)
+        if mapping:
+            if mapping.external_id != new_external_id:
+                mapping.external_id = new_external_id
+            if scim_username is not None:
+                mapping.scim_username = scim_username
+            if fields is not None:
+                mapping.department = fields.department
+                mapping.manager = fields.manager
+                mapping.given_name = fields.given_name
+                mapping.family_name = fields.family_name
+                mapping.scim_emails_json = fields.scim_emails_json
+        elif new_external_id:
+            self.create_user_mapping(
+                external_id=new_external_id,
+                user_id=user_id,
+                scim_username=scim_username,
+                fields=fields,
+            )
 
     def _get_user_mappings_batch(
         self, user_ids: list[UUID]

backend/ee/onyx/db/user_group.py

@@ -15,6 +15,7 @@ from sqlalchemy.orm import Session
 from ee.onyx.server.user_group.models import SetCuratorRequest
 from ee.onyx.server.user_group.models import UserGroupCreate
 from ee.onyx.server.user_group.models import UserGroupUpdate
+from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.enums import AccessType
 from onyx.db.enums import ConnectorCredentialPairStatus
@@ -471,7 +472,9 @@ def _add_user_group__cc_pair_relationships__no_commit(
 
 def insert_user_group(db_session: Session, user_group: UserGroupCreate) -> UserGroup:
     db_user_group = UserGroup(
-        name=user_group.name, time_last_modified_by_user=func.now()
+        name=user_group.name,
+        time_last_modified_by_user=func.now(),
+        is_up_to_date=DISABLE_VECTOR_DB,
     )
     db_session.add(db_user_group)
     db_session.flush()  # give the group an ID
@@ -774,8 +777,7 @@ def update_user_group(
             cc_pair_ids=user_group_update.cc_pair_ids,
         )
 
-    # only needs to sync with Vespa if the cc_pairs have been updated
-    if cc_pairs_updated:
+    if cc_pairs_updated and not DISABLE_VECTOR_DB:
         db_user_group.is_up_to_date = False
 
     removed_users = db_session.scalars(

backend/ee/onyx/external_permissions/sharepoint/permission_utils.py

@@ -4,7 +4,6 @@ from collections import deque
 from collections.abc import Callable
 from collections.abc import Generator
 from typing import Any
-from urllib.parse import unquote
 from urllib.parse import urlparse
 
 import requests as _requests
@@ -598,8 +597,12 @@ def get_external_access_from_sharepoint(
         )
     elif site_page:
         site_url = site_page.get("webUrl")
-        # Prefer server-relative URL to avoid OData filters that break on apostrophes
-        server_relative_url = unquote(urlparse(site_url).path)
+        # Keep percent-encoding intact so the path matches the encoding
+        # used by the Office365 library's SPResPath.create_relative(),
+        # which compares against urlparse(context.base_url).path.
+        # Decoding (e.g. %27 → ') causes a mismatch that duplicates
+        # the site prefix in the constructed URL.
+        server_relative_url = urlparse(site_url).path
         file_obj = client_context.web.get_file_by_server_relative_url(
             server_relative_url
         )

backend/ee/onyx/main.py

@@ -31,6 +31,7 @@ from ee.onyx.server.query_and_chat.query_backend import (
 from ee.onyx.server.query_and_chat.search_backend import router as search_router
 from ee.onyx.server.query_history.api import router as query_history_router
 from ee.onyx.server.reporting.usage_export_api import router as usage_export_router
+from ee.onyx.server.scim.api import register_scim_exception_handlers
 from ee.onyx.server.scim.api import scim_router
 from ee.onyx.server.seeding import seed_db
 from ee.onyx.server.tenants.api import router as tenants_router
@@ -167,6 +168,7 @@ def get_application() -> FastAPI:
     # they use their own SCIM bearer token auth).
     # Not behind APP_API_PREFIX because IdPs expect /scim/v2/... directly.
     application.include_router(scim_router)
+    register_scim_exception_handlers(application)
 
     # Ensure all routes have auth enabled or are explicitly marked as public
     check_ee_router_auth(application)

backend/ee/onyx/server/billing/api.py

@@ -26,7 +26,6 @@ import asyncio
 import httpx
 from fastapi import APIRouter
 from fastapi import Depends
-from fastapi import HTTPException
 from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
@@ -42,7 +41,6 @@ from ee.onyx.server.billing.models import SeatUpdateRequest
 from ee.onyx.server.billing.models import SeatUpdateResponse
 from ee.onyx.server.billing.models import StripePublishableKeyResponse
 from ee.onyx.server.billing.models import SubscriptionStatusResponse
-from ee.onyx.server.billing.service import BillingServiceError
 from ee.onyx.server.billing.service import (
     create_checkout_session as create_checkout_service,
 )
@@ -58,6 +56,8 @@ from onyx.configs.app_configs import STRIPE_PUBLISHABLE_KEY_OVERRIDE
 from onyx.configs.app_configs import STRIPE_PUBLISHABLE_KEY_URL
 from onyx.configs.app_configs import WEB_DOMAIN
 from onyx.db.engine.sql_engine import get_session
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
 from onyx.redis.redis_pool import get_shared_redis_client
 from onyx.utils.logger import setup_logger
 from shared_configs.configs import MULTI_TENANT
@@ -169,26 +169,23 @@ async def create_checkout_session(
     if seats is not None:
         used_seats = get_used_seats(tenant_id)
         if seats < used_seats:
-            raise HTTPException(
-                status_code=400,
-                detail=f"Cannot subscribe with fewer seats than current usage. "
+            raise OnyxError(
+                OnyxErrorCode.VALIDATION_ERROR,
+                f"Cannot subscribe with fewer seats than current usage. "
                 f"You have {used_seats} active users/integrations but requested {seats} seats.",
             )
 
     # Build redirect URL for after checkout completion
     redirect_url = f"{WEB_DOMAIN}/admin/billing?checkout=success"
 
-    try:
-        return await create_checkout_service(
-            billing_period=billing_period,
-            seats=seats,
-            email=email,
-            license_data=license_data,
-            redirect_url=redirect_url,
-            tenant_id=tenant_id,
-        )
-    except BillingServiceError as e:
-        raise HTTPException(status_code=e.status_code, detail=e.message)
+    return await create_checkout_service(
+        billing_period=billing_period,
+        seats=seats,
+        email=email,
+        license_data=license_data,
+        redirect_url=redirect_url,
+        tenant_id=tenant_id,
+    )
 
 
 @router.post("/create-customer-portal-session")
@@ -206,18 +203,15 @@ async def create_customer_portal_session(
 
     # Self-hosted requires license
     if not MULTI_TENANT and not license_data:
-        raise HTTPException(status_code=400, detail="No license found")
+        raise OnyxError(OnyxErrorCode.VALIDATION_ERROR, "No license found")
 
     return_url = request.return_url if request else f"{WEB_DOMAIN}/admin/billing"
 
-    try:
-        return await create_portal_service(
-            license_data=license_data,
-            return_url=return_url,
-            tenant_id=tenant_id,
-        )
-    except BillingServiceError as e:
-        raise HTTPException(status_code=e.status_code, detail=e.message)
+    return await create_portal_service(
+        license_data=license_data,
+        return_url=return_url,
+        tenant_id=tenant_id,
+    )
 
 
 @router.get("/billing-information")
@@ -240,9 +234,9 @@ async def get_billing_information(
 
     # Check circuit breaker (self-hosted only)
     if _is_billing_circuit_open():
-        raise HTTPException(
-            status_code=503,
-            detail="Stripe connection temporarily disabled. Click 'Connect to Stripe' to retry.",
+        raise OnyxError(
+            OnyxErrorCode.SERVICE_UNAVAILABLE,
+            "Stripe connection temporarily disabled. Click 'Connect to Stripe' to retry.",
         )
 
     try:
@@ -250,11 +244,11 @@ async def get_billing_information(
             license_data=license_data,
             tenant_id=tenant_id,
         )
-    except BillingServiceError as e:
+    except OnyxError as e:
         # Open circuit breaker on connection failures (self-hosted only)
         if e.status_code in (502, 503, 504):
             _open_billing_circuit()
-        raise HTTPException(status_code=e.status_code, detail=e.message)
+        raise
 
 
 @router.post("/seats/update")
@@ -274,31 +268,25 @@ async def update_seats(
 
     # Self-hosted requires license
     if not MULTI_TENANT and not license_data:
-        raise HTTPException(status_code=400, detail="No license found")
+        raise OnyxError(OnyxErrorCode.VALIDATION_ERROR, "No license found")
 
     # Validate that new seat count is not less than current used seats
     used_seats = get_used_seats(tenant_id)
     if request.new_seat_count < used_seats:
-        raise HTTPException(
-            status_code=400,
-            detail=f"Cannot reduce seats below current usage. "
+        raise OnyxError(
+            OnyxErrorCode.VALIDATION_ERROR,
+            f"Cannot reduce seats below current usage. "
             f"You have {used_seats} active users/integrations but requested {request.new_seat_count} seats.",
         )
 
-    try:
-        result = await update_seat_service(
-            new_seat_count=request.new_seat_count,
-            license_data=license_data,
-            tenant_id=tenant_id,
-        )
-
-        # Note: Don't store license here - the control plane may still be processing
-        # the subscription update. The frontend should call /license/claim after a
-        # short delay to get the freshly generated license.
-
-        return result
-    except BillingServiceError as e:
-        raise HTTPException(status_code=e.status_code, detail=e.message)
+    # Note: Don't store license here - the control plane may still be processing
+    # the subscription update. The frontend should call /license/claim after a
+    # short delay to get the freshly generated license.
+    return await update_seat_service(
+        new_seat_count=request.new_seat_count,
+        license_data=license_data,
+        tenant_id=tenant_id,
+    )
 
 
 @router.get("/stripe-publishable-key")
@@ -329,18 +317,18 @@ async def get_stripe_publishable_key() -> StripePublishableKeyResponse:
         if STRIPE_PUBLISHABLE_KEY_OVERRIDE:
             key = STRIPE_PUBLISHABLE_KEY_OVERRIDE.strip()
             if not key.startswith("pk_"):
-                raise HTTPException(
-                    status_code=500,
-                    detail="Invalid Stripe publishable key format",
+                raise OnyxError(
+                    OnyxErrorCode.INTERNAL_ERROR,
+                    "Invalid Stripe publishable key format",
                 )
             _stripe_publishable_key_cache = key
             return StripePublishableKeyResponse(publishable_key=key)
 
         # Fall back to S3 bucket
         if not STRIPE_PUBLISHABLE_KEY_URL:
-            raise HTTPException(
-                status_code=500,
-                detail="Stripe publishable key is not configured",
+            raise OnyxError(
+                OnyxErrorCode.INTERNAL_ERROR,
+                "Stripe publishable key is not configured",
             )
 
         try:
@@ -351,17 +339,17 @@ async def get_stripe_publishable_key() -> StripePublishableKeyResponse:
 
                 # Validate key format
                 if not key.startswith("pk_"):
-                    raise HTTPException(
-                        status_code=500,
-                        detail="Invalid Stripe publishable key format",
+                    raise OnyxError(
+                        OnyxErrorCode.INTERNAL_ERROR,
+                        "Invalid Stripe publishable key format",
                     )
 
                 _stripe_publishable_key_cache = key
                 return StripePublishableKeyResponse(publishable_key=key)
         except httpx.HTTPError:
-            raise HTTPException(
-                status_code=500,
-                detail="Failed to fetch Stripe publishable key",
+            raise OnyxError(
+                OnyxErrorCode.INTERNAL_ERROR,
+                "Failed to fetch Stripe publishable key",
             )
 
 

backend/ee/onyx/server/billing/service.py

@@ -22,6 +22,8 @@ from ee.onyx.server.billing.models import SeatUpdateResponse
 from ee.onyx.server.billing.models import SubscriptionStatusResponse
 from ee.onyx.server.tenants.access import generate_data_plane_token
 from onyx.configs.app_configs import CONTROL_PLANE_API_BASE_URL
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
 from onyx.utils.logger import setup_logger
 from shared_configs.configs import MULTI_TENANT
 
@@ -31,15 +33,6 @@ logger = setup_logger()
 _REQUEST_TIMEOUT = 30.0
 
 
-class BillingServiceError(Exception):
-    """Exception raised for billing service errors."""
-
-    def __init__(self, message: str, status_code: int = 500):
-        self.message = message
-        self.status_code = status_code
-        super().__init__(self.message)
-
-
 def _get_proxy_headers(license_data: str | None) -> dict[str, str]:
     """Build headers for proxy requests (self-hosted).
 
@@ -101,7 +94,7 @@ async def _make_billing_request(
         Response JSON as dict
 
     Raises:
-        BillingServiceError: If request fails
+        OnyxError: If request fails
     """
 
     base_url = _get_base_url()
@@ -128,11 +121,17 @@ async def _make_billing_request(
         except Exception:
             pass
         logger.error(f"{error_message}: {e.response.status_code} - {detail}")
-        raise BillingServiceError(detail, e.response.status_code)
+        raise OnyxError(
+            OnyxErrorCode.BAD_GATEWAY,
+            detail,
+            status_code_override=e.response.status_code,
+        )
 
     except httpx.RequestError:
         logger.exception("Failed to connect to billing service")
-        raise BillingServiceError("Failed to connect to billing service", 502)
+        raise OnyxError(
+            OnyxErrorCode.BAD_GATEWAY, "Failed to connect to billing service"
+        )
 
 
 async def create_checkout_session(

backend/ee/onyx/server/enterprise_settings/api.py

@@ -223,6 +223,15 @@ def get_active_scim_token(
     token = dal.get_active_token()
     if not token:
         raise HTTPException(status_code=404, detail="No active SCIM token")
+
+    # Derive the IdP domain from the first synced user as a heuristic.
+    idp_domain: str | None = None
+    mappings, _total = dal.list_user_mappings(start_index=1, count=1)
+    if mappings:
+        user = dal.get_user(mappings[0].user_id)
+        if user and "@" in user.email:
+            idp_domain = user.email.rsplit("@", 1)[1]
+
     return ScimTokenResponse(
         id=token.id,
         name=token.name,
@@ -230,6 +239,7 @@ def get_active_scim_token(
         is_active=token.is_active,
         created_at=token.created_at,
         last_used_at=token.last_used_at,
+        idp_domain=idp_domain,
     )
 
 

backend/ee/onyx/server/license/api.py

@@ -14,7 +14,6 @@ import requests
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import File
-from fastapi import HTTPException
 from fastapi import UploadFile
 from sqlalchemy.orm import Session
 
@@ -35,6 +34,8 @@ from ee.onyx.server.license.models import SeatUsageResponse
 from ee.onyx.utils.license import verify_license_signature
 from onyx.auth.users import User
 from onyx.db.engine.sql_engine import get_session
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
 from onyx.utils.logger import setup_logger
 from shared_configs.configs import MULTI_TENANT
 
@@ -127,9 +128,9 @@ async def claim_license(
     2. Without session_id: Re-claim using existing license for auth
     """
     if MULTI_TENANT:
-        raise HTTPException(
-            status_code=400,
-            detail="License claiming is only available for self-hosted deployments",
+        raise OnyxError(
+            OnyxErrorCode.VALIDATION_ERROR,
+            "License claiming is only available for self-hosted deployments",
         )
 
     try:
@@ -146,15 +147,16 @@ async def claim_license(
             # Re-claim using existing license for auth
             metadata = get_license_metadata(db_session)
             if not metadata or not metadata.tenant_id:
-                raise HTTPException(
-                    status_code=400,
-                    detail="No license found. Provide session_id after checkout.",
+                raise OnyxError(
+                    OnyxErrorCode.VALIDATION_ERROR,
+                    "No license found. Provide session_id after checkout.",
                 )
 
             license_row = get_license(db_session)
             if not license_row or not license_row.license_data:
-                raise HTTPException(
-                    status_code=400, detail="No license found in database"
+                raise OnyxError(
+                    OnyxErrorCode.VALIDATION_ERROR,
+                    "No license found in database",
                 )
 
             url = f"{CLOUD_DATA_PLANE_URL}/proxy/license/{metadata.tenant_id}"
@@ -173,7 +175,7 @@ async def claim_license(
         license_data = data.get("license")
 
         if not license_data:
-            raise HTTPException(status_code=404, detail="No license in response")
+            raise OnyxError(OnyxErrorCode.NOT_FOUND, "No license in response")
 
         # Verify signature before persisting
         payload = verify_license_signature(license_data)
@@ -199,12 +201,14 @@ async def claim_license(
             detail = error_data.get("detail", detail)
         except Exception:
             pass
-        raise HTTPException(status_code=status_code, detail=detail)
+        raise OnyxError(
+            OnyxErrorCode.BAD_GATEWAY, detail, status_code_override=status_code
+        )
     except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
+        raise OnyxError(OnyxErrorCode.VALIDATION_ERROR, str(e))
     except requests.RequestException:
-        raise HTTPException(
-            status_code=502, detail="Failed to connect to license server"
+        raise OnyxError(
+            OnyxErrorCode.BAD_GATEWAY, "Failed to connect to license server"
         )
 
 
@@ -221,9 +225,9 @@ async def upload_license(
     The license file must be cryptographically signed by Onyx.
     """
     if MULTI_TENANT:
-        raise HTTPException(
-            status_code=400,
-            detail="License upload is only available for self-hosted deployments",
+        raise OnyxError(
+            OnyxErrorCode.VALIDATION_ERROR,
+            "License upload is only available for self-hosted deployments",
         )
 
     try:
@@ -234,14 +238,14 @@ async def upload_license(
         # Remove any stray whitespace/newlines from user input
         license_data = license_data.strip()
     except UnicodeDecodeError:
-        raise HTTPException(status_code=400, detail="Invalid license file format")
+        raise OnyxError(OnyxErrorCode.INVALID_INPUT, "Invalid license file format")
 
     # Verify cryptographic signature - this is the only validation needed
     # The license's tenant_id identifies the customer in control plane, not locally
     try:
         payload = verify_license_signature(license_data)
     except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
+        raise OnyxError(OnyxErrorCode.VALIDATION_ERROR, str(e))
 
     # Persist to DB and update cache
     upsert_license(db_session, license_data)
@@ -297,9 +301,9 @@ async def delete_license(
     Admin only - removes license from database and invalidates cache.
     """
     if MULTI_TENANT:
-        raise HTTPException(
-            status_code=400,
-            detail="License deletion is only available for self-hosted deployments",
+        raise OnyxError(
+            OnyxErrorCode.VALIDATION_ERROR,
+            "License deletion is only available for self-hosted deployments",
         )
 
     try:

backend/ee/onyx/server/middleware/license_enforcement.py

@@ -46,7 +46,6 @@ from fastapi import FastAPI
 from fastapi import Request
 from fastapi import Response
 from fastapi.responses import JSONResponse
-from redis.exceptions import RedisError
 from sqlalchemy.exc import SQLAlchemyError
 
 from ee.onyx.configs.app_configs import LICENSE_ENFORCEMENT_ENABLED
@@ -56,6 +55,7 @@ from ee.onyx.configs.license_enforcement_config import (
 )
 from ee.onyx.db.license import get_cached_license_metadata
 from ee.onyx.db.license import refresh_license_cache
+from onyx.cache.interface import CACHE_TRANSIENT_ERRORS
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
 from onyx.server.settings.models import ApplicationStatus
 from shared_configs.contextvars import get_current_tenant_id
@@ -164,9 +164,9 @@ def add_license_enforcement_middleware(
                     "[license_enforcement] No license, allowing community features"
                 )
                 is_gated = False
-        except RedisError as e:
+        except CACHE_TRANSIENT_ERRORS as e:
             logger.warning(f"Failed to check license metadata: {e}")
-            # Fail open - don't block users due to Redis connectivity issues
+            # Fail open - don't block users due to cache connectivity issues
             is_gated = False
 
         if is_gated:

backend/ee/onyx/server/scim/api.py

@@ -15,7 +15,9 @@ from uuid import UUID
 
 from fastapi import APIRouter
 from fastapi import Depends
+from fastapi import FastAPI
 from fastapi import Query
+from fastapi import Request
 from fastapi import Response
 from fastapi.responses import JSONResponse
 from fastapi_users.password import PasswordHelper
@@ -24,6 +26,7 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from ee.onyx.db.scim import ScimDAL
+from ee.onyx.server.scim.auth import ScimAuthError
 from ee.onyx.server.scim.auth import verify_scim_token
 from ee.onyx.server.scim.filtering import parse_scim_filter
 from ee.onyx.server.scim.models import SCIM_LIST_RESPONSE_SCHEMA
@@ -31,6 +34,7 @@ from ee.onyx.server.scim.models import ScimError
 from ee.onyx.server.scim.models import ScimGroupMember
 from ee.onyx.server.scim.models import ScimGroupResource
 from ee.onyx.server.scim.models import ScimListResponse
+from ee.onyx.server.scim.models import ScimMappingFields
 from ee.onyx.server.scim.models import ScimName
 from ee.onyx.server.scim.models import ScimPatchRequest
 from ee.onyx.server.scim.models import ScimServiceProviderConfig
@@ -40,6 +44,8 @@ from ee.onyx.server.scim.patch import apply_user_patch
 from ee.onyx.server.scim.patch import ScimPatchError
 from ee.onyx.server.scim.providers.base import get_default_provider
 from ee.onyx.server.scim.providers.base import ScimProvider
+from ee.onyx.server.scim.providers.base import serialize_emails
+from ee.onyx.server.scim.schema_definitions import ENTERPRISE_USER_SCHEMA_DEF
 from ee.onyx.server.scim.schema_definitions import GROUP_RESOURCE_TYPE
 from ee.onyx.server.scim.schema_definitions import GROUP_SCHEMA_DEF
 from ee.onyx.server.scim.schema_definitions import SERVICE_PROVIDER_CONFIG
@@ -47,6 +53,7 @@ from ee.onyx.server.scim.schema_definitions import USER_RESOURCE_TYPE
 from ee.onyx.server.scim.schema_definitions import USER_SCHEMA_DEF
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.models import ScimToken
+from onyx.db.models import ScimUserMapping
 from onyx.db.models import User
 from onyx.db.models import UserGroup
 from onyx.db.models import UserRole
@@ -73,6 +80,22 @@ scim_router = APIRouter(prefix="/scim/v2", tags=["SCIM"])
 _pw_helper = PasswordHelper()
 
 
+def register_scim_exception_handlers(app: FastAPI) -> None:
+    """Register SCIM-specific exception handlers on the FastAPI app.
+
+    Call this after ``app.include_router(scim_router)`` so that auth
+    failures from ``verify_scim_token`` return RFC 7644 §3.12 error
+    envelopes (with ``schemas`` and ``status`` fields) instead of
+    FastAPI's default ``{"detail": "..."}`` format.
+    """
+
+    @app.exception_handler(ScimAuthError)
+    async def _handle_scim_auth_error(
+        _request: Request, exc: ScimAuthError
+    ) -> ScimJSONResponse:
+        return _scim_error_response(exc.status_code, exc.detail)
+
+
 def _get_provider(
     _token: ScimToken = Depends(verify_scim_token),
 ) -> ScimProvider:
@@ -122,7 +145,7 @@ def get_schemas() -> ScimJSONResponse:
     Wrapped in a ListResponse envelope (RFC 7644 §3.4.2) because IdPs
     like Entra ID expect a JSON object, not a bare array.
     """
-    schemas = [USER_SCHEMA_DEF, GROUP_SCHEMA_DEF]
+    schemas = [USER_SCHEMA_DEF, GROUP_SCHEMA_DEF, ENTERPRISE_USER_SCHEMA_DEF]
     return ScimJSONResponse(
         content={
             "schemas": [SCIM_LIST_RESPONSE_SCHEMA],
@@ -261,6 +284,45 @@ def _build_list_response(
     )
 
 
+def _extract_enterprise_fields(
+    resource: ScimUserResource,
+) -> tuple[str | None, str | None]:
+    """Extract department and manager from enterprise extension."""
+    ext = resource.enterprise_extension
+    if not ext:
+        return None, None
+    department = ext.department
+    manager = ext.manager.value if ext.manager else None
+    return department, manager
+
+
+def _mapping_to_fields(
+    mapping: ScimUserMapping | None,
+) -> ScimMappingFields | None:
+    """Extract round-trip fields from a SCIM user mapping."""
+    if not mapping:
+        return None
+    return ScimMappingFields(
+        department=mapping.department,
+        manager=mapping.manager,
+        given_name=mapping.given_name,
+        family_name=mapping.family_name,
+        scim_emails_json=mapping.scim_emails_json,
+    )
+
+
+def _fields_from_resource(resource: ScimUserResource) -> ScimMappingFields:
+    """Build mapping fields from an incoming SCIM user resource."""
+    department, manager = _extract_enterprise_fields(resource)
+    return ScimMappingFields(
+        department=department,
+        manager=manager,
+        given_name=resource.name.givenName if resource.name else None,
+        family_name=resource.name.familyName if resource.name else None,
+        scim_emails_json=serialize_emails(resource.emails),
+    )
+
+
 # ---------------------------------------------------------------------------
 # User CRUD (RFC 7644 §3)
 # ---------------------------------------------------------------------------
@@ -279,6 +341,7 @@ def list_users(
     """List users with optional SCIM filter and pagination."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
+    dal.commit()
 
     try:
         scim_filter = parse_scim_filter(filter)
@@ -297,6 +360,7 @@ def list_users(
             mapping.external_id if mapping else None,
             groups=user_groups_map.get(user.id, []),
             scim_username=mapping.scim_username if mapping else None,
+            fields=_mapping_to_fields(mapping),
         )
         for user, mapping in users_with_mappings
     ]
@@ -321,6 +385,7 @@ def get_user(
     """Get a single user by ID."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
+    dal.commit()
 
     result = _fetch_user_or_404(user_id, dal)
     if isinstance(result, ScimJSONResponse):
@@ -334,6 +399,7 @@ def get_user(
         mapping.external_id if mapping else None,
         groups=dal.get_user_groups(user.id),
         scim_username=mapping.scim_username if mapping else None,
+        fields=_mapping_to_fields(mapping),
     )
 
     # RFC 7644 §3.4.2.5 — IdP may request certain attributes be omitted
@@ -357,21 +423,63 @@ def create_user(
 
     email = user_resource.userName.strip()
 
-    # externalId is how the IdP correlates this user on subsequent requests.
-    # Without it, the IdP can't find the user and will try to re-create,
-    # hitting a 409 conflict — so we require it up front.
-    if not user_resource.externalId:
-        return _scim_error_response(400, "externalId is required")
+    # Check for existing user — if they exist but aren't SCIM-managed yet,
+    # link them to the IdP rather than rejecting with 409.
+    external_id: str | None = user_resource.externalId
+    scim_username: str = user_resource.userName.strip()
+    fields: ScimMappingFields = _fields_from_resource(user_resource)
 
-    # Enforce seat limit
+    existing_user = dal.get_user_by_email(email)
+    if existing_user:
+        existing_mapping = dal.get_user_mapping_by_user_id(existing_user.id)
+        if existing_mapping:
+            return _scim_error_response(409, f"User with email {email} already exists")
+
+        # Adopt pre-existing user into SCIM management.
+        # Reactivating a deactivated user consumes a seat, so enforce the
+        # seat limit the same way replace_user does.
+        if user_resource.active and not existing_user.is_active:
+            seat_error = _check_seat_availability(dal)
+            if seat_error:
+                return _scim_error_response(403, seat_error)
+
+        personal_name = _scim_name_to_str(user_resource.name)
+        dal.update_user(
+            existing_user,
+            is_active=user_resource.active,
+            **({"personal_name": personal_name} if personal_name else {}),
+        )
+
+        try:
+            dal.create_user_mapping(
+                external_id=external_id,
+                user_id=existing_user.id,
+                scim_username=scim_username,
+                fields=fields,
+            )
+            dal.commit()
+        except IntegrityError:
+            dal.rollback()
+            return _scim_error_response(
+                409, f"User with email {email} already has a SCIM mapping"
+            )
+
+        return _scim_resource_response(
+            provider.build_user_resource(
+                existing_user,
+                external_id,
+                scim_username=scim_username,
+                fields=fields,
+            ),
+            status_code=201,
+        )
+
+    # Only enforce seat limit for net-new users — adopting a pre-existing
+    # user doesn't consume a new seat.
     seat_error = _check_seat_availability(dal)
     if seat_error:
         return _scim_error_response(403, seat_error)
 
-    # Check for existing user
-    if dal.get_user_by_email(email):
-        return _scim_error_response(409, f"User with email {email} already exists")
-
     # Create user with a random password (SCIM users authenticate via IdP)
     personal_name = _scim_name_to_str(user_resource.name)
     user = User(
@@ -389,17 +497,29 @@ def create_user(
         dal.rollback()
         return _scim_error_response(409, f"User with email {email} already exists")
 
-    # Create SCIM mapping (externalId is validated above, always present)
-    external_id = user_resource.externalId
-    scim_username = user_resource.userName.strip()
-    dal.create_user_mapping(
-        external_id=external_id, user_id=user.id, scim_username=scim_username
-    )
-
-    dal.commit()
+    # Always create a SCIM mapping so that the user is marked as
+    # SCIM-managed. externalId may be None (RFC 7643 says it's optional).
+    try:
+        dal.create_user_mapping(
+            external_id=external_id,
+            user_id=user.id,
+            scim_username=scim_username,
+            fields=fields,
+        )
+        dal.commit()
+    except IntegrityError:
+        dal.rollback()
+        return _scim_error_response(
+            409, f"User with email {email} already has a SCIM mapping"
+        )
 
     return _scim_resource_response(
-        provider.build_user_resource(user, external_id, scim_username=scim_username),
+        provider.build_user_resource(
+            user,
+            external_id,
+            scim_username=scim_username,
+            fields=fields,
+        ),
         status_code=201,
     )
 
@@ -438,7 +558,13 @@ def replace_user(
 
     new_external_id = user_resource.externalId
     scim_username = user_resource.userName.strip()
-    dal.sync_user_external_id(user.id, new_external_id, scim_username=scim_username)
+    fields = _fields_from_resource(user_resource)
+    dal.sync_user_external_id(
+        user.id,
+        new_external_id,
+        scim_username=scim_username,
+        fields=fields,
+    )
 
     dal.commit()
 
@@ -448,6 +574,7 @@ def replace_user(
             new_external_id,
             groups=dal.get_user_groups(user.id),
             scim_username=scim_username,
+            fields=fields,
         )
     )
 
@@ -476,16 +603,18 @@ def patch_user(
     mapping = dal.get_user_mapping_by_user_id(user.id)
     external_id = mapping.external_id if mapping else None
     current_scim_username = mapping.scim_username if mapping else None
+    current_fields = _mapping_to_fields(mapping)
 
     current = provider.build_user_resource(
         user,
         external_id,
         groups=dal.get_user_groups(user.id),
         scim_username=current_scim_username,
+        fields=current_fields,
     )
 
     try:
-        patched = apply_user_patch(
+        patched, ent_data = apply_user_patch(
             patch_request.Operations, current, provider.ignored_patch_paths
         )
     except ScimPatchError as e:
@@ -520,8 +649,25 @@ def patch_user(
         personal_name=personal_name,
     )
 
+    # Build updated fields by merging PATCH enterprise data with current values
+    cf = current_fields or ScimMappingFields()
+    fields = ScimMappingFields(
+        department=ent_data.get("department", cf.department),
+        manager=ent_data.get("manager", cf.manager),
+        given_name=patched.name.givenName if patched.name else cf.given_name,
+        family_name=patched.name.familyName if patched.name else cf.family_name,
+        scim_emails_json=(
+            serialize_emails(patched.emails)
+            if patched.emails is not None
+            else cf.scim_emails_json
+        ),
+    )
+
     dal.sync_user_external_id(
-        user.id, patched.externalId, scim_username=new_scim_username
+        user.id,
+        patched.externalId,
+        scim_username=new_scim_username,
+        fields=fields,
     )
 
     dal.commit()
@@ -532,6 +678,7 @@ def patch_user(
             patched.externalId,
             groups=dal.get_user_groups(user.id),
             scim_username=new_scim_username,
+            fields=fields,
         )
     )
 
@@ -640,6 +787,7 @@ def list_groups(
     """List groups with optional SCIM filter and pagination."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
+    dal.commit()
 
     try:
         scim_filter = parse_scim_filter(filter)
@@ -676,6 +824,7 @@ def get_group(
     """Get a single group by ID."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
+    dal.commit()
 
     result = _fetch_group_or_404(group_id, dal)
     if isinstance(result, ScimJSONResponse):

backend/ee/onyx/server/scim/auth.py

@@ -19,7 +19,6 @@ import hashlib
 import secrets
 
 from fastapi import Depends
-from fastapi import HTTPException
 from fastapi import Request
 from sqlalchemy.orm import Session
 
@@ -28,6 +27,21 @@ from onyx.auth.utils import get_hashed_bearer_token_from_request
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.models import ScimToken
 
+
+class ScimAuthError(Exception):
+    """Raised when SCIM bearer token authentication fails.
+
+    Unlike HTTPException, this carries the status and detail so the SCIM
+    exception handler can wrap them in an RFC 7644 §3.12 error envelope
+    with ``schemas`` and ``status`` fields.
+    """
+
+    def __init__(self, status_code: int, detail: str) -> None:
+        self.status_code = status_code
+        self.detail = detail
+        super().__init__(detail)
+
+
 SCIM_TOKEN_PREFIX = "onyx_scim_"
 SCIM_TOKEN_LENGTH = 48
 
@@ -82,23 +96,14 @@ def verify_scim_token(
     """
     hashed = _get_hashed_scim_token_from_request(request)
     if not hashed:
-        raise HTTPException(
-            status_code=401,
-            detail="Missing or invalid SCIM bearer token",
-        )
+        raise ScimAuthError(401, "Missing or invalid SCIM bearer token")
 
     token = dal.get_token_by_hash(hashed)
 
     if not token:
-        raise HTTPException(
-            status_code=401,
-            detail="Invalid SCIM bearer token",
-        )
+        raise ScimAuthError(401, "Invalid SCIM bearer token")
 
     if not token.is_active:
-        raise HTTPException(
-            status_code=401,
-            detail="SCIM token has been revoked",
-        )
+        raise ScimAuthError(401, "SCIM token has been revoked")
 
     return token

backend/ee/onyx/server/scim/models.py

@@ -7,6 +7,7 @@ SCIM protocol schemas follow the wire format defined in:
 Admin API schemas are internal to Onyx and used for SCIM token management.
 """
 
+from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
 
@@ -32,6 +33,9 @@ SCIM_SERVICE_PROVIDER_CONFIG_SCHEMA = (
 )
 SCIM_RESOURCE_TYPE_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:ResourceType"
 SCIM_SCHEMA_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:Schema"
+SCIM_ENTERPRISE_USER_SCHEMA = (
+    "urn:ietf:params:scim:schemas:extension:enterprise:2.0:User"
+)
 
 
 # ---------------------------------------------------------------------------
@@ -71,6 +75,36 @@ class ScimUserGroupRef(BaseModel):
     display: str | None = None
 
 
+class ScimManagerRef(BaseModel):
+    """Manager sub-attribute for the enterprise extension (RFC 7643 §4.3)."""
+
+    value: str | None = None
+
+
+class ScimEnterpriseExtension(BaseModel):
+    """Enterprise User extension attributes (RFC 7643 §4.3)."""
+
+    department: str | None = None
+    manager: ScimManagerRef | None = None
+
+
+@dataclass
+class ScimMappingFields:
+    """Stored SCIM mapping fields that need to round-trip through the IdP.
+
+    Entra ID sends structured name components, email metadata, and enterprise
+    extension attributes that must be returned verbatim in subsequent GET
+    responses. These fields are persisted on ScimUserMapping and threaded
+    through the DAL, provider, and endpoint layers.
+    """
+
+    department: str | None = None
+    manager: str | None = None
+    given_name: str | None = None
+    family_name: str | None = None
+    scim_emails_json: str | None = None
+
+
 class ScimUserResource(BaseModel):
     """SCIM User resource representation (RFC 7643 §4.1).
 
@@ -79,6 +113,8 @@ class ScimUserResource(BaseModel):
     to match the SCIM wire format (not Python convention).
     """
 
+    model_config = ConfigDict(populate_by_name=True)
+
     schemas: list[str] = Field(default_factory=lambda: [SCIM_USER_SCHEMA])
     id: str | None = None  # Onyx's internal user ID, set on responses
     externalId: str | None = None  # IdP's identifier for this user
@@ -89,6 +125,10 @@ class ScimUserResource(BaseModel):
     active: bool = True
     groups: list[ScimUserGroupRef] = Field(default_factory=list)
     meta: ScimMeta | None = None
+    enterprise_extension: ScimEnterpriseExtension | None = Field(
+        default=None,
+        alias="urn:ietf:params:scim:schemas:extension:enterprise:2.0:User",
+    )
 
 
 class ScimGroupMember(BaseModel):
@@ -325,6 +365,7 @@ class ScimTokenResponse(BaseModel):
     is_active: bool
     created_at: datetime
     last_used_at: datetime | None = None
+    idp_domain: str | None = None
 
 
 class ScimTokenCreatedResponse(ScimTokenResponse):

backend/ee/onyx/server/scim/patch.py

@@ -14,10 +14,13 @@ responsible for persisting changes.
 
 from __future__ import annotations
 
+import logging
 import re
 from dataclasses import dataclass
+from dataclasses import field
 from typing import Any
 
+from ee.onyx.server.scim.models import SCIM_ENTERPRISE_USER_SCHEMA
 from ee.onyx.server.scim.models import ScimGroupMember
 from ee.onyx.server.scim.models import ScimGroupResource
 from ee.onyx.server.scim.models import ScimPatchOperation
@@ -26,6 +29,11 @@ from ee.onyx.server.scim.models import ScimPatchResourceValue
 from ee.onyx.server.scim.models import ScimPatchValue
 from ee.onyx.server.scim.models import ScimUserResource
 
+logger = logging.getLogger(__name__)
+
+# Lowercased enterprise extension URN for case-insensitive matching
+_ENTERPRISE_URN_LOWER = SCIM_ENTERPRISE_USER_SCHEMA.lower()
+
 # Pattern for email filter paths, e.g.:
 #   emails[primary eq true].value  (Okta)
 #   emails[type eq "work"].value   (Azure AD / Entra ID)
@@ -86,6 +94,7 @@ class _UserPatchCtx:
 
     data: dict[str, Any]
     name_data: dict[str, Any]
+    ent_data: dict[str, str | None] = field(default_factory=dict)
 
 
 # ---------------------------------------------------------------------------
@@ -97,7 +106,7 @@ def apply_user_patch(
     operations: list[ScimPatchOperation],
     current: ScimUserResource,
     ignored_paths: frozenset[str] = frozenset(),
-) -> ScimUserResource:
+) -> tuple[ScimUserResource, dict[str, str | None]]:
     """Apply SCIM PATCH operations to a user resource.
 
     Args:
@@ -105,8 +114,10 @@ def apply_user_patch(
         current: The current user resource state.
         ignored_paths: SCIM attribute paths to silently skip (from provider).
 
-    Returns a new ``ScimUserResource`` with the modifications applied.
-    The original object is not mutated.
+    Returns:
+        A tuple of (modified user resource, enterprise extension data dict).
+        The enterprise dict has keys ``"department"`` and ``"manager"``
+        with values set only when a PATCH operation touched them.
 
     Raises:
         ScimPatchError: If an operation targets an unsupported path.
@@ -125,7 +136,7 @@ def apply_user_patch(
             )
 
     ctx.data["name"] = ctx.name_data
-    return ScimUserResource.model_validate(ctx.data)
+    return ScimUserResource.model_validate(ctx.data), ctx.ent_data
 
 
 def _apply_user_replace(
@@ -209,6 +220,8 @@ def _set_user_field(
             ctx.data["emails"] = value
     elif _EMAIL_FILTER_RE.match(path):
         _update_primary_email(ctx.data, value)
+    elif path.startswith(_ENTERPRISE_URN_LOWER):
+        _set_enterprise_field(path, value, ctx.ent_data)
     elif not strict:
         return
     else:
@@ -227,6 +240,54 @@ def _update_primary_email(data: dict[str, Any], value: ScimPatchValue) -> None:
     data["emails"] = emails
 
 
+def _to_dict(value: ScimPatchValue) -> dict | None:
+    """Coerce a SCIM patch value to a plain dict if possible.
+
+    Pydantic may parse raw dicts as ``ScimPatchResourceValue`` (which uses
+    ``extra="allow"``), so we also dump those back to a dict.
+    """
+    if isinstance(value, dict):
+        return value
+    if isinstance(value, ScimPatchResourceValue):
+        return value.model_dump(exclude_unset=True)
+    return None
+
+
+def _set_enterprise_field(
+    path: str,
+    value: ScimPatchValue,
+    ent_data: dict[str, str | None],
+) -> None:
+    """Handle enterprise extension URN paths or value dicts."""
+    # Full URN as key with dict value (path-less PATCH)
+    # e.g. key="urn:...:user", value={"department": "Eng", "manager": {...}}
+    if path == _ENTERPRISE_URN_LOWER:
+        d = _to_dict(value)
+        if d is not None:
+            if "department" in d:
+                ent_data["department"] = d["department"]
+            if "manager" in d:
+                mgr = d["manager"]
+                if isinstance(mgr, dict):
+                    ent_data["manager"] = mgr.get("value")
+        return
+
+    # Dotted URN path, e.g. "urn:...:user:department"
+    suffix = path[len(_ENTERPRISE_URN_LOWER) :].lstrip(":").lower()
+    if suffix == "department":
+        ent_data["department"] = str(value) if value is not None else None
+    elif suffix == "manager":
+        d = _to_dict(value)
+        if d is not None:
+            ent_data["manager"] = d.get("value")
+        elif isinstance(value, str):
+            ent_data["manager"] = value
+    else:
+        # Unknown enterprise attributes are silently ignored rather than
+        # rejected — IdPs may send attributes we don't model yet.
+        logger.warning("Ignoring unknown enterprise extension attribute '%s'", suffix)
+
+
 # ---------------------------------------------------------------------------
 # Group PATCH
 # ---------------------------------------------------------------------------

backend/ee/onyx/server/scim/providers/base.py

@@ -2,13 +2,22 @@
 
 from __future__ import annotations
 
+import json
+import logging
 from abc import ABC
 from abc import abstractmethod
 from uuid import UUID
 
+from pydantic import ValidationError
+
+from ee.onyx.server.scim.models import SCIM_ENTERPRISE_USER_SCHEMA
+from ee.onyx.server.scim.models import SCIM_USER_SCHEMA
 from ee.onyx.server.scim.models import ScimEmail
+from ee.onyx.server.scim.models import ScimEnterpriseExtension
 from ee.onyx.server.scim.models import ScimGroupMember
 from ee.onyx.server.scim.models import ScimGroupResource
+from ee.onyx.server.scim.models import ScimManagerRef
+from ee.onyx.server.scim.models import ScimMappingFields
 from ee.onyx.server.scim.models import ScimMeta
 from ee.onyx.server.scim.models import ScimName
 from ee.onyx.server.scim.models import ScimUserGroupRef
@@ -16,6 +25,9 @@ from ee.onyx.server.scim.models import ScimUserResource
 from onyx.db.models import User
 from onyx.db.models import UserGroup
 
+
+logger = logging.getLogger(__name__)
+
 COMMON_IGNORED_PATCH_PATHS: frozenset[str] = frozenset(
     {
         "id",
@@ -49,12 +61,22 @@ class ScimProvider(ABC):
         """
         ...
 
+    @property
+    def user_schemas(self) -> list[str]:
+        """Schema URIs to include in User resource responses.
+
+        Override in subclasses to advertise additional schemas (e.g. the
+        enterprise extension for Entra ID).
+        """
+        return [SCIM_USER_SCHEMA]
+
     def build_user_resource(
         self,
         user: User,
         external_id: str | None = None,
         groups: list[tuple[int, str]] | None = None,
         scim_username: str | None = None,
+        fields: ScimMappingFields | None = None,
     ) -> ScimUserResource:
         """Build a SCIM User response from an Onyx User.
 
@@ -66,27 +88,48 @@ class ScimProvider(ABC):
                 for newly-created users.
             scim_username: The original-case userName from the IdP. Falls
                 back to ``user.email`` (lowercase) when not available.
+            fields: Stored mapping fields that the IdP expects round-tripped.
         """
+        f = fields or ScimMappingFields()
         group_refs = [
             ScimUserGroupRef(value=str(gid), display=gname)
             for gid, gname in (groups or [])
         ]
 
-        # Use original-case userName if stored, otherwise fall back to the
-        # lowercased email from the User model.
         username = scim_username or user.email
 
-        return ScimUserResource(
+        # Build enterprise extension when at least one value is present.
+        # Dynamically add the enterprise URN to schemas per RFC 7643 §3.0.
+        enterprise_ext: ScimEnterpriseExtension | None = None
+        schemas = list(self.user_schemas)
+        if f.department is not None or f.manager is not None:
+            manager_ref = (
+                ScimManagerRef(value=f.manager) if f.manager is not None else None
+            )
+            enterprise_ext = ScimEnterpriseExtension(
+                department=f.department,
+                manager=manager_ref,
+            )
+            if SCIM_ENTERPRISE_USER_SCHEMA not in schemas:
+                schemas.append(SCIM_ENTERPRISE_USER_SCHEMA)
+
+        name = self.build_scim_name(user, f)
+        emails = _deserialize_emails(f.scim_emails_json, username)
+
+        resource = ScimUserResource(
+            schemas=schemas,
             id=str(user.id),
             externalId=external_id,
             userName=username,
-            name=self._build_scim_name(user),
+            name=name,
             displayName=user.personal_name,
-            emails=[ScimEmail(value=username, type="work", primary=True)],
+            emails=emails,
             active=user.is_active,
             groups=group_refs,
             meta=ScimMeta(resourceType="User"),
         )
+        resource.enterprise_extension = enterprise_ext
+        return resource
 
     def build_group_resource(
         self,
@@ -106,19 +149,60 @@ class ScimProvider(ABC):
             meta=ScimMeta(resourceType="Group"),
         )
 
-    @staticmethod
-    def _build_scim_name(user: User) -> ScimName | None:
-        """Extract SCIM name components from a user's personal name."""
+    def build_scim_name(
+        self,
+        user: User,
+        fields: ScimMappingFields,
+    ) -> ScimName:
+        """Build SCIM name components for the response.
+
+        Round-trips stored ``given_name``/``family_name`` when available (so
+        the IdP gets back what it sent). Falls back to splitting
+        ``personal_name`` for users provisioned before we stored components.
+        Always returns a ScimName — Okta's spec tests expect ``name``
+        (with ``givenName``/``familyName``) on every user resource.
+        Providers may override for custom behavior.
+        """
+        if fields.given_name is not None or fields.family_name is not None:
+            return ScimName(
+                givenName=fields.given_name or "",
+                familyName=fields.family_name or "",
+                formatted=user.personal_name or "",
+            )
         if not user.personal_name:
-            return None
+            # Derive a reasonable name from the email so that SCIM spec tests
+            # see non-empty givenName / familyName for every user resource.
+            local = user.email.split("@")[0] if user.email else ""
+            return ScimName(givenName=local, familyName="", formatted=local)
         parts = user.personal_name.split(" ", 1)
         return ScimName(
             givenName=parts[0],
-            familyName=parts[1] if len(parts) > 1 else None,
+            familyName=parts[1] if len(parts) > 1 else "",
             formatted=user.personal_name,
         )
 
 
+def _deserialize_emails(stored_json: str | None, username: str) -> list[ScimEmail]:
+    """Deserialize stored email entries or build a default work email."""
+    if stored_json:
+        try:
+            entries = json.loads(stored_json)
+            if isinstance(entries, list) and entries:
+                return [ScimEmail(**e) for e in entries]
+        except (json.JSONDecodeError, TypeError, ValidationError):
+            logger.warning(
+                "Corrupt scim_emails_json, falling back to default: %s", stored_json
+            )
+    return [ScimEmail(value=username, type="work", primary=True)]
+
+
+def serialize_emails(emails: list[ScimEmail]) -> str | None:
+    """Serialize SCIM email entries to JSON for storage."""
+    if not emails:
+        return None
+    return json.dumps([e.model_dump(exclude_none=True) for e in emails])
+
+
 def get_default_provider() -> ScimProvider:
     """Return the default SCIM provider.
 

backend/ee/onyx/server/scim/providers/entra.py

@@ -0,0 +1,36 @@
+"""Entra ID (Azure AD) SCIM provider."""
+
+from __future__ import annotations
+
+from ee.onyx.server.scim.models import SCIM_ENTERPRISE_USER_SCHEMA
+from ee.onyx.server.scim.models import SCIM_USER_SCHEMA
+from ee.onyx.server.scim.providers.base import COMMON_IGNORED_PATCH_PATHS
+from ee.onyx.server.scim.providers.base import ScimProvider
+
+_ENTRA_IGNORED_PATCH_PATHS = COMMON_IGNORED_PATCH_PATHS
+
+
+class EntraProvider(ScimProvider):
+    """Entra ID (Azure AD) SCIM provider.
+
+    Entra behavioral notes:
+      - Sends capitalized PATCH ops (``"Add"``, ``"Replace"``, ``"Remove"``)
+        — handled by ``ScimPatchOperation.normalize_op`` validator.
+      - Sends the enterprise extension URN as a key in path-less PATCH value
+        dicts — handled by ``_set_enterprise_field`` in ``patch.py`` to
+        store department/manager values.
+      - Expects the enterprise extension schema in ``schemas`` arrays and
+        ``/Schemas`` + ``/ResourceTypes`` discovery endpoints.
+    """
+
+    @property
+    def name(self) -> str:
+        return "entra"
+
+    @property
+    def ignored_patch_paths(self) -> frozenset[str]:
+        return _ENTRA_IGNORED_PATCH_PATHS
+
+    @property
+    def user_schemas(self) -> list[str]:
+        return [SCIM_USER_SCHEMA, SCIM_ENTERPRISE_USER_SCHEMA]

backend/ee/onyx/server/scim/schema_definitions.py

@@ -4,6 +4,7 @@ Pre-built at import time — these never change at runtime. Separated from
 api.py to keep the endpoint module focused on request handling.
 """
 
+from ee.onyx.server.scim.models import SCIM_ENTERPRISE_USER_SCHEMA
 from ee.onyx.server.scim.models import SCIM_GROUP_SCHEMA
 from ee.onyx.server.scim.models import SCIM_USER_SCHEMA
 from ee.onyx.server.scim.models import ScimResourceType
@@ -20,6 +21,9 @@ USER_RESOURCE_TYPE = ScimResourceType.model_validate(
         "endpoint": "/scim/v2/Users",
         "description": "SCIM User resource",
         "schema": SCIM_USER_SCHEMA,
+        "schemaExtensions": [
+            {"schema": SCIM_ENTERPRISE_USER_SCHEMA, "required": False}
+        ],
     }
 )
 
@@ -104,6 +108,31 @@ USER_SCHEMA_DEF = ScimSchemaDefinition(
     ],
 )
 
+ENTERPRISE_USER_SCHEMA_DEF = ScimSchemaDefinition(
+    id=SCIM_ENTERPRISE_USER_SCHEMA,
+    name="EnterpriseUser",
+    description="Enterprise User extension (RFC 7643 §4.3)",
+    attributes=[
+        ScimSchemaAttribute(
+            name="department",
+            type="string",
+            description="Department.",
+        ),
+        ScimSchemaAttribute(
+            name="manager",
+            type="complex",
+            description="The user's manager.",
+            subAttributes=[
+                ScimSchemaAttribute(
+                    name="value",
+                    type="string",
+                    description="Manager user ID.",
+                ),
+            ],
+        ),
+    ],
+)
+
 GROUP_SCHEMA_DEF = ScimSchemaDefinition(
     id=SCIM_GROUP_SCHEMA,
     name="Group",

backend/ee/onyx/server/seeding.py

@@ -18,8 +18,8 @@ from ee.onyx.server.enterprise_settings.store import (
     store_settings as store_ee_settings,
 )
 from ee.onyx.server.enterprise_settings.store import upload_logo
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
+from onyx.db.llm import fetch_existing_llm_provider
 from onyx.db.llm import update_default_provider
 from onyx.db.llm import upsert_llm_provider
 from onyx.db.models import Tool
@@ -117,27 +117,38 @@ def _seed_custom_tools(db_session: Session, tools: List[CustomToolSeed]) -> None
 def _seed_llms(
     db_session: Session, llm_upsert_requests: list[LLMProviderUpsertRequest]
 ) -> None:
-    if llm_upsert_requests:
-        logger.notice("Seeding LLMs")
-        seeded_providers = [
-            upsert_llm_provider(llm_upsert_request, db_session)
-            for llm_upsert_request in llm_upsert_requests
-        ]
+    if not llm_upsert_requests:
+        return
 
-        if len(seeded_providers[0].model_configurations) > 0:
-            default_model = next(
-                (
-                    mc
-                    for mc in seeded_providers[0].model_configurations
-                    if mc.is_visible
-                ),
-                seeded_providers[0].model_configurations[0],
-            ).name
-            update_default_provider(
-                provider_id=seeded_providers[0].id,
-                model_name=default_model,
-                db_session=db_session,
-            )
+    logger.notice("Seeding LLMs")
+    for request in llm_upsert_requests:
+        existing = fetch_existing_llm_provider(name=request.name, db_session=db_session)
+        if existing:
+            request.id = existing.id
+    seeded_providers = [
+        upsert_llm_provider(llm_upsert_request, db_session)
+        for llm_upsert_request in llm_upsert_requests
+    ]
+
+    default_provider = next(
+        (p for p in seeded_providers if p.model_configurations), None
+    )
+    if not default_provider:
+        return
+
+    visible_configs = [
+        mc for mc in default_provider.model_configurations if mc.is_visible
+    ]
+    default_config = (
+        visible_configs[0]
+        if visible_configs
+        else default_provider.model_configurations[0]
+    )
+    update_default_provider(
+        provider_id=default_provider.id,
+        model_name=default_config.name,
+        db_session=db_session,
+    )
 
 
 def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) -> None:
@@ -149,12 +160,6 @@ def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) ->
                     user=None,  # Seeding is done as admin
                     name=persona.name,
                     description=persona.description,
-                    num_chunks=(
-                        persona.num_chunks if persona.num_chunks is not None else 0.0
-                    ),
-                    llm_relevance_filter=persona.llm_relevance_filter,
-                    llm_filter_extraction=persona.llm_filter_extraction,
-                    recency_bias=RecencyBiasSetting.AUTO,
                     document_set_ids=persona.document_set_ids,
                     llm_model_provider_override=persona.llm_model_provider_override,
                     llm_model_version_override=persona.llm_model_version_override,
@@ -166,6 +171,7 @@ def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) ->
                     system_prompt=persona.system_prompt,
                     task_prompt=persona.task_prompt,
                     datetime_aware=persona.datetime_aware,
+                    featured=persona.featured,
                     commit=False,
                 )
             db_session.commit()

backend/ee/onyx/server/settings/api.py

@@ -6,6 +6,7 @@ from sqlalchemy.exc import SQLAlchemyError
 from ee.onyx.configs.app_configs import LICENSE_ENFORCEMENT_ENABLED
 from ee.onyx.db.license import get_cached_license_metadata
 from ee.onyx.db.license import refresh_license_cache
+from onyx.cache.interface import CACHE_TRANSIENT_ERRORS
 from onyx.configs.app_configs import ENTERPRISE_EDITION_ENABLED
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
 from onyx.server.settings.models import ApplicationStatus
@@ -109,6 +110,12 @@ def apply_license_status_to_settings(settings: Settings) -> Settings:
             if metadata.status == _BLOCKING_STATUS:
                 settings.application_status = metadata.status
                 settings.ee_features_enabled = False
+            elif metadata.used_seats > metadata.seats:
+                # License is valid but seat limit exceeded
+                settings.application_status = ApplicationStatus.SEAT_LIMIT_EXCEEDED
+                settings.seat_count = metadata.seats
+                settings.used_seats = metadata.used_seats
+                settings.ee_features_enabled = True
             else:
                 # Has a valid license (GRACE_PERIOD/PAYMENT_REMINDER still allow EE features)
                 settings.ee_features_enabled = True
@@ -119,7 +126,7 @@ def apply_license_status_to_settings(settings: Settings) -> Settings:
                 # syncing) means indexed data may need protection.
                 settings.application_status = _BLOCKING_STATUS
             settings.ee_features_enabled = False
-    except RedisError as e:
+    except CACHE_TRANSIENT_ERRORS as e:
         logger.warning(f"Failed to check license metadata for settings: {e}")
         # Fail closed - disable EE features if we can't verify license
         settings.ee_features_enabled = False

backend/ee/onyx/server/tenants/billing_api.py

@@ -21,7 +21,6 @@ import asyncio
 import httpx
 from fastapi import APIRouter
 from fastapi import Depends
-from fastapi import HTTPException
 
 from ee.onyx.auth.users import current_admin_user
 from ee.onyx.server.tenants.access import control_plane_dep
@@ -43,6 +42,8 @@ from onyx.auth.users import User
 from onyx.configs.app_configs import STRIPE_PUBLISHABLE_KEY_OVERRIDE
 from onyx.configs.app_configs import STRIPE_PUBLISHABLE_KEY_URL
 from onyx.configs.app_configs import WEB_DOMAIN
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
 from onyx.utils.logger import setup_logger
 from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.contextvars import get_current_tenant_id
@@ -116,9 +117,14 @@ async def create_customer_portal_session(
     try:
         portal_url = fetch_customer_portal_session(tenant_id, return_url)
         return {"stripe_customer_portal_url": portal_url}
-    except Exception as e:
+    except OnyxError:
+        raise
+    except Exception:
         logger.exception("Failed to create customer portal session")
-        raise HTTPException(status_code=500, detail=str(e))
+        raise OnyxError(
+            OnyxErrorCode.INTERNAL_ERROR,
+            "Failed to create customer portal session",
+        )
 
 
 @router.post("/create-checkout-session")
@@ -134,9 +140,14 @@ async def create_checkout_session(
     try:
         checkout_url = fetch_stripe_checkout_session(tenant_id, billing_period, seats)
         return {"stripe_checkout_url": checkout_url}
-    except Exception as e:
+    except OnyxError:
+        raise
+    except Exception:
         logger.exception("Failed to create checkout session")
-        raise HTTPException(status_code=500, detail=str(e))
+        raise OnyxError(
+            OnyxErrorCode.INTERNAL_ERROR,
+            "Failed to create checkout session",
+        )
 
 
 @router.post("/create-subscription-session")
@@ -147,15 +158,20 @@ async def create_subscription_session(
     try:
         tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
         if not tenant_id:
-            raise HTTPException(status_code=400, detail="Tenant ID not found")
+            raise OnyxError(OnyxErrorCode.VALIDATION_ERROR, "Tenant ID not found")
 
         billing_period = request.billing_period if request else "monthly"
         session_id = fetch_stripe_checkout_session(tenant_id, billing_period)
         return SubscriptionSessionResponse(sessionId=session_id)
 
-    except Exception as e:
+    except OnyxError:
+        raise
+    except Exception:
         logger.exception("Failed to create subscription session")
-        raise HTTPException(status_code=500, detail=str(e))
+        raise OnyxError(
+            OnyxErrorCode.INTERNAL_ERROR,
+            "Failed to create subscription session",
+        )
 
 
 @router.get("/stripe-publishable-key")
@@ -186,18 +202,18 @@ async def get_stripe_publishable_key() -> StripePublishableKeyResponse:
         if STRIPE_PUBLISHABLE_KEY_OVERRIDE:
             key = STRIPE_PUBLISHABLE_KEY_OVERRIDE.strip()
             if not key.startswith("pk_"):
-                raise HTTPException(
-                    status_code=500,
-                    detail="Invalid Stripe publishable key format",
+                raise OnyxError(
+                    OnyxErrorCode.INTERNAL_ERROR,
+                    "Invalid Stripe publishable key format",
                 )
             _stripe_publishable_key_cache = key
             return StripePublishableKeyResponse(publishable_key=key)
 
         # Fall back to S3 bucket
         if not STRIPE_PUBLISHABLE_KEY_URL:
-            raise HTTPException(
-                status_code=500,
-                detail="Stripe publishable key is not configured",
+            raise OnyxError(
+                OnyxErrorCode.INTERNAL_ERROR,
+                "Stripe publishable key is not configured",
             )
 
         try:
@@ -208,15 +224,15 @@ async def get_stripe_publishable_key() -> StripePublishableKeyResponse:
 
                 # Validate key format
                 if not key.startswith("pk_"):
-                    raise HTTPException(
-                        status_code=500,
-                        detail="Invalid Stripe publishable key format",
+                    raise OnyxError(
+                        OnyxErrorCode.INTERNAL_ERROR,
+                        "Invalid Stripe publishable key format",
                     )
 
                 _stripe_publishable_key_cache = key
                 return StripePublishableKeyResponse(publishable_key=key)
         except httpx.HTTPError:
-            raise HTTPException(
-                status_code=500,
-                detail="Failed to fetch Stripe publishable key",
+            raise OnyxError(
+                OnyxErrorCode.INTERNAL_ERROR,
+                "Failed to fetch Stripe publishable key",
             )

backend/ee/onyx/server/tenants/provisioning.py

@@ -33,6 +33,7 @@ from onyx.configs.constants import MilestoneRecordType
 from onyx.db.engine.sql_engine import get_session_with_shared_schema
 from onyx.db.engine.sql_engine import get_session_with_tenant
 from onyx.db.image_generation import create_default_image_gen_config_from_api_key
+from onyx.db.llm import fetch_existing_llm_provider
 from onyx.db.llm import update_default_provider
 from onyx.db.llm import upsert_cloud_embedding_provider
 from onyx.db.llm import upsert_llm_provider
@@ -305,6 +306,11 @@ def configure_default_api_keys(db_session: Session) -> None:
     def _upsert(request: LLMProviderUpsertRequest, default_model: str) -> None:
         nonlocal has_set_default_provider
         try:
+            existing = fetch_existing_llm_provider(
+                name=request.name, db_session=db_session
+            )
+            if existing:
+                request.id = existing.id
             provider = upsert_llm_provider(request, db_session)
             if not has_set_default_provider:
                 update_default_provider(provider.id, default_model, db_session)

backend/ee/onyx/server/user_group/api.py

@@ -5,6 +5,8 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from ee.onyx.db.user_group import add_users_to_user_group
+from ee.onyx.db.user_group import delete_user_group as db_delete_user_group
+from ee.onyx.db.user_group import fetch_user_group
 from ee.onyx.db.user_group import fetch_user_groups
 from ee.onyx.db.user_group import fetch_user_groups_for_user
 from ee.onyx.db.user_group import insert_user_group
@@ -20,6 +22,7 @@ from ee.onyx.server.user_group.models import UserGroupUpdate
 from onyx.auth.users import current_admin_user
 from onyx.auth.users import current_curator_or_admin_user
 from onyx.auth.users import current_user
+from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.configs.constants import PUBLIC_API_TAGS
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.models import User
@@ -153,3 +156,8 @@ def delete_user_group(
         prepare_user_group_for_deletion(db_session, user_group_id)
     except ValueError as e:
         raise HTTPException(status_code=404, detail=str(e))
+
+    if DISABLE_VECTOR_DB:
+        user_group = fetch_user_group(db_session, user_group_id)
+        if user_group:
+            db_delete_user_group(db_session, user_group)

backend/onyx/auth/oauth_token_manager.py

@@ -58,16 +58,27 @@ class OAuthTokenManager:
         if not user_token.token_data:
             raise ValueError("No token data available for refresh")
 
+        if (
+            self.oauth_config.client_id is None
+            or self.oauth_config.client_secret is None
+        ):
+            raise ValueError(
+                "OAuth client_id and client_secret are required for token refresh"
+            )
+
         token_data = self._unwrap_token_data(user_token.token_data)
 
+        data: dict[str, str] = {
+            "grant_type": "refresh_token",
+            "refresh_token": token_data["refresh_token"],
+            "client_id": self._unwrap_sensitive_str(self.oauth_config.client_id),
+            "client_secret": self._unwrap_sensitive_str(
+                self.oauth_config.client_secret
+            ),
+        }
         response = requests.post(
             self.oauth_config.token_url,
-            data={
-                "grant_type": "refresh_token",
-                "refresh_token": token_data["refresh_token"],
-                "client_id": self.oauth_config.client_id,
-                "client_secret": self.oauth_config.client_secret,
-            },
+            data=data,
             headers={"Accept": "application/json"},
         )
         response.raise_for_status()
@@ -115,15 +126,26 @@ class OAuthTokenManager:
 
     def exchange_code_for_token(self, code: str, redirect_uri: str) -> dict[str, Any]:
         """Exchange authorization code for access token"""
+        if (
+            self.oauth_config.client_id is None
+            or self.oauth_config.client_secret is None
+        ):
+            raise ValueError(
+                "OAuth client_id and client_secret are required for code exchange"
+            )
+
+        data: dict[str, str] = {
+            "grant_type": "authorization_code",
+            "code": code,
+            "client_id": self._unwrap_sensitive_str(self.oauth_config.client_id),
+            "client_secret": self._unwrap_sensitive_str(
+                self.oauth_config.client_secret
+            ),
+            "redirect_uri": redirect_uri,
+        }
         response = requests.post(
             self.oauth_config.token_url,
-            data={
-                "grant_type": "authorization_code",
-                "code": code,
-                "client_id": self.oauth_config.client_id,
-                "client_secret": self.oauth_config.client_secret,
-                "redirect_uri": redirect_uri,
-            },
+            data=data,
             headers={"Accept": "application/json"},
         )
         response.raise_for_status()
@@ -141,8 +163,13 @@ class OAuthTokenManager:
         oauth_config: OAuthConfig, redirect_uri: str, state: str
     ) -> str:
         """Build OAuth authorization URL"""
+        if oauth_config.client_id is None:
+            raise ValueError("OAuth client_id is required to build authorization URL")
+
         params: dict[str, Any] = {
-            "client_id": oauth_config.client_id,
+            "client_id": OAuthTokenManager._unwrap_sensitive_str(
+                oauth_config.client_id
+            ),
             "redirect_uri": redirect_uri,
             "response_type": "code",
             "state": state,
@@ -161,6 +188,12 @@ class OAuthTokenManager:
 
         return f"{oauth_config.authorization_url}{separator}{urlencode(params)}"
 
+    @staticmethod
+    def _unwrap_sensitive_str(value: SensitiveValue[str] | str) -> str:
+        if isinstance(value, SensitiveValue):
+            return value.get_value(apply_mask=False)
+        return value
+
     @staticmethod
     def _unwrap_token_data(
         token_data: SensitiveValue[dict[str, Any]] | dict[str, Any],

backend/onyx/auth/users.py

@@ -120,7 +120,6 @@ from onyx.db.models import User
 from onyx.db.pat import fetch_user_for_pat
 from onyx.db.users import get_user_by_email
 from onyx.redis.redis_pool import get_async_redis_connection
-from onyx.redis.redis_pool import get_redis_client
 from onyx.server.settings.store import load_settings
 from onyx.server.utils import BasicAuthenticationError
 from onyx.utils.logger import setup_logger
@@ -201,13 +200,14 @@ def user_needs_to_be_verified() -> bool:
 
 
 def anonymous_user_enabled(*, tenant_id: str | None = None) -> bool:
-    redis_client = get_redis_client(tenant_id=tenant_id)
-    value = redis_client.get(OnyxRedisLocks.ANONYMOUS_USER_ENABLED)
+    from onyx.cache.factory import get_cache_backend
+
+    cache = get_cache_backend(tenant_id=tenant_id)
+    value = cache.get(OnyxRedisLocks.ANONYMOUS_USER_ENABLED)
 
     if value is None:
         return False
 
-    assert isinstance(value, bytes)
     return int(value.decode("utf-8")) == 1
 
 
@@ -543,7 +543,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
         result = await db_session.execute(
             select(Persona.id)
             .where(
-                Persona.is_default_persona.is_(True),
+                Persona.featured.is_(True),
                 Persona.is_public.is_(True),
                 Persona.is_visible.is_(True),
                 Persona.deleted.is_(False),
@@ -725,11 +725,19 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
                         if user_by_session:
                             user = user_by_session
 
+                # If the user is inactive, check seat availability before
+                # upgrading role — otherwise they'd become an inactive BASIC
+                # user who still can't log in.
+                if not user.is_active:
+                    with get_session_with_current_tenant() as sync_db:
+                        enforce_seat_limit(sync_db)
+
                 await self.user_db.update(
                     user,
                     {
                         "is_verified": is_verified_by_default,
                         "role": UserRole.BASIC,
+                        **({"is_active": True} if not user.is_active else {}),
                     },
                 )
 

backend/onyx/background/celery/apps/background.py

@@ -1,142 +0,0 @@
-from typing import Any
-from typing import cast
-
-from celery import Celery
-from celery import signals
-from celery import Task
-from celery.apps.worker import Worker
-from celery.signals import celeryd_init
-from celery.signals import worker_init
-from celery.signals import worker_process_init
-from celery.signals import worker_ready
-from celery.signals import worker_shutdown
-
-import onyx.background.celery.apps.app_base as app_base
-from onyx.background.celery.celery_utils import httpx_init_vespa_pool
-from onyx.configs.app_configs import MANAGED_VESPA
-from onyx.configs.app_configs import VESPA_CLOUD_CERT_PATH
-from onyx.configs.app_configs import VESPA_CLOUD_KEY_PATH
-from onyx.configs.constants import POSTGRES_CELERY_WORKER_BACKGROUND_APP_NAME
-from onyx.db.engine.sql_engine import SqlEngine
-from onyx.utils.logger import setup_logger
-from shared_configs.configs import MULTI_TENANT
-
-
-logger = setup_logger()
-
-celery_app = Celery(__name__)
-celery_app.config_from_object("onyx.background.celery.configs.background")
-celery_app.Task = app_base.TenantAwareTask  # type: ignore [misc]
-
-
-@signals.task_prerun.connect
-def on_task_prerun(
-    sender: Any | None = None,
-    task_id: str | None = None,
-    task: Task | None = None,
-    args: tuple | None = None,
-    kwargs: dict | None = None,
-    **kwds: Any,
-) -> None:
-    app_base.on_task_prerun(sender, task_id, task, args, kwargs, **kwds)
-
-
-@signals.task_postrun.connect
-def on_task_postrun(
-    sender: Any | None = None,
-    task_id: str | None = None,
-    task: Task | None = None,
-    args: tuple | None = None,
-    kwargs: dict | None = None,
-    retval: Any | None = None,
-    state: str | None = None,
-    **kwds: Any,
-) -> None:
-    app_base.on_task_postrun(sender, task_id, task, args, kwargs, retval, state, **kwds)
-
-
-@celeryd_init.connect
-def on_celeryd_init(sender: str, conf: Any = None, **kwargs: Any) -> None:
-    app_base.on_celeryd_init(sender, conf, **kwargs)
-
-
-@worker_init.connect
-def on_worker_init(sender: Worker, **kwargs: Any) -> None:
-    EXTRA_CONCURRENCY = 8  # small extra fudge factor for connection limits
-
-    logger.info("worker_init signal received for consolidated background worker.")
-
-    SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_BACKGROUND_APP_NAME)
-    pool_size = cast(int, sender.concurrency)  # type: ignore
-    SqlEngine.init_engine(pool_size=pool_size, max_overflow=EXTRA_CONCURRENCY)
-
-    # Initialize Vespa httpx pool (needed for light worker tasks)
-    if MANAGED_VESPA:
-        httpx_init_vespa_pool(
-            sender.concurrency + EXTRA_CONCURRENCY,  # type: ignore
-            ssl_cert=VESPA_CLOUD_CERT_PATH,
-            ssl_key=VESPA_CLOUD_KEY_PATH,
-        )
-    else:
-        httpx_init_vespa_pool(sender.concurrency + EXTRA_CONCURRENCY)  # type: ignore
-
-    app_base.wait_for_redis(sender, **kwargs)
-    app_base.wait_for_db(sender, **kwargs)
-    app_base.wait_for_vespa_or_shutdown(sender, **kwargs)
-
-    # Less startup checks in multi-tenant case
-    if MULTI_TENANT:
-        return
-
-    app_base.on_secondary_worker_init(sender, **kwargs)
-
-
-@worker_ready.connect
-def on_worker_ready(sender: Any, **kwargs: Any) -> None:
-    app_base.on_worker_ready(sender, **kwargs)
-
-
-@worker_shutdown.connect
-def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
-    app_base.on_worker_shutdown(sender, **kwargs)
-
-
-@worker_process_init.connect
-def init_worker(**kwargs: Any) -> None:  # noqa: ARG001
-    SqlEngine.reset_engine()
-
-
-@signals.setup_logging.connect
-def on_setup_logging(
-    loglevel: Any, logfile: Any, format: Any, colorize: Any, **kwargs: Any
-) -> None:
-    app_base.on_setup_logging(loglevel, logfile, format, colorize, **kwargs)
-
-
-base_bootsteps = app_base.get_bootsteps()
-for bootstep in base_bootsteps:
-    celery_app.steps["worker"].add(bootstep)
-
-celery_app.autodiscover_tasks(
-    app_base.filter_task_modules(
-        [
-            # Original background worker tasks
-            "onyx.background.celery.tasks.pruning",
-            "onyx.background.celery.tasks.monitoring",
-            "onyx.background.celery.tasks.user_file_processing",
-            "onyx.background.celery.tasks.llm_model_update",
-            # Light worker tasks
-            "onyx.background.celery.tasks.shared",
-            "onyx.background.celery.tasks.vespa",
-            "onyx.background.celery.tasks.connector_deletion",
-            "onyx.background.celery.tasks.doc_permission_syncing",
-            "onyx.background.celery.tasks.opensearch_migration",
-            # Docprocessing worker tasks
-            "onyx.background.celery.tasks.docprocessing",
-            # Docfetching worker tasks
-            "onyx.background.celery.tasks.docfetching",
-            # Sandbox cleanup tasks (isolated in build feature)
-            "onyx.server.features.build.sandbox.tasks",
-        ]
-    )
-)

backend/onyx/background/celery/configs/background.py

@@ -1,23 +0,0 @@
-import onyx.background.celery.configs.base as shared_config
-from onyx.configs.app_configs import CELERY_WORKER_BACKGROUND_CONCURRENCY
-
-broker_url = shared_config.broker_url
-broker_connection_retry_on_startup = shared_config.broker_connection_retry_on_startup
-broker_pool_limit = shared_config.broker_pool_limit
-broker_transport_options = shared_config.broker_transport_options
-
-redis_socket_keepalive = shared_config.redis_socket_keepalive
-redis_retry_on_timeout = shared_config.redis_retry_on_timeout
-redis_backend_health_check_interval = shared_config.redis_backend_health_check_interval
-
-result_backend = shared_config.result_backend
-result_expires = shared_config.result_expires  # 86400 seconds is the default
-
-task_default_priority = shared_config.task_default_priority
-task_acks_late = shared_config.task_acks_late
-
-worker_concurrency = CELERY_WORKER_BACKGROUND_CONCURRENCY
-worker_pool = "threads"
-# Increased from 1 to 4 to handle fast light worker tasks more efficiently
-# This allows the worker to prefetch multiple tasks per thread
-worker_prefetch_multiplier = 4

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -241,8 +241,7 @@ _VECTOR_DB_BEAT_TASK_NAMES: set[str] = {
     "check-for-index-attempt-cleanup",
     "check-for-doc-permissions-sync",
     "check-for-external-group-sync",
-    "check-for-documents-for-opensearch-migration",
-    "migrate-documents-from-vespa-to-opensearch",
+    "migrate-chunks-from-vespa-to-opensearch",
 }
 
 if DISABLE_VECTOR_DB:

backend/onyx/background/celery/tasks/opensearch_migration/tasks.py

@@ -30,6 +30,7 @@ from onyx.background.celery.tasks.opensearch_migration.transformer import (
     transform_vespa_chunks_to_opensearch_chunks,
 )
 from onyx.configs.app_configs import ENABLE_OPENSEARCH_INDEXING_FOR_ONYX
+from onyx.configs.app_configs import VESPA_MIGRATION_REQUEST_TIMEOUT_S
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
@@ -47,7 +48,9 @@ from onyx.document_index.interfaces_new import TenantState
 from onyx.document_index.opensearch.opensearch_document_index import (
     OpenSearchDocumentIndex,
 )
+from onyx.document_index.vespa.shared_utils.utils import get_vespa_http_client
 from onyx.document_index.vespa.vespa_document_index import VespaDocumentIndex
+from onyx.indexing.models import IndexingSetting
 from onyx.redis.redis_pool import get_redis_client
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.contextvars import get_current_tenant_id
@@ -145,17 +148,27 @@ def migrate_chunks_from_vespa_to_opensearch_task(
             task_logger.error(err_str)
             return False
 
-        with get_session_with_current_tenant() as db_session:
+        with (
+            get_session_with_current_tenant() as db_session,
+            get_vespa_http_client(
+                timeout=VESPA_MIGRATION_REQUEST_TIMEOUT_S
+            ) as vespa_client,
+        ):
             try_insert_opensearch_tenant_migration_record_with_commit(db_session)
             search_settings = get_current_search_settings(db_session)
             tenant_state = TenantState(tenant_id=tenant_id, multitenant=MULTI_TENANT)
+            indexing_setting = IndexingSetting.from_db_model(search_settings)
             opensearch_document_index = OpenSearchDocumentIndex(
-                index_name=search_settings.index_name, tenant_state=tenant_state
+                tenant_state=tenant_state,
+                index_name=search_settings.index_name,
+                embedding_dim=indexing_setting.final_embedding_dim,
+                embedding_precision=indexing_setting.embedding_precision,
             )
             vespa_document_index = VespaDocumentIndex(
                 index_name=search_settings.index_name,
                 tenant_state=tenant_state,
                 large_chunks_enabled=False,
+                httpx_client=vespa_client,
             )
 
             sanitized_doc_start_time = time.monotonic()

backend/onyx/background/celery/tasks/user_file_processing/tasks.py

@@ -76,7 +76,7 @@ def _user_file_queued_key(user_file_id: str | UUID) -> str:
     return f"{OnyxRedisLocks.USER_FILE_QUEUED_PREFIX}:{user_file_id}"
 
 
-def _user_file_project_sync_lock_key(user_file_id: str | UUID) -> str:
+def user_file_project_sync_lock_key(user_file_id: str | UUID) -> str:
     return f"{OnyxRedisLocks.USER_FILE_PROJECT_SYNC_LOCK_PREFIX}:{user_file_id}"
 
 
@@ -414,34 +414,31 @@ def _process_user_file_with_indexing(
         raise RuntimeError(f"Indexing pipeline failed for user file {user_file_id}")
 
 
-@shared_task(
-    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
-    bind=True,
-    ignore_result=True,
-)
-def process_single_user_file(
-    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+def process_user_file_impl(
+    *, user_file_id: str, tenant_id: str, redis_locking: bool
 ) -> None:
-    task_logger.info(f"process_single_user_file - Starting id={user_file_id}")
+    """Core implementation for processing a single user file.
+
+    When redis_locking=True, acquires a per-file Redis lock and clears the
+    queued-key guard (Celery path).  When redis_locking=False, skips all Redis
+    operations (BackgroundTask path).
+    """
+    task_logger.info(f"process_user_file_impl - Starting id={user_file_id}")
     start = time.monotonic()
 
-    redis_client = get_redis_client(tenant_id=tenant_id)
-
-    # Clear the "queued" guard set by the beat generator so that the next beat
-    # cycle can re-enqueue this file if it is still in PROCESSING state after
-    # this task completes or fails.
-    redis_client.delete(_user_file_queued_key(user_file_id))
-
-    file_lock: RedisLock = redis_client.lock(
-        _user_file_lock_key(user_file_id),
-        timeout=CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT,
-    )
-
-    if not file_lock.acquire(blocking=False):
-        task_logger.info(
-            f"process_single_user_file - Lock held, skipping user_file_id={user_file_id}"
+    file_lock: RedisLock | None = None
+    if redis_locking:
+        redis_client = get_redis_client(tenant_id=tenant_id)
+        redis_client.delete(_user_file_queued_key(user_file_id))
+        file_lock = redis_client.lock(
+            _user_file_lock_key(user_file_id),
+            timeout=CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT,
         )
-        return None
+        if file_lock is not None and not file_lock.acquire(blocking=False):
+            task_logger.info(
+                f"process_user_file_impl - Lock held, skipping user_file_id={user_file_id}"
+            )
+            return
 
     documents: list[Document] = []
     try:
@@ -449,15 +446,18 @@ def process_single_user_file(
             uf = db_session.get(UserFile, _as_uuid(user_file_id))
             if not uf:
                 task_logger.warning(
-                    f"process_single_user_file - UserFile not found id={user_file_id}"
+                    f"process_user_file_impl - UserFile not found id={user_file_id}"
                 )
-                return None
+                return
 
-            if uf.status != UserFileStatus.PROCESSING:
+            if uf.status not in (
+                UserFileStatus.PROCESSING,
+                UserFileStatus.INDEXING,
+            ):
                 task_logger.info(
-                    f"process_single_user_file - Skipping id={user_file_id} status={uf.status}"
+                    f"process_user_file_impl - Skipping id={user_file_id} status={uf.status}"
                 )
-                return None
+                return
 
             connector = LocalFileConnector(
                 file_locations=[uf.file_id],
@@ -471,7 +471,6 @@ def process_single_user_file(
                         [doc for doc in batch if not isinstance(doc, HierarchyNode)]
                     )
 
-                # update the document id to userfile id in the documents
                 for document in documents:
                     document.id = str(user_file_id)
                     document.source = DocumentSource.USER_FILE
@@ -493,9 +492,8 @@ def process_single_user_file(
 
             except Exception as e:
                 task_logger.exception(
-                    f"process_single_user_file - Error processing file id={user_file_id} - {e.__class__.__name__}"
+                    f"process_user_file_impl - Error processing file id={user_file_id} - {e.__class__.__name__}"
                 )
-                # don't update the status if the user file is being deleted
                 current_user_file = db_session.get(UserFile, _as_uuid(user_file_id))
                 if (
                     current_user_file
@@ -504,33 +502,43 @@ def process_single_user_file(
                     uf.status = UserFileStatus.FAILED
                     db_session.add(uf)
                     db_session.commit()
-                return None
+                return
 
         elapsed = time.monotonic() - start
         task_logger.info(
-            f"process_single_user_file - Finished id={user_file_id} docs={len(documents)} elapsed={elapsed:.2f}s"
+            f"process_user_file_impl - Finished id={user_file_id} docs={len(documents)} elapsed={elapsed:.2f}s"
         )
-        return None
     except Exception as e:
-        # Attempt to mark the file as failed
         with get_session_with_current_tenant() as db_session:
             uf = db_session.get(UserFile, _as_uuid(user_file_id))
             if uf:
-                # don't update the status if the user file is being deleted
                 if uf.status != UserFileStatus.DELETING:
                     uf.status = UserFileStatus.FAILED
                 db_session.add(uf)
                 db_session.commit()
 
         task_logger.exception(
-            f"process_single_user_file - Error processing file id={user_file_id} - {e.__class__.__name__}"
+            f"process_user_file_impl - Error processing file id={user_file_id} - {e.__class__.__name__}"
         )
-        return None
+        raise
     finally:
-        if file_lock.owned():
+        if file_lock is not None and file_lock.owned():
             file_lock.release()
 
 
+@shared_task(
+    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file(
+    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+) -> None:
+    process_user_file_impl(
+        user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True
+    )
+
+
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_USER_FILE_DELETE,
     soft_time_limit=300,
@@ -581,36 +589,38 @@ def check_for_user_file_delete(self: Task, *, tenant_id: str) -> None:
     return None
 
 
-@shared_task(
-    name=OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
-    bind=True,
-    ignore_result=True,
-)
-def process_single_user_file_delete(
-    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+def delete_user_file_impl(
+    *, user_file_id: str, tenant_id: str, redis_locking: bool
 ) -> None:
-    """Process a single user file delete."""
-    task_logger.info(f"process_single_user_file_delete - Starting id={user_file_id}")
-    redis_client = get_redis_client(tenant_id=tenant_id)
-    file_lock: RedisLock = redis_client.lock(
-        _user_file_delete_lock_key(user_file_id),
-        timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
-    )
-    if not file_lock.acquire(blocking=False):
-        task_logger.info(
-            f"process_single_user_file_delete - Lock held, skipping user_file_id={user_file_id}"
+    """Core implementation for deleting a single user file.
+
+    When redis_locking=True, acquires a per-file Redis lock (Celery path).
+    When redis_locking=False, skips Redis operations (BackgroundTask path).
+    """
+    task_logger.info(f"delete_user_file_impl - Starting id={user_file_id}")
+
+    file_lock: RedisLock | None = None
+    if redis_locking:
+        redis_client = get_redis_client(tenant_id=tenant_id)
+        file_lock = redis_client.lock(
+            _user_file_delete_lock_key(user_file_id),
+            timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
         )
-        return None
+        if file_lock is not None and not file_lock.acquire(blocking=False):
+            task_logger.info(
+                f"delete_user_file_impl - Lock held, skipping user_file_id={user_file_id}"
+            )
+            return
+
     try:
         with get_session_with_current_tenant() as db_session:
             user_file = db_session.get(UserFile, _as_uuid(user_file_id))
             if not user_file:
                 task_logger.info(
-                    f"process_single_user_file_delete - User file not found id={user_file_id}"
+                    f"delete_user_file_impl - User file not found id={user_file_id}"
                 )
-                return None
+                return
 
-            # 1) Delete vector DB chunks (skip when disabled)
             if not DISABLE_VECTOR_DB:
                 if MANAGED_VESPA:
                     httpx_init_vespa_pool(
@@ -648,7 +658,6 @@ def process_single_user_file_delete(
                         chunk_count=chunk_count,
                     )
 
-            # 2) Delete the user-uploaded file content from filestore (blob + metadata)
             file_store = get_default_file_store()
             try:
                 file_store.delete_file(user_file.file_id)
@@ -656,26 +665,34 @@ def process_single_user_file_delete(
                     user_file_id_to_plaintext_file_name(user_file.id)
                 )
             except Exception as e:
-                # This block executed only if the file is not found in the filestore
                 task_logger.exception(
-                    f"process_single_user_file_delete - Error deleting file id={user_file.id} - {e.__class__.__name__}"
+                    f"delete_user_file_impl - Error deleting file id={user_file.id} - {e.__class__.__name__}"
                 )
 
-            # 3) Finally, delete the UserFile row
             db_session.delete(user_file)
             db_session.commit()
-            task_logger.info(
-                f"process_single_user_file_delete - Completed id={user_file_id}"
-            )
+            task_logger.info(f"delete_user_file_impl - Completed id={user_file_id}")
     except Exception as e:
         task_logger.exception(
-            f"process_single_user_file_delete - Error processing file id={user_file_id} - {e.__class__.__name__}"
+            f"delete_user_file_impl - Error processing file id={user_file_id} - {e.__class__.__name__}"
         )
-        return None
+        raise
     finally:
-        if file_lock.owned():
+        if file_lock is not None and file_lock.owned():
             file_lock.release()
-    return None
+
+
+@shared_task(
+    name=OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file_delete(
+    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+) -> None:
+    delete_user_file_impl(
+        user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True
+    )
 
 
 @shared_task(
@@ -747,32 +764,30 @@ def check_for_user_file_project_sync(self: Task, *, tenant_id: str) -> None:
     return None
 
 
-@shared_task(
-    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE_PROJECT_SYNC,
-    bind=True,
-    ignore_result=True,
-)
-def process_single_user_file_project_sync(
-    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+def project_sync_user_file_impl(
+    *, user_file_id: str, tenant_id: str, redis_locking: bool
 ) -> None:
-    """Process a single user file project sync."""
-    task_logger.info(
-        f"process_single_user_file_project_sync - Starting id={user_file_id}"
-    )
+    """Core implementation for syncing a user file's project/persona metadata.
 
-    redis_client = get_redis_client(tenant_id=tenant_id)
-    redis_client.delete(_user_file_project_sync_queued_key(user_file_id))
+    When redis_locking=True, acquires a per-file Redis lock and clears the
+    queued-key guard (Celery path).  When redis_locking=False, skips Redis
+    operations (BackgroundTask path).
+    """
+    task_logger.info(f"project_sync_user_file_impl - Starting id={user_file_id}")
 
-    file_lock: RedisLock = redis_client.lock(
-        _user_file_project_sync_lock_key(user_file_id),
-        timeout=CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT,
-    )
-
-    if not file_lock.acquire(blocking=False):
-        task_logger.info(
-            f"process_single_user_file_project_sync - Lock held, skipping user_file_id={user_file_id}"
+    file_lock: RedisLock | None = None
+    if redis_locking:
+        redis_client = get_redis_client(tenant_id=tenant_id)
+        redis_client.delete(_user_file_project_sync_queued_key(user_file_id))
+        file_lock = redis_client.lock(
+            user_file_project_sync_lock_key(user_file_id),
+            timeout=CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT,
         )
-        return None
+        if file_lock is not None and not file_lock.acquire(blocking=False):
+            task_logger.info(
+                f"project_sync_user_file_impl - Lock held, skipping user_file_id={user_file_id}"
+            )
+            return
 
     try:
         with get_session_with_current_tenant() as db_session:
@@ -783,11 +798,10 @@ def process_single_user_file_project_sync(
             ).scalar_one_or_none()
             if not user_file:
                 task_logger.info(
-                    f"process_single_user_file_project_sync - User file not found id={user_file_id}"
+                    f"project_sync_user_file_impl - User file not found id={user_file_id}"
                 )
-                return None
+                return
 
-            # Sync project metadata to vector DB (skip when disabled)
             if not DISABLE_VECTOR_DB:
                 if MANAGED_VESPA:
                     httpx_init_vespa_pool(
@@ -822,7 +836,7 @@ def process_single_user_file_project_sync(
                     )
 
             task_logger.info(
-                f"process_single_user_file_project_sync - User file id={user_file_id}"
+                f"project_sync_user_file_impl - User file id={user_file_id}"
             )
 
             user_file.needs_project_sync = False
@@ -835,11 +849,22 @@ def process_single_user_file_project_sync(
 
     except Exception as e:
         task_logger.exception(
-            f"process_single_user_file_project_sync - Error syncing project for file id={user_file_id} - {e.__class__.__name__}"
+            f"project_sync_user_file_impl - Error syncing project for file id={user_file_id} - {e.__class__.__name__}"
         )
-        return None
+        raise
     finally:
-        if file_lock.owned():
+        if file_lock is not None and file_lock.owned():
             file_lock.release()
 
-    return None
+
+@shared_task(
+    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE_PROJECT_SYNC,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file_project_sync(
+    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+) -> None:
+    project_sync_user_file_impl(
+        user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True
+    )

backend/onyx/background/celery/versioned_apps/background.py

@@ -1,10 +0,0 @@
-from celery import Celery
-
-from onyx.utils.variable_functionality import fetch_versioned_implementation
-from onyx.utils.variable_functionality import set_is_ee_based_on_env_variable
-
-set_is_ee_based_on_env_variable()
-app: Celery = fetch_versioned_implementation(
-    "onyx.background.celery.apps.background",
-    "celery_app",
-)

backend/onyx/background/periodic_poller.py

@@ -0,0 +1,307 @@
+"""Periodic poller for NO_VECTOR_DB deployments.
+
+Replaces Celery Beat and background workers with a lightweight daemon thread
+that runs from the API server process.  Two responsibilities:
+
+1. Recovery polling (every 30 s): re-processes user files stuck in
+   PROCESSING / DELETING / needs_sync states via the drain loops defined
+   in ``task_utils.py``.
+
+2. Periodic task execution (configurable intervals): runs LLM model updates
+   and scheduled evals at their configured cadences, with Postgres advisory
+   lock deduplication across multiple API server instances.
+"""
+
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from dataclasses import field
+
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+RECOVERY_INTERVAL_SECONDS = 30
+PERIODIC_TASK_LOCK_BASE = 20_000
+PERIODIC_TASK_KV_PREFIX = "periodic_poller:last_claimed:"
+
+
+# ------------------------------------------------------------------
+# Periodic task definitions
+# ------------------------------------------------------------------
+
+
+_NEVER_RAN: float = -1e18
+
+
+@dataclass
+class _PeriodicTaskDef:
+    name: str
+    interval_seconds: float
+    lock_id: int
+    run_fn: Callable[[], None]
+    last_run_at: float = field(default=_NEVER_RAN)
+
+
+def _run_auto_llm_update() -> None:
+    from onyx.configs.app_configs import AUTO_LLM_CONFIG_URL
+
+    if not AUTO_LLM_CONFIG_URL:
+        return
+
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+    from onyx.llm.well_known_providers.auto_update_service import (
+        sync_llm_models_from_github,
+    )
+
+    with get_session_with_current_tenant() as db_session:
+        sync_llm_models_from_github(db_session)
+
+
+def _run_cache_cleanup() -> None:
+    from onyx.cache.postgres_backend import cleanup_expired_cache_entries
+
+    cleanup_expired_cache_entries()
+
+
+def _run_scheduled_eval() -> None:
+    from onyx.configs.app_configs import BRAINTRUST_API_KEY
+    from onyx.configs.app_configs import SCHEDULED_EVAL_DATASET_NAMES
+    from onyx.configs.app_configs import SCHEDULED_EVAL_PERMISSIONS_EMAIL
+    from onyx.configs.app_configs import SCHEDULED_EVAL_PROJECT
+
+    if not all(
+        [
+            BRAINTRUST_API_KEY,
+            SCHEDULED_EVAL_PROJECT,
+            SCHEDULED_EVAL_DATASET_NAMES,
+            SCHEDULED_EVAL_PERMISSIONS_EMAIL,
+        ]
+    ):
+        return
+
+    from datetime import datetime
+    from datetime import timezone
+
+    from onyx.evals.eval import run_eval
+    from onyx.evals.models import EvalConfigurationOptions
+
+    run_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    for dataset_name in SCHEDULED_EVAL_DATASET_NAMES:
+        try:
+            run_eval(
+                configuration=EvalConfigurationOptions(
+                    search_permissions_email=SCHEDULED_EVAL_PERMISSIONS_EMAIL,
+                    dataset_name=dataset_name,
+                    no_send_logs=False,
+                    braintrust_project=SCHEDULED_EVAL_PROJECT,
+                    experiment_name=f"{dataset_name} - {run_timestamp}",
+                ),
+                remote_dataset_name=dataset_name,
+            )
+        except Exception:
+            logger.exception(
+                f"Periodic poller - Failed scheduled eval for dataset {dataset_name}"
+            )
+
+
+_CACHE_CLEANUP_INTERVAL_SECONDS = 300
+
+
+def _build_periodic_tasks() -> list[_PeriodicTaskDef]:
+    from onyx.cache.interface import CacheBackendType
+    from onyx.configs.app_configs import AUTO_LLM_CONFIG_URL
+    from onyx.configs.app_configs import AUTO_LLM_UPDATE_INTERVAL_SECONDS
+    from onyx.configs.app_configs import CACHE_BACKEND
+    from onyx.configs.app_configs import SCHEDULED_EVAL_DATASET_NAMES
+
+    tasks: list[_PeriodicTaskDef] = []
+    if CACHE_BACKEND == CacheBackendType.POSTGRES:
+        tasks.append(
+            _PeriodicTaskDef(
+                name="cache-cleanup",
+                interval_seconds=_CACHE_CLEANUP_INTERVAL_SECONDS,
+                lock_id=PERIODIC_TASK_LOCK_BASE + 2,
+                run_fn=_run_cache_cleanup,
+            )
+        )
+    if AUTO_LLM_CONFIG_URL:
+        tasks.append(
+            _PeriodicTaskDef(
+                name="auto-llm-update",
+                interval_seconds=AUTO_LLM_UPDATE_INTERVAL_SECONDS,
+                lock_id=PERIODIC_TASK_LOCK_BASE,
+                run_fn=_run_auto_llm_update,
+            )
+        )
+    if SCHEDULED_EVAL_DATASET_NAMES:
+        tasks.append(
+            _PeriodicTaskDef(
+                name="scheduled-eval",
+                interval_seconds=7 * 24 * 3600,
+                lock_id=PERIODIC_TASK_LOCK_BASE + 1,
+                run_fn=_run_scheduled_eval,
+            )
+        )
+    return tasks
+
+
+# ------------------------------------------------------------------
+# Periodic task runner with advisory-lock-guarded claim
+# ------------------------------------------------------------------
+
+
+def _try_claim_task(task_def: _PeriodicTaskDef) -> bool:
+    """Atomically check whether *task_def* should run and record a claim.
+
+    Uses a transaction-scoped advisory lock for atomicity combined with a
+    ``KVStore`` timestamp for cross-instance dedup.  The DB session is held
+    only for this brief claim transaction, not during task execution.
+    """
+    from datetime import datetime
+    from datetime import timezone
+
+    from sqlalchemy import text
+
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+    from onyx.db.models import KVStore
+
+    kv_key = PERIODIC_TASK_KV_PREFIX + task_def.name
+
+    with get_session_with_current_tenant() as db_session:
+        acquired = db_session.execute(
+            text("SELECT pg_try_advisory_xact_lock(:id)"),
+            {"id": task_def.lock_id},
+        ).scalar()
+        if not acquired:
+            return False
+
+        row = db_session.query(KVStore).filter_by(key=kv_key).first()
+        if row and row.value is not None:
+            last_claimed = datetime.fromisoformat(str(row.value))
+            elapsed = (datetime.now(timezone.utc) - last_claimed).total_seconds()
+            if elapsed < task_def.interval_seconds:
+                return False
+
+        now_ts = datetime.now(timezone.utc).isoformat()
+        if row:
+            row.value = now_ts
+        else:
+            db_session.add(KVStore(key=kv_key, value=now_ts))
+        db_session.commit()
+
+    return True
+
+
+def _try_run_periodic_task(task_def: _PeriodicTaskDef) -> None:
+    """Run *task_def* if its interval has elapsed and no peer holds the lock."""
+    now = time.monotonic()
+    if now - task_def.last_run_at < task_def.interval_seconds:
+        return
+
+    if not _try_claim_task(task_def):
+        return
+
+    try:
+        task_def.run_fn()
+        task_def.last_run_at = now
+    except Exception:
+        logger.exception(
+            f"Periodic poller - Error running periodic task {task_def.name}"
+        )
+
+
+# ------------------------------------------------------------------
+# Recovery / drain loop runner
+# ------------------------------------------------------------------
+
+
+def _run_drain_loops(tenant_id: str) -> None:
+    from onyx.background.task_utils import drain_delete_loop
+    from onyx.background.task_utils import drain_processing_loop
+    from onyx.background.task_utils import drain_project_sync_loop
+
+    drain_processing_loop(tenant_id)
+    drain_delete_loop(tenant_id)
+    drain_project_sync_loop(tenant_id)
+
+
+# ------------------------------------------------------------------
+# Startup recovery (10g)
+# ------------------------------------------------------------------
+
+
+def recover_stuck_user_files(tenant_id: str) -> None:
+    """Run all drain loops once to re-process files left in intermediate states.
+
+    Called from ``lifespan()`` on startup when ``DISABLE_VECTOR_DB`` is set.
+    """
+    logger.info("recover_stuck_user_files - Checking for stuck user files")
+    try:
+        _run_drain_loops(tenant_id)
+    except Exception:
+        logger.exception("recover_stuck_user_files - Error during recovery")
+
+
+# ------------------------------------------------------------------
+# Daemon thread (10f)
+# ------------------------------------------------------------------
+
+_shutdown_event = threading.Event()
+_poller_thread: threading.Thread | None = None
+
+
+def _poller_loop(tenant_id: str) -> None:
+    from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
+
+    CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
+
+    periodic_tasks = _build_periodic_tasks()
+    logger.info(
+        f"Periodic poller started with {len(periodic_tasks)} periodic task(s): "
+        f"{[t.name for t in periodic_tasks]}"
+    )
+
+    while not _shutdown_event.is_set():
+        try:
+            _run_drain_loops(tenant_id)
+        except Exception:
+            logger.exception("Periodic poller - Error in recovery polling")
+
+        for task_def in periodic_tasks:
+            try:
+                _try_run_periodic_task(task_def)
+            except Exception:
+                logger.exception(
+                    f"Periodic poller - Unhandled error checking task {task_def.name}"
+                )
+
+        _shutdown_event.wait(RECOVERY_INTERVAL_SECONDS)
+
+
+def start_periodic_poller(tenant_id: str) -> None:
+    """Start the periodic poller daemon thread."""
+    global _poller_thread  # noqa: PLW0603
+    _shutdown_event.clear()
+    _poller_thread = threading.Thread(
+        target=_poller_loop,
+        args=(tenant_id,),
+        daemon=True,
+        name="no-vectordb-periodic-poller",
+    )
+    _poller_thread.start()
+    logger.info("Periodic poller thread started")
+
+
+def stop_periodic_poller() -> None:
+    """Signal the periodic poller to stop and wait for it to exit."""
+    global _poller_thread  # noqa: PLW0603
+    if _poller_thread is None:
+        return
+    _shutdown_event.set()
+    _poller_thread.join(timeout=10)
+    if _poller_thread.is_alive():
+        logger.warning("Periodic poller thread did not stop within timeout")
+    _poller_thread = None
+    logger.info("Periodic poller thread stopped")

backend/onyx/background/task_utils.py

@@ -1,3 +1,33 @@
+"""Background task utilities.
+
+Contains query-history report helpers (used by all deployment modes) and
+in-process background task execution helpers for NO_VECTOR_DB mode:
+
+- Atomic claim-and-mark helpers that prevent duplicate processing
+- Drain loops that process all pending user file work
+
+Each claim function runs a short-lived transaction: SELECT ... FOR UPDATE
+SKIP LOCKED, UPDATE the row to remove it from future queries, COMMIT.
+After the commit the row lock is released, but the row is no longer
+eligible for re-claiming.  No long-lived sessions or advisory locks.
+"""
+
+from uuid import UUID
+
+import sqlalchemy as sa
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from onyx.db.enums import UserFileStatus
+from onyx.db.models import UserFile
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+# ------------------------------------------------------------------
+# Query-history report helpers (pre-existing, used by all modes)
+# ------------------------------------------------------------------
+
 QUERY_REPORT_NAME_PREFIX = "query-history"
 
 
@@ -9,3 +39,168 @@ def construct_query_history_report_name(
 
 def extract_task_id_from_query_history_report_name(name: str) -> str:
     return name.removeprefix(f"{QUERY_REPORT_NAME_PREFIX}-").removesuffix(".csv")
+
+
+# ------------------------------------------------------------------
+# Atomic claim-and-mark helpers
+# ------------------------------------------------------------------
+# Each function runs inside a single short-lived session/transaction:
+#   1. SELECT ... FOR UPDATE SKIP LOCKED  (locks one eligible row)
+#   2. UPDATE the row so it is no longer eligible
+#   3. COMMIT  (releases the row lock)
+# After the commit, no other drain loop can claim the same row.
+
+
+def _claim_next_processing_file(db_session: Session) -> UUID | None:
+    """Claim the next PROCESSING file by transitioning it to INDEXING.
+
+    Returns the file id, or None when no eligible files remain.
+    """
+    file_id = db_session.execute(
+        select(UserFile.id)
+        .where(UserFile.status == UserFileStatus.PROCESSING)
+        .order_by(UserFile.created_at)
+        .limit(1)
+        .with_for_update(skip_locked=True)
+    ).scalar_one_or_none()
+    if file_id is None:
+        return None
+
+    db_session.execute(
+        sa.update(UserFile)
+        .where(UserFile.id == file_id)
+        .values(status=UserFileStatus.INDEXING)
+    )
+    db_session.commit()
+    return file_id
+
+
+def _claim_next_deleting_file(
+    db_session: Session,
+    exclude_ids: set[UUID] | None = None,
+) -> UUID | None:
+    """Claim the next DELETING file.
+
+    No status transition needed — the impl deletes the row on success.
+    The short-lived FOR UPDATE lock prevents concurrent claims.
+    *exclude_ids* prevents re-processing the same file if the impl fails.
+    """
+    stmt = (
+        select(UserFile.id)
+        .where(UserFile.status == UserFileStatus.DELETING)
+        .order_by(UserFile.created_at)
+        .limit(1)
+        .with_for_update(skip_locked=True)
+    )
+    if exclude_ids:
+        stmt = stmt.where(UserFile.id.notin_(exclude_ids))
+    file_id = db_session.execute(stmt).scalar_one_or_none()
+    db_session.commit()
+    return file_id
+
+
+def _claim_next_sync_file(
+    db_session: Session,
+    exclude_ids: set[UUID] | None = None,
+) -> UUID | None:
+    """Claim the next file needing project/persona sync.
+
+    No status transition needed — the impl clears the sync flags on
+    success.  The short-lived FOR UPDATE lock prevents concurrent claims.
+    *exclude_ids* prevents re-processing the same file if the impl fails.
+    """
+    stmt = (
+        select(UserFile.id)
+        .where(
+            sa.and_(
+                sa.or_(
+                    UserFile.needs_project_sync.is_(True),
+                    UserFile.needs_persona_sync.is_(True),
+                ),
+                UserFile.status == UserFileStatus.COMPLETED,
+            )
+        )
+        .order_by(UserFile.created_at)
+        .limit(1)
+        .with_for_update(skip_locked=True)
+    )
+    if exclude_ids:
+        stmt = stmt.where(UserFile.id.notin_(exclude_ids))
+    file_id = db_session.execute(stmt).scalar_one_or_none()
+    db_session.commit()
+    return file_id
+
+
+# ------------------------------------------------------------------
+# Drain loops — process *all* pending work of each type
+# ------------------------------------------------------------------
+
+
+def drain_processing_loop(tenant_id: str) -> None:
+    """Process all pending PROCESSING user files."""
+    from onyx.background.celery.tasks.user_file_processing.tasks import (
+        process_user_file_impl,
+    )
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+
+    while True:
+        with get_session_with_current_tenant() as session:
+            file_id = _claim_next_processing_file(session)
+        if file_id is None:
+            break
+        try:
+            process_user_file_impl(
+                user_file_id=str(file_id),
+                tenant_id=tenant_id,
+                redis_locking=False,
+            )
+        except Exception:
+            logger.exception(f"Failed to process user file {file_id}")
+
+
+def drain_delete_loop(tenant_id: str) -> None:
+    """Delete all pending DELETING user files."""
+    from onyx.background.celery.tasks.user_file_processing.tasks import (
+        delete_user_file_impl,
+    )
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+
+    failed: set[UUID] = set()
+    while True:
+        with get_session_with_current_tenant() as session:
+            file_id = _claim_next_deleting_file(session, exclude_ids=failed)
+        if file_id is None:
+            break
+        try:
+            delete_user_file_impl(
+                user_file_id=str(file_id),
+                tenant_id=tenant_id,
+                redis_locking=False,
+            )
+        except Exception:
+            logger.exception(f"Failed to delete user file {file_id}")
+            failed.add(file_id)
+
+
+def drain_project_sync_loop(tenant_id: str) -> None:
+    """Sync all pending project/persona metadata for user files."""
+    from onyx.background.celery.tasks.user_file_processing.tasks import (
+        project_sync_user_file_impl,
+    )
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+
+    failed: set[UUID] = set()
+    while True:
+        with get_session_with_current_tenant() as session:
+            file_id = _claim_next_sync_file(session, exclude_ids=failed)
+        if file_id is None:
+            break
+        try:
+            project_sync_user_file_impl(
+                user_file_id=str(file_id),
+                tenant_id=tenant_id,
+                redis_locking=False,
+            )
+        except Exception:
+            logger.exception(f"Failed to sync user file {file_id}")
+            failed.add(file_id)

backend/onyx/cache/factory.py

@@ -0,0 +1,51 @@
+from collections.abc import Callable
+
+from onyx.cache.interface import CacheBackend
+from onyx.cache.interface import CacheBackendType
+from onyx.configs.app_configs import CACHE_BACKEND
+
+
+def _build_redis_backend(tenant_id: str) -> CacheBackend:
+    from onyx.cache.redis_backend import RedisCacheBackend
+    from onyx.redis.redis_pool import redis_pool
+
+    return RedisCacheBackend(redis_pool.get_client(tenant_id))
+
+
+def _build_postgres_backend(tenant_id: str) -> CacheBackend:
+    from onyx.cache.postgres_backend import PostgresCacheBackend
+
+    return PostgresCacheBackend(tenant_id)
+
+
+_BACKEND_BUILDERS: dict[CacheBackendType, Callable[[str], CacheBackend]] = {
+    CacheBackendType.REDIS: _build_redis_backend,
+    CacheBackendType.POSTGRES: _build_postgres_backend,
+}
+
+
+def get_cache_backend(*, tenant_id: str | None = None) -> CacheBackend:
+    """Return a tenant-aware ``CacheBackend``.
+
+    If *tenant_id* is ``None``, the current tenant is read from the
+    thread-local context variable (same behaviour as ``get_redis_client``).
+    """
+    if tenant_id is None:
+        from shared_configs.contextvars import get_current_tenant_id
+
+        tenant_id = get_current_tenant_id()
+
+    builder = _BACKEND_BUILDERS.get(CACHE_BACKEND)
+    if builder is None:
+        raise ValueError(
+            f"Unsupported CACHE_BACKEND={CACHE_BACKEND!r}. "
+            f"Supported values: {[t.value for t in CacheBackendType]}"
+        )
+    return builder(tenant_id)
+
+
+def get_shared_cache_backend() -> CacheBackend:
+    """Return a ``CacheBackend`` in the shared (cross-tenant) namespace."""
+    from shared_configs.configs import DEFAULT_REDIS_PREFIX
+
+    return get_cache_backend(tenant_id=DEFAULT_REDIS_PREFIX)

backend/onyx/cache/interface.py

@@ -0,0 +1,115 @@
+import abc
+from enum import Enum
+
+from redis.exceptions import RedisError
+from sqlalchemy.exc import SQLAlchemyError
+
+TTL_KEY_NOT_FOUND = -2
+TTL_NO_EXPIRY = -1
+
+CACHE_TRANSIENT_ERRORS: tuple[type[Exception], ...] = (RedisError, SQLAlchemyError)
+"""Exception types that represent transient cache connectivity / operational
+failures.  Callers that want to fail-open (or fail-closed) on cache errors
+should catch this tuple instead of bare ``Exception``.
+
+When adding a new ``CacheBackend`` implementation, add its transient error
+base class(es) here so all call-sites pick it up automatically."""
+
+
+class CacheBackendType(str, Enum):
+    REDIS = "redis"
+    POSTGRES = "postgres"
+
+
+class CacheLock(abc.ABC):
+    """Abstract distributed lock returned by CacheBackend.lock()."""
+
+    @abc.abstractmethod
+    def acquire(
+        self,
+        blocking: bool = True,
+        blocking_timeout: float | None = None,
+    ) -> bool:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def release(self) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def owned(self) -> bool:
+        raise NotImplementedError
+
+    def __enter__(self) -> "CacheLock":
+        if not self.acquire():
+            raise RuntimeError("Failed to acquire lock")
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.release()
+
+
+class CacheBackend(abc.ABC):
+    """Thin abstraction over a key-value cache with TTL, locks, and blocking lists.
+
+    Covers the subset of Redis operations used outside of Celery. When
+    CACHE_BACKEND=postgres, a PostgreSQL-backed implementation is used instead.
+    """
+
+    # -- basic key/value ---------------------------------------------------
+
+    @abc.abstractmethod
+    def get(self, key: str) -> bytes | None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def set(
+        self,
+        key: str,
+        value: str | bytes | int | float,
+        ex: int | None = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def delete(self, key: str) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def exists(self, key: str) -> bool:
+        raise NotImplementedError
+
+    # -- TTL ---------------------------------------------------------------
+
+    @abc.abstractmethod
+    def expire(self, key: str, seconds: int) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def ttl(self, key: str) -> int:
+        """Return remaining TTL in seconds.
+
+        Returns ``TTL_NO_EXPIRY`` (-1) if key exists without expiry,
+        ``TTL_KEY_NOT_FOUND`` (-2) if key is missing or expired.
+        """
+        raise NotImplementedError
+
+    # -- distributed lock --------------------------------------------------
+
+    @abc.abstractmethod
+    def lock(self, name: str, timeout: float | None = None) -> CacheLock:
+        raise NotImplementedError
+
+    # -- blocking list (used by MCP OAuth BLPOP pattern) -------------------
+
+    @abc.abstractmethod
+    def rpush(self, key: str, value: str | bytes) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def blpop(self, keys: list[str], timeout: int = 0) -> tuple[bytes, bytes] | None:
+        """Block until a value is available on one of *keys*, or *timeout* expires.
+
+        Returns ``(key, value)`` or ``None`` on timeout.
+        """
+        raise NotImplementedError

backend/onyx/cache/postgres_backend.py

@@ -0,0 +1,323 @@
+"""PostgreSQL-backed ``CacheBackend`` for NO_VECTOR_DB deployments.
+
+Uses the ``cache_store`` table for key-value storage, PostgreSQL advisory locks
+for distributed locking, and a polling loop for the BLPOP pattern.
+"""
+
+import hashlib
+import struct
+import time
+import uuid
+from contextlib import AbstractContextManager
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+
+from sqlalchemy import delete
+from sqlalchemy import func
+from sqlalchemy import or_
+from sqlalchemy import select
+from sqlalchemy import update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.orm import Session
+
+from onyx.cache.interface import CacheBackend
+from onyx.cache.interface import CacheLock
+from onyx.cache.interface import TTL_KEY_NOT_FOUND
+from onyx.cache.interface import TTL_NO_EXPIRY
+from onyx.db.models import CacheStore
+
+_LIST_KEY_PREFIX = "_q:"
+# ASCII: ':' (0x3A) < ';' (0x3B). Upper bound for range queries so [prefix+, prefix;)
+# captures all list-item keys (e.g. _q:mylist:123:uuid) without including other
+# lists whose names share a prefix (e.g. _q:mylist2:...).
+_LIST_KEY_RANGE_TERMINATOR = ";"
+_LIST_ITEM_TTL_SECONDS = 3600
+_LOCK_POLL_INTERVAL = 0.1
+_BLPOP_POLL_INTERVAL = 0.25
+
+
+def _list_item_key(key: str) -> str:
+    """Unique key for a list item. Timestamp for FIFO ordering; UUID prevents
+    collision when concurrent rpush calls occur within the same nanosecond.
+    """
+    return f"{_LIST_KEY_PREFIX}{key}:{time.time_ns()}:{uuid.uuid4().hex}"
+
+
+def _to_bytes(value: str | bytes | int | float) -> bytes:
+    if isinstance(value, bytes):
+        return value
+    return str(value).encode()
+
+
+# ------------------------------------------------------------------
+# Lock
+# ------------------------------------------------------------------
+
+
+class PostgresCacheLock(CacheLock):
+    """Advisory-lock-based distributed lock.
+
+    Uses ``get_session_with_tenant`` for connection lifecycle.  The lock is tied
+    to the session's connection; releasing or closing the session frees it.
+
+    NOTE: Unlike Redis locks, advisory locks do not auto-expire after
+    ``timeout`` seconds.  They are released when ``release()`` is
+    called or when the session is closed.
+    """
+
+    def __init__(self, lock_id: int, timeout: float | None, tenant_id: str) -> None:
+        self._lock_id = lock_id
+        self._timeout = timeout
+        self._tenant_id = tenant_id
+        self._session_cm: AbstractContextManager[Session] | None = None
+        self._session: Session | None = None
+        self._acquired = False
+
+    def acquire(
+        self,
+        blocking: bool = True,
+        blocking_timeout: float | None = None,
+    ) -> bool:
+        from onyx.db.engine.sql_engine import get_session_with_tenant
+
+        self._session_cm = get_session_with_tenant(tenant_id=self._tenant_id)
+        self._session = self._session_cm.__enter__()
+        try:
+            if not blocking:
+                return self._try_lock()
+
+            effective_timeout = blocking_timeout or self._timeout
+            deadline = (
+                (time.monotonic() + effective_timeout) if effective_timeout else None
+            )
+            while True:
+                if self._try_lock():
+                    return True
+                if deadline is not None and time.monotonic() >= deadline:
+                    return False
+                time.sleep(_LOCK_POLL_INTERVAL)
+        finally:
+            if not self._acquired:
+                self._close_session()
+
+    def release(self) -> None:
+        if not self._acquired or self._session is None:
+            return
+        try:
+            self._session.execute(select(func.pg_advisory_unlock(self._lock_id)))
+        finally:
+            self._acquired = False
+            self._close_session()
+
+    def owned(self) -> bool:
+        return self._acquired
+
+    def _close_session(self) -> None:
+        if self._session_cm is not None:
+            try:
+                self._session_cm.__exit__(None, None, None)
+            finally:
+                self._session_cm = None
+                self._session = None
+
+    def _try_lock(self) -> bool:
+        assert self._session is not None
+        result = self._session.execute(
+            select(func.pg_try_advisory_lock(self._lock_id))
+        ).scalar()
+        if result:
+            self._acquired = True
+            return True
+        return False
+
+
+# ------------------------------------------------------------------
+# Backend
+# ------------------------------------------------------------------
+
+
+class PostgresCacheBackend(CacheBackend):
+    """``CacheBackend`` backed by the ``cache_store`` table in PostgreSQL.
+
+    Each operation opens and closes its own database session so the backend
+    is safe to share across threads.  Tenant isolation is handled by
+    SQLAlchemy's ``schema_translate_map`` (set by ``get_session_with_tenant``).
+    """
+
+    def __init__(self, tenant_id: str) -> None:
+        self._tenant_id = tenant_id
+
+    # -- basic key/value ---------------------------------------------------
+
+    def get(self, key: str) -> bytes | None:
+        from onyx.db.engine.sql_engine import get_session_with_tenant
+
+        stmt = select(CacheStore.value).where(
+            CacheStore.key == key,
+            or_(CacheStore.expires_at.is_(None), CacheStore.expires_at > func.now()),
+        )
+        with get_session_with_tenant(tenant_id=self._tenant_id) as session:
+            value = session.execute(stmt).scalar_one_or_none()
+        if value is None:
+            return None
+        return bytes(value)
+
+    def set(
+        self,
+        key: str,
+        value: str | bytes | int | float,
+        ex: int | None = None,
+    ) -> None:
+        from onyx.db.engine.sql_engine import get_session_with_tenant
+
+        value_bytes = _to_bytes(value)
+        expires_at = (
+            datetime.now(timezone.utc) + timedelta(seconds=ex)
+            if ex is not None
+            else None
+        )
+        stmt = (
+            pg_insert(CacheStore)
+            .values(key=key, value=value_bytes, expires_at=expires_at)
+            .on_conflict_do_update(
+                index_elements=[CacheStore.key],
+                set_={"value": value_bytes, "expires_at": expires_at},
+            )
+        )
+        with get_session_with_tenant(tenant_id=self._tenant_id) as session:
+            session.execute(stmt)
+            session.commit()
+
+    def delete(self, key: str) -> None:
+        from onyx.db.engine.sql_engine import get_session_with_tenant
+
+        with get_session_with_tenant(tenant_id=self._tenant_id) as session:
+            session.execute(delete(CacheStore).where(CacheStore.key == key))
+            session.commit()
+
+    def exists(self, key: str) -> bool:
+        from onyx.db.engine.sql_engine import get_session_with_tenant
+
+        stmt = (
+            select(CacheStore.key)
+            .where(
+                CacheStore.key == key,
+                or_(
+                    CacheStore.expires_at.is_(None),
+                    CacheStore.expires_at > func.now(),
+                ),
+            )
+            .limit(1)
+        )
+        with get_session_with_tenant(tenant_id=self._tenant_id) as session:
+            return session.execute(stmt).first() is not None
+
+    # -- TTL ---------------------------------------------------------------
+
+    def expire(self, key: str, seconds: int) -> None:
+        from onyx.db.engine.sql_engine import get_session_with_tenant
+
+        new_exp = datetime.now(timezone.utc) + timedelta(seconds=seconds)
+        stmt = (
+            update(CacheStore).where(CacheStore.key == key).values(expires_at=new_exp)
+        )
+        with get_session_with_tenant(tenant_id=self._tenant_id) as session:
+            session.execute(stmt)
+            session.commit()
+
+    def ttl(self, key: str) -> int:
+        from onyx.db.engine.sql_engine import get_session_with_tenant
+
+        stmt = select(CacheStore.expires_at).where(CacheStore.key == key)
+        with get_session_with_tenant(tenant_id=self._tenant_id) as session:
+            result = session.execute(stmt).first()
+        if result is None:
+            return TTL_KEY_NOT_FOUND
+        expires_at: datetime | None = result[0]
+        if expires_at is None:
+            return TTL_NO_EXPIRY
+        remaining = (expires_at - datetime.now(timezone.utc)).total_seconds()
+        if remaining <= 0:
+            return TTL_KEY_NOT_FOUND
+        return int(remaining)
+
+    # -- distributed lock --------------------------------------------------
+
+    def lock(self, name: str, timeout: float | None = None) -> CacheLock:
+        return PostgresCacheLock(
+            self._lock_id_for(name), timeout, tenant_id=self._tenant_id
+        )
+
+    # -- blocking list (MCP OAuth BLPOP pattern) ---------------------------
+
+    def rpush(self, key: str, value: str | bytes) -> None:
+        self.set(_list_item_key(key), value, ex=_LIST_ITEM_TTL_SECONDS)
+
+    def blpop(self, keys: list[str], timeout: int = 0) -> tuple[bytes, bytes] | None:
+        if timeout <= 0:
+            raise ValueError(
+                "PostgresCacheBackend.blpop requires timeout > 0. "
+                "timeout=0 would block the calling thread indefinitely "
+                "with no way to interrupt short of process termination."
+            )
+        from onyx.db.engine.sql_engine import get_session_with_tenant
+
+        deadline = time.monotonic() + timeout
+        while True:
+            for key in keys:
+                lower = f"{_LIST_KEY_PREFIX}{key}:"
+                upper = f"{_LIST_KEY_PREFIX}{key}{_LIST_KEY_RANGE_TERMINATOR}"
+                stmt = (
+                    select(CacheStore)
+                    .where(
+                        CacheStore.key >= lower,
+                        CacheStore.key < upper,
+                        or_(
+                            CacheStore.expires_at.is_(None),
+                            CacheStore.expires_at > func.now(),
+                        ),
+                    )
+                    .order_by(CacheStore.key)
+                    .limit(1)
+                    .with_for_update(skip_locked=True)
+                )
+                with get_session_with_tenant(tenant_id=self._tenant_id) as session:
+                    row = session.execute(stmt).scalars().first()
+                    if row is not None:
+                        value = bytes(row.value) if row.value else b""
+                        session.delete(row)
+                        session.commit()
+                        return (key.encode(), value)
+            if time.monotonic() >= deadline:
+                return None
+            time.sleep(_BLPOP_POLL_INTERVAL)
+
+    # -- helpers -----------------------------------------------------------
+
+    def _lock_id_for(self, name: str) -> int:
+        """Map *name* to a 64-bit signed int for ``pg_advisory_lock``."""
+        h = hashlib.md5(f"{self._tenant_id}:{name}".encode()).digest()
+        return struct.unpack("q", h[:8])[0]
+
+
+# ------------------------------------------------------------------
+# Periodic cleanup
+# ------------------------------------------------------------------
+
+
+def cleanup_expired_cache_entries() -> None:
+    """Delete rows whose ``expires_at`` is in the past.
+
+    Called by the periodic poller every 5 minutes.
+    """
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+
+    with get_session_with_current_tenant() as session:
+        session.execute(
+            delete(CacheStore).where(
+                CacheStore.expires_at.is_not(None),
+                CacheStore.expires_at < func.now(),
+            )
+        )
+        session.commit()

backend/onyx/cache/redis_backend.py

@@ -0,0 +1,92 @@
+from typing import cast
+
+from redis.client import Redis
+from redis.lock import Lock as RedisLock
+
+from onyx.cache.interface import CacheBackend
+from onyx.cache.interface import CacheLock
+
+
+class RedisCacheLock(CacheLock):
+    """Wraps ``redis.lock.Lock`` behind the ``CacheLock`` interface."""
+
+    def __init__(self, lock: RedisLock) -> None:
+        self._lock = lock
+
+    def acquire(
+        self,
+        blocking: bool = True,
+        blocking_timeout: float | None = None,
+    ) -> bool:
+        return bool(
+            self._lock.acquire(
+                blocking=blocking,
+                blocking_timeout=blocking_timeout,
+            )
+        )
+
+    def release(self) -> None:
+        self._lock.release()
+
+    def owned(self) -> bool:
+        return bool(self._lock.owned())
+
+
+class RedisCacheBackend(CacheBackend):
+    """``CacheBackend`` implementation that delegates to a ``redis.Redis`` client.
+
+    This is a thin pass-through — every method maps 1-to-1 to the underlying
+    Redis command.  ``TenantRedis`` key-prefixing is handled by the client
+    itself (provided by ``get_redis_client``).
+    """
+
+    def __init__(self, redis_client: Redis) -> None:
+        self._r = redis_client
+
+    # -- basic key/value ---------------------------------------------------
+
+    def get(self, key: str) -> bytes | None:
+        val = self._r.get(key)
+        if val is None:
+            return None
+        if isinstance(val, bytes):
+            return val
+        return str(val).encode()
+
+    def set(
+        self,
+        key: str,
+        value: str | bytes | int | float,
+        ex: int | None = None,
+    ) -> None:
+        self._r.set(key, value, ex=ex)
+
+    def delete(self, key: str) -> None:
+        self._r.delete(key)
+
+    def exists(self, key: str) -> bool:
+        return bool(self._r.exists(key))
+
+    # -- TTL ---------------------------------------------------------------
+
+    def expire(self, key: str, seconds: int) -> None:
+        self._r.expire(key, seconds)
+
+    def ttl(self, key: str) -> int:
+        return cast(int, self._r.ttl(key))
+
+    # -- distributed lock --------------------------------------------------
+
+    def lock(self, name: str, timeout: float | None = None) -> CacheLock:
+        return RedisCacheLock(self._r.lock(name, timeout=timeout))
+
+    # -- blocking list (MCP OAuth BLPOP pattern) ---------------------------
+
+    def rpush(self, key: str, value: str | bytes) -> None:
+        self._r.rpush(key, value)
+
+    def blpop(self, keys: list[str], timeout: int = 0) -> tuple[bytes, bytes] | None:
+        result = cast(list[bytes] | None, self._r.blpop(keys, timeout=timeout))
+        if result is None:
+            return None
+        return (result[0], result[1])

backend/onyx/chat/chat_processing_checker.py

@@ -1,57 +1,52 @@
 from uuid import UUID
 
-from redis.client import Redis
+from onyx.cache.interface import CacheBackend
 
-# Redis key prefixes for chat message processing
 PREFIX = "chatprocessing"
 FENCE_PREFIX = f"{PREFIX}_fence"
 FENCE_TTL = 30 * 60  # 30 minutes
 
 
 def _get_fence_key(chat_session_id: UUID) -> str:
-    """
-    Generate the Redis key for a chat session processing a message.
+    """Generate the cache key for a chat session processing fence.
 
     Args:
         chat_session_id: The UUID of the chat session
 
     Returns:
-        The fence key string (tenant_id is automatically added by the Redis client)
+        The fence key string. Tenant isolation is handled automatically
+        by the cache backend (Redis key-prefixing or Postgres schema routing).
     """
     return f"{FENCE_PREFIX}_{chat_session_id}"
 
 
 def set_processing_status(
-    chat_session_id: UUID, redis_client: Redis, value: bool
+    chat_session_id: UUID, cache: CacheBackend, value: bool
 ) -> None:
-    """
-    Set or clear the fence for a chat session processing a message.
+    """Set or clear the fence for a chat session processing a message.
 
-    If the key exists, we are processing a message. If the key does not exist, we are not processing a message.
+    If the key exists, a message is being processed.
 
     Args:
         chat_session_id: The UUID of the chat session
-        redis_client: The Redis client to use
+        cache: Tenant-aware cache backend
         value: True to set the fence, False to clear it
     """
     fence_key = _get_fence_key(chat_session_id)
-
     if value:
-        redis_client.set(fence_key, 0, ex=FENCE_TTL)
+        cache.set(fence_key, 0, ex=FENCE_TTL)
     else:
-        redis_client.delete(fence_key)
+        cache.delete(fence_key)
 
 
-def is_chat_session_processing(chat_session_id: UUID, redis_client: Redis) -> bool:
-    """
-    Check if the chat session is processing a message.
+def is_chat_session_processing(chat_session_id: UUID, cache: CacheBackend) -> bool:
+    """Check if the chat session is processing a message.
 
     Args:
         chat_session_id: The UUID of the chat session
-        redis_client: The Redis client to use
+        cache: Tenant-aware cache backend
 
     Returns:
         True if the chat session is processing a message, False otherwise
     """
-    fence_key = _get_fence_key(chat_session_id)
-    return bool(redis_client.exists(fence_key))
+    return cache.exists(_get_fence_key(chat_session_id))

backend/onyx/chat/chat_state.py

@@ -3,7 +3,6 @@ import time
 from collections.abc import Callable
 from collections.abc import Generator
 from queue import Empty
-from typing import Any
 
 from onyx.chat.citation_processor import CitationMapping
 from onyx.chat.emitter import Emitter
@@ -163,13 +162,11 @@ class ChatStateContainer:
 
 
 def run_chat_loop_with_state_containers(
-    func: Callable[..., None],
+    chat_loop_func: Callable[[Emitter, ChatStateContainer], None],
     completion_callback: Callable[[ChatStateContainer], None],
     is_connected: Callable[[], bool],
     emitter: Emitter,
     state_container: ChatStateContainer,
-    *args: Any,
-    **kwargs: Any,
 ) -> Generator[Packet, None]:
     """
     Explicit wrapper function that runs a function in a background thread
@@ -180,19 +177,18 @@ def run_chat_loop_with_state_containers(
 
     Args:
         func: The function to wrap (should accept emitter and state_container as first and second args)
+        completion_callback: Callback function to call when the function completes
         emitter: Emitter instance for sending packets
         state_container: ChatStateContainer instance for accumulating state
         is_connected: Callable that returns False when stop signal is set
-        *args: Additional positional arguments for func
-        **kwargs: Additional keyword arguments for func
 
     Usage:
         packets = run_chat_loop_with_state_containers(
             my_func,
+            completion_callback=completion_callback,
             emitter=emitter,
             state_container=state_container,
             is_connected=check_func,
-            arg1, arg2, kwarg1=value1
         )
         for packet in packets:
             # Process packets
@@ -201,9 +197,7 @@ def run_chat_loop_with_state_containers(
 
     def run_with_exception_capture() -> None:
         try:
-            # Ensure state_container is passed explicitly, removing it from kwargs if present
-            kwargs_with_state = {**kwargs, "state_container": state_container}
-            func(emitter, *args, **kwargs_with_state)
+            chat_loop_func(emitter, state_container)
         except Exception as e:
             # If execution fails, emit an exception packet
             emitter.emit(

backend/onyx/chat/chat_utils.py

@@ -461,7 +461,7 @@ def _build_tool_call_response_history_message(
 def convert_chat_history(
     chat_history: list[ChatMessage],
     files: list[ChatLoadedFile],
-    project_image_files: list[ChatLoadedFile],
+    context_image_files: list[ChatLoadedFile],
     additional_context: str | None,
     token_counter: Callable[[str], int],
     tool_id_to_name_map: dict[int, str],
@@ -541,11 +541,11 @@ def convert_chat_history(
             )
 
             # Add the user message with image files attached
-            # If this is the last USER message, also include project_image_files
-            # Note: project image file tokens are NOT counted in the token count
+            # If this is the last USER message, also include context_image_files
+            # Note: context image file tokens are NOT counted in the token count
             if idx == last_user_message_idx:
-                if project_image_files:
-                    image_files.extend(project_image_files)
+                if context_image_files:
+                    image_files.extend(context_image_files)
 
                 if additional_context:
                     simple_messages.append(

backend/onyx/chat/llm_loop.py

@@ -1,6 +1,7 @@
 import json
 import time
 from collections.abc import Callable
+from typing import Any
 from typing import Literal
 
 from sqlalchemy.orm import Session
@@ -15,10 +16,10 @@ from onyx.chat.emitter import Emitter
 from onyx.chat.llm_step import extract_tool_calls_from_response_text
 from onyx.chat.llm_step import run_llm_step
 from onyx.chat.models import ChatMessageSimple
-from onyx.chat.models import ExtractedProjectFiles
+from onyx.chat.models import ContextFileMetadata
+from onyx.chat.models import ExtractedContextFiles
 from onyx.chat.models import FileToolMetadata
 from onyx.chat.models import LlmStepResult
-from onyx.chat.models import ProjectFileMetadata
 from onyx.chat.models import ToolCallSimple
 from onyx.chat.prompt_utils import build_reminder_message
 from onyx.chat.prompt_utils import build_system_prompt
@@ -51,6 +52,7 @@ from onyx.tools.built_in_tools import STOPPING_TOOLS_NAMES
 from onyx.tools.interface import Tool
 from onyx.tools.models import ChatFile
 from onyx.tools.models import MemoryToolResponseSnapshot
+from onyx.tools.models import PythonToolRichResponse
 from onyx.tools.models import ToolCallInfo
 from onyx.tools.models import ToolCallKickoff
 from onyx.tools.models import ToolResponse
@@ -203,17 +205,17 @@ def _try_fallback_tool_extraction(
 MAX_LLM_CYCLES = 6
 
 
-def _build_project_file_citation_mapping(
-    project_file_metadata: list[ProjectFileMetadata],
+def _build_context_file_citation_mapping(
+    file_metadata: list[ContextFileMetadata],
     starting_citation_num: int = 1,
 ) -> CitationMapping:
-    """Build citation mapping for project files.
+    """Build citation mapping for context files.
 
-    Converts project file metadata into SearchDoc objects that can be cited.
+    Converts context file metadata into SearchDoc objects that can be cited.
     Citation numbers start from the provided starting number.
 
     Args:
-        project_file_metadata: List of project file metadata
+        file_metadata: List of context file metadata
         starting_citation_num: Starting citation number (default: 1)
 
     Returns:
@@ -221,8 +223,7 @@ def _build_project_file_citation_mapping(
     """
     citation_mapping: CitationMapping = {}
 
-    for idx, file_meta in enumerate(project_file_metadata, start=starting_citation_num):
-        # Create a SearchDoc for each project file
+    for idx, file_meta in enumerate(file_metadata, start=starting_citation_num):
         search_doc = SearchDoc(
             document_id=file_meta.file_id,
             chunk_ind=0,
@@ -242,29 +243,28 @@ def _build_project_file_citation_mapping(
 
 
 def _build_project_message(
-    project_files: ExtractedProjectFiles | None,
+    context_files: ExtractedContextFiles | None,
     token_counter: Callable[[str], int] | None,
 ) -> list[ChatMessageSimple]:
-    """Build messages for project / tool-backed files.
+    """Build messages for context-injected / tool-backed files.
 
     Returns up to two messages:
-    1. The full-text project files message (if project_file_texts is populated).
+    1. The full-text files message (if file_texts is populated).
     2. A lightweight metadata message for files the LLM should access via the
-       FileReaderTool (e.g. oversized chat-attached files or project files that
-       don't fit in context).
+       FileReaderTool (e.g. oversized files that don't fit in context).
     """
-    if not project_files:
+    if not context_files:
         return []
 
     messages: list[ChatMessageSimple] = []
-    if project_files.project_file_texts:
+    if context_files.file_texts:
         messages.append(
-            _create_project_files_message(project_files, token_counter=None)
+            _create_context_files_message(context_files, token_counter=None)
         )
-    if project_files.file_metadata_for_tool and token_counter:
+    if context_files.file_metadata_for_tool and token_counter:
         messages.append(
             _create_file_tool_metadata_message(
-                project_files.file_metadata_for_tool, token_counter
+                context_files.file_metadata_for_tool, token_counter
             )
         )
     return messages
@@ -275,7 +275,7 @@ def construct_message_history(
     custom_agent_prompt: ChatMessageSimple | None,
     simple_chat_history: list[ChatMessageSimple],
     reminder_message: ChatMessageSimple | None,
-    project_files: ExtractedProjectFiles | None,
+    context_files: ExtractedContextFiles | None,
     available_tokens: int,
     last_n_user_messages: int | None = None,
     token_counter: Callable[[str], int] | None = None,
@@ -289,7 +289,7 @@ def construct_message_history(
 
     # Build the project / file-metadata messages up front so we can use their
     # actual token counts for the budget.
-    project_messages = _build_project_message(project_files, token_counter)
+    project_messages = _build_project_message(context_files, token_counter)
     project_messages_tokens = sum(m.token_count for m in project_messages)
 
     history_token_budget = available_tokens
@@ -445,17 +445,17 @@ def construct_message_history(
                     )
 
     # Attach project images to the last user message
-    if project_files and project_files.project_image_files:
+    if context_files and context_files.image_files:
         existing_images = last_user_message.image_files or []
         last_user_message = ChatMessageSimple(
             message=last_user_message.message,
             token_count=last_user_message.token_count,
             message_type=last_user_message.message_type,
-            image_files=existing_images + project_files.project_image_files,
+            image_files=existing_images + context_files.image_files,
         )
 
     # Build the final message list according to README ordering:
-    # [system], [history_before_last_user], [custom_agent], [project_files],
+    # [system], [history_before_last_user], [custom_agent], [context_files],
     # [forgotten_files], [last_user_message], [messages_after_last_user], [reminder]
     result = [system_prompt] if system_prompt else []
 
@@ -466,14 +466,14 @@ def construct_message_history(
     if custom_agent_prompt:
         result.append(custom_agent_prompt)
 
-    # 3. Add project files / file-metadata messages (inserted before last user message)
+    # 3. Add context files / file-metadata messages (inserted before last user message)
     result.extend(project_messages)
 
     # 4. Add forgotten-files metadata (right before the user's question)
     if forgotten_files_message:
         result.append(forgotten_files_message)
 
-    # 5. Add last user message (with project images attached)
+    # 5. Add last user message (with context images attached)
     result.append(last_user_message)
 
     # 6. Add messages after last user message (tool calls, responses, etc.)
@@ -532,11 +532,13 @@ def _create_file_tool_metadata_message(
     """
     lines = [
         "You have access to the following files. Use the read_file tool to "
-        "read sections of any file:"
+        "read sections of any file. You MUST pass the file_id UUID (not the "
+        "filename) to read_file:"
     ]
     for meta in file_metadata:
         lines.append(
-            f'- {meta.file_id}: "{meta.filename}" (~{meta.approx_char_count:,} chars)'
+            f'- file_id="{meta.file_id}" filename="{meta.filename}" '
+            f"(~{meta.approx_char_count:,} chars)"
         )
 
     message_content = "\n".join(lines)
@@ -547,11 +549,11 @@ def _create_file_tool_metadata_message(
     )
 
 
-def _create_project_files_message(
-    project_files: ExtractedProjectFiles,
+def _create_context_files_message(
+    context_files: ExtractedContextFiles,
     token_counter: Callable[[str], int] | None,  # noqa: ARG001
 ) -> ChatMessageSimple:
-    """Convert project files to a ChatMessageSimple message.
+    """Convert context files to a ChatMessageSimple message.
 
     Format follows the README specification for document representation.
     """
@@ -559,21 +561,25 @@ def _create_project_files_message(
 
     # Format as documents JSON as described in README
     documents_list = []
-    for idx, file_text in enumerate(project_files.project_file_texts, start=1):
-        documents_list.append(
-            {
-                "document": idx,
-                "contents": file_text,
-            }
+    for idx, file_text in enumerate(context_files.file_texts, start=1):
+        title = (
+            context_files.file_metadata[idx - 1].filename
+            if idx - 1 < len(context_files.file_metadata)
+            else None
         )
+        entry: dict[str, Any] = {"document": idx}
+        if title:
+            entry["title"] = title
+        entry["contents"] = file_text
+        documents_list.append(entry)
 
     documents_json = json.dumps({"documents": documents_list}, indent=2)
     message_content = f"Here are some documents provided for context, they may not all be relevant:\n{documents_json}"
 
-    # Use pre-calculated token count from project_files
+    # Use pre-calculated token count from context_files
     return ChatMessageSimple(
         message=message_content,
-        token_count=project_files.total_token_count,
+        token_count=context_files.total_token_count,
         message_type=MessageType.USER,
     )
 
@@ -584,7 +590,7 @@ def run_llm_loop(
     simple_chat_history: list[ChatMessageSimple],
     tools: list[Tool],
     custom_agent_prompt: str | None,
-    project_files: ExtractedProjectFiles,
+    context_files: ExtractedContextFiles,
     persona: Persona | None,
     user_memory_context: UserMemoryContext | None,
     llm: LLM,
@@ -627,9 +633,9 @@ def run_llm_loop(
 
         # Add project file citation mappings if project files are present
         project_citation_mapping: CitationMapping = {}
-        if project_files.project_file_metadata:
-            project_citation_mapping = _build_project_file_citation_mapping(
-                project_files.project_file_metadata
+        if context_files.file_metadata:
+            project_citation_mapping = _build_context_file_citation_mapping(
+                context_files.file_metadata
             )
             citation_processor.update_citation_mapping(project_citation_mapping)
 
@@ -647,7 +653,7 @@ def run_llm_loop(
         # TODO allow citing of images in Projects. Since attached to the last user message, it has no text associated with it.
         # One future workaround is to include the images as separate user messages with citation information and process those.
         always_cite_documents: bool = bool(
-            project_files.project_as_filter or project_files.project_file_texts
+            context_files.use_as_search_filter or context_files.file_texts
         )
         should_cite_documents: bool = False
         ran_image_gen: bool = False
@@ -788,7 +794,7 @@ def run_llm_loop(
                 custom_agent_prompt=custom_agent_prompt_msg,
                 simple_chat_history=simple_chat_history,
                 reminder_message=reminder_msg,
-                project_files=project_files,
+                context_files=context_files,
                 available_tokens=available_tokens,
                 token_counter=token_counter,
                 all_injected_file_metadata=all_injected_file_metadata,
@@ -961,6 +967,13 @@ def run_llm_loop(
                 ):
                     generated_images = tool_response.rich_response.generated_images
 
+                # Extract generated_files if this is a code interpreter response
+                generated_files = None
+                if isinstance(tool_response.rich_response, PythonToolRichResponse):
+                    generated_files = (
+                        tool_response.rich_response.generated_files or None
+                    )
+
                 # Persist memory if this is a memory tool response
                 memory_snapshot: MemoryToolResponseSnapshot | None = None
                 if isinstance(tool_response.rich_response, MemoryToolResponse):
@@ -1012,6 +1025,7 @@ def run_llm_loop(
                     tool_call_response=saved_response,
                     search_docs=displayed_docs or search_docs,
                     generated_images=generated_images,
+                    generated_files=generated_files,
                 )
                 # Add to state container for partial save support
                 state_container.add_tool_call(tool_call_info)

backend/onyx/chat/llm_step.py

@@ -15,6 +15,7 @@ from onyx.chat.citation_processor import DynamicCitationProcessor
 from onyx.chat.emitter import Emitter
 from onyx.chat.models import ChatMessageSimple
 from onyx.chat.models import LlmStepResult
+from onyx.chat.tool_call_args_streaming import maybe_emit_argument_delta
 from onyx.configs.app_configs import LOG_ONYX_MODEL_INTERACTIONS
 from onyx.configs.app_configs import PROMPT_CACHE_CHAT_HISTORY
 from onyx.configs.constants import MessageType
@@ -1018,6 +1019,7 @@ def run_llm_step_pkt_generator(
         )
 
     id_to_tool_call_map: dict[int, dict[str, Any]] = {}
+    arg_scan_offsets: dict[int, int] = {}
     reasoning_start = False
     answer_start = False
     accumulated_reasoning = ""
@@ -1224,7 +1226,14 @@ def run_llm_step_pkt_generator(
                 yield from _close_reasoning_if_active()
 
                 for tool_call_delta in delta.tool_calls:
+                    # maybe_emit depends and update being called first and attaching the delta
                     _update_tool_call_with_delta(id_to_tool_call_map, tool_call_delta)
+                    yield from maybe_emit_argument_delta(
+                        tool_calls_in_progress=id_to_tool_call_map,
+                        tool_call_delta=tool_call_delta,
+                        placement=_current_placement(),
+                        scan_offsets=arg_scan_offsets,
+                    )
 
         # Flush any tail text buffered while checking for split "<function_calls" markers.
         filtered_content_tail = xml_tool_call_content_filter.flush()

backend/onyx/chat/models.py

@@ -31,13 +31,6 @@ class CustomToolResponse(BaseModel):
     tool_name: str
 
 
-class ProjectSearchConfig(BaseModel):
-    """Configuration for search tool availability in project context."""
-
-    search_usage: SearchToolUsage
-    disable_forced_tool: bool
-
-
 class CreateChatSessionID(BaseModel):
     chat_session_id: UUID
 
@@ -132,8 +125,8 @@ class ChatMessageSimple(BaseModel):
     file_id: str | None = None
 
 
-class ProjectFileMetadata(BaseModel):
-    """Metadata for a project file to enable citation support."""
+class ContextFileMetadata(BaseModel):
+    """Metadata for a context-injected file to enable citation support."""
 
     file_id: str
     filename: str
@@ -167,20 +160,28 @@ class ChatHistoryResult(BaseModel):
     all_injected_file_metadata: dict[str, FileToolMetadata]
 
 
-class ExtractedProjectFiles(BaseModel):
-    project_file_texts: list[str]
-    project_image_files: list[ChatLoadedFile]
-    project_as_filter: bool
+class ExtractedContextFiles(BaseModel):
+    """Result of attempting to load user files (from a project or persona) into context."""
+
+    file_texts: list[str]
+    image_files: list[ChatLoadedFile]
+    use_as_search_filter: bool
     total_token_count: int
-    # Metadata for project files to enable citations
-    project_file_metadata: list[ProjectFileMetadata]
-    # None if not a project
-    project_uncapped_token_count: int | None
     # Lightweight metadata for files exposed via FileReaderTool
-    # (populated when files don't fit in context and vector DB is disabled)
+    # (populated when files don't fit in context and vector DB is disabled).
+    file_metadata: list[ContextFileMetadata]
+    uncapped_token_count: int | None
     file_metadata_for_tool: list[FileToolMetadata] = []
 
 
+class SearchParams(BaseModel):
+    """Resolved search filter IDs and search-tool usage for a chat turn."""
+
+    search_project_id: int | None
+    search_persona_id: int | None
+    search_usage: SearchToolUsage
+
+
 class LlmStepResult(BaseModel):
     reasoning: str | None
     answer: str | None

backend/onyx/chat/process_message.py

@@ -3,6 +3,7 @@ IMPORTANT: familiarize yourself with the design concepts prior to contributing t
 An overview can be found in the README.md file in this directory.
 """
 
+import io
 import re
 import traceback
 from collections.abc import Callable
@@ -10,9 +11,10 @@ from contextvars import Token
 from uuid import UUID
 
 from pydantic import BaseModel
-from redis.client import Redis
 from sqlalchemy.orm import Session
 
+from onyx.cache.factory import get_cache_backend
+from onyx.cache.interface import CacheBackend
 from onyx.chat.chat_processing_checker import set_processing_status
 from onyx.chat.chat_state import ChatStateContainer
 from onyx.chat.chat_state import run_chat_loop_with_state_containers
@@ -33,11 +35,11 @@ from onyx.chat.models import ChatBasicResponse
 from onyx.chat.models import ChatFullResponse
 from onyx.chat.models import ChatLoadedFile
 from onyx.chat.models import ChatMessageSimple
+from onyx.chat.models import ContextFileMetadata
 from onyx.chat.models import CreateChatSessionID
-from onyx.chat.models import ExtractedProjectFiles
+from onyx.chat.models import ExtractedContextFiles
 from onyx.chat.models import FileToolMetadata
-from onyx.chat.models import ProjectFileMetadata
-from onyx.chat.models import ProjectSearchConfig
+from onyx.chat.models import SearchParams
 from onyx.chat.models import StreamingError
 from onyx.chat.models import ToolCallResponse
 from onyx.chat.prompt_utils import calculate_reserved_tokens
@@ -62,11 +64,12 @@ from onyx.db.models import ChatSession
 from onyx.db.models import Persona
 from onyx.db.models import User
 from onyx.db.models import UserFile
-from onyx.db.projects import get_project_token_count
 from onyx.db.projects import get_user_files_from_project
 from onyx.db.tools import get_tools
 from onyx.deep_research.dr_loop import run_deep_research_llm_loop
+from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_store.models import ChatFileType
+from onyx.file_store.models import InMemoryChatFile
 from onyx.file_store.utils import load_in_memory_chat_files
 from onyx.file_store.utils import verify_user_files
 from onyx.llm.factory import get_llm_for_persona
@@ -77,7 +80,6 @@ from onyx.llm.request_context import reset_llm_mock_response
 from onyx.llm.request_context import set_llm_mock_response
 from onyx.llm.utils import litellm_exception_to_error_msg
 from onyx.onyxbot.slack.models import SlackContext
-from onyx.redis.redis_pool import get_redis_client
 from onyx.server.query_and_chat.models import AUTO_PLACE_AFTER_LATEST_MESSAGE
 from onyx.server.query_and_chat.models import MessageResponseIDInfo
 from onyx.server.query_and_chat.models import SendMessageRequest
@@ -139,12 +141,12 @@ def _collect_available_file_ids(
                 pass
 
     if project_id:
-        project_files = get_user_files_from_project(
+        user_files = get_user_files_from_project(
             project_id=project_id,
             user_id=user_id,
             db_session=db_session,
         )
-        for uf in project_files:
+        for uf in user_files:
             user_file_ids.add(uf.id)
 
     return _AvailableFiles(
@@ -192,9 +194,67 @@ def _convert_loaded_files_to_chat_files(
     return chat_files
 
 
-def _extract_project_file_texts_and_images(
+def resolve_context_user_files(
+    persona: Persona,
     project_id: int | None,
     user_id: UUID | None,
+    db_session: Session,
+) -> list[UserFile]:
+    """Apply the precedence rule to decide which user files to load.
+
+    A custom persona fully supersedes the project.  When a chat uses a
+    custom persona, the project is purely organisational — its files are
+    never loaded and never made searchable.
+
+    Custom persona → persona's own user_files (may be empty).
+    Default persona inside a project → project files.
+    Otherwise → empty list.
+    """
+    if persona.id != DEFAULT_PERSONA_ID:
+        return list(persona.user_files) if persona.user_files else []
+    if project_id:
+        return get_user_files_from_project(
+            project_id=project_id,
+            user_id=user_id,
+            db_session=db_session,
+        )
+    return []
+
+
+def _empty_extracted_context_files() -> ExtractedContextFiles:
+    return ExtractedContextFiles(
+        file_texts=[],
+        image_files=[],
+        use_as_search_filter=False,
+        total_token_count=0,
+        file_metadata=[],
+        uncapped_token_count=None,
+    )
+
+
+def _extract_text_from_in_memory_file(f: InMemoryChatFile) -> str | None:
+    """Extract text content from an InMemoryChatFile.
+
+    PLAIN_TEXT: the content is pre-extracted UTF-8 plaintext stored during
+    ingestion — decode directly.
+    DOC / CSV / other text types: the content is the original file bytes —
+    use extract_file_text which handles encoding detection and format parsing.
+    """
+    try:
+        if f.file_type == ChatFileType.PLAIN_TEXT:
+            return f.content.decode("utf-8", errors="ignore").replace("\x00", "")
+        return extract_file_text(
+            file=io.BytesIO(f.content),
+            file_name=f.filename or "",
+            break_on_unprocessable=False,
+        )
+    except Exception:
+        logger.warning(f"Failed to extract text from file {f.file_id}", exc_info=True)
+        return None
+
+
+def extract_context_files(
+    user_files: list[UserFile],
     llm_max_context_window: int,
     reserved_token_count: int,
     db_session: Session,
@@ -203,8 +263,12 @@ def _extract_project_file_texts_and_images(
     # 60% of the LLM's max context window. The other benefit is that for projects with
     # more files, this makes it so that we don't throw away the history too quickly every time.
     max_llm_context_percentage: float = 0.6,
-) -> ExtractedProjectFiles:
-    """Extract text content from project files if they fit within the context window.
+) -> ExtractedContextFiles:
+    """Load user files into context if they fit; otherwise flag for search.
+
+    The caller is responsible for deciding *which* user files to pass in
+    (project files, persona files, etc.).  This function only cares about
+    the all-or-nothing fit check and the actual content loading.
 
     Args:
         project_id: The project ID to load files from
@@ -213,160 +277,95 @@ def _extract_project_file_texts_and_images(
         reserved_token_count: Number of tokens to reserve for other content
         db_session: Database session
         max_llm_context_percentage: Maximum percentage of the LLM context window to use.
-
     Returns:
-        ExtractedProjectFiles containing:
-        - List of text content strings from project files (text files only)
-        - List of image files from project (ChatLoadedFile objects)
-        - Project id if the the project should be provided as a filter in search or None if not.
+        ExtractedContextFiles containing:
+        - List of text content strings from context files (text files only)
+        - List of image files from context (ChatLoadedFile objects)
         - Total token count of all extracted files
+        - File metadata for context files
+        - Uncapped token count of all extracted files
+        - File metadata for files that don't fit in context and vector DB is disabled
     """
-    # TODO I believe this is not handling all file types correctly.
-    project_as_filter = False
-    if not project_id:
-        return ExtractedProjectFiles(
-            project_file_texts=[],
-            project_image_files=[],
-            project_as_filter=False,
-            total_token_count=0,
-            project_file_metadata=[],
-            project_uncapped_token_count=None,
-        )
+    # TODO(yuhong): I believe this is not handling all file types correctly.
 
+    if not user_files:
+        return _empty_extracted_context_files()
+
+    aggregate_tokens = sum(uf.token_count or 0 for uf in user_files)
     max_actual_tokens = (
         llm_max_context_window - reserved_token_count
     ) * max_llm_context_percentage
 
-    # Calculate total token count for all user files in the project
-    project_tokens = get_project_token_count(
-        project_id=project_id,
-        user_id=user_id,
+    if aggregate_tokens >= max_actual_tokens:
+        tool_metadata = []
+        use_as_search_filter = not DISABLE_VECTOR_DB
+        if DISABLE_VECTOR_DB:
+            tool_metadata = _build_file_tool_metadata_for_user_files(user_files)
+        return ExtractedContextFiles(
+            file_texts=[],
+            image_files=[],
+            use_as_search_filter=use_as_search_filter,
+            total_token_count=0,
+            file_metadata=[],
+            uncapped_token_count=aggregate_tokens,
+            file_metadata_for_tool=tool_metadata,
+        )
+
+    # Files fit — load them into context
+    user_file_map = {str(uf.id): uf for uf in user_files}
+    in_memory_files = load_in_memory_chat_files(
+        user_file_ids=[uf.id for uf in user_files],
         db_session=db_session,
     )
 
-    project_file_texts: list[str] = []
-    project_image_files: list[ChatLoadedFile] = []
-    project_file_metadata: list[ProjectFileMetadata] = []
+    file_texts: list[str] = []
+    image_files: list[ChatLoadedFile] = []
+    file_metadata: list[ContextFileMetadata] = []
     total_token_count = 0
-    if project_tokens < max_actual_tokens:
-        # Load project files into memory using cached plaintext when available
-        project_user_files = get_user_files_from_project(
-            project_id=project_id,
-            user_id=user_id,
-            db_session=db_session,
-        )
-        if project_user_files:
-            # Create a mapping from file_id to UserFile for token count lookup
-            user_file_map = {str(file.id): file for file in project_user_files}
 
-            project_file_ids = [file.id for file in project_user_files]
-            in_memory_project_files = load_in_memory_chat_files(
-                user_file_ids=project_file_ids,
-                db_session=db_session,
+    for f in in_memory_files:
+        uf = user_file_map.get(str(f.file_id))
+        if f.file_type.is_text_file():
+            text_content = _extract_text_from_in_memory_file(f)
+            if not text_content:
+                continue
+            file_texts.append(text_content)
+            file_metadata.append(
+                ContextFileMetadata(
+                    file_id=str(f.file_id),
+                    filename=f.filename or f"file_{f.file_id}",
+                    file_content=text_content,
+                )
+            )
+            if uf and uf.token_count:
+                total_token_count += uf.token_count
+        elif f.file_type == ChatFileType.IMAGE:
+            token_count = uf.token_count if uf and uf.token_count else 0
+            total_token_count += token_count
+            image_files.append(
+                ChatLoadedFile(
+                    file_id=f.file_id,
+                    content=f.content,
+                    file_type=f.file_type,
+                    filename=f.filename,
+                    content_text=None,
+                    token_count=token_count,
+                )
             )
 
-            # Extract text content from loaded files
-            for file in in_memory_project_files:
-                if file.file_type.is_text_file():
-                    try:
-                        text_content = file.content.decode("utf-8", errors="ignore")
-                        # Strip null bytes
-                        text_content = text_content.replace("\x00", "")
-                        if text_content:
-                            project_file_texts.append(text_content)
-                            # Add metadata for citation support
-                            project_file_metadata.append(
-                                ProjectFileMetadata(
-                                    file_id=str(file.file_id),
-                                    filename=file.filename or f"file_{file.file_id}",
-                                    file_content=text_content,
-                                )
-                            )
-                            # Add token count for text file
-                            user_file = user_file_map.get(str(file.file_id))
-                            if user_file and user_file.token_count:
-                                total_token_count += user_file.token_count
-                    except Exception:
-                        # Skip files that can't be decoded
-                        pass
-                elif file.file_type == ChatFileType.IMAGE:
-                    # Convert InMemoryChatFile to ChatLoadedFile
-                    user_file = user_file_map.get(str(file.file_id))
-                    token_count = (
-                        user_file.token_count
-                        if user_file and user_file.token_count
-                        else 0
-                    )
-                    total_token_count += token_count
-                    chat_loaded_file = ChatLoadedFile(
-                        file_id=file.file_id,
-                        content=file.content,
-                        file_type=file.file_type,
-                        filename=file.filename,
-                        content_text=None,  # Images don't have text content
-                        token_count=token_count,
-                    )
-                    project_image_files.append(chat_loaded_file)
-    else:
-        if DISABLE_VECTOR_DB:
-            # Without a vector DB we can't use project-as-filter search.
-            # Instead, build lightweight metadata so the LLM can call the
-            # FileReaderTool to inspect individual files on demand.
-            file_metadata_for_tool = _build_file_tool_metadata_for_project(
-                project_id=project_id,
-                user_id=user_id,
-                db_session=db_session,
-            )
-            return ExtractedProjectFiles(
-                project_file_texts=[],
-                project_image_files=[],
-                project_as_filter=False,
-                total_token_count=0,
-                project_file_metadata=[],
-                project_uncapped_token_count=project_tokens,
-                file_metadata_for_tool=file_metadata_for_tool,
-            )
-        project_as_filter = True
-
-    return ExtractedProjectFiles(
-        project_file_texts=project_file_texts,
-        project_image_files=project_image_files,
-        project_as_filter=project_as_filter,
+    return ExtractedContextFiles(
+        file_texts=file_texts,
+        image_files=image_files,
+        use_as_search_filter=False,
         total_token_count=total_token_count,
-        project_file_metadata=project_file_metadata,
-        project_uncapped_token_count=project_tokens,
+        file_metadata=file_metadata,
+        uncapped_token_count=aggregate_tokens,
     )
 
 
 APPROX_CHARS_PER_TOKEN = 4
 
 
-def _build_file_tool_metadata_for_project(
-    project_id: int,
-    user_id: UUID | None,
-    db_session: Session,
-) -> list[FileToolMetadata]:
-    """Build lightweight FileToolMetadata for every file in a project.
-
-    Used when files are too large to fit in context and the vector DB is
-    disabled, so the LLM needs to know which files it can read via the
-    FileReaderTool.
-    """
-    project_user_files = get_user_files_from_project(
-        project_id=project_id,
-        user_id=user_id,
-        db_session=db_session,
-    )
-    return [
-        FileToolMetadata(
-            file_id=str(uf.id),
-            filename=uf.name,
-            approx_char_count=(uf.token_count or 0) * APPROX_CHARS_PER_TOKEN,
-        )
-        for uf in project_user_files
-    ]
-
-
 def _build_file_tool_metadata_for_user_files(
     user_files: list[UserFile],
 ) -> list[FileToolMetadata]:
@@ -381,55 +380,46 @@ def _build_file_tool_metadata_for_user_files(
     ]
 
 
-def _get_project_search_availability(
+def determine_search_params(
+    persona_id: int,
     project_id: int | None,
-    persona_id: int | None,
-    loaded_project_files: bool,
-    project_has_files: bool,
-    forced_tool_id: int | None,
-    search_tool_id: int | None,
-) -> ProjectSearchConfig:
-    """Determine search tool availability based on project context.
+    extracted_context_files: ExtractedContextFiles,
+) -> SearchParams:
+    """Decide which search filter IDs and search-tool usage apply for a chat turn.
 
-    Search is disabled when ALL of the following are true:
-    - User is in a project
-    - Using the default persona (not a custom agent)
-    - Project files are already loaded in context
+    A custom persona fully supersedes the project — project files are never
+    searchable and the search tool config is entirely controlled by the
+    persona.  The project_id filter is only set for the default persona.
 
-    When search is disabled and the user tried to force the search tool,
-    that forcing is also disabled.
-
-    Returns AUTO (follow persona config) in all other cases.
+    For the default persona inside a project:
+      - Files overflow  → ENABLED  (vector DB scopes to these files)
+      - Files fit       → DISABLED (content already in prompt)
+      - No files at all → DISABLED (nothing to search)
     """
-    # Not in a project, this should have no impact on search tool availability
-    if not project_id:
-        return ProjectSearchConfig(
-            search_usage=SearchToolUsage.AUTO, disable_forced_tool=False
-        )
+    is_custom_persona = persona_id != DEFAULT_PERSONA_ID
 
-    # Custom persona in project - let persona config decide
-    # Even if there are no files in the project, it's still guided by the persona config.
-    if persona_id != DEFAULT_PERSONA_ID:
-        return ProjectSearchConfig(
-            search_usage=SearchToolUsage.AUTO, disable_forced_tool=False
-        )
+    search_project_id: int | None = None
+    search_persona_id: int | None = None
+    if extracted_context_files.use_as_search_filter:
+        if is_custom_persona:
+            search_persona_id = persona_id
+        else:
+            search_project_id = project_id
 
-    # If in a project with the default persona and the files have been already loaded into the context or
-    # there are no files in the project, disable search as there is nothing to search for.
-    if loaded_project_files or not project_has_files:
-        user_forced_search = (
-            forced_tool_id is not None
-            and search_tool_id is not None
-            and forced_tool_id == search_tool_id
-        )
-        return ProjectSearchConfig(
-            search_usage=SearchToolUsage.DISABLED,
-            disable_forced_tool=user_forced_search,
-        )
+    search_usage = SearchToolUsage.AUTO
+    if not is_custom_persona and project_id:
+        has_context_files = bool(extracted_context_files.uncapped_token_count)
+        files_loaded_in_context = bool(extracted_context_files.file_texts)
 
-    # Default persona in a project with files, but also the files have not been loaded into the context already.
-    return ProjectSearchConfig(
-        search_usage=SearchToolUsage.ENABLED, disable_forced_tool=False
+        if extracted_context_files.use_as_search_filter:
+            search_usage = SearchToolUsage.ENABLED
+        elif files_loaded_in_context or not has_context_files:
+            search_usage = SearchToolUsage.DISABLED
+
+    return SearchParams(
+        search_project_id=search_project_id,
+        search_persona_id=search_persona_id,
+        search_usage=search_usage,
     )
 
 
@@ -458,7 +448,7 @@ def handle_stream_message_objects(
 
     llm: LLM | None = None
     chat_session: ChatSession | None = None
-    redis_client: Redis | None = None
+    cache: CacheBackend | None = None
 
     user_id = user.id
     if user.is_anonymous:
@@ -661,26 +651,37 @@ def handle_stream_message_objects(
             user_memory_context=prompt_memory_context,
         )
 
-        # Process projects, if all of the files fit in the context, it doesn't need to use RAG
-        extracted_project_files = _extract_project_file_texts_and_images(
+        # Determine which user files to use.  A custom persona fully
+        # supersedes the project — project files are never loaded or
+        # searchable when a custom persona is in play.  Only the default
+        # persona inside a project uses the project's files.
+        context_user_files = resolve_context_user_files(
+            persona=persona,
             project_id=chat_session.project_id,
             user_id=user_id,
+            db_session=db_session,
+        )
+
+        extracted_context_files = extract_context_files(
+            user_files=context_user_files,
             llm_max_context_window=llm.config.max_input_tokens,
             reserved_token_count=reserved_token_count,
             db_session=db_session,
         )
 
-        # When the vector DB is disabled, persona-attached user_files have no
-        # search pipeline path. Inject them as file_metadata_for_tool so the
-        # LLM can read them via the FileReaderTool.
-        if DISABLE_VECTOR_DB and persona.user_files:
-            persona_file_metadata = _build_file_tool_metadata_for_user_files(
-                persona.user_files
-            )
-            # Merge persona file metadata into the extracted project files
-            extracted_project_files.file_metadata_for_tool.extend(persona_file_metadata)
+        search_params = determine_search_params(
+            persona_id=persona.id,
+            project_id=chat_session.project_id,
+            extracted_context_files=extracted_context_files,
+        )
+
+        # Also grant access to persona-attached user files for FileReaderTool
+        if persona.user_files:
+            existing = set(available_files.user_file_ids)
+            for uf in persona.user_files:
+                if uf.id not in existing:
+                    available_files.user_file_ids.append(uf.id)
 
-        # Build a mapping of tool_id to tool_name for history reconstruction
         all_tools = get_tools(db_session)
         tool_id_to_name_map = {tool.id: tool.name for tool in all_tools}
 
@@ -689,30 +690,17 @@ def handle_stream_message_objects(
             None,
         )
 
-        # Determine if search should be disabled for this project context
         forced_tool_id = new_msg_req.forced_tool_id
-        project_search_config = _get_project_search_availability(
-            project_id=chat_session.project_id,
-            persona_id=persona.id,
-            loaded_project_files=bool(extracted_project_files.project_file_texts),
-            project_has_files=bool(
-                extracted_project_files.project_uncapped_token_count
-            ),
-            forced_tool_id=new_msg_req.forced_tool_id,
-            search_tool_id=search_tool_id,
-        )
-        if project_search_config.disable_forced_tool:
+        if (
+            search_params.search_usage == SearchToolUsage.DISABLED
+            and forced_tool_id is not None
+            and search_tool_id is not None
+            and forced_tool_id == search_tool_id
+        ):
             forced_tool_id = None
 
         emitter = get_default_emitter()
 
-        # Also grant access to persona-attached user files
-        if persona.user_files:
-            existing = set(available_files.user_file_ids)
-            for uf in persona.user_files:
-                if uf.id not in existing:
-                    available_files.user_file_ids.append(uf.id)
-
         # Construct tools based on the persona configurations
         tool_dict = construct_tools(
             persona=persona,
@@ -722,11 +710,8 @@ def handle_stream_message_objects(
             llm=llm,
             search_tool_config=SearchToolConfig(
                 user_selected_filters=new_msg_req.internal_search_filters,
-                project_id=(
-                    chat_session.project_id
-                    if extracted_project_files.project_as_filter
-                    else None
-                ),
+                project_id=search_params.search_project_id,
+                persona_id=search_params.search_persona_id,
                 bypass_acl=bypass_acl,
                 slack_context=slack_context,
                 enable_slack_search=_should_enable_slack_search(
@@ -744,7 +729,7 @@ def handle_stream_message_objects(
                 chat_file_ids=available_files.chat_file_ids,
             ),
             allowed_tool_ids=new_msg_req.allowed_tool_ids,
-            search_usage_forcing_setting=project_search_config.search_usage,
+            search_usage_forcing_setting=search_params.search_usage,
         )
         tools: list[Tool] = []
         for tool_list in tool_dict.values():
@@ -783,7 +768,7 @@ def handle_stream_message_objects(
         chat_history_result = convert_chat_history(
             chat_history=chat_history,
             files=files,
-            project_image_files=extracted_project_files.project_image_files,
+            context_image_files=extracted_context_files.image_files,
             additional_context=additional_context,
             token_counter=token_counter,
             tool_id_to_name_map=tool_id_to_name_map,
@@ -824,19 +809,19 @@ def handle_stream_message_objects(
             )
             simple_chat_history.insert(0, summary_simple)
 
-        redis_client = get_redis_client()
+        cache = get_cache_backend()
 
         reset_cancel_status(
             chat_session.id,
-            redis_client,
+            cache,
         )
 
         def check_is_connected() -> bool:
-            return check_stop_signal(chat_session.id, redis_client)
+            return check_stop_signal(chat_session.id, cache)
 
         set_processing_status(
             chat_session_id=chat_session.id,
-            redis_client=redis_client,
+            cache=cache,
             value=True,
         )
 
@@ -879,46 +864,54 @@ def handle_stream_message_objects(
             # (user has already responded to a clarification question)
             skip_clarification = is_last_assistant_message_clarification(chat_history)
 
+            # NOTE: we _could_ pass in a zero argument function since emitter and state_container
+            # are just passed in immediately anyways, but the abstraction is cleaner this way.
             yield from run_chat_loop_with_state_containers(
-                run_deep_research_llm_loop,
+                lambda emitter, state_container: run_deep_research_llm_loop(
+                    emitter=emitter,
+                    state_container=state_container,
+                    simple_chat_history=simple_chat_history,
+                    tools=tools,
+                    custom_agent_prompt=custom_agent_prompt,
+                    llm=llm,
+                    token_counter=token_counter,
+                    db_session=db_session,
+                    skip_clarification=skip_clarification,
+                    user_identity=user_identity,
+                    chat_session_id=str(chat_session.id),
+                    all_injected_file_metadata=all_injected_file_metadata,
+                ),
                 llm_loop_completion_callback,
                 is_connected=check_is_connected,
                 emitter=emitter,
                 state_container=state_container,
-                simple_chat_history=simple_chat_history,
-                tools=tools,
-                custom_agent_prompt=custom_agent_prompt,
-                llm=llm,
-                token_counter=token_counter,
-                db_session=db_session,
-                skip_clarification=skip_clarification,
-                user_identity=user_identity,
-                chat_session_id=str(chat_session.id),
-                all_injected_file_metadata=all_injected_file_metadata,
             )
         else:
             yield from run_chat_loop_with_state_containers(
-                run_llm_loop,
+                lambda emitter, state_container: run_llm_loop(
+                    emitter=emitter,
+                    state_container=state_container,
+                    simple_chat_history=simple_chat_history,
+                    tools=tools,
+                    custom_agent_prompt=custom_agent_prompt,
+                    context_files=extracted_context_files,
+                    persona=persona,
+                    user_memory_context=user_memory_context,
+                    llm=llm,
+                    token_counter=token_counter,
+                    db_session=db_session,
+                    forced_tool_id=forced_tool_id,
+                    user_identity=user_identity,
+                    chat_session_id=str(chat_session.id),
+                    chat_files=chat_files_for_tools,
+                    include_citations=new_msg_req.include_citations,
+                    all_injected_file_metadata=all_injected_file_metadata,
+                    inject_memories_in_prompt=user.use_memories,
+                ),
                 llm_loop_completion_callback,
                 is_connected=check_is_connected,  # Not passed through to run_llm_loop
                 emitter=emitter,
                 state_container=state_container,
-                simple_chat_history=simple_chat_history,
-                tools=tools,
-                custom_agent_prompt=custom_agent_prompt,
-                project_files=extracted_project_files,
-                persona=persona,
-                user_memory_context=user_memory_context,
-                llm=llm,
-                token_counter=token_counter,
-                db_session=db_session,
-                forced_tool_id=forced_tool_id,
-                user_identity=user_identity,
-                chat_session_id=str(chat_session.id),
-                chat_files=chat_files_for_tools,
-                include_citations=new_msg_req.include_citations,
-                all_injected_file_metadata=all_injected_file_metadata,
-                inject_memories_in_prompt=user.use_memories,
             )
 
     except ValueError as e:
@@ -975,10 +968,10 @@ def handle_stream_message_objects(
             reset_llm_mock_response(mock_response_token)
 
         try:
-            if redis_client is not None and chat_session is not None:
+            if cache is not None and chat_session is not None:
                 set_processing_status(
                     chat_session_id=chat_session.id,
-                    redis_client=redis_client,
+                    cache=cache,
                     value=False,
                 )
         except Exception:

backend/onyx/chat/save_chat.py

@@ -1,4 +1,5 @@
 import json
+import mimetypes
 
 from sqlalchemy.orm import Session
 
@@ -12,14 +13,41 @@ from onyx.db.chat import create_db_search_doc
 from onyx.db.models import ChatMessage
 from onyx.db.models import ToolCall
 from onyx.db.tools import create_tool_call_no_commit
+from onyx.file_store.models import FileDescriptor
 from onyx.natural_language_processing.utils import BaseTokenizer
 from onyx.natural_language_processing.utils import get_tokenizer
+from onyx.server.query_and_chat.chat_utils import mime_type_to_chat_file_type
 from onyx.tools.models import ToolCallInfo
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
 
+def _extract_referenced_file_descriptors(
+    tool_calls: list[ToolCallInfo],
+    message_text: str,
+) -> list[FileDescriptor]:
+    """Extract FileDescriptors for code interpreter files referenced in the message text."""
+    descriptors: list[FileDescriptor] = []
+    for tool_call_info in tool_calls:
+        if not tool_call_info.generated_files:
+            continue
+        for gen_file in tool_call_info.generated_files:
+            file_id = (
+                gen_file.file_link.rsplit("/", 1)[-1] if gen_file.file_link else ""
+            )
+            if file_id and file_id in message_text:
+                mime_type, _ = mimetypes.guess_type(gen_file.filename)
+                descriptors.append(
+                    FileDescriptor(
+                        id=file_id,
+                        type=mime_type_to_chat_file_type(mime_type),
+                        name=gen_file.filename,
+                    )
+                )
+    return descriptors
+
+
 def _create_and_link_tool_calls(
     tool_calls: list[ToolCallInfo],
     assistant_message: ChatMessage,
@@ -297,5 +325,14 @@ def save_chat_turn(
         citation_number_to_search_doc_id if citation_number_to_search_doc_id else None
     )
 
+    # 8. Attach code interpreter generated files that the assistant actually
+    # referenced in its response, so they are available via load_all_chat_files
+    # on subsequent turns. Files not mentioned are intermediate artifacts.
+    if message_text:
+        referenced = _extract_referenced_file_descriptors(tool_calls, message_text)
+        if referenced:
+            existing_files = assistant_message.files or []
+            assistant_message.files = existing_files + referenced
+
     # Finally save the messages, tool calls, and docs
     db_session.commit()

backend/onyx/chat/stop_signal_checker.py

@@ -1,65 +1,58 @@
 from uuid import UUID
 
-from redis.client import Redis
+from onyx.cache.interface import CacheBackend
 
-# Redis key prefixes for chat session stop signals
 PREFIX = "chatsessionstop"
 FENCE_PREFIX = f"{PREFIX}_fence"
-FENCE_TTL = 10 * 60  # 10 minutes - defensive TTL to prevent memory leaks
+FENCE_TTL = 10 * 60  # 10 minutes
 
 
 def _get_fence_key(chat_session_id: UUID) -> str:
-    """
-    Generate the Redis key for a chat session stop signal fence.
+    """Generate the cache key for a chat session stop signal fence.
 
     Args:
         chat_session_id: The UUID of the chat session
 
     Returns:
-        The fence key string (tenant_id is automatically added by the Redis client)
+        The fence key string. Tenant isolation is handled automatically
+        by the cache backend (Redis key-prefixing or Postgres schema routing).
     """
     return f"{FENCE_PREFIX}_{chat_session_id}"
 
 
-def set_fence(chat_session_id: UUID, redis_client: Redis, value: bool) -> None:
-    """
-    Set or clear the stop signal fence for a chat session.
+def set_fence(chat_session_id: UUID, cache: CacheBackend, value: bool) -> None:
+    """Set or clear the stop signal fence for a chat session.
 
     Args:
         chat_session_id: The UUID of the chat session
-        redis_client: Redis client to use (tenant-aware client that auto-prefixes keys)
+        cache: Tenant-aware cache backend
         value: True to set the fence (stop signal), False to clear it
     """
     fence_key = _get_fence_key(chat_session_id)
     if not value:
-        redis_client.delete(fence_key)
+        cache.delete(fence_key)
         return
-
-    redis_client.set(fence_key, 0, ex=FENCE_TTL)
+    cache.set(fence_key, 0, ex=FENCE_TTL)
 
 
-def is_connected(chat_session_id: UUID, redis_client: Redis) -> bool:
-    """
-    Check if the chat session should continue (not stopped).
+def is_connected(chat_session_id: UUID, cache: CacheBackend) -> bool:
+    """Check if the chat session should continue (not stopped).
 
     Args:
         chat_session_id: The UUID of the chat session to check
-        redis_client: Redis client to use for checking the stop signal (tenant-aware client that auto-prefixes keys)
+        cache: Tenant-aware cache backend
 
     Returns:
         True if the session should continue, False if it should stop
     """
-    fence_key = _get_fence_key(chat_session_id)
-    return not bool(redis_client.exists(fence_key))
+    return not cache.exists(_get_fence_key(chat_session_id))
 
 
-def reset_cancel_status(chat_session_id: UUID, redis_client: Redis) -> None:
-    """
-    Clear the stop signal for a chat session.
+def reset_cancel_status(chat_session_id: UUID, cache: CacheBackend) -> None:
+    """Clear the stop signal for a chat session.
 
     Args:
         chat_session_id: The UUID of the chat session
-        redis_client: Redis client to use (tenant-aware client that auto-prefixes keys)
+        cache: Tenant-aware cache backend
     """
-    fence_key = _get_fence_key(chat_session_id)
-    redis_client.delete(fence_key)
+    cache.delete(_get_fence_key(chat_session_id))

backend/onyx/chat/tool_call_args_streaming.py

@@ -0,0 +1,236 @@
+import json
+from collections.abc import Generator
+from collections.abc import Mapping
+from typing import Any
+from typing import NamedTuple
+from typing import Type
+
+from onyx.llm.model_response import ChatCompletionDeltaToolCall
+from onyx.server.query_and_chat.placement import Placement
+from onyx.server.query_and_chat.streaming_models import Packet
+from onyx.server.query_and_chat.streaming_models import ToolCallArgumentDelta
+from onyx.tools.built_in_tools import TOOL_NAME_TO_CLASS
+from onyx.tools.interface import Tool
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def _get_tool_class(
+    tool_calls_in_progress: Mapping[int, Mapping[str, Any]],
+    tool_call_delta: ChatCompletionDeltaToolCall,
+) -> Type[Tool] | None:
+    """Look up the Tool subclass for a streaming tool call delta."""
+    tool_name = tool_calls_in_progress.get(tool_call_delta.index, {}).get("name")
+    if not tool_name:
+        return None
+    return TOOL_NAME_TO_CLASS.get(tool_name)
+
+
+class _Token(NamedTuple):
+    """A parsed JSON string with position info."""
+
+    value: str  # raw content between the quotes
+    start: int  # index of first char inside the quotes
+    end: int  # index of closing quote, or len(text) if incomplete
+    complete: bool  # whether the closing quote was found
+
+
+def _parse_json_string(text: str, pos: int) -> _Token:
+    """Parse a JSON string starting at the opening quote at ``pos``."""
+    i = pos + 1
+    while i < len(text):
+        if text[i] == "\\":
+            i += 2
+        elif text[i] == '"':
+            return _Token(text[pos + 1 : i], pos + 1, i, complete=True)
+        else:
+            i += 1
+    return _Token(text[pos + 1 :], pos + 1, len(text), complete=False)
+
+
+def _skip_json_value(text: str, pos: int) -> int:
+    """Skip past a non-string JSON value (number, bool, null, array, object).
+
+    Tracks ``[]`` / ``{}`` nesting depth and skips over embedded strings so
+    that internal commas and braces don't terminate the scan early.  Stops
+    at the next top-level ``,`` or ``}`` (not consumed).
+    """
+    depth = 0
+    while pos < len(text):
+        ch = text[pos]
+        if ch == '"':
+            tok = _parse_json_string(text, pos)
+            pos = tok.end + 1 if tok.complete else tok.end
+            continue
+        if ch in ("{", "["):
+            depth += 1
+        elif ch in ("}", "]"):
+            if depth == 0:
+                break
+            depth -= 1
+        elif ch == "," and depth == 0:
+            break
+        pos += 1
+    return pos
+
+
+def _skip(text: str, pos: int, chars: str = " \t\n\r,") -> int:
+    """Advance ``pos`` past any characters in ``chars``."""
+    while pos < len(text) and text[pos] in chars:
+        pos += 1
+    return pos
+
+
+def _decode_partial_json_string(raw: str) -> str:
+    """Decode JSON escapes (``\\n`` → newline) from a possibly incomplete value.
+
+    Progressively trims up to 6 trailing chars to handle partial escape
+    sequences (the longest JSON escape is ``\\uXXXX``).
+    """
+    for trim in range(min(7, len(raw) + 1)):
+        candidate = raw[: len(raw) - trim] if trim else raw
+        try:
+            result = json.loads('"' + candidate + '"')
+            if trim > 0 and not result and raw:
+                logger.warning(
+                    "Dropped %d chars from partial JSON string value (trim=%d)",
+                    len(raw),
+                    trim,
+                )
+            return result
+        except (json.JSONDecodeError, ValueError):
+            continue
+    logger.warning(
+        "Failed to decode partial JSON string value; dropping %d chars", len(raw)
+    )
+    return ""
+
+
+def _extract_delta_args(
+    pre: str, delta: str, scan_offset: int = 0
+) -> tuple[dict[str, str], int]:
+    """Extract decoded argument values contributed by ``delta``.
+
+    Walks ``pre + delta`` as a partial JSON object (``{"k": "v", ...}``),
+    and for each string value returns only the decoded content that falls
+    within the ``delta`` portion. Escape sequences that straddle the
+    boundary are handled correctly.
+
+    Returns ``(argument_deltas, next_scan_offset)`` where
+    ``next_scan_offset`` should be passed to the next call to skip
+    completed key-value pairs, reducing cost from O(accumulated) to
+    O(delta) per call.
+    """
+    full = pre + delta
+    delta_start = len(pre)
+
+    result: dict[str, str] = {}
+
+    if scan_offset > 0:
+        pos = scan_offset
+    else:
+        pos = full.find("{")
+        if pos == -1:
+            return result, 0
+        pos += 1
+
+    resume = pos
+
+    while pos < len(full):
+        pos = _skip(full, pos)
+        if pos >= len(full) or full[pos] == "}":
+            break
+
+        resume = pos  # remember start of this key-value pair
+
+        # Key
+        if full[pos] != '"':
+            break
+        key = _parse_json_string(full, pos)
+        if not key.complete:
+            break
+        pos = key.end + 1
+
+        # Colon
+        pos = _skip(full, pos, " \t\n\r")
+        if pos >= len(full) or full[pos] != ":":
+            break
+        pos += 1
+
+        # Value
+        pos = _skip(full, pos, " \t\n\r")
+        if pos >= len(full):
+            break
+        if full[pos] != '"':
+            # Skip non-string values (number, boolean, null, array, object).
+            # They are available in the final tool-call kickoff packet;
+            # emitting them here as strings would be ambiguous for consumers
+            # (e.g. the number 30 vs the string "30").
+            pos = _skip_json_value(full, pos)
+            continue
+        val = _parse_json_string(full, pos)
+
+        # Only include the portion of this value that overlaps with delta
+        lo = max(val.start, delta_start)
+        hi = val.end
+        if lo < hi:
+            # Decode from value start through both boundaries so escape
+            # sequences straddling the delta edge are handled correctly.
+            decoded_before = _decode_partial_json_string(full[val.start : lo])
+            decoded_through = _decode_partial_json_string(full[val.start : hi])
+            new_content = decoded_through[len(decoded_before) :]
+            if new_content:
+                result[key.value] = new_content
+
+        if not val.complete:
+            break
+        pos = val.end + 1
+
+    return result, resume
+
+
+def maybe_emit_argument_delta(
+    tool_calls_in_progress: Mapping[int, Mapping[str, Any]],
+    tool_call_delta: ChatCompletionDeltaToolCall,
+    placement: Placement,
+    scan_offsets: dict[int, int],
+) -> Generator[Packet, None, None]:
+    """Emit decoded tool-call argument deltas to the frontend.
+
+    NOTE: Currently skips non-string arguments
+
+    ``scan_offsets`` is a mutable dict keyed by tool-call index that allows
+    each call to skip past already-processed key-value pairs, reducing
+    per-call cost from O(accumulated) to O(delta).
+    """
+    tool_cls = _get_tool_class(tool_calls_in_progress, tool_call_delta)
+    if not tool_cls or not tool_cls.do_emit_argument_deltas():
+        return
+
+    fn = tool_call_delta.function
+    delta_fragment = fn.arguments if fn else None
+    if not delta_fragment:
+        return
+
+    tc_data = tool_calls_in_progress[tool_call_delta.index]
+    accumulated_args = tc_data["arguments"]
+    prev_args = accumulated_args[: -len(delta_fragment)]
+
+    idx = tool_call_delta.index
+    offset = scan_offsets.get(idx, 0)
+
+    argument_deltas, new_offset = _extract_delta_args(prev_args, delta_fragment, offset)
+    scan_offsets[idx] = new_offset
+
+    if not argument_deltas:
+        return
+
+    yield Packet(
+        placement=placement,
+        obj=ToolCallArgumentDelta(
+            tool_type=tc_data.get("name", ""),
+            tool_id=tc_data.get("id", ""),
+            argument_deltas=argument_deltas,
+        ),
+    )

backend/onyx/configs/app_configs.py

@@ -6,6 +6,7 @@ from datetime import timezone
 from typing import cast
 
 from onyx.auth.schemas import AuthBackend
+from onyx.cache.interface import CacheBackendType
 from onyx.configs.constants import AuthType
 from onyx.configs.constants import QueryHistoryType
 from onyx.file_processing.enums import HtmlBasedConnectorTransformLinksStrategy
@@ -54,6 +55,12 @@ DISABLE_USER_KNOWLEDGE = os.environ.get("DISABLE_USER_KNOWLEDGE", "").lower() ==
 # are disabled but core chat, tools, user file uploads, and Projects still work.
 DISABLE_VECTOR_DB = os.environ.get("DISABLE_VECTOR_DB", "").lower() == "true"
 
+# Which backend to use for caching, locks, and ephemeral state.
+# "redis" (default) or "postgres" (only valid when DISABLE_VECTOR_DB=true).
+CACHE_BACKEND = CacheBackendType(
+    os.environ.get("CACHE_BACKEND", CacheBackendType.REDIS)
+)
+
 # Maximum token count for a single uploaded file. Files exceeding this are rejected.
 # Defaults to 100k tokens (or 10M when vector DB is disabled).
 _DEFAULT_FILE_TOKEN_LIMIT = 10_000_000 if DISABLE_VECTOR_DB else 100_000
@@ -294,6 +301,12 @@ ENABLE_OPENSEARCH_RETRIEVAL_FOR_ONYX = (
     ENABLE_OPENSEARCH_INDEXING_FOR_ONYX
     and os.environ.get("ENABLE_OPENSEARCH_RETRIEVAL_FOR_ONYX", "").lower() == "true"
 )
+# Whether we should check for and create an index if necessary every time we
+# instantiate an OpenSearchDocumentIndex on multitenant cloud. Defaults to True.
+VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT = (
+    os.environ.get("VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT", "true").lower()
+    == "true"
+)
 
 VESPA_HOST = os.environ.get("VESPA_HOST") or "localhost"
 # NOTE: this is used if and only if the vespa config server is accessible via a
@@ -482,14 +495,7 @@ CELERY_WORKER_PRIMARY_POOL_OVERFLOW = int(
     os.environ.get("CELERY_WORKER_PRIMARY_POOL_OVERFLOW") or 4
 )
 
-# Consolidated background worker (light, docprocessing, docfetching, heavy, monitoring, user_file_processing)
-# separate workers' defaults: light=24, docprocessing=6, docfetching=1, heavy=4, kg=2, monitoring=1, user_file=2
-# Total would be 40, but we use a more conservative default of 20 for the consolidated worker
-CELERY_WORKER_BACKGROUND_CONCURRENCY = int(
-    os.environ.get("CELERY_WORKER_BACKGROUND_CONCURRENCY") or 20
-)
-
-# Individual worker concurrency settings (used when USE_LIGHTWEIGHT_BACKGROUND_WORKER is False or on Kuberenetes deployments)
+# Individual worker concurrency settings
 CELERY_WORKER_HEAVY_CONCURRENCY = int(
     os.environ.get("CELERY_WORKER_HEAVY_CONCURRENCY") or 4
 )
@@ -806,7 +812,9 @@ RERANK_COUNT = int(os.environ.get("RERANK_COUNT") or 1000)
 # Tool Configs
 #####
 # Code Interpreter Service Configuration
-CODE_INTERPRETER_BASE_URL = os.environ.get("CODE_INTERPRETER_BASE_URL")
+CODE_INTERPRETER_BASE_URL = os.environ.get(
+    "CODE_INTERPRETER_BASE_URL", "http://localhost:8000"
+)
 
 CODE_INTERPRETER_DEFAULT_TIMEOUT_MS = int(
     os.environ.get("CODE_INTERPRETER_DEFAULT_TIMEOUT_MS") or 60_000
@@ -887,6 +895,9 @@ CUSTOM_ANSWER_VALIDITY_CONDITIONS = json.loads(
 )
 
 VESPA_REQUEST_TIMEOUT = int(os.environ.get("VESPA_REQUEST_TIMEOUT") or "15")
+VESPA_MIGRATION_REQUEST_TIMEOUT_S = int(
+    os.environ.get("VESPA_MIGRATION_REQUEST_TIMEOUT_S") or "120"
+)
 
 SYSTEM_RECURSION_LIMIT = int(os.environ.get("SYSTEM_RECURSION_LIMIT") or "1000")
 

backend/onyx/configs/constants.py

@@ -84,7 +84,6 @@ POSTGRES_CELERY_WORKER_LIGHT_APP_NAME = "celery_worker_light"
 POSTGRES_CELERY_WORKER_DOCPROCESSING_APP_NAME = "celery_worker_docprocessing"
 POSTGRES_CELERY_WORKER_DOCFETCHING_APP_NAME = "celery_worker_docfetching"
 POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME = "celery_worker_indexing_child"
-POSTGRES_CELERY_WORKER_BACKGROUND_APP_NAME = "celery_worker_background"
 POSTGRES_CELERY_WORKER_HEAVY_APP_NAME = "celery_worker_heavy"
 POSTGRES_CELERY_WORKER_MONITORING_APP_NAME = "celery_worker_monitoring"
 POSTGRES_CELERY_WORKER_USER_FILE_PROCESSING_APP_NAME = (

backend/onyx/connectors/gong/connector.py

@@ -32,6 +32,8 @@ class GongConnector(LoadConnector, PollConnector):
     BASE_URL = "https://api.gong.io"
     MAX_CALL_DETAILS_ATTEMPTS = 6
     CALL_DETAILS_DELAY = 30  # in seconds
+    # Gong API limit is 3 calls/sec — stay safely under it
+    MIN_REQUEST_INTERVAL = 0.5  # seconds between requests
 
     def __init__(
         self,
@@ -45,9 +47,13 @@ class GongConnector(LoadConnector, PollConnector):
         self.continue_on_fail = continue_on_fail
         self.auth_token_basic: str | None = None
         self.hide_user_info = hide_user_info
+        self._last_request_time: float = 0.0
 
+        # urllib3 Retry already respects the Retry-After header by default
+        # (respect_retry_after_header=True), so on 429 it will sleep for the
+        # duration Gong specifies before retrying.
         retry_strategy = Retry(
-            total=5,
+            total=10,
             backoff_factor=2,
             status_forcelist=[429, 500, 502, 503, 504],
         )
@@ -61,8 +67,24 @@ class GongConnector(LoadConnector, PollConnector):
         url = f"{GongConnector.BASE_URL}{endpoint}"
         return url
 
+    def _throttled_request(
+        self, method: str, url: str, **kwargs: Any
+    ) -> requests.Response:
+        """Rate-limited request wrapper. Enforces MIN_REQUEST_INTERVAL between
+        calls to stay under Gong's 3 calls/sec limit and avoid triggering 429s."""
+        now = time.monotonic()
+        elapsed = now - self._last_request_time
+        if elapsed < self.MIN_REQUEST_INTERVAL:
+            time.sleep(self.MIN_REQUEST_INTERVAL - elapsed)
+
+        response = self._session.request(method, url, **kwargs)
+        self._last_request_time = time.monotonic()
+        return response
+
     def _get_workspace_id_map(self) -> dict[str, str]:
-        response = self._session.get(GongConnector.make_url("/v2/workspaces"))
+        response = self._throttled_request(
+            "GET", GongConnector.make_url("/v2/workspaces")
+        )
         response.raise_for_status()
 
         workspaces_details = response.json().get("workspaces")
@@ -106,8 +128,8 @@ class GongConnector(LoadConnector, PollConnector):
                     del body["filter"]["workspaceId"]
 
             while True:
-                response = self._session.post(
-                    GongConnector.make_url("/v2/calls/transcript"), json=body
+                response = self._throttled_request(
+                    "POST", GongConnector.make_url("/v2/calls/transcript"), json=body
                 )
                 # If no calls in the range, just break out
                 if response.status_code == 404:
@@ -142,8 +164,8 @@ class GongConnector(LoadConnector, PollConnector):
             "contentSelector": {"exposedFields": {"parties": True}},
         }
 
-        response = self._session.post(
-            GongConnector.make_url("/v2/calls/extensive"), json=body
+        response = self._throttled_request(
+            "POST", GongConnector.make_url("/v2/calls/extensive"), json=body
         )
         response.raise_for_status()
 
@@ -194,7 +216,8 @@ class GongConnector(LoadConnector, PollConnector):
             # There's a likely race condition in the API where a transcript will have a
             # call id but the call to v2/calls/extensive will not return all of the id's
             # retry with exponential backoff has been observed to mitigate this
-            # in ~2 minutes
+            # in ~2 minutes. After max attempts, proceed with whatever we have —
+            # the per-call loop below will skip missing IDs gracefully.
             current_attempt = 0
             while True:
                 current_attempt += 1
@@ -213,11 +236,14 @@ class GongConnector(LoadConnector, PollConnector):
                     f"missing_call_ids={missing_call_ids}"
                 )
                 if current_attempt >= self.MAX_CALL_DETAILS_ATTEMPTS:
-                    raise RuntimeError(
-                        f"Attempt count exceeded for _get_call_details_by_ids: "
-                        f"missing_call_ids={missing_call_ids} "
-                        f"max_attempts={self.MAX_CALL_DETAILS_ATTEMPTS}"
+                    logger.error(
+                        f"Giving up on missing call id's after "
+                        f"{self.MAX_CALL_DETAILS_ATTEMPTS} attempts: "
+                        f"missing_call_ids={missing_call_ids} — "
+                        f"proceeding with {len(call_details_map)} of "
+                        f"{len(transcript_call_ids)} calls"
                     )
+                    break
 
                 wait_seconds = self.CALL_DETAILS_DELAY * pow(2, current_attempt - 1)
                 logger.warning(

backend/onyx/connectors/sharepoint/connector.py

@@ -23,7 +23,6 @@ from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.serialization import pkcs12
 from office365.graph_client import GraphClient  # type: ignore[import-untyped]
-from office365.intune.organizations.organization import Organization  # type: ignore[import-untyped]
 from office365.onedrive.driveitems.driveItem import DriveItem  # type: ignore[import-untyped]
 from office365.onedrive.sites.site import Site  # type: ignore[import-untyped]
 from office365.onedrive.sites.sites_with_root import SitesWithRoot  # type: ignore[import-untyped]
@@ -872,6 +871,56 @@ class SharepointConnector(
                     "Site URLs must be full Sharepoint URLs (e.g. https://your-tenant.sharepoint.com/sites/your-site or https://your-tenant.sharepoint.com/teams/your-team)"
                 )
 
+    def _extract_tenant_domain_from_sites(self) -> str | None:
+        """Extract the tenant domain from configured site URLs.
+
+        Site URLs look like https://{tenant}.sharepoint.com/sites/... so the
+        tenant domain is the first label of the hostname.
+        """
+        for site_url in self.sites:
+            try:
+                hostname = urlsplit(site_url.strip()).hostname
+            except ValueError:
+                continue
+            if not hostname:
+                continue
+            tenant = hostname.split(".")[0]
+            if tenant:
+                return tenant
+        logger.warning(f"No tenant domain found from {len(self.sites)} sites")
+        return None
+
+    def _resolve_tenant_domain_from_root_site(self) -> str:
+        """Resolve tenant domain via GET /v1.0/sites/root which only requires
+        Sites.Read.All (a permission the connector already needs)."""
+        root_site = self.graph_client.sites.root.get().execute_query()
+        hostname = root_site.site_collection.hostname
+        if not hostname:
+            raise ConnectorValidationError(
+                "Could not determine tenant domain from root site"
+            )
+        tenant_domain = hostname.split(".")[0]
+        logger.info(
+            "Resolved tenant domain '%s' from root site hostname '%s'",
+            tenant_domain,
+            hostname,
+        )
+        return tenant_domain
+
+    def _resolve_tenant_domain(self) -> str:
+        """Determine the tenant domain, preferring site URLs over a Graph API
+        call to avoid needing extra permissions."""
+        from_sites = self._extract_tenant_domain_from_sites()
+        if from_sites:
+            logger.info(
+                "Resolved tenant domain '%s' from site URLs",
+                from_sites,
+            )
+            return from_sites
+
+        logger.info("No site URLs available; resolving tenant domain from root site")
+        return self._resolve_tenant_domain_from_root_site()
+
     @property
     def graph_client(self) -> GraphClient:
         if self._graph_client is None:
@@ -1589,6 +1638,11 @@ class SharepointConnector(
         sp_private_key = credentials.get("sp_private_key")
         sp_certificate_password = credentials.get("sp_certificate_password")
 
+        if not sp_client_id:
+            raise ConnectorValidationError("Client ID is required")
+        if not sp_directory_id:
+            raise ConnectorValidationError("Directory (tenant) ID is required")
+
         authority_url = f"{self.authority_host}/{sp_directory_id}"
 
         if auth_method == SharepointAuthMethod.CERTIFICATE.value:
@@ -1641,21 +1695,7 @@ class SharepointConnector(
             _acquire_token_for_graph, environment=self._azure_environment
         )
         if auth_method == SharepointAuthMethod.CERTIFICATE.value:
-            org = self.graph_client.organization.get().execute_query()
-            if not org or len(org) == 0:
-                raise ConnectorValidationError("No organization found")
-
-            tenant_info: Organization = org[
-                0
-            ]  # Access first item directly from collection
-            if not tenant_info.verified_domains:
-                raise ConnectorValidationError("No verified domains found for tenant")
-
-            sp_tenant_domain = tenant_info.verified_domains[0].name
-            if not sp_tenant_domain:
-                raise ConnectorValidationError("No verified domains found for tenant")
-            # remove the .onmicrosoft.com part
-            self.sp_tenant_domain = sp_tenant_domain.split(".")[0]
+            self.sp_tenant_domain = self._resolve_tenant_domain()
         return None
 
     def _get_drive_names_for_site(self, site_url: str) -> list[str]:

backend/onyx/context/search/models.py

@@ -72,6 +72,7 @@ class BaseFilters(BaseModel):
 class UserFileFilters(BaseModel):
     user_file_ids: list[UUID] | None = None
     project_id: int | None = None
+    persona_id: int | None = None
 
 
 class AssistantKnowledgeFilters(BaseModel):

backend/onyx/context/search/pipeline.py

@@ -40,6 +40,7 @@ def _build_index_filters(
     user_provided_filters: BaseFilters | None,
     user: User,  # Used for ACLs, anonymous users only see public docs
     project_id: int | None,
+    persona_id: int | None,
     user_file_ids: list[UUID] | None,
     persona_document_sets: list[str] | None,
     persona_time_cutoff: datetime | None,
@@ -118,6 +119,7 @@ def _build_index_filters(
     final_filters = IndexFilters(
         user_file_ids=user_file_ids,
         project_id=project_id,
+        persona_id=persona_id,
         source_type=source_filter,
         document_set=document_set_filter,
         time_cutoff=time_filter,
@@ -265,6 +267,8 @@ def search_pipeline(
     llm: LLM | None = None,
     # If a project ID is provided, it will be exclusively scoped to that project
     project_id: int | None = None,
+    # If a persona_id is provided, search scopes to files attached to this persona
+    persona_id: int | None = None,
     # Pre-fetched data — when provided, avoids DB queries (no session needed)
     acl_filters: list[str] | None = None,
     embedding_model: EmbeddingModel | None = None,
@@ -299,6 +303,7 @@ def search_pipeline(
         user_provided_filters=chunk_search_request.user_selected_filters,
         user=user,
         project_id=project_id,
+        persona_id=persona_id,
         user_file_ids=user_uploaded_persona_files,
         persona_document_sets=persona_document_sets,
         persona_time_cutoff=persona_time_cutoff,

backend/onyx/db/chat.py

@@ -98,6 +98,7 @@ def get_chat_sessions_by_user(
     db_session: Session,
     include_onyxbot_flows: bool = False,
     limit: int = 50,
+    before: datetime | None = None,
     project_id: int | None = None,
     only_non_project_chats: bool = False,
     include_failed_chats: bool = False,
@@ -112,6 +113,9 @@ def get_chat_sessions_by_user(
     if deleted is not None:
         stmt = stmt.where(ChatSession.deleted == deleted)
 
+    if before is not None:
+        stmt = stmt.where(ChatSession.time_updated < before)
+
     if limit:
         stmt = stmt.limit(limit)
 

backend/onyx/db/document_set.py

@@ -13,6 +13,7 @@ from sqlalchemy.orm import aliased
 from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
+from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.db.connector_credential_pair import get_cc_pair_groups_for_ids
 from onyx.db.connector_credential_pair import get_connector_credential_pairs
 from onyx.db.enums import AccessType
@@ -246,6 +247,7 @@ def insert_document_set(
             description=document_set_creation_request.description,
             user_id=user_id,
             is_public=document_set_creation_request.is_public,
+            is_up_to_date=DISABLE_VECTOR_DB,
             time_last_modified_by_user=func.now(),
         )
         db_session.add(new_document_set_row)
@@ -336,7 +338,8 @@ def update_document_set(
             )
 
         document_set_row.description = document_set_update_request.description
-        document_set_row.is_up_to_date = False
+        if not DISABLE_VECTOR_DB:
+            document_set_row.is_up_to_date = False
         document_set_row.is_public = document_set_update_request.is_public
         document_set_row.time_last_modified_by_user = func.now()
         versioned_private_doc_set_fn = fetch_versioned_implementation(

backend/onyx/db/engine/async_sql_engine.py

@@ -21,8 +21,8 @@ from onyx.configs.app_configs import POSTGRES_POOL_RECYCLE
 from onyx.configs.app_configs import POSTGRES_PORT
 from onyx.configs.app_configs import POSTGRES_USE_NULL_POOL
 from onyx.configs.app_configs import POSTGRES_USER
+from onyx.db.engine.iam_auth import create_ssl_context_if_iam
 from onyx.db.engine.iam_auth import get_iam_auth_token
-from onyx.db.engine.iam_auth import ssl_context
 from onyx.db.engine.sql_engine import ASYNC_DB_API
 from onyx.db.engine.sql_engine import build_connection_string
 from onyx.db.engine.sql_engine import is_valid_schema_name
@@ -66,7 +66,7 @@ def get_sqlalchemy_async_engine() -> AsyncEngine:
         if app_name:
             connect_args["server_settings"] = {"application_name": app_name}
 
-        connect_args["ssl"] = ssl_context
+        connect_args["ssl"] = create_ssl_context_if_iam()
 
         engine_kwargs = {
             "connect_args": connect_args,
@@ -97,7 +97,7 @@ def get_sqlalchemy_async_engine() -> AsyncEngine:
                 user = POSTGRES_USER
                 token = get_iam_auth_token(host, port, user, AWS_REGION_NAME)
                 cparams["password"] = token
-                cparams["ssl"] = ssl_context
+                cparams["ssl"] = create_ssl_context_if_iam()
 
     return _ASYNC_ENGINE
 

backend/onyx/db/engine/iam_auth.py

@@ -1,3 +1,4 @@
+import functools
 import os
 import ssl
 from typing import Any
@@ -48,11 +49,9 @@ def provide_iam_token(
         configure_psycopg2_iam_auth(cparams, host, port, user, region)
 
 
+@functools.cache
 def create_ssl_context_if_iam() -> ssl.SSLContext | None:
     """Create an SSL context if IAM authentication is enabled, else return None."""
     if USE_IAM_AUTH:
         return ssl.create_default_context(cafile=SSL_CERT_FILE)
     return None
-
-
-ssl_context = create_ssl_context_if_iam()

backend/onyx/db/enums.py

@@ -186,6 +186,7 @@ class EmbeddingPrecision(str, PyEnum):
 
 class UserFileStatus(str, PyEnum):
     PROCESSING = "PROCESSING"
+    INDEXING = "INDEXING"
     COMPLETED = "COMPLETED"
     FAILED = "FAILED"
     CANCELED = "CANCELED"

backend/onyx/db/image_generation.py

@@ -202,7 +202,6 @@ def create_default_image_gen_config_from_api_key(
             api_key=api_key,
             api_base=None,
             api_version=None,
-            default_model_name=model_name,
             deployment_name=None,
             is_public=True,
         )

backend/onyx/db/llm.py

@@ -109,45 +109,38 @@ def can_user_access_llm_provider(
         is_admin: If True, bypass user group restrictions but still respect persona restrictions
 
     Access logic:
-    1. If is_public=True → everyone has access (public override)
-    2. If is_public=False:
-       - Both groups AND personas set → must satisfy BOTH (AND logic, admins bypass group check)
-       - Only groups set → must be in one of the groups (OR across groups, admins bypass)
-       - Only personas set → must use one of the personas (OR across personas, applies to admins)
-       - Neither set → NOBODY has access unless admin (locked, admin-only)
+    - is_public controls USER access (group bypass): when True, all users can access
+      regardless of group membership. When False, user must be in a whitelisted group
+      (or be admin).
+    - Persona restrictions are ALWAYS enforced when set, regardless of is_public.
+      This allows admins to make a provider available to all users while still
+      restricting which personas (assistants) can use it.
+
+    Decision matrix:
+    1. is_public=True, no personas set → everyone has access
+    2. is_public=True, personas set → all users, but only whitelisted personas
+    3. is_public=False, groups+personas set → must satisfy BOTH (admins bypass groups)
+    4. is_public=False, only groups set → must be in group (admins bypass)
+    5. is_public=False, only personas set → must use whitelisted persona
+    6. is_public=False, neither set → admin-only (locked)
     """
-    # Public override - everyone has access
-    if provider.is_public:
-        return True
-
-    # Extract IDs once to avoid multiple iterations
-    provider_group_ids = (
-        {group.id for group in provider.groups} if provider.groups else set()
-    )
-    provider_persona_ids = (
-        {p.id for p in provider.personas} if provider.personas else set()
-    )
-
+    provider_group_ids = {g.id for g in (provider.groups or [])}
+    provider_persona_ids = {p.id for p in (provider.personas or [])}
     has_groups = bool(provider_group_ids)
     has_personas = bool(provider_persona_ids)
 
-    # Both groups AND personas set → AND logic (must satisfy both)
-    if has_groups and has_personas:
-        # Admins bypass group check but still must satisfy persona restrictions
-        user_in_group = is_admin or bool(user_group_ids & provider_group_ids)
-        persona_allowed = persona.id in provider_persona_ids if persona else False
-        return user_in_group and persona_allowed
+    # Persona restrictions are always enforced when set, regardless of is_public
+    if has_personas and not (persona and persona.id in provider_persona_ids):
+        return False
+
+    if provider.is_public:
+        return True
 
-    # Only groups set → user must be in one of the groups (admins bypass)
     if has_groups:
         return is_admin or bool(user_group_ids & provider_group_ids)
 
-    # Only personas set → persona must be in allowed list (applies to admins too)
-    if has_personas:
-        return persona.id in provider_persona_ids if persona else False
-
-    # Neither groups nor personas set, and not public → admins can access
-    return is_admin
+    # No groups: either persona-whitelisted (already passed) or admin-only if locked
+    return has_personas or is_admin
 
 
 def validate_persona_ids_exist(
@@ -213,15 +206,29 @@ def upsert_llm_provider(
     llm_provider_upsert_request: LLMProviderUpsertRequest,
     db_session: Session,
 ) -> LLMProviderView:
-    existing_llm_provider = (
-        fetch_existing_llm_provider_by_id(
+    existing_llm_provider: LLMProviderModel | None = None
+    if llm_provider_upsert_request.id:
+        existing_llm_provider = fetch_existing_llm_provider_by_id(
             id=llm_provider_upsert_request.id, db_session=db_session
         )
-        if llm_provider_upsert_request.id
-        else None
-    )
+        if not existing_llm_provider:
+            raise ValueError(
+                f"LLM provider with id {llm_provider_upsert_request.id} not found"
+            )
 
-    if not existing_llm_provider:
+        if existing_llm_provider.name != llm_provider_upsert_request.name:
+            raise ValueError(
+                f"LLM provider with id {llm_provider_upsert_request.id} name change not allowed"
+            )
+    else:
+        existing_llm_provider = fetch_existing_llm_provider(
+            name=llm_provider_upsert_request.name, db_session=db_session
+        )
+        if existing_llm_provider:
+            raise ValueError(
+                f"LLM provider with name '{llm_provider_upsert_request.name}'"
+                " already exists"
+            )
         existing_llm_provider = LLMProviderModel(name=llm_provider_upsert_request.name)
         db_session.add(existing_llm_provider)
 
@@ -242,6 +249,7 @@ def upsert_llm_provider(
     existing_llm_provider.api_base = api_base
     existing_llm_provider.api_version = llm_provider_upsert_request.api_version
     existing_llm_provider.custom_config = custom_config
+
     existing_llm_provider.is_public = llm_provider_upsert_request.is_public
     existing_llm_provider.is_auto_mode = llm_provider_upsert_request.is_auto_mode
     existing_llm_provider.deployment_name = llm_provider_upsert_request.deployment_name
@@ -250,10 +258,6 @@ def upsert_llm_provider(
         # If its not already in the db, we need to generate an ID by flushing
         db_session.flush()
 
-    models_to_exist = {
-        mc.name for mc in llm_provider_upsert_request.model_configurations
-    }
-
     # Build a lookup of existing model configurations by name (single iteration)
     existing_by_name = {
         mc.name: mc for mc in existing_llm_provider.model_configurations
@@ -528,6 +532,7 @@ def fetch_default_model(
 ) -> ModelConfiguration | None:
     model_config = db_session.scalar(
         select(ModelConfiguration)
+        .options(selectinload(ModelConfiguration.llm_provider))
         .join(LLMModelFlow)
         .where(
             ModelConfiguration.is_visible == True,  # noqa: E712
@@ -861,6 +866,7 @@ def insert_new_model_configuration__no_commit(
             is_visible=is_visible,
             max_input_tokens=max_input_tokens,
             display_name=display_name,
+            supports_image_input=LLMModelFlowType.VISION in supported_flows,
         )
         .on_conflict_do_nothing()
         .returning(ModelConfiguration.id)
@@ -895,6 +901,7 @@ def update_model_configuration__no_commit(
             is_visible=is_visible,
             max_input_tokens=max_input_tokens,
             display_name=display_name,
+            supports_image_input=LLMModelFlowType.VISION in supported_flows,
         )
         .where(ModelConfiguration.id == model_configuration_id)
         .returning(ModelConfiguration)

backend/onyx/db/models.py

@@ -103,7 +103,6 @@ from onyx.utils.encryption import encrypt_string_to_bytes
 from onyx.utils.sensitive import SensitiveValue
 from onyx.utils.headers import HeaderItemDict
 from shared_configs.enums import EmbeddingProvider
-from onyx.context.search.enums import RecencyBiasSetting
 
 # TODO: After anonymous user migration has been deployed, make user_id columns NOT NULL
 # and update Mapped[User | None] relationships to Mapped[User] where needed.
@@ -2823,8 +2822,17 @@ class LLMProvider(Base):
         postgresql.JSONB(), nullable=True
     )
 
+    # Deprecated: use LLMModelFlow with CHAT flow type instead
+    default_model_name: Mapped[str | None] = mapped_column(String, nullable=True)
+
     deployment_name: Mapped[str | None] = mapped_column(String, nullable=True)
 
+    # Deprecated: use LLMModelFlow.is_default with CHAT flow type instead
+    is_default_provider: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
+    # Deprecated: use LLMModelFlow.is_default with VISION flow type instead
+    is_default_vision_provider: Mapped[bool | None] = mapped_column(Boolean)
+    # Deprecated: use LLMModelFlow with VISION flow type instead
+    default_vision_model: Mapped[str | None] = mapped_column(String, nullable=True)
     # EE only
     is_public: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
     # Auto mode: models, visibility, and defaults are managed by GitHub config
@@ -2874,6 +2882,9 @@ class ModelConfiguration(Base):
     # - The end-user is configuring a model and chooses not to set a max-input-tokens limit.
     max_input_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
 
+    # Deprecated: use LLMModelFlow with VISION flow type instead
+    supports_image_input: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
+
     # Human-readable display name for the model.
     # For dynamic providers (OpenRouter, Bedrock, Ollama), this comes from the source API.
     # For static providers (OpenAI, Anthropic), this may be null and will fall back to LiteLLM.
@@ -3253,19 +3264,6 @@ class Persona(Base):
     )
     name: Mapped[str] = mapped_column(String)
     description: Mapped[str] = mapped_column(String)
-    # Number of chunks to pass to the LLM for generation.
-    num_chunks: Mapped[float | None] = mapped_column(Float, nullable=True)
-    chunks_above: Mapped[int] = mapped_column(Integer)
-    chunks_below: Mapped[int] = mapped_column(Integer)
-    # Pass every chunk through LLM for evaluation, fairly expensive
-    # Can be turned off globally by admin, in which case, this setting is ignored
-    llm_relevance_filter: Mapped[bool] = mapped_column(Boolean)
-    # Enables using LLM to extract time and source type filters
-    # Can also be admin disabled globally
-    llm_filter_extraction: Mapped[bool] = mapped_column(Boolean)
-    recency_bias: Mapped[RecencyBiasSetting] = mapped_column(
-        Enum(RecencyBiasSetting, native_enum=False)
-    )
 
     # Allows the persona to specify a specific default LLM model
     # NOTE: only is applied on the actual response generation - is not used for things like
@@ -3292,11 +3290,8 @@ class Persona(Base):
     # Treated specially (cannot be user edited etc.)
     builtin_persona: Mapped[bool] = mapped_column(Boolean, default=False)
 
-    # Default personas are personas created by admins and are automatically added
-    # to all users' assistants list.
-    is_default_persona: Mapped[bool] = mapped_column(
-        Boolean, default=False, nullable=False
-    )
+    # Featured personas are highlighted in the UI
+    featured: Mapped[bool] = mapped_column(Boolean, default=False)
     # controls whether the persona is available to be selected by users
     is_visible: Mapped[bool] = mapped_column(Boolean, default=True)
     # controls the ordering of personas in the UI
@@ -4931,7 +4926,9 @@ class ScimUserMapping(Base):
     __tablename__ = "scim_user_mapping"
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    external_id: Mapped[str] = mapped_column(String, unique=True, index=True)
+    external_id: Mapped[str | None] = mapped_column(
+        String, unique=True, index=True, nullable=True
+    )
     user_id: Mapped[UUID] = mapped_column(
         ForeignKey("user.id", ondelete="CASCADE"), unique=True, nullable=False
     )
@@ -4988,3 +4985,25 @@ class CodeInterpreterServer(Base):
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
     server_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+
+
+class CacheStore(Base):
+    """Key-value cache table used by ``PostgresCacheBackend``.
+
+    Replaces Redis for simple KV caching, locks, and list operations
+    when ``CACHE_BACKEND=postgres`` (NO_VECTOR_DB deployments).
+
+    Intentionally separate from ``KVStore``:
+    - Stores raw bytes (LargeBinary) vs JSONB, matching Redis semantics.
+    - Has ``expires_at`` for TTL; rows are periodically garbage-collected.
+    - Holds ephemeral data (tokens, stop signals, lock state) not
+      persistent application config, so cleanup can be aggressive.
+    """
+
+    __tablename__ = "cache_store"
+
+    key: Mapped[str] = mapped_column(String, primary_key=True)
+    value: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True)
+    expires_at: Mapped[datetime.datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )

backend/onyx/db/persona.py

@@ -18,11 +18,8 @@ from sqlalchemy.orm import Session
 from onyx.access.hierarchy_access import get_user_external_group_ids
 from onyx.auth.schemas import UserRole
 from onyx.configs.app_configs import CURATORS_CANNOT_VIEW_OR_EDIT_NON_OWNED_ASSISTANTS
-from onyx.configs.chat_configs import CONTEXT_CHUNKS_ABOVE
-from onyx.configs.chat_configs import CONTEXT_CHUNKS_BELOW
 from onyx.configs.constants import DEFAULT_PERSONA_ID
 from onyx.configs.constants import NotificationType
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.constants import SLACK_BOT_PERSONA_PREFIX
 from onyx.db.document_access import get_accessible_documents_by_ids
 from onyx.db.models import ConnectorCredentialPair
@@ -254,16 +251,15 @@ def create_update_persona(
     # Permission to actually use these is checked later
 
     try:
-        # Default persona validation
-        if create_persona_request.is_default_persona:
-            if not create_persona_request.is_public:
-                raise ValueError("Cannot make a default persona non public")
+        # Featured persona validation
+        if create_persona_request.featured:
 
-            # Curators can edit default personas, but not make them
+            # Curators can edit featured personas, but not make them
+            # TODO this will be reworked soon with RBAC permissions feature
             if user.role == UserRole.CURATOR or user.role == UserRole.GLOBAL_CURATOR:
                 pass
             elif user.role != UserRole.ADMIN:
-                raise ValueError("Only admins can make a default persona")
+                raise ValueError("Only admins can make a featured persona")
 
         # Convert incoming string UUIDs to UUID objects for DB operations
         converted_user_file_ids = None
@@ -284,7 +280,6 @@ def create_update_persona(
             document_set_ids=create_persona_request.document_set_ids,
             tool_ids=create_persona_request.tool_ids,
             is_public=create_persona_request.is_public,
-            recency_bias=create_persona_request.recency_bias,
             llm_model_provider_override=create_persona_request.llm_model_provider_override,
             llm_model_version_override=create_persona_request.llm_model_version_override,
             starter_messages=create_persona_request.starter_messages,
@@ -298,10 +293,7 @@ def create_update_persona(
             remove_image=create_persona_request.remove_image,
             search_start_date=create_persona_request.search_start_date,
             label_ids=create_persona_request.label_ids,
-            num_chunks=create_persona_request.num_chunks,
-            llm_relevance_filter=create_persona_request.llm_relevance_filter,
-            llm_filter_extraction=create_persona_request.llm_filter_extraction,
-            is_default_persona=create_persona_request.is_default_persona,
+            featured=create_persona_request.featured,
             user_file_ids=converted_user_file_ids,
             commit=False,
             hierarchy_node_ids=create_persona_request.hierarchy_node_ids,
@@ -335,6 +327,7 @@ def update_persona_shared(
     db_session: Session,
     group_ids: list[int] | None = None,
     is_public: bool | None = None,
+    label_ids: list[int] | None = None,
 ) -> None:
     """Simplified version of `create_update_persona` which only touches the
     accessibility rather than any of the logic (e.g. prompt, connected data sources,
@@ -344,9 +337,7 @@ def update_persona_shared(
     )
 
     if user and user.role != UserRole.ADMIN and persona.user_id != user.id:
-        raise HTTPException(
-            status_code=403, detail="You don't have permission to modify this persona"
-        )
+        raise PermissionError("You don't have permission to modify this persona")
 
     versioned_update_persona_access = fetch_versioned_implementation(
         "onyx.db.persona", "update_persona_access"
@@ -360,6 +351,15 @@ def update_persona_shared(
         group_ids=group_ids,
     )
 
+    if label_ids is not None:
+        labels = (
+            db_session.query(PersonaLabel).filter(PersonaLabel.id.in_(label_ids)).all()
+        )
+        if len(labels) != len(label_ids):
+            raise ValueError("Some label IDs were not found in the database")
+        persona.labels.clear()
+        persona.labels = labels
+
     db_session.commit()
 
 
@@ -869,10 +869,6 @@ def upsert_persona(
     user: User | None,
     name: str,
     description: str,
-    num_chunks: float,
-    llm_relevance_filter: bool,
-    llm_filter_extraction: bool,
-    recency_bias: RecencyBiasSetting,
     llm_model_provider_override: str | None,
     llm_model_version_override: str | None,
     starter_messages: list[StarterMessage] | None,
@@ -893,13 +889,11 @@ def upsert_persona(
     remove_image: bool | None = None,
     search_start_date: datetime | None = None,
     builtin_persona: bool = False,
-    is_default_persona: bool | None = None,
+    featured: bool | None = None,
     label_ids: list[int] | None = None,
     user_file_ids: list[UUID] | None = None,
     hierarchy_node_ids: list[int] | None = None,
     document_ids: list[str] | None = None,
-    chunks_above: int = CONTEXT_CHUNKS_ABOVE,
-    chunks_below: int = CONTEXT_CHUNKS_BELOW,
     replace_base_system_prompt: bool = False,
 ) -> Persona:
     """
@@ -965,6 +959,8 @@ def upsert_persona(
         labels = (
             db_session.query(PersonaLabel).filter(PersonaLabel.id.in_(label_ids)).all()
         )
+        if len(labels) != len(label_ids):
+            raise ValueError("Some label IDs were not found in the database")
 
     # Fetch and attach hierarchy_nodes by IDs
     hierarchy_nodes = None
@@ -1008,12 +1004,6 @@ def upsert_persona(
         # `default` and `built-in` properties can only be set when creating a persona.
         existing_persona.name = name
         existing_persona.description = description
-        existing_persona.num_chunks = num_chunks
-        existing_persona.chunks_above = chunks_above
-        existing_persona.chunks_below = chunks_below
-        existing_persona.llm_relevance_filter = llm_relevance_filter
-        existing_persona.llm_filter_extraction = llm_filter_extraction
-        existing_persona.recency_bias = recency_bias
         existing_persona.llm_model_provider_override = llm_model_provider_override
         existing_persona.llm_model_version_override = llm_model_version_override
         existing_persona.starter_messages = starter_messages
@@ -1027,10 +1017,8 @@ def upsert_persona(
         if label_ids is not None:
             existing_persona.labels.clear()
             existing_persona.labels = labels or []
-        existing_persona.is_default_persona = (
-            is_default_persona
-            if is_default_persona is not None
-            else existing_persona.is_default_persona
+        existing_persona.featured = (
+            featured if featured is not None else existing_persona.featured
         )
         # Update embedded prompt fields if provided
         if system_prompt is not None:
@@ -1083,12 +1071,6 @@ def upsert_persona(
             is_public=is_public,
             name=name,
             description=description,
-            num_chunks=num_chunks,
-            chunks_above=chunks_above,
-            chunks_below=chunks_below,
-            llm_relevance_filter=llm_relevance_filter,
-            llm_filter_extraction=llm_filter_extraction,
-            recency_bias=recency_bias,
             builtin_persona=builtin_persona,
             system_prompt=system_prompt or "",
             task_prompt=task_prompt or "",
@@ -1104,9 +1086,7 @@ def upsert_persona(
             display_priority=display_priority,
             is_visible=is_visible,
             search_start_date=search_start_date,
-            is_default_persona=(
-                is_default_persona if is_default_persona is not None else False
-            ),
+            featured=(featured if featured is not None else False),
             user_files=user_files or [],
             labels=labels or [],
             hierarchy_nodes=hierarchy_nodes or [],
@@ -1151,9 +1131,9 @@ def delete_old_default_personas(
     db_session.commit()
 
 
-def update_persona_is_default(
+def update_persona_featured(
     persona_id: int,
-    is_default: bool,
+    featured: bool,
     db_session: Session,
     user: User,
 ) -> None:
@@ -1161,10 +1141,7 @@ def update_persona_is_default(
         db_session=db_session, persona_id=persona_id, user=user, get_editable=True
     )
 
-    if not persona.is_public:
-        persona.is_public = True
-
-    persona.is_default_persona = is_default
+    persona.featured = featured
     db_session.commit()
 
 

backend/onyx/db/projects.py

@@ -9,8 +9,9 @@ from pydantic import BaseModel
 from pydantic import ConfigDict
 from sqlalchemy import func
 from sqlalchemy.orm import Session
+from starlette.background import BackgroundTasks
 
-from onyx.background.celery.versioned_apps.client import app as client_app
+from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
@@ -51,7 +52,7 @@ def create_user_files(
 ) -> CategorizedFilesResult:
 
     # Categorize the files
-    categorized_files = categorize_uploaded_files(files)
+    categorized_files = categorize_uploaded_files(files, db_session)
     # NOTE: At the moment, zip metadata is not used for user files.
     # Should revisit to decide whether this should be a feature.
     upload_response = upload_files(categorized_files.acceptable, FileOrigin.USER_FILE)
@@ -105,8 +106,8 @@ def upload_files_to_user_files_with_indexing(
     user: User,
     temp_id_map: dict[str, str] | None,
     db_session: Session,
+    background_tasks: BackgroundTasks | None = None,
 ) -> CategorizedFilesResult:
-    # Validate project ownership if a project_id is provided
     if project_id is not None and user is not None:
         if not check_project_ownership(project_id, user.id, db_session):
             raise HTTPException(status_code=404, detail="Project not found")
@@ -127,16 +128,27 @@ def upload_files_to_user_files_with_indexing(
         logger.warning(
             f"File {rejected_file.filename} rejected for {rejected_file.reason}"
         )
-    for user_file in user_files:
-        task = client_app.send_task(
-            OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
-            kwargs={"user_file_id": user_file.id, "tenant_id": tenant_id},
-            queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
-            priority=OnyxCeleryPriority.HIGH,
-        )
-        logger.info(
-            f"Triggered indexing for user_file_id={user_file.id} with task_id={task.id}"
-        )
+
+    if DISABLE_VECTOR_DB and background_tasks is not None:
+        from onyx.background.task_utils import drain_processing_loop
+
+        background_tasks.add_task(drain_processing_loop, tenant_id)
+        for user_file in user_files:
+            logger.info(f"Queued in-process processing for user_file_id={user_file.id}")
+    else:
+        from onyx.background.celery.versioned_apps.client import app as client_app
+
+        for user_file in user_files:
+            task = client_app.send_task(
+                OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
+                kwargs={"user_file_id": user_file.id, "tenant_id": tenant_id},
+                queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
+                priority=OnyxCeleryPriority.HIGH,
+            )
+            logger.info(
+                f"Triggered indexing for user_file_id={user_file.id} "
+                f"with task_id={task.id}"
+            )
 
     return CategorizedFilesResult(
         user_files=user_files,

backend/onyx/db/slack_channel_config.py

@@ -5,8 +5,6 @@ from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm import Session
 
-from onyx.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.constants import DEFAULT_PERSONA_SLACK_CHANNEL_NAME
 from onyx.db.constants import SLACK_BOT_PERSONA_PREFIX
 from onyx.db.models import ChannelConfig
@@ -45,8 +43,6 @@ def create_slack_channel_persona(
     channel_name: str | None,
     document_set_ids: list[int],
     existing_persona_id: int | None = None,
-    num_chunks: float = MAX_CHUNKS_FED_TO_CHAT,
-    enable_auto_filters: bool = False,
 ) -> Persona:
     """NOTE: does not commit changes"""
 
@@ -73,17 +69,13 @@ def create_slack_channel_persona(
         system_prompt="",
         task_prompt="",
         datetime_aware=True,
-        num_chunks=num_chunks,
-        llm_relevance_filter=True,
-        llm_filter_extraction=enable_auto_filters,
-        recency_bias=RecencyBiasSetting.AUTO,
         tool_ids=[search_tool.id],
         document_set_ids=document_set_ids,
         llm_model_provider_override=None,
         llm_model_version_override=None,
         starter_messages=None,
         is_public=True,
-        is_default_persona=False,
+        featured=False,
         db_session=db_session,
         commit=False,
     )

backend/onyx/db/user_file.py

@@ -6,6 +6,7 @@ from sqlalchemy import select
 from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
+from onyx.db.models import Project__UserFile
 from onyx.db.models import UserFile
 
 
@@ -57,12 +58,19 @@ def fetch_user_project_ids_for_user_files(
     db_session: Session,
 ) -> dict[str, list[int]]:
     """Fetch user project ids for specified user files"""
-    stmt = select(UserFile).where(UserFile.id.in_(user_file_ids))
-    results = db_session.execute(stmt).scalars().all()
-    return {
-        str(user_file.id): [project.id for project in user_file.projects]
-        for user_file in results
+    user_file_uuid_ids = [UUID(user_file_id) for user_file_id in user_file_ids]
+    stmt = select(Project__UserFile.user_file_id, Project__UserFile.project_id).where(
+        Project__UserFile.user_file_id.in_(user_file_uuid_ids)
+    )
+    rows = db_session.execute(stmt).all()
+
+    user_file_id_to_project_ids: dict[str, list[int]] = {
+        user_file_id: [] for user_file_id in user_file_ids
     }
+    for user_file_id, project_id in rows:
+        user_file_id_to_project_ids[str(user_file_id)].append(project_id)
+
+    return user_file_id_to_project_ids
 
 
 def fetch_persona_ids_for_user_files(

backend/onyx/deep_research/dr_loop.py

@@ -139,7 +139,7 @@ def generate_final_report(
             custom_agent_prompt=None,
             simple_chat_history=history,
             reminder_message=reminder_message,
-            project_files=None,
+            context_files=None,
             available_tokens=llm.config.max_input_tokens,
             all_injected_file_metadata=all_injected_file_metadata,
         )
@@ -257,7 +257,7 @@ def run_deep_research_llm_loop(
                     custom_agent_prompt=None,
                     simple_chat_history=simple_chat_history,
                     reminder_message=None,
-                    project_files=None,
+                    context_files=None,
                     available_tokens=available_tokens,
                     last_n_user_messages=MAX_USER_MESSAGES_FOR_CONTEXT,
                     all_injected_file_metadata=all_injected_file_metadata,
@@ -321,7 +321,7 @@ def run_deep_research_llm_loop(
                 custom_agent_prompt=None,
                 simple_chat_history=simple_chat_history + [reminder_message],
                 reminder_message=None,
-                project_files=None,
+                context_files=None,
                 available_tokens=available_tokens,
                 last_n_user_messages=MAX_USER_MESSAGES_FOR_CONTEXT + 1,
                 all_injected_file_metadata=all_injected_file_metadata,
@@ -485,7 +485,7 @@ def run_deep_research_llm_loop(
                     custom_agent_prompt=None,
                     simple_chat_history=simple_chat_history,
                     reminder_message=first_cycle_reminder_message,
-                    project_files=None,
+                    context_files=None,
                     available_tokens=available_tokens,
                     last_n_user_messages=MAX_USER_MESSAGES_FOR_CONTEXT,
                     all_injected_file_metadata=all_injected_file_metadata,

backend/onyx/document_index/factory.py

@@ -11,6 +11,7 @@ from onyx.document_index.opensearch.opensearch_document_index import (
     OpenSearchOldDocumentIndex,
 )
 from onyx.document_index.vespa.index import VespaIndex
+from onyx.indexing.models import IndexingSetting
 from shared_configs.configs import MULTI_TENANT
 
 
@@ -49,8 +50,11 @@ def get_default_document_index(
 
     opensearch_retrieval_enabled = get_opensearch_retrieval_state(db_session)
     if opensearch_retrieval_enabled:
+        indexing_setting = IndexingSetting.from_db_model(search_settings)
         return OpenSearchOldDocumentIndex(
             index_name=search_settings.index_name,
+            embedding_dim=indexing_setting.final_embedding_dim,
+            embedding_precision=indexing_setting.embedding_precision,
             secondary_index_name=secondary_index_name,
             large_chunks_enabled=search_settings.large_chunks_enabled,
             secondary_large_chunks_enabled=secondary_large_chunks_enabled,
@@ -118,8 +122,11 @@ def get_all_document_indices(
     )
     opensearch_document_index: OpenSearchOldDocumentIndex | None = None
     if ENABLE_OPENSEARCH_INDEXING_FOR_ONYX:
+        indexing_setting = IndexingSetting.from_db_model(search_settings)
         opensearch_document_index = OpenSearchOldDocumentIndex(
             index_name=search_settings.index_name,
+            embedding_dim=indexing_setting.final_embedding_dim,
+            embedding_precision=indexing_setting.embedding_precision,
             secondary_index_name=None,
             large_chunks_enabled=False,
             secondary_large_chunks_enabled=None,

backend/onyx/document_index/opensearch/client.py

@@ -1,5 +1,7 @@
 import logging
 import time
+from contextlib import AbstractContextManager
+from contextlib import nullcontext
 from typing import Any
 from typing import Generic
 from typing import TypeVar
@@ -83,22 +85,26 @@ def get_new_body_without_vectors(body: dict[str, Any]) -> dict[str, Any]:
     return new_body
 
 
-class OpenSearchClient:
-    """Client for interacting with OpenSearch.
+class OpenSearchClient(AbstractContextManager):
+    """Client for interacting with OpenSearch for cluster-level operations.
 
-    OpenSearch's Python module has pretty bad typing support so this client
-    attempts to protect the rest of the codebase from this. As a consequence,
-    most methods here return the minimum data needed for the rest of Onyx, and
-    tend to rely on Exceptions to handle errors.
-
-    TODO(andrei): This class currently assumes the structure of the database
-    schema when it returns a DocumentChunk. Make the class, or at least the
-    search method, templated on the structure the caller can expect.
+    Args:
+        host: The host of the OpenSearch cluster.
+        port: The port of the OpenSearch cluster.
+        auth: The authentication credentials for the OpenSearch cluster. A tuple
+            of (username, password).
+        use_ssl: Whether to use SSL for the OpenSearch cluster. Defaults to
+            True.
+        verify_certs: Whether to verify the SSL certificates for the OpenSearch
+            cluster. Defaults to False.
+        ssl_show_warn: Whether to show warnings for SSL certificates. Defaults
+            to False.
+        timeout: The timeout for the OpenSearch cluster. Defaults to
+            DEFAULT_OPENSEARCH_CLIENT_TIMEOUT_S.
     """
 
     def __init__(
         self,
-        index_name: str,
         host: str = OPENSEARCH_HOST,
         port: int = OPENSEARCH_REST_API_PORT,
         auth: tuple[str, str] = (OPENSEARCH_ADMIN_USERNAME, OPENSEARCH_ADMIN_PASSWORD),
@@ -107,9 +113,8 @@ class OpenSearchClient:
         ssl_show_warn: bool = False,
         timeout: int = DEFAULT_OPENSEARCH_CLIENT_TIMEOUT_S,
     ):
-        self._index_name = index_name
         logger.debug(
-            f"Creating OpenSearch client for index {index_name} with host {host} and port {port} and timeout {timeout} seconds."
+            f"Creating OpenSearch client with host {host}, port {port} and timeout {timeout} seconds."
         )
         self._client = OpenSearch(
             hosts=[{"host": host, "port": port}],
@@ -125,6 +130,142 @@ class OpenSearchClient:
             # your request body that is less than this value.
             timeout=timeout,
         )
+
+    def __exit__(self, *_: Any) -> None:
+        self.close()
+
+    def __del__(self) -> None:
+        try:
+            self.close()
+        except Exception:
+            pass
+
+    @log_function_time(print_only=True, debug_only=True, include_args=True)
+    def create_search_pipeline(
+        self,
+        pipeline_id: str,
+        pipeline_body: dict[str, Any],
+    ) -> None:
+        """Creates a search pipeline.
+
+        See the OpenSearch documentation for more information on the search
+        pipeline body.
+        https://docs.opensearch.org/latest/search-plugins/search-pipelines/index/
+
+        Args:
+            pipeline_id: The ID of the search pipeline to create.
+            pipeline_body: The body of the search pipeline to create.
+
+        Raises:
+            Exception: There was an error creating the search pipeline.
+        """
+        result = self._client.search_pipeline.put(id=pipeline_id, body=pipeline_body)
+        if not result.get("acknowledged", False):
+            raise RuntimeError(f"Failed to create search pipeline {pipeline_id}.")
+
+    @log_function_time(print_only=True, debug_only=True, include_args=True)
+    def delete_search_pipeline(self, pipeline_id: str) -> None:
+        """Deletes a search pipeline.
+
+        Args:
+            pipeline_id: The ID of the search pipeline to delete.
+
+        Raises:
+            Exception: There was an error deleting the search pipeline.
+        """
+        result = self._client.search_pipeline.delete(id=pipeline_id)
+        if not result.get("acknowledged", False):
+            raise RuntimeError(f"Failed to delete search pipeline {pipeline_id}.")
+
+    @log_function_time(print_only=True, debug_only=True, include_args=True)
+    def put_cluster_settings(self, settings: dict[str, Any]) -> bool:
+        """Puts cluster settings.
+
+        Args:
+            settings: The settings to put.
+
+        Raises:
+            Exception: There was an error putting the cluster settings.
+
+        Returns:
+            True if the settings were put successfully, False otherwise.
+        """
+        response = self._client.cluster.put_settings(body=settings)
+        if response.get("acknowledged", False):
+            logger.info("Successfully put cluster settings.")
+            return True
+        else:
+            logger.error(f"Failed to put cluster settings: {response}.")
+            return False
+
+    @log_function_time(print_only=True, debug_only=True)
+    def ping(self) -> bool:
+        """Pings the OpenSearch cluster.
+
+        Returns:
+            True if OpenSearch could be reached, False if it could not.
+        """
+        return self._client.ping()
+
+    @log_function_time(print_only=True, debug_only=True)
+    def close(self) -> None:
+        """Closes the client.
+
+        Raises:
+            Exception: There was an error closing the client.
+        """
+        self._client.close()
+
+
+class OpenSearchIndexClient(OpenSearchClient):
+    """Client for interacting with OpenSearch for index-level operations.
+
+    OpenSearch's Python module has pretty bad typing support so this client
+    attempts to protect the rest of the codebase from this. As a consequence,
+    most methods here return the minimum data needed for the rest of Onyx, and
+    tend to rely on Exceptions to handle errors.
+
+    TODO(andrei): This class currently assumes the structure of the database
+    schema when it returns a DocumentChunk. Make the class, or at least the
+    search method, templated on the structure the caller can expect.
+
+    Args:
+        index_name: The name of the index to interact with.
+        host: The host of the OpenSearch cluster.
+        port: The port of the OpenSearch cluster.
+        auth: The authentication credentials for the OpenSearch cluster. A tuple
+            of (username, password).
+        use_ssl: Whether to use SSL for the OpenSearch cluster. Defaults to
+            True.
+        verify_certs: Whether to verify the SSL certificates for the OpenSearch
+            cluster. Defaults to False.
+        ssl_show_warn: Whether to show warnings for SSL certificates. Defaults
+            to False.
+        timeout: The timeout for the OpenSearch cluster. Defaults to
+            DEFAULT_OPENSEARCH_CLIENT_TIMEOUT_S.
+    """
+
+    def __init__(
+        self,
+        index_name: str,
+        host: str = OPENSEARCH_HOST,
+        port: int = OPENSEARCH_REST_API_PORT,
+        auth: tuple[str, str] = (OPENSEARCH_ADMIN_USERNAME, OPENSEARCH_ADMIN_PASSWORD),
+        use_ssl: bool = True,
+        verify_certs: bool = False,
+        ssl_show_warn: bool = False,
+        timeout: int = DEFAULT_OPENSEARCH_CLIENT_TIMEOUT_S,
+    ):
+        super().__init__(
+            host=host,
+            port=port,
+            auth=auth,
+            use_ssl=use_ssl,
+            verify_certs=verify_certs,
+            ssl_show_warn=ssl_show_warn,
+            timeout=timeout,
+        )
+        self._index_name = index_name
         logger.debug(
             f"OpenSearch client created successfully for index {self._index_name}."
         )
@@ -192,6 +333,38 @@ class OpenSearchClient:
         """
         return self._client.indices.exists(index=self._index_name)
 
+    @log_function_time(print_only=True, debug_only=True, include_args=True)
+    def put_mapping(self, mappings: dict[str, Any]) -> None:
+        """Updates the index mapping in an idempotent manner.
+
+        - Existing fields with the same definition: No-op (succeeds silently).
+        - New fields: Added to the index.
+        - Existing fields with different types: Raises exception (requires
+          reindex).
+
+        See the OpenSearch documentation for more information:
+        https://docs.opensearch.org/latest/api-reference/index-apis/put-mapping/
+
+        Args:
+            mappings: The complete mapping definition to apply. This will be
+                merged with existing mappings in the index.
+
+        Raises:
+            Exception: There was an error updating the mappings, such as
+                attempting to change the type of an existing field.
+        """
+        logger.debug(
+            f"Putting mappings for index {self._index_name} with mappings {mappings}."
+        )
+        response = self._client.indices.put_mapping(
+            index=self._index_name, body=mappings
+        )
+        if not response.get("acknowledged", False):
+            raise RuntimeError(
+                f"Failed to put the mapping update for index {self._index_name}."
+            )
+        logger.debug(f"Successfully put mappings for index {self._index_name}.")
+
     @log_function_time(print_only=True, debug_only=True, include_args=True)
     def validate_index(self, expected_mappings: dict[str, Any]) -> bool:
         """Validates the index.
@@ -610,43 +783,6 @@ class OpenSearchClient:
         )
         return DocumentChunk.model_validate(document_chunk_source)
 
-    @log_function_time(print_only=True, debug_only=True, include_args=True)
-    def create_search_pipeline(
-        self,
-        pipeline_id: str,
-        pipeline_body: dict[str, Any],
-    ) -> None:
-        """Creates a search pipeline.
-
-        See the OpenSearch documentation for more information on the search
-        pipeline body.
-        https://docs.opensearch.org/latest/search-plugins/search-pipelines/index/
-
-        Args:
-            pipeline_id: The ID of the search pipeline to create.
-            pipeline_body: The body of the search pipeline to create.
-
-        Raises:
-            Exception: There was an error creating the search pipeline.
-        """
-        result = self._client.search_pipeline.put(id=pipeline_id, body=pipeline_body)
-        if not result.get("acknowledged", False):
-            raise RuntimeError(f"Failed to create search pipeline {pipeline_id}.")
-
-    @log_function_time(print_only=True, debug_only=True, include_args=True)
-    def delete_search_pipeline(self, pipeline_id: str) -> None:
-        """Deletes a search pipeline.
-
-        Args:
-            pipeline_id: The ID of the search pipeline to delete.
-
-        Raises:
-            Exception: There was an error deleting the search pipeline.
-        """
-        result = self._client.search_pipeline.delete(id=pipeline_id)
-        if not result.get("acknowledged", False):
-            raise RuntimeError(f"Failed to delete search pipeline {pipeline_id}.")
-
     @log_function_time(print_only=True, debug_only=True)
     def search(
         self, body: dict[str, Any], search_pipeline_id: str | None
@@ -807,48 +943,6 @@ class OpenSearchClient:
         """
         self._client.indices.refresh(index=self._index_name)
 
-    @log_function_time(print_only=True, debug_only=True, include_args=True)
-    def put_cluster_settings(self, settings: dict[str, Any]) -> bool:
-        """Puts cluster settings.
-
-        Args:
-            settings: The settings to put.
-
-        Raises:
-            Exception: There was an error putting the cluster settings.
-
-        Returns:
-            True if the settings were put successfully, False otherwise.
-        """
-        response = self._client.cluster.put_settings(body=settings)
-        if response.get("acknowledged", False):
-            logger.info("Successfully put cluster settings.")
-            return True
-        else:
-            logger.error(f"Failed to put cluster settings: {response}.")
-            return False
-
-    @log_function_time(print_only=True, debug_only=True)
-    def ping(self) -> bool:
-        """Pings the OpenSearch cluster.
-
-        Returns:
-            True if OpenSearch could be reached, False if it could not.
-        """
-        return self._client.ping()
-
-    @log_function_time(print_only=True, debug_only=True)
-    def close(self) -> None:
-        """Closes the client.
-
-        TODO(andrei): Can we have some way to auto close when the client no
-        longer has any references?
-
-        Raises:
-            Exception: There was an error closing the client.
-        """
-        self._client.close()
-
     def _get_hits_and_profile_from_search_result(
         self, result: dict[str, Any]
     ) -> tuple[list[Any], int | None, bool | None, dict[str, Any], dict[str, Any]]:
@@ -945,14 +1039,7 @@ def wait_for_opensearch_with_timeout(
     Returns:
         True if OpenSearch is ready, False otherwise.
     """
-    made_client = False
-    try:
-        if client is None:
-            # NOTE: index_name does not matter because we are only using this object
-            # to ping.
-            # TODO(andrei): Make this better.
-            client = OpenSearchClient(index_name="")
-            made_client = True
+    with nullcontext(client) if client else OpenSearchClient() as client:
         time_start = time.monotonic()
         while True:
             if client.ping():
@@ -969,7 +1056,3 @@ def wait_for_opensearch_with_timeout(
                 f"[OpenSearch] Readiness probe ongoing. elapsed={time_elapsed:.1f} timeout={wait_limit_s:.1f}"
             )
             time.sleep(wait_interval_s)
-    finally:
-        if made_client:
-            assert client is not None
-            client.close()

backend/onyx/document_index/opensearch/opensearch_document_index.py

@@ -6,7 +6,7 @@ import httpx
 from opensearchpy import NotFoundError
 
 from onyx.access.models import DocumentAccess
-from onyx.configs.app_configs import USING_AWS_MANAGED_OPENSEARCH
+from onyx.configs.app_configs import VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT
 from onyx.configs.chat_configs import NUM_RETURNED_HITS
 from onyx.configs.chat_configs import TITLE_CONTENT_RATIO
 from onyx.configs.constants import PUBLIC_DOC_PAT
@@ -40,6 +40,7 @@ from onyx.document_index.interfaces_new import IndexingMetadata
 from onyx.document_index.interfaces_new import MetadataUpdateRequest
 from onyx.document_index.interfaces_new import TenantState
 from onyx.document_index.opensearch.client import OpenSearchClient
+from onyx.document_index.opensearch.client import OpenSearchIndexClient
 from onyx.document_index.opensearch.client import SearchHit
 from onyx.document_index.opensearch.cluster_settings import OPENSEARCH_CLUSTER_SETTINGS
 from onyx.document_index.opensearch.schema import ACCESS_CONTROL_LIST_FIELD_NAME
@@ -93,6 +94,25 @@ def generate_opensearch_filtered_access_control_list(
     return list(access_control_list)
 
 
+def set_cluster_state(client: OpenSearchClient) -> None:
+    if not client.put_cluster_settings(settings=OPENSEARCH_CLUSTER_SETTINGS):
+        logger.error(
+            "Failed to put cluster settings. If the settings have never been set before, "
+            "this may cause unexpected index creation when indexing documents into an "
+            "index that does not exist, or may cause expected logs to not appear. If this "
+            "is not the first time running Onyx against this instance of OpenSearch, these "
+            "settings have likely already been set. Not taking any further action..."
+        )
+    client.create_search_pipeline(
+        pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME,
+        pipeline_body=MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
+    )
+    client.create_search_pipeline(
+        pipeline_id=ZSCORE_NORMALIZATION_PIPELINE_NAME,
+        pipeline_body=ZSCORE_NORMALIZATION_PIPELINE_CONFIG,
+    )
+
+
 def _convert_retrieved_opensearch_chunk_to_inference_chunk_uncleaned(
     chunk: DocumentChunk,
     score: float | None,
@@ -248,6 +268,8 @@ class OpenSearchOldDocumentIndex(OldDocumentIndex):
     def __init__(
         self,
         index_name: str,
+        embedding_dim: int,
+        embedding_precision: EmbeddingPrecision,
         secondary_index_name: str | None,
         large_chunks_enabled: bool,  # noqa: ARG002
         secondary_large_chunks_enabled: bool | None,  # noqa: ARG002
@@ -258,10 +280,6 @@ class OpenSearchOldDocumentIndex(OldDocumentIndex):
             index_name=index_name,
             secondary_index_name=secondary_index_name,
         )
-        if multitenant:
-            raise ValueError(
-                "Bug: OpenSearch is not yet ready for multitenant environments but something tried to use it."
-            )
         if multitenant != MULTI_TENANT:
             raise ValueError(
                 "Bug: Multitenant mismatch when initializing an OpenSearchDocumentIndex. "
@@ -269,8 +287,10 @@ class OpenSearchOldDocumentIndex(OldDocumentIndex):
             )
         tenant_id = get_current_tenant_id()
         self._real_index = OpenSearchDocumentIndex(
-            index_name=index_name,
             tenant_state=TenantState(tenant_id=tenant_id, multitenant=multitenant),
+            index_name=index_name,
+            embedding_dim=embedding_dim,
+            embedding_precision=embedding_precision,
         )
 
     @staticmethod
@@ -279,9 +299,8 @@ class OpenSearchOldDocumentIndex(OldDocumentIndex):
         embedding_dims: list[int],
         embedding_precisions: list[EmbeddingPrecision],
     ) -> None:
-        # TODO(andrei): Implement.
         raise NotImplementedError(
-            "Multitenant index registration is not yet implemented for OpenSearch."
+            "Bug: Multitenant index registration is not supported for OpenSearch."
         )
 
     def ensure_indices_exist(
@@ -471,19 +490,37 @@ class OpenSearchDocumentIndex(DocumentIndex):
     for an OpenSearch search engine instance. It handles the complete lifecycle
     of document chunks within a specific OpenSearch index/schema.
 
-    Although not yet used in this way in the codebase, each kind of embedding
-    used should correspond to a different instance of this class, and therefore
-    a different index in OpenSearch.
+    Each kind of embedding used should correspond to a different instance of
+    this class, and therefore a different index in OpenSearch.
+
+    If in a multitenant environment and
+    VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT, will verify and create the index
+    if necessary on initialization. This is because there is no logic which runs
+    on cluster restart which scans through all search settings over all tenants
+    and creates the relevant indices.
+
+    Args:
+        tenant_state: The tenant state of the caller.
+        index_name: The name of the index to interact with.
+        embedding_dim: The dimensionality of the embeddings used for the index.
+        embedding_precision: The precision of the embeddings used for the index.
     """
 
     def __init__(
         self,
-        index_name: str,
         tenant_state: TenantState,
+        index_name: str,
+        embedding_dim: int,
+        embedding_precision: EmbeddingPrecision,
     ) -> None:
         self._index_name: str = index_name
         self._tenant_state: TenantState = tenant_state
-        self._os_client = OpenSearchClient(index_name=self._index_name)
+        self._client = OpenSearchIndexClient(index_name=self._index_name)
+
+        if self._tenant_state.multitenant and VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT:
+            self.verify_and_create_index_if_necessary(
+                embedding_dim=embedding_dim, embedding_precision=embedding_precision
+            )
 
     def verify_and_create_index_if_necessary(
         self,
@@ -492,10 +529,15 @@ class OpenSearchDocumentIndex(DocumentIndex):
     ) -> None:
         """Verifies and creates the index if necessary.
 
-        Also puts the desired cluster settings.
+        Also puts the desired cluster settings if not in a multitenant
+        environment.
 
-        Also puts the desired search pipeline state, creating the pipelines if
-        they do not exist and updating them otherwise.
+        Also puts the desired search pipeline state if not in a multitenant
+        environment, creating the pipelines if they do not exist and updating
+        them otherwise.
+
+        In a multitenant environment, the above steps happen explicitly on
+        setup.
 
         Args:
             embedding_dim: Vector dimensionality for the vector similarity part
@@ -508,47 +550,33 @@ class OpenSearchDocumentIndex(DocumentIndex):
                 search pipelines.
         """
         logger.debug(
-            f"[OpenSearchDocumentIndex] Verifying and creating index {self._index_name} if necessary, "
-            f"with embedding dimension {embedding_dim}."
+            f"[OpenSearchDocumentIndex] Verifying and creating index {self._index_name} if "
+            f"necessary, with embedding dimension {embedding_dim}."
         )
+
+        if not self._tenant_state.multitenant:
+            set_cluster_state(self._client)
+
         expected_mappings = DocumentSchema.get_document_schema(
             embedding_dim, self._tenant_state.multitenant
         )
-        if not self._os_client.put_cluster_settings(
-            settings=OPENSEARCH_CLUSTER_SETTINGS
-        ):
-            logger.error(
-                f"Failed to put cluster settings for index {self._index_name}. If the settings have never been set before this "
-                "may cause unexpected index creation when indexing documents into an index that does not exist, or may cause "
-                "expected logs to not appear. If this is not the first time running Onyx against this instance of OpenSearch, "
-                "these settings have likely already been set. Not taking any further action..."
-            )
-        if not self._os_client.index_exists():
-            if USING_AWS_MANAGED_OPENSEARCH:
-                index_settings = (
-                    DocumentSchema.get_index_settings_for_aws_managed_opensearch()
-                )
-            else:
-                index_settings = DocumentSchema.get_index_settings()
-            self._os_client.create_index(
+
+        if not self._client.index_exists():
+            index_settings = DocumentSchema.get_index_settings_based_on_environment()
+            self._client.create_index(
                 mappings=expected_mappings,
                 settings=index_settings,
             )
-        if not self._os_client.validate_index(
-            expected_mappings=expected_mappings,
-        ):
-            raise RuntimeError(
-                f"The index {self._index_name} is not valid. The expected mappings do not match the actual mappings."
-            )
-
-        self._os_client.create_search_pipeline(
-            pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME,
-            pipeline_body=MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
-        )
-        self._os_client.create_search_pipeline(
-            pipeline_id=ZSCORE_NORMALIZATION_PIPELINE_NAME,
-            pipeline_body=ZSCORE_NORMALIZATION_PIPELINE_CONFIG,
-        )
+        else:
+            # Ensure schema is up to date by applying the current mappings.
+            try:
+                self._client.put_mapping(expected_mappings)
+            except Exception as e:
+                logger.error(
+                    f"Failed to update mappings for index {self._index_name}. This likely means a "
+                    f"field type was changed which requires reindexing. Error: {e}"
+                )
+                raise
 
     def index(
         self,
@@ -620,7 +648,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
             )
             # Now index. This will raise if a chunk of the same ID exists, which
             # we do not expect because we should have deleted all chunks.
-            self._os_client.bulk_index_documents(
+            self._client.bulk_index_documents(
                 documents=chunk_batch,
                 tenant_state=self._tenant_state,
             )
@@ -660,7 +688,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
             tenant_state=self._tenant_state,
         )
 
-        return self._os_client.delete_by_query(query_body)
+        return self._client.delete_by_query(query_body)
 
     def update(
         self,
@@ -760,7 +788,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
                         document_id=doc_id,
                         chunk_index=chunk_index,
                     )
-                    self._os_client.update_document(
+                    self._client.update_document(
                         document_chunk_id=document_chunk_id,
                         properties_to_update=properties_to_update,
                     )
@@ -799,7 +827,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
                 min_chunk_index=chunk_request.min_chunk_ind,
                 max_chunk_index=chunk_request.max_chunk_ind,
             )
-            search_hits = self._os_client.search(
+            search_hits = self._client.search(
                 body=query_body,
                 search_pipeline_id=None,
             )
@@ -849,7 +877,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
         # NOTE: Using z-score normalization here because it's better for hybrid search from a theoretical standpoint.
         # Empirically on a small dataset of up to 10K docs, it's not very different. Likely more impactful at scale.
         # https://opensearch.org/blog/introducing-the-z-score-normalization-technique-for-hybrid-search/
-        search_hits: list[SearchHit[DocumentChunk]] = self._os_client.search(
+        search_hits: list[SearchHit[DocumentChunk]] = self._client.search(
             body=query_body,
             search_pipeline_id=ZSCORE_NORMALIZATION_PIPELINE_NAME,
         )
@@ -881,7 +909,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
             index_filters=filters,
             num_to_retrieve=num_to_retrieve,
         )
-        search_hits: list[SearchHit[DocumentChunk]] = self._os_client.search(
+        search_hits: list[SearchHit[DocumentChunk]] = self._client.search(
             body=query_body,
             search_pipeline_id=None,
         )
@@ -909,6 +937,6 @@ class OpenSearchDocumentIndex(DocumentIndex):
         # Do not raise if the document already exists, just update. This is
         # because the document may already have been indexed during the
         # OpenSearch transition period.
-        self._os_client.bulk_index_documents(
+        self._client.bulk_index_documents(
             documents=chunks, tenant_state=self._tenant_state, update_if_exists=True
         )

backend/onyx/document_index/opensearch/schema.py

@@ -12,6 +12,7 @@ from pydantic import model_validator
 from pydantic import SerializerFunctionWrapHandler
 
 from onyx.configs.app_configs import OPENSEARCH_TEXT_ANALYZER
+from onyx.configs.app_configs import USING_AWS_MANAGED_OPENSEARCH
 from onyx.document_index.interfaces_new import TenantState
 from onyx.document_index.opensearch.constants import DEFAULT_MAX_CHUNK_SIZE
 from onyx.document_index.opensearch.constants import EF_CONSTRUCTION
@@ -525,7 +526,7 @@ class DocumentSchema:
         }
 
     @staticmethod
-    def get_index_settings_for_aws_managed_opensearch() -> dict[str, Any]:
+    def get_index_settings_for_aws_managed_opensearch_st_dev() -> dict[str, Any]:
         """
         Settings for AWS-managed OpenSearch.
 
@@ -546,3 +547,41 @@ class DocumentSchema:
                 "knn.algo_param.ef_search": EF_SEARCH,
             }
         }
+
+    @staticmethod
+    def get_index_settings_for_aws_managed_opensearch_mt_cloud() -> dict[str, Any]:
+        """
+        Settings for AWS-managed OpenSearch in multi-tenant cloud.
+
+        324 shards very roughly targets a storage load of ~30Gb per shard, which
+        according to AWS OpenSearch documentation is within a good target range.
+
+        As documented above we need 2 replicas for a total of 3 copies of the
+        data because the cluster is configured with 3-AZ awareness.
+        """
+        return {
+            "index": {
+                "number_of_shards": 324,
+                "number_of_replicas": 2,
+                # Required for vector search.
+                "knn": True,
+                "knn.algo_param.ef_search": EF_SEARCH,
+            }
+        }
+
+    @staticmethod
+    def get_index_settings_based_on_environment() -> dict[str, Any]:
+        """
+        Returns the index settings based on the environment.
+        """
+        if USING_AWS_MANAGED_OPENSEARCH:
+            if MULTI_TENANT:
+                return (
+                    DocumentSchema.get_index_settings_for_aws_managed_opensearch_mt_cloud()
+                )
+            else:
+                return (
+                    DocumentSchema.get_index_settings_for_aws_managed_opensearch_st_dev()
+                )
+        else:
+            return DocumentSchema.get_index_settings()

backend/onyx/document_index/opensearch/search.py

@@ -28,6 +28,7 @@ from onyx.document_index.opensearch.schema import HIDDEN_FIELD_NAME
 from onyx.document_index.opensearch.schema import LAST_UPDATED_FIELD_NAME
 from onyx.document_index.opensearch.schema import MAX_CHUNK_SIZE_FIELD_NAME
 from onyx.document_index.opensearch.schema import METADATA_LIST_FIELD_NAME
+from onyx.document_index.opensearch.schema import PERSONAS_FIELD_NAME
 from onyx.document_index.opensearch.schema import PUBLIC_FIELD_NAME
 from onyx.document_index.opensearch.schema import set_or_convert_timezone_to_utc
 from onyx.document_index.opensearch.schema import SOURCE_TYPE_FIELD_NAME
@@ -144,6 +145,7 @@ class DocumentQuery:
             document_sets=index_filters.document_set or [],
             user_file_ids=index_filters.user_file_ids or [],
             project_id=index_filters.project_id,
+            persona_id=index_filters.persona_id,
             time_cutoff=index_filters.time_cutoff,
             min_chunk_index=min_chunk_index,
             max_chunk_index=max_chunk_index,
@@ -202,6 +204,7 @@ class DocumentQuery:
             document_sets=[],
             user_file_ids=[],
             project_id=None,
+            persona_id=None,
             time_cutoff=None,
             min_chunk_index=None,
             max_chunk_index=None,
@@ -267,6 +270,7 @@ class DocumentQuery:
             document_sets=index_filters.document_set or [],
             user_file_ids=index_filters.user_file_ids or [],
             project_id=index_filters.project_id,
+            persona_id=index_filters.persona_id,
             time_cutoff=index_filters.time_cutoff,
             min_chunk_index=None,
             max_chunk_index=None,
@@ -334,6 +338,7 @@ class DocumentQuery:
             document_sets=index_filters.document_set or [],
             user_file_ids=index_filters.user_file_ids or [],
             project_id=index_filters.project_id,
+            persona_id=index_filters.persona_id,
             time_cutoff=index_filters.time_cutoff,
             min_chunk_index=None,
             max_chunk_index=None,
@@ -496,6 +501,7 @@ class DocumentQuery:
         document_sets: list[str],
         user_file_ids: list[UUID],
         project_id: int | None,
+        persona_id: int | None,
         time_cutoff: datetime | None,
         min_chunk_index: int | None,
         max_chunk_index: int | None,
@@ -530,6 +536,8 @@ class DocumentQuery:
                 retrieved.
             project_id: If not None, only documents with this project ID in user
                 projects will be retrieved.
+            persona_id: If not None, only documents whose personas array
+                contains this persona ID will be retrieved.
             time_cutoff: Time cutoff for the documents to retrieve. If not None,
                 Documents which were last updated before this date will not be
                 returned. For documents which do not have a value for their last
@@ -627,6 +635,9 @@ class DocumentQuery:
             )
             return user_project_filter
 
+        def _get_persona_filter(persona_id: int) -> dict[str, Any]:
+            return {"term": {PERSONAS_FIELD_NAME: {"value": persona_id}}}
+
         def _get_time_cutoff_filter(time_cutoff: datetime) -> dict[str, Any]:
             # Convert to UTC if not already so the cutoff is comparable to the
             # document data.
@@ -780,6 +791,9 @@ class DocumentQuery:
             # document's user projects list.
             filter_clauses.append(_get_user_project_filter(project_id))
 
+        if persona_id is not None:
+            filter_clauses.append(_get_persona_filter(persona_id))
+
         if time_cutoff is not None:
             # If a time cutoff is provided, the caller will only retrieve
             # documents where the document was last updated at or after the time

backend/onyx/document_index/vespa/chunk_retrieval.py

@@ -1,5 +1,6 @@
 import json
 import string
+import time
 from collections.abc import Callable
 from collections.abc import Mapping
 from datetime import datetime
@@ -18,6 +19,7 @@ from onyx.background.celery.tasks.opensearch_migration.transformer import (
 )
 from onyx.configs.app_configs import LOG_VESPA_TIMING_INFORMATION
 from onyx.configs.app_configs import VESPA_LANGUAGE_OVERRIDE
+from onyx.configs.app_configs import VESPA_MIGRATION_REQUEST_TIMEOUT_S
 from onyx.context.search.models import IndexFilters
 from onyx.context.search.models import InferenceChunkUncleaned
 from onyx.document_index.interfaces import VespaChunkRequest
@@ -338,12 +340,18 @@ def get_all_chunks_paginated(
             params["continuation"] = continuation_token
 
         response: httpx.Response | None = None
+        start_time = time.monotonic()
         try:
-            with get_vespa_http_client() as http_client:
+            with get_vespa_http_client(
+                timeout=VESPA_MIGRATION_REQUEST_TIMEOUT_S
+            ) as http_client:
                 response = http_client.get(url, params=params)
                 response.raise_for_status()
         except httpx.HTTPError as e:
-            error_base = f"Failed to get chunks from Vespa slice {slice_id} with continuation token {continuation_token}."
+            error_base = (
+                f"Failed to get chunks from Vespa slice {slice_id} with continuation token "
+                f"{continuation_token} in {time.monotonic() - start_time:.3f} seconds."
+            )
             logger.exception(
                 f"Request URL: {e.request.url}\n"
                 f"Request Headers: {e.request.headers}\n"

backend/onyx/document_index/vespa/shared_utils/utils.py

@@ -52,7 +52,9 @@ def replace_invalid_doc_id_characters(text: str) -> str:
     return text.replace("'", "_")
 
 
-def get_vespa_http_client(no_timeout: bool = False, http2: bool = True) -> httpx.Client:
+def get_vespa_http_client(
+    no_timeout: bool = False, http2: bool = True, timeout: int | None = None
+) -> httpx.Client:
     """
     Configures and returns an HTTP client for communicating with Vespa,
     including authentication if needed.
@@ -64,7 +66,7 @@ def get_vespa_http_client(no_timeout: bool = False, http2: bool = True) -> httpx
             else None
         ),
         verify=False if not MANAGED_VESPA else True,
-        timeout=None if no_timeout else VESPA_REQUEST_TIMEOUT,
+        timeout=None if no_timeout else (timeout or VESPA_REQUEST_TIMEOUT),
         http2=http2,
     )
 

backend/onyx/document_index/vespa/shared_utils/vespa_request_builders.py

@@ -12,6 +12,7 @@ from onyx.document_index.vespa_constants import DOCUMENT_ID
 from onyx.document_index.vespa_constants import DOCUMENT_SETS
 from onyx.document_index.vespa_constants import HIDDEN
 from onyx.document_index.vespa_constants import METADATA_LIST
+from onyx.document_index.vespa_constants import PERSONAS
 from onyx.document_index.vespa_constants import SOURCE_TYPE
 from onyx.document_index.vespa_constants import TENANT_ID
 from onyx.document_index.vespa_constants import USER_PROJECT
@@ -149,6 +150,18 @@ def build_vespa_filters(
         # Vespa YQL 'contains' expects a string literal; quote the integer
         return f'({USER_PROJECT} contains "{pid}") and '
 
+    def _build_persona_filter(
+        persona_id: int | None,
+    ) -> str:
+        if persona_id is None:
+            return ""
+        try:
+            pid = int(persona_id)
+        except Exception:
+            logger.warning(f"Invalid persona ID: {persona_id}")
+            return ""
+        return f'({PERSONAS} contains "{pid}") and '
+
     # Start building the filter string
     filter_str = f"!({HIDDEN}=true) and " if not include_hidden else ""
 
@@ -192,6 +205,9 @@ def build_vespa_filters(
     # User project filter (array<int> attribute membership)
     filter_str += _build_user_project_filter(filters.project_id)
 
+    # Persona filter (array<int> attribute membership)
+    filter_str += _build_persona_filter(filters.persona_id)
+
     # Time filter
     filter_str += _build_time_filter(filters.time_cutoff)
 

backend/onyx/error_handling/error_codes.py

@@ -0,0 +1,101 @@
+"""
+Standardized error codes for the Onyx backend.
+
+Usage:
+    from onyx.error_handling.error_codes import OnyxErrorCode
+    from onyx.error_handling.exceptions import OnyxError
+
+    raise OnyxError(OnyxErrorCode.UNAUTHENTICATED, "Token expired")
+"""
+
+from enum import Enum
+
+
+class OnyxErrorCode(Enum):
+    """
+    Each member is a tuple of (error_code_string, http_status_code).
+
+    The error_code_string is a stable, machine-readable identifier that
+    API consumers can match on. The http_status_code is the default HTTP
+    status to return.
+    """
+
+    # ------------------------------------------------------------------
+    # Authentication (401)
+    # ------------------------------------------------------------------
+    UNAUTHENTICATED = ("UNAUTHENTICATED", 401)
+    INVALID_TOKEN = ("INVALID_TOKEN", 401)
+    TOKEN_EXPIRED = ("TOKEN_EXPIRED", 401)
+    CSRF_FAILURE = ("CSRF_FAILURE", 403)
+
+    # ------------------------------------------------------------------
+    # Authorization (403)
+    # ------------------------------------------------------------------
+    UNAUTHORIZED = ("UNAUTHORIZED", 403)
+    INSUFFICIENT_PERMISSIONS = ("INSUFFICIENT_PERMISSIONS", 403)
+    ADMIN_ONLY = ("ADMIN_ONLY", 403)
+    EE_REQUIRED = ("EE_REQUIRED", 403)
+
+    # ------------------------------------------------------------------
+    # Validation / Bad Request (400)
+    # ------------------------------------------------------------------
+    VALIDATION_ERROR = ("VALIDATION_ERROR", 400)
+    INVALID_INPUT = ("INVALID_INPUT", 400)
+    MISSING_REQUIRED_FIELD = ("MISSING_REQUIRED_FIELD", 400)
+
+    # ------------------------------------------------------------------
+    # Not Found (404)
+    # ------------------------------------------------------------------
+    NOT_FOUND = ("NOT_FOUND", 404)
+    CONNECTOR_NOT_FOUND = ("CONNECTOR_NOT_FOUND", 404)
+    CREDENTIAL_NOT_FOUND = ("CREDENTIAL_NOT_FOUND", 404)
+    PERSONA_NOT_FOUND = ("PERSONA_NOT_FOUND", 404)
+    DOCUMENT_NOT_FOUND = ("DOCUMENT_NOT_FOUND", 404)
+    SESSION_NOT_FOUND = ("SESSION_NOT_FOUND", 404)
+    USER_NOT_FOUND = ("USER_NOT_FOUND", 404)
+
+    # ------------------------------------------------------------------
+    # Conflict (409)
+    # ------------------------------------------------------------------
+    CONFLICT = ("CONFLICT", 409)
+    DUPLICATE_RESOURCE = ("DUPLICATE_RESOURCE", 409)
+
+    # ------------------------------------------------------------------
+    # Rate Limiting / Quotas (429 / 402)
+    # ------------------------------------------------------------------
+    RATE_LIMITED = ("RATE_LIMITED", 429)
+    SEAT_LIMIT_EXCEEDED = ("SEAT_LIMIT_EXCEEDED", 402)
+
+    # ------------------------------------------------------------------
+    # Connector / Credential Errors (400-range)
+    # ------------------------------------------------------------------
+    CONNECTOR_VALIDATION_FAILED = ("CONNECTOR_VALIDATION_FAILED", 400)
+    CREDENTIAL_INVALID = ("CREDENTIAL_INVALID", 400)
+    CREDENTIAL_EXPIRED = ("CREDENTIAL_EXPIRED", 401)
+
+    # ------------------------------------------------------------------
+    # Server Errors (5xx)
+    # ------------------------------------------------------------------
+    INTERNAL_ERROR = ("INTERNAL_ERROR", 500)
+    NOT_IMPLEMENTED = ("NOT_IMPLEMENTED", 501)
+    SERVICE_UNAVAILABLE = ("SERVICE_UNAVAILABLE", 503)
+    BAD_GATEWAY = ("BAD_GATEWAY", 502)
+    LLM_PROVIDER_ERROR = ("LLM_PROVIDER_ERROR", 502)
+    GATEWAY_TIMEOUT = ("GATEWAY_TIMEOUT", 504)
+
+    def __init__(self, code: str, status_code: int) -> None:
+        self.code = code
+        self.status_code = status_code
+
+    def detail(self, message: str | None = None) -> dict[str, str]:
+        """Build a structured error detail dict.
+
+        Returns a dict like:
+            {"error_code": "UNAUTHENTICATED", "message": "Token expired"}
+
+        If no message is supplied, the error code itself is used as the message.
+        """
+        return {
+            "error_code": self.code,
+            "message": message or self.code,
+        }

backend/onyx/error_handling/exceptions.py

@@ -0,0 +1,82 @@
+"""OnyxError — the single exception type for all Onyx business errors.
+
+Raise ``OnyxError`` instead of ``HTTPException`` in business code.  A global
+FastAPI exception handler (registered via ``register_onyx_exception_handlers``)
+converts it into a JSON response with the standard
+``{"error_code": "...", "message": "..."}`` shape.
+
+Usage::
+
+    from onyx.error_handling.error_codes import OnyxErrorCode
+    from onyx.error_handling.exceptions import OnyxError
+
+    raise OnyxError(OnyxErrorCode.NOT_FOUND, "Session not found")
+
+For upstream errors with a dynamic HTTP status (e.g. billing service),
+use ``status_code_override``::
+
+    raise OnyxError(
+        OnyxErrorCode.BAD_GATEWAY,
+        detail,
+        status_code_override=upstream_status,
+    )
+"""
+
+from fastapi import FastAPI
+from fastapi import Request
+from fastapi.responses import JSONResponse
+
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+class OnyxError(Exception):
+    """Structured error that maps to a specific ``OnyxErrorCode``.
+
+    Attributes:
+        error_code: The ``OnyxErrorCode`` enum member.
+        message: Human-readable message (defaults to the error code string).
+        status_code: HTTP status — either overridden or from the error code.
+    """
+
+    def __init__(
+        self,
+        error_code: OnyxErrorCode,
+        message: str | None = None,
+        *,
+        status_code_override: int | None = None,
+    ) -> None:
+        self.error_code = error_code
+        self.message = message or error_code.code
+        self._status_code_override = status_code_override
+        super().__init__(self.message)
+
+    @property
+    def status_code(self) -> int:
+        return self._status_code_override or self.error_code.status_code
+
+
+def register_onyx_exception_handlers(app: FastAPI) -> None:
+    """Register a global handler that converts ``OnyxError`` to JSON responses.
+
+    Must be called *after* the app is created but *before* it starts serving.
+    The handler logs at WARNING for 4xx and ERROR for 5xx.
+    """
+
+    @app.exception_handler(OnyxError)
+    async def _handle_onyx_error(
+        request: Request,  # noqa: ARG001
+        exc: OnyxError,
+    ) -> JSONResponse:
+        status_code = exc.status_code
+        if status_code >= 500:
+            logger.error(f"OnyxError {exc.error_code.code}: {exc.message}")
+        elif status_code >= 400:
+            logger.warning(f"OnyxError {exc.error_code.code}: {exc.message}")
+
+        return JSONResponse(
+            status_code=status_code,
+            content=exc.error_code.detail(exc.message),
+        )

backend/onyx/federated_connectors/oauth_utils.py

@@ -4,39 +4,33 @@ import base64
 import json
 import uuid
 from typing import Any
-from typing import cast
-from typing import Dict
-from typing import Optional
 
+from onyx.cache.factory import get_cache_backend
 from onyx.configs.app_configs import WEB_DOMAIN
-from onyx.redis.redis_pool import get_redis_client
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
-# Redis key prefix for OAuth state
 OAUTH_STATE_PREFIX = "federated_oauth"
-# Default TTL for OAuth state (5 minutes)
-OAUTH_STATE_TTL = 300
+OAUTH_STATE_TTL = 300  # 5 minutes
 
 
 class OAuthSession:
-    """Represents an OAuth session stored in Redis."""
+    """Represents an OAuth session stored in the cache backend."""
 
     def __init__(
         self,
         federated_connector_id: int,
         user_id: str,
-        redirect_uri: Optional[str] = None,
-        additional_data: Optional[Dict[str, Any]] = None,
+        redirect_uri: str | None = None,
+        additional_data: dict[str, Any] | None = None,
     ):
         self.federated_connector_id = federated_connector_id
         self.user_id = user_id
         self.redirect_uri = redirect_uri
         self.additional_data = additional_data or {}
 
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary for Redis storage."""
+    def to_dict(self) -> dict[str, Any]:
         return {
             "federated_connector_id": self.federated_connector_id,
             "user_id": self.user_id,
@@ -45,8 +39,7 @@ class OAuthSession:
         }
 
     @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> "OAuthSession":
-        """Create from dictionary retrieved from Redis."""
+    def from_dict(cls, data: dict[str, Any]) -> "OAuthSession":
         return cls(
             federated_connector_id=data["federated_connector_id"],
             user_id=data["user_id"],
@@ -58,31 +51,27 @@ class OAuthSession:
 def generate_oauth_state(
     federated_connector_id: int,
     user_id: str,
-    redirect_uri: Optional[str] = None,
-    additional_data: Optional[Dict[str, Any]] = None,
+    redirect_uri: str | None = None,
+    additional_data: dict[str, Any] | None = None,
     ttl: int = OAUTH_STATE_TTL,
 ) -> str:
     """
-    Generate a secure state parameter and store session data in Redis.
+    Generate a secure state parameter and store session data in the cache backend.
 
     Args:
         federated_connector_id: ID of the federated connector
         user_id: ID of the user initiating OAuth
         redirect_uri: Optional redirect URI after OAuth completion
         additional_data: Any additional data to store with the session
-        ttl: Time-to-live in seconds for the Redis key
+        ttl: Time-to-live in seconds for the cache key
 
     Returns:
         Base64-encoded state parameter
     """
     # Generate a random UUID for the state
     state_uuid = uuid.uuid4()
+    state_b64 = base64.urlsafe_b64encode(state_uuid.bytes).decode("utf-8").rstrip("=")
 
-    # Convert UUID to base64 for URL-safe state parameter
-    state_bytes = state_uuid.bytes
-    state_b64 = base64.urlsafe_b64encode(state_bytes).decode("utf-8").rstrip("=")
-
-    # Create session object
     session = OAuthSession(
         federated_connector_id=federated_connector_id,
         user_id=user_id,
@@ -90,15 +79,9 @@ def generate_oauth_state(
         additional_data=additional_data,
     )
 
-    # Store in Redis with TTL
-    redis_client = get_redis_client()
-    redis_key = f"{OAUTH_STATE_PREFIX}:{state_uuid}"
-
-    redis_client.set(
-        redis_key,
-        json.dumps(session.to_dict()),
-        ex=ttl,
-    )
+    cache = get_cache_backend()
+    cache_key = f"{OAUTH_STATE_PREFIX}:{state_uuid}"
+    cache.set(cache_key, json.dumps(session.to_dict()), ex=ttl)
 
     logger.info(
         f"Generated OAuth state for federated_connector_id={federated_connector_id}, "
@@ -125,18 +108,15 @@ def verify_oauth_state(state: str) -> OAuthSession:
     state_bytes = base64.urlsafe_b64decode(padded_state)
     state_uuid = uuid.UUID(bytes=state_bytes)
 
-    # Look up in Redis
-    redis_client = get_redis_client()
-    redis_key = f"{OAUTH_STATE_PREFIX}:{state_uuid}"
+    cache = get_cache_backend()
+    cache_key = f"{OAUTH_STATE_PREFIX}:{state_uuid}"
 
-    session_data = cast(bytes, redis_client.get(redis_key))
+    session_data = cache.get(cache_key)
     if not session_data:
-        raise ValueError(f"OAuth state not found in Redis: {state}")
+        raise ValueError(f"OAuth state not found: {state}")
 
-    # Delete the key after retrieval (one-time use)
-    redis_client.delete(redis_key)
+    cache.delete(cache_key)
 
-    # Parse and return session
     session_dict = json.loads(session_data)
     return OAuthSession.from_dict(session_dict)
 

backend/onyx/key_value_store/store.py

@@ -1,13 +1,11 @@
 import json
 from typing import cast
 
-from redis.client import Redis
-
+from onyx.cache.interface import CacheBackend
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
 from onyx.db.models import KVStore
 from onyx.key_value_store.interface import KeyValueStore
 from onyx.key_value_store.interface import KvKeyNotFoundError
-from onyx.redis.redis_pool import get_redis_client
 from onyx.utils.logger import setup_logger
 from onyx.utils.special_types import JSON_ro
 
@@ -20,22 +18,27 @@ KV_REDIS_KEY_EXPIRATION = 60 * 60 * 24  # 1 Day
 
 
 class PgRedisKVStore(KeyValueStore):
-    def __init__(self, redis_client: Redis | None = None) -> None:
-        # If no redis_client is provided, fall back to the context var
-        if redis_client is not None:
-            self.redis_client = redis_client
-        else:
-            self.redis_client = get_redis_client()
+    def __init__(self, cache: CacheBackend | None = None) -> None:
+        self._cache = cache
+
+    def _get_cache(self) -> CacheBackend:
+        if self._cache is None:
+            from onyx.cache.factory import get_cache_backend
+
+            self._cache = get_cache_backend()
+        return self._cache
 
     def store(self, key: str, val: JSON_ro, encrypt: bool = False) -> None:
-        # Not encrypted in Redis, but encrypted in Postgres
+        # Not encrypted in Cache backend (typically Redis), but encrypted in Postgres
         try:
-            self.redis_client.set(
+            self._get_cache().set(
                 REDIS_KEY_PREFIX + key, json.dumps(val), ex=KV_REDIS_KEY_EXPIRATION
             )
         except Exception as e:
-            # Fallback gracefully to Postgres if Redis fails
-            logger.error(f"Failed to set value in Redis for key '{key}': {str(e)}")
+            # Fallback gracefully to Postgres if Cache backend fails
+            logger.error(
+                f"Failed to set value in Cache backend for key '{key}': {str(e)}"
+            )
 
         encrypted_val = val if encrypt else None
         plain_val = val if not encrypt else None
@@ -53,16 +56,12 @@ class PgRedisKVStore(KeyValueStore):
     def load(self, key: str, refresh_cache: bool = False) -> JSON_ro:
         if not refresh_cache:
             try:
-                redis_value = self.redis_client.get(REDIS_KEY_PREFIX + key)
-                if redis_value:
-                    if not isinstance(redis_value, bytes):
-                        raise ValueError(
-                            f"Redis value for key '{key}' is not a bytes object"
-                        )
-                    return json.loads(redis_value.decode("utf-8"))
+                cached = self._get_cache().get(REDIS_KEY_PREFIX + key)
+                if cached is not None:
+                    return json.loads(cached.decode("utf-8"))
             except Exception as e:
                 logger.error(
-                    f"Failed to get value from Redis for key '{key}': {str(e)}"
+                    f"Failed to get value from cache for key '{key}': {str(e)}"
                 )
 
         with get_session_with_current_tenant() as db_session:
@@ -79,21 +78,21 @@ class PgRedisKVStore(KeyValueStore):
                 value = None
 
             try:
-                self.redis_client.set(
+                self._get_cache().set(
                     REDIS_KEY_PREFIX + key,
                     json.dumps(value),
                     ex=KV_REDIS_KEY_EXPIRATION,
                 )
             except Exception as e:
-                logger.error(f"Failed to set value in Redis for key '{key}': {str(e)}")
+                logger.error(f"Failed to set value in cache for key '{key}': {str(e)}")
 
             return cast(JSON_ro, value)
 
     def delete(self, key: str) -> None:
         try:
-            self.redis_client.delete(REDIS_KEY_PREFIX + key)
+            self._get_cache().delete(REDIS_KEY_PREFIX + key)
         except Exception as e:
-            logger.error(f"Failed to delete value from Redis for key '{key}': {str(e)}")
+            logger.error(f"Failed to delete value from cache for key '{key}': {str(e)}")
 
         with get_session_with_current_tenant() as db_session:
             result = db_session.query(KVStore).filter_by(key=key).delete()

backend/onyx/llm/litellm_singleton/monkey_patches.py

@@ -67,6 +67,18 @@ Status checked against LiteLLM v1.81.6-nightly (2026-02-02):
    STATUS: STILL NEEDED - litellm_core_utils/litellm_logging.py lines 3185-3199 set
            usage as a dict with chat completion format instead of keeping it as
            ResponseAPIUsage. Our patch creates a deep copy before modification.
+
+7. Responses API metadata=None TypeError (_patch_responses_metadata_none):
+   - LiteLLM's @client decorator wrapper in utils.py uses kwargs.get("metadata", {})
+     to check for router calls, but when metadata is explicitly None (key exists with
+     value None), the default {} is not used
+   - This causes "argument of type 'NoneType' is not iterable" TypeError which swallows
+     the real exception (e.g. AuthenticationError for wrong API key)
+   - Surfaces as: APIConnectionError: OpenAIException - argument of type 'NoneType' is
+     not iterable
+   STATUS: STILL NEEDED - litellm/utils.py wrapper function (line 1721) does not guard
+           against metadata being explicitly None. Triggered when Responses API bridge
+           passes **litellm_params containing metadata=None.
 """
 
 import time
@@ -725,6 +737,44 @@ def _patch_logging_assembled_streaming_response() -> None:
     LiteLLMLoggingObj._get_assembled_streaming_response = _patched_get_assembled_streaming_response  # type: ignore[method-assign]
 
 
+def _patch_responses_metadata_none() -> None:
+    """
+    Patches litellm.responses to normalize metadata=None to metadata={} in kwargs.
+
+    LiteLLM's @client decorator wrapper in utils.py (line 1721) does:
+        _is_litellm_router_call = "model_group" in kwargs.get("metadata", {})
+    When metadata is explicitly None in kwargs, kwargs.get("metadata", {}) returns
+    None (the key exists, so the default is not used), causing:
+        TypeError: argument of type 'NoneType' is not iterable
+
+    This swallows the real exception (e.g. AuthenticationError) and surfaces as:
+        APIConnectionError: OpenAIException - argument of type 'NoneType' is not iterable
+
+    This happens when the Responses API bridge calls litellm.responses() with
+    **litellm_params which may contain metadata=None.
+
+    STATUS: STILL NEEDED - litellm/utils.py wrapper function uses kwargs.get("metadata", {})
+            which does not guard against metadata being explicitly None. Same pattern exists
+            on line 1407 for async path.
+    """
+    import litellm as _litellm
+    from functools import wraps
+
+    original_responses = _litellm.responses
+
+    if getattr(original_responses, "_metadata_patched", False):
+        return
+
+    @wraps(original_responses)
+    def _patched_responses(*args: Any, **kwargs: Any) -> Any:
+        if kwargs.get("metadata") is None:
+            kwargs["metadata"] = {}
+        return original_responses(*args, **kwargs)
+
+    _patched_responses._metadata_patched = True  # type: ignore[attr-defined]
+    _litellm.responses = _patched_responses
+
+
 def apply_monkey_patches() -> None:
     """
     Apply all necessary monkey patches to LiteLLM for compatibility.
@@ -736,6 +786,7 @@ def apply_monkey_patches() -> None:
     - Patching AzureOpenAIResponsesAPIConfig.should_fake_stream to enable native streaming
     - Patching ResponsesAPIResponse.model_construct to fix usage format in all code paths
     - Patching LiteLLMLoggingObj._get_assembled_streaming_response to avoid mutating original response
+    - Patching litellm.responses to fix metadata=None causing TypeError in error handling
     """
     _patch_ollama_chunk_parser()
     _patch_openai_responses_parallel_tool_calls()
@@ -743,3 +794,4 @@ def apply_monkey_patches() -> None:
     _patch_azure_responses_should_fake_stream()
     _patch_responses_api_usage_format()
     _patch_logging_assembled_streaming_response()
+    _patch_responses_metadata_none()

backend/onyx/llm/well_known_providers/auto_update_service.py

@@ -13,44 +13,38 @@ from datetime import datetime
 import httpx
 from sqlalchemy.orm import Session
 
+from onyx.cache.factory import get_cache_backend
 from onyx.configs.app_configs import AUTO_LLM_CONFIG_URL
 from onyx.db.llm import fetch_auto_mode_providers
 from onyx.db.llm import sync_auto_mode_models
 from onyx.llm.well_known_providers.auto_update_models import LLMRecommendations
-from onyx.redis.redis_pool import get_redis_client
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
-# Redis key for caching the last updated timestamp (per-tenant)
-_REDIS_KEY_LAST_UPDATED_AT = "auto_llm_update:last_updated_at"
+_CACHE_KEY_LAST_UPDATED_AT = "auto_llm_update:last_updated_at"
+_CACHE_TTL_SECONDS = 60 * 60 * 24  # 24 hours
 
 
 def _get_cached_last_updated_at() -> datetime | None:
-    """Get the cached last_updated_at timestamp from Redis."""
     try:
-        redis_client = get_redis_client()
-        value = redis_client.get(_REDIS_KEY_LAST_UPDATED_AT)
-        if value and isinstance(value, bytes):
-            # Value is bytes, decode to string then parse as ISO format
+        value = get_cache_backend().get(_CACHE_KEY_LAST_UPDATED_AT)
+        if value is not None:
             return datetime.fromisoformat(value.decode("utf-8"))
     except Exception as e:
-        logger.warning(f"Failed to get cached last_updated_at from Redis: {e}")
+        logger.warning(f"Failed to get cached last_updated_at: {e}")
     return None
 
 
 def _set_cached_last_updated_at(updated_at: datetime) -> None:
-    """Set the cached last_updated_at timestamp in Redis."""
     try:
-        redis_client = get_redis_client()
-        # Store as ISO format string, with 24 hour expiration
-        redis_client.set(
-            _REDIS_KEY_LAST_UPDATED_AT,
+        get_cache_backend().set(
+            _CACHE_KEY_LAST_UPDATED_AT,
             updated_at.isoformat(),
-            ex=60 * 60 * 24,  # 24 hours
+            ex=_CACHE_TTL_SECONDS,
         )
     except Exception as e:
-        logger.warning(f"Failed to set cached last_updated_at in Redis: {e}")
+        logger.warning(f"Failed to set cached last_updated_at: {e}")
 
 
 def fetch_llm_recommendations_from_github(
@@ -148,9 +142,8 @@ def sync_llm_models_from_github(
 
 
 def reset_cache() -> None:
-    """Reset the cache timestamp in Redis. Useful for testing."""
+    """Reset the cache timestamp. Useful for testing."""
     try:
-        redis_client = get_redis_client()
-        redis_client.delete(_REDIS_KEY_LAST_UPDATED_AT)
+        get_cache_backend().delete(_CACHE_KEY_LAST_UPDATED_AT)
     except Exception as e:
-        logger.warning(f"Failed to reset cache in Redis: {e}")
+        logger.warning(f"Failed to reset cache: {e}")