Hide deprecated columns

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jamison Lahman <jamison@lahman.dev>

.github/actions/build-integration-image/action.yml

@@ -54,6 +54,7 @@ runs:
       shell: bash
       env:
         RUNS_ON_ECR_CACHE: ${{ inputs.runs-on-ecr-cache }}
+        INTEGRATION_REPOSITORY: ${{ inputs.runs-on-ecr-cache }}
         TAG: nightly-llm-it-${{ inputs.run-id }}
         CACHE_SUFFIX: ${{ steps.format-branch.outputs.cache-suffix }}
         HEAD_SHA: ${{ inputs.github-sha }}

.github/actions/run-nightly-provider-chat-test/action.yml

@@ -9,7 +9,8 @@ inputs:
     required: true
   provider-api-key:
     description: "API key for NIGHTLY_LLM_API_KEY"
-    required: true
+    required: false
+    default: ""
   strict:
     description: "String true/false for NIGHTLY_LLM_STRICT"
     required: true
@@ -17,6 +18,14 @@ inputs:
     description: "Optional NIGHTLY_LLM_API_BASE"
     required: false
     default: ""
+  api-version:
+    description: "Optional NIGHTLY_LLM_API_VERSION"
+    required: false
+    default: ""
+  deployment-name:
+    description: "Optional NIGHTLY_LLM_DEPLOYMENT_NAME"
+    required: false
+    default: ""
   custom-config-json:
     description: "Optional NIGHTLY_LLM_CUSTOM_CONFIG_JSON"
     required: false
@@ -59,6 +68,7 @@ runs:
         DISABLE_TELEMETRY=true
         INTEGRATION_TESTS_MODE=true
         AUTO_LLM_UPDATE_INTERVAL_SECONDS=10
+        AWS_REGION_NAME=us-west-2
         ONYX_BACKEND_IMAGE=${ECR_CACHE}:nightly-llm-it-backend-${RUN_ID}
         ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:nightly-llm-it-model-server-${RUN_ID}
         EOF2
@@ -82,6 +92,8 @@ runs:
         NIGHTLY_LLM_PROVIDER: ${{ inputs.provider }}
         NIGHTLY_LLM_API_KEY: ${{ inputs.provider-api-key }}
         NIGHTLY_LLM_API_BASE: ${{ inputs.api-base }}
+        NIGHTLY_LLM_API_VERSION: ${{ inputs.api-version }}
+        NIGHTLY_LLM_DEPLOYMENT_NAME: ${{ inputs.deployment-name }}
         NIGHTLY_LLM_CUSTOM_CONFIG_JSON: ${{ inputs.custom-config-json }}
         NIGHTLY_LLM_STRICT: ${{ inputs.strict }}
         RUNS_ON_ECR_CACHE: ${{ inputs.runs-on-ecr-cache }}
@@ -91,11 +103,6 @@ runs:
         max_attempts: 2
         retry_wait_seconds: 10
         command: |
-          if [ -z "${MODELS}" ]; then
-            echo "Input 'models' must be non-empty for provider '${NIGHTLY_LLM_PROVIDER}'."
-            exit 1
-          fi
-
           docker run --rm --network onyx_default \
             --name test-runner \
             -e POSTGRES_HOST=relational_db \
@@ -110,10 +117,13 @@ runs:
             -e REDIS_HOST=cache \
             -e API_SERVER_HOST=api_server \
             -e TEST_WEB_HOSTNAME=test-runner \
+            -e AWS_REGION_NAME=us-west-2 \
             -e NIGHTLY_LLM_PROVIDER="${NIGHTLY_LLM_PROVIDER}" \
             -e NIGHTLY_LLM_MODELS="${MODELS}" \
             -e NIGHTLY_LLM_API_KEY="${NIGHTLY_LLM_API_KEY}" \
             -e NIGHTLY_LLM_API_BASE="${NIGHTLY_LLM_API_BASE}" \
+            -e NIGHTLY_LLM_API_VERSION="${NIGHTLY_LLM_API_VERSION}" \
+            -e NIGHTLY_LLM_DEPLOYMENT_NAME="${NIGHTLY_LLM_DEPLOYMENT_NAME}" \
             -e NIGHTLY_LLM_CUSTOM_CONFIG_JSON="${NIGHTLY_LLM_CUSTOM_CONFIG_JSON}" \
             -e NIGHTLY_LLM_STRICT="${NIGHTLY_LLM_STRICT}" \
             ${RUNS_ON_ECR_CACHE}:nightly-llm-it-${RUN_ID} \

.github/workflows/deployment.yml

@@ -426,8 +426,9 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-amd64,mode=max
@@ -499,8 +500,9 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-arm64,mode=max
@@ -646,8 +648,8 @@ jobs:
             NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-amd64,mode=max
@@ -728,8 +730,8 @@ jobs:
             NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-arm64,mode=max
@@ -862,8 +864,9 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-amd64,mode=max
@@ -934,8 +937,9 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-arm64,mode=max
@@ -1072,8 +1076,8 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             ENABLE_CRAFT=true
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-amd64,mode=max
@@ -1145,8 +1149,8 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             ENABLE_CRAFT=true
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-arm64,mode=max
@@ -1287,8 +1291,9 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-amd64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-amd64,mode=max
@@ -1366,8 +1371,9 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-arm64
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-arm64,mode=max

.github/workflows/nightly-llm-provider-chat-openai.yml

@@ -1,44 +0,0 @@
-name: Nightly LLM Provider Chat Tests (OpenAI)
-concurrency:
-  group: Nightly-LLM-Provider-Chat-OpenAI-${{ github.workflow }}-${{ github.ref_name }}
-  cancel-in-progress: true
-
-on:
-  schedule:
-    # Runs daily at 10:30 UTC (2:30 AM PST / 3:30 AM PDT)
-    - cron: "30 10 * * *"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  openai-provider-chat-test:
-    uses: ./.github/workflows/reusable-nightly-llm-provider-chat.yml
-    with:
-      provider: openai
-      models: ${{ vars.NIGHTLY_LLM_OPENAI_MODELS }}
-      strict: true
-    secrets:
-      provider_api_key: ${{ secrets.OPENAI_API_KEY }}
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
-
-  notify-slack-on-failure:
-    needs: [openai-provider-chat-test]
-    if: failure() && github.event_name == 'schedule'
-    runs-on: ubuntu-slim
-    timeout-minutes: 5
-    steps:
-      - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Send Slack notification
-        uses: ./.github/actions/slack-notify
-        with:
-          webhook-url: ${{ secrets.SLACK_WEBHOOK }}
-          failed-jobs: openai-provider-chat-test
-          title: "🚨 Scheduled OpenAI Provider Chat Tests failed!"
-          ref-name: ${{ github.ref_name }}

.github/workflows/nightly-llm-provider-chat.yml

@@ -0,0 +1,49 @@
+name: Nightly LLM Provider Chat Tests
+concurrency:
+  group: Nightly-LLM-Provider-Chat-${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+on:
+  schedule:
+    # Runs daily at 10:30 UTC (2:30 AM PST / 3:30 AM PDT)
+    - cron: "30 10 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  provider-chat-test:
+    uses: ./.github/workflows/reusable-nightly-llm-provider-chat.yml
+    permissions:
+      contents: read
+      id-token: write
+    with:
+      openai_models: ${{ vars.NIGHTLY_LLM_OPENAI_MODELS }}
+      anthropic_models: ${{ vars.NIGHTLY_LLM_ANTHROPIC_MODELS }}
+      bedrock_models: ${{ vars.NIGHTLY_LLM_BEDROCK_MODELS }}
+      vertex_ai_models: ${{ vars.NIGHTLY_LLM_VERTEX_AI_MODELS }}
+      azure_models: ${{ vars.NIGHTLY_LLM_AZURE_MODELS }}
+      azure_api_base: ${{ vars.NIGHTLY_LLM_AZURE_API_BASE }}
+      ollama_models: ${{ vars.NIGHTLY_LLM_OLLAMA_MODELS }}
+      openrouter_models: ${{ vars.NIGHTLY_LLM_OPENROUTER_MODELS }}
+      strict: true
+
+  notify-slack-on-failure:
+    needs: [provider-chat-test]
+    if: failure() && github.event_name == 'schedule'
+    runs-on: ubuntu-slim
+    timeout-minutes: 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Send Slack notification
+        uses: ./.github/actions/slack-notify
+        with:
+          webhook-url: ${{ secrets.SLACK_WEBHOOK }}
+          failed-jobs: provider-chat-test
+          title: "🚨 Scheduled LLM Provider Chat Tests failed!"
+          ref-name: ${{ github.ref_name }}

.github/workflows/post-merge-beta-cherry-pick.yml

@@ -114,8 +114,10 @@ jobs:
 
       - name: Mark workflow as failed if cherry-pick failed
         if: steps.gate.outputs.should_cherrypick == 'true' && steps.run_cherry_pick.outputs.status == 'failure'
+        env:
+          CHERRY_PICK_REASON: ${{ steps.run_cherry_pick.outputs.reason }}
         run: |
-          echo "::error::Automated cherry-pick failed (${{ steps.run_cherry_pick.outputs.reason }})."
+          echo "::error::Automated cherry-pick failed (${CHERRY_PICK_REASON})."
           exit 1
 
   notify-slack-on-cherry-pick-failure:

.github/workflows/pr-external-dependency-unit-tests.yml

@@ -160,7 +160,7 @@ jobs:
           cd deployment/docker_compose
 
           # Get list of running containers
-          containers=$(docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.opensearch.yml ps -q)
+          containers=$(docker compose -f docker-compose.yml -f docker-compose.dev.yml ps -q)
 
           # Collect logs from each container
           for container in $containers; do

.github/workflows/pr-playwright-tests.yml

@@ -603,7 +603,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Download visual diff summaries
-        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # ratchet:actions/download-artifact@v4
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
         with:
           pattern: screenshot-diff-summary-*
           path: summaries/

.github/workflows/pr-python-checks.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
   push:
     tags:
       - "v*.*.*"
@@ -21,7 +21,13 @@ jobs:
     # See https://runs-on.com/runners/linux/
     # Note: Mypy seems quite optimized for x64 compared to arm64.
     # Similarly, mypy is single-threaded and incremental, so 2cpu is sufficient.
-    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}-mypy-check", "extras=s3-cache"]
+    runs-on:
+      [
+        runs-on,
+        runner=2cpu-linux-x64,
+        "run-id=${{ github.run_id }}-mypy-check",
+        "extras=s3-cache",
+      ]
     timeout-minutes: 45
 
     steps:
@@ -52,21 +58,14 @@ jobs:
         if: ${{ vars.DISABLE_MYPY_CACHE != 'true' }}
         uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
         with:
-          path: backend/.mypy_cache
-          key: mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-${{ hashFiles('**/*.py', '**/*.pyi', 'backend/pyproject.toml') }}
+          path: .mypy_cache
+          key: mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-${{ hashFiles('**/*.py', '**/*.pyi', 'pyproject.toml') }}
           restore-keys: |
             mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-
             mypy-${{ runner.os }}-
 
       - name: Run MyPy
-        working-directory: ./backend
         env:
           MYPY_FORCE_COLOR: 1
           TERM: xterm-256color
         run: mypy .
-
-      - name: Run MyPy (tools/)
-        env:
-          MYPY_FORCE_COLOR: 1
-          TERM: xterm-256color
-        run: mypy tools/

.github/workflows/pr-python-connector-tests.yml

@@ -89,6 +89,10 @@ env:
   SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
   SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ vars.SHAREPOINT_CLIENT_DIRECTORY_ID }}
   SHAREPOINT_SITE: ${{ vars.SHAREPOINT_SITE }}
+  PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
+  PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
+  PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
+  PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
 
   # Github
   ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}

.github/workflows/reusable-nightly-llm-provider-chat.yml

@@ -3,64 +3,58 @@ name: Reusable Nightly LLM Provider Chat Tests
 on:
   workflow_call:
     inputs:
-      provider:
-        description: "Provider slug passed to NIGHTLY_LLM_PROVIDER (e.g. openai, anthropic)"
-        required: true
+      openai_models:
+        description: "Comma-separated models for openai"
+        required: false
+        default: ""
         type: string
-      models:
-        description: "Comma-separated model list passed to NIGHTLY_LLM_MODELS"
-        required: true
+      anthropic_models:
+        description: "Comma-separated models for anthropic"
+        required: false
+        default: ""
+        type: string
+      bedrock_models:
+        description: "Comma-separated models for bedrock"
+        required: false
+        default: ""
+        type: string
+      vertex_ai_models:
+        description: "Comma-separated models for vertex_ai"
+        required: false
+        default: ""
+        type: string
+      azure_models:
+        description: "Comma-separated models for azure"
+        required: false
+        default: ""
+        type: string
+      ollama_models:
+        description: "Comma-separated models for ollama_chat"
+        required: false
+        default: ""
+        type: string
+      openrouter_models:
+        description: "Comma-separated models for openrouter"
+        required: false
+        default: ""
+        type: string
+      azure_api_base:
+        description: "API base for azure provider"
+        required: false
+        default: ""
         type: string
       strict:
-        description: "Pass-through value for NIGHTLY_LLM_STRICT"
+        description: "Default NIGHTLY_LLM_STRICT passed to tests"
         required: false
         default: true
         type: boolean
-      api_base:
-        description: "Optional NIGHTLY_LLM_API_BASE override"
-        required: false
-        default: ""
-        type: string
-      custom_config_json:
-        description: "Optional NIGHTLY_LLM_CUSTOM_CONFIG_JSON override"
-        required: false
-        default: ""
-        type: string
-    secrets:
-      provider_api_key:
-        description: "Provider API key passed to NIGHTLY_LLM_API_KEY"
-        required: true
-      DOCKER_USERNAME:
-        required: true
-      DOCKER_TOKEN:
-        required: true
 
 permissions:
   contents: read
-
-env:
-  NIGHTLY_LLM_PROVIDER: ${{ inputs.provider }}
-  NIGHTLY_LLM_MODELS: ${{ inputs.models }}
-  NIGHTLY_LLM_API_KEY: ${{ secrets.provider_api_key }}
-  NIGHTLY_LLM_API_BASE: ${{ inputs.api_base }}
-  NIGHTLY_LLM_CUSTOM_CONFIG_JSON: ${{ inputs.custom_config_json }}
-  NIGHTLY_LLM_STRICT: ${{ inputs.strict && 'true' || 'false' }}
+  id-token: write
 
 jobs:
-  validate-inputs:
-    # NOTE: Keep this cheap and fail before image builds if required inputs are missing.
-    runs-on: ubuntu-slim
-    timeout-minutes: 5
-    steps:
-      - name: Validate required nightly provider inputs
-        run: |
-          if [ -z "${NIGHTLY_LLM_MODELS}" ]; then
-            echo "Input 'models' must be non-empty for provider '${NIGHTLY_LLM_PROVIDER}'."
-            exit 1
-          fi
-
   build-backend-image:
-    needs: [validate-inputs]
     runs-on:
       [
         runs-on,
@@ -69,6 +63,7 @@ jobs:
         "extras=ecr-cache",
       ]
     timeout-minutes: 45
+    environment: ci-protected
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
@@ -77,6 +72,19 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+
+      - name: Get AWS Secrets
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
+        with:
+          secret-ids: |
+            DOCKER_USERNAME, test/docker-username
+            DOCKER_TOKEN, test/docker-token
+
       - name: Build backend image
         uses: ./.github/actions/build-backend-image
         with:
@@ -85,12 +93,11 @@ jobs:
           pr-number: ${{ github.event.pull_request.number }}
           github-sha: ${{ github.sha }}
           run-id: ${{ github.run_id }}
-          docker-username: ${{ secrets.DOCKER_USERNAME }}
-          docker-token: ${{ secrets.DOCKER_TOKEN }}
+          docker-username: ${{ env.DOCKER_USERNAME }}
+          docker-token: ${{ env.DOCKER_TOKEN }}
           docker-no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' && 'true' || 'false' }}
 
   build-model-server-image:
-    needs: [validate-inputs]
     runs-on:
       [
         runs-on,
@@ -99,6 +106,7 @@ jobs:
         "extras=ecr-cache",
       ]
     timeout-minutes: 45
+    environment: ci-protected
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
@@ -107,6 +115,19 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+
+      - name: Get AWS Secrets
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
+        with:
+          secret-ids: |
+            DOCKER_USERNAME, test/docker-username
+            DOCKER_TOKEN, test/docker-token
+
       - name: Build model server image
         uses: ./.github/actions/build-model-server-image
         with:
@@ -115,11 +136,10 @@ jobs:
           pr-number: ${{ github.event.pull_request.number }}
           github-sha: ${{ github.sha }}
           run-id: ${{ github.run_id }}
-          docker-username: ${{ secrets.DOCKER_USERNAME }}
-          docker-token: ${{ secrets.DOCKER_TOKEN }}
+          docker-username: ${{ env.DOCKER_USERNAME }}
+          docker-token: ${{ env.DOCKER_TOKEN }}
 
   build-integration-image:
-    needs: [validate-inputs]
     runs-on:
       [
         runs-on,
@@ -128,6 +148,7 @@ jobs:
         "extras=ecr-cache",
       ]
     timeout-minutes: 45
+    environment: ci-protected
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
@@ -136,6 +157,19 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+
+      - name: Get AWS Secrets
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
+        with:
+          secret-ids: |
+            DOCKER_USERNAME, test/docker-username
+            DOCKER_TOKEN, test/docker-token
+
       - name: Build integration image
         uses: ./.github/actions/build-integration-image
         with:
@@ -144,18 +178,83 @@ jobs:
           pr-number: ${{ github.event.pull_request.number }}
           github-sha: ${{ github.sha }}
           run-id: ${{ github.run_id }}
-          docker-username: ${{ secrets.DOCKER_USERNAME }}
-          docker-token: ${{ secrets.DOCKER_TOKEN }}
+          docker-username: ${{ env.DOCKER_USERNAME }}
+          docker-token: ${{ env.DOCKER_TOKEN }}
 
   provider-chat-test:
     needs:
-      [build-backend-image, build-model-server-image, build-integration-image]
+      [
+        build-backend-image,
+        build-model-server-image,
+        build-integration-image,
+      ]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - provider: openai
+            models: ${{ inputs.openai_models }}
+            api_key_env: OPENAI_API_KEY
+            custom_config_env: ""
+            api_base: ""
+            api_version: ""
+            deployment_name: ""
+            required: true
+          - provider: anthropic
+            models: ${{ inputs.anthropic_models }}
+            api_key_env: ANTHROPIC_API_KEY
+            custom_config_env: ""
+            api_base: ""
+            api_version: ""
+            deployment_name: ""
+            required: true
+          - provider: bedrock
+            models: ${{ inputs.bedrock_models }}
+            api_key_env: BEDROCK_API_KEY
+            custom_config_env: ""
+            api_base: ""
+            api_version: ""
+            deployment_name: ""
+            required: false
+          - provider: vertex_ai
+            models: ${{ inputs.vertex_ai_models }}
+            api_key_env: ""
+            custom_config_env: NIGHTLY_LLM_VERTEX_AI_CUSTOM_CONFIG_JSON
+            api_base: ""
+            api_version: ""
+            deployment_name: ""
+            required: false
+          - provider: azure
+            models: ${{ inputs.azure_models }}
+            api_key_env: AZURE_API_KEY
+            custom_config_env: ""
+            api_base: ${{ inputs.azure_api_base }}
+            api_version: "2025-04-01-preview"
+            deployment_name: ""
+            required: false
+          - provider: ollama_chat
+            models: ${{ inputs.ollama_models }}
+            api_key_env: OLLAMA_API_KEY
+            custom_config_env: ""
+            api_base: "https://ollama.com"
+            api_version: ""
+            deployment_name: ""
+            required: false
+          - provider: openrouter
+            models: ${{ inputs.openrouter_models }}
+            api_key_env: OPENROUTER_API_KEY
+            custom_config_env: ""
+            api_base: "https://openrouter.ai/api/v1"
+            api_version: ""
+            deployment_name: ""
+            required: false
     runs-on:
       - runs-on
       - runner=4cpu-linux-arm64
-      - "run-id=${{ github.run_id }}-nightly-${{ inputs.provider }}-provider-chat-test"
+      - "run-id=${{ github.run_id }}-nightly-${{ matrix.provider }}-provider-chat-test"
       - extras=ecr-cache
     timeout-minutes: 45
+    environment: ci-protected
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
@@ -164,19 +263,43 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+
+      - name: Get AWS Secrets
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
+        with:
+          # Keep JSON values unparsed so vertex custom config is passed as raw JSON.
+          parse-json-secrets: false
+          secret-ids: |
+            DOCKER_USERNAME, test/docker-username
+            DOCKER_TOKEN, test/docker-token
+            OPENAI_API_KEY, test/openai-api-key
+            ANTHROPIC_API_KEY, test/anthropic-api-key
+            BEDROCK_API_KEY, test/bedrock-api-key
+            NIGHTLY_LLM_VERTEX_AI_CUSTOM_CONFIG_JSON, test/nightly-llm-vertex-ai-custom-config-json
+            AZURE_API_KEY, test/azure-api-key
+            OLLAMA_API_KEY, test/ollama-api-key
+            OPENROUTER_API_KEY, test/openrouter-api-key
+
       - name: Run nightly provider chat test
         uses: ./.github/actions/run-nightly-provider-chat-test
         with:
-          provider: ${{ env.NIGHTLY_LLM_PROVIDER }}
-          models: ${{ env.NIGHTLY_LLM_MODELS }}
-          provider-api-key: ${{ secrets.provider_api_key }}
-          strict: ${{ env.NIGHTLY_LLM_STRICT }}
-          api-base: ${{ env.NIGHTLY_LLM_API_BASE }}
-          custom-config-json: ${{ env.NIGHTLY_LLM_CUSTOM_CONFIG_JSON }}
+          provider: ${{ matrix.provider }}
+          models: ${{ matrix.models }}
+          provider-api-key: ${{ matrix.api_key_env && env[matrix.api_key_env] || '' }}
+          strict: ${{ inputs.strict && 'true' || 'false' }}
+          api-base: ${{ matrix.api_base }}
+          api-version: ${{ matrix.api_version }}
+          deployment-name: ${{ matrix.deployment_name }}
+          custom-config-json: ${{ matrix.custom_config_env && env[matrix.custom_config_env] || '' }}
           runs-on-ecr-cache: ${{ env.RUNS_ON_ECR_CACHE }}
           run-id: ${{ github.run_id }}
-          docker-username: ${{ secrets.DOCKER_USERNAME }}
-          docker-token: ${{ secrets.DOCKER_TOKEN }}
+          docker-username: ${{ env.DOCKER_USERNAME }}
+          docker-token: ${{ env.DOCKER_TOKEN }}
 
       - name: Dump API server logs
         if: always()
@@ -194,7 +317,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
-          name: docker-all-logs-nightly-${{ inputs.provider }}-llm-provider
+          name: docker-all-logs-nightly-${{ matrix.provider }}-llm-provider
           path: |
             ${{ github.workspace }}/api_server.log
             ${{ github.workspace }}/docker-compose.log

backend/alembic/versions/4a1e4b1c89d2_add_indexing_to_userfilestatus.py

@@ -0,0 +1,51 @@
+"""Add INDEXING to UserFileStatus
+
+Revision ID: 4a1e4b1c89d2
+Revises: 6b3b4083c5aa
+Create Date: 2026-02-28 00:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "4a1e4b1c89d2"
+down_revision = "6b3b4083c5aa"
+branch_labels = None
+depends_on = None
+
+TABLE = "user_file"
+COLUMN = "status"
+CONSTRAINT_NAME = "ck_user_file_status"
+
+OLD_VALUES = ("PROCESSING", "COMPLETED", "FAILED", "CANCELED", "DELETING")
+NEW_VALUES = ("PROCESSING", "INDEXING", "COMPLETED", "FAILED", "CANCELED", "DELETING")
+
+
+def _drop_status_check_constraint() -> None:
+    """Drop the existing CHECK constraint on user_file.status.
+
+    The constraint name is auto-generated by SQLAlchemy and unknown,
+    so we look it up via the inspector.
+    """
+    inspector = sa.inspect(op.get_bind())
+    for constraint in inspector.get_check_constraints(TABLE):
+        if COLUMN in constraint.get("sqltext", ""):
+            constraint_name = constraint["name"]
+            if constraint_name is not None:
+                op.drop_constraint(constraint_name, TABLE, type_="check")
+
+
+def upgrade() -> None:
+    _drop_status_check_constraint()
+    in_clause = ", ".join(f"'{v}'" for v in NEW_VALUES)
+    op.create_check_constraint(CONSTRAINT_NAME, TABLE, f"{COLUMN} IN ({in_clause})")
+
+
+def downgrade() -> None:
+    op.execute(
+        f"UPDATE {TABLE} SET {COLUMN} = 'PROCESSING' WHERE {COLUMN} = 'INDEXING'"
+    )
+    op.drop_constraint(CONSTRAINT_NAME, TABLE, type_="check")
+    in_clause = ", ".join(f"'{v}'" for v in OLD_VALUES)
+    op.create_check_constraint(CONSTRAINT_NAME, TABLE, f"{COLUMN} IN ({in_clause})")

backend/alembic/versions/57122d037335_add_python_tool_on_default.py

@@ -0,0 +1,69 @@
+"""add python tool on default
+
+Revision ID: 57122d037335
+Revises: c0c937d5c9e5
+Create Date: 2026-02-27 10:10:40.124925
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "57122d037335"
+down_revision = "c0c937d5c9e5"
+branch_labels = None
+depends_on = None
+
+
+PYTHON_TOOL_NAME = "python"
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+
+    # Look up the PythonTool id
+    result = conn.execute(
+        sa.text("SELECT id FROM tool WHERE name = :name"),
+        {"name": PYTHON_TOOL_NAME},
+    ).fetchone()
+
+    if not result:
+        return
+
+    tool_id = result[0]
+
+    # Attach to the default persona (id=0) if not already attached
+    conn.execute(
+        sa.text(
+            """
+            INSERT INTO persona__tool (persona_id, tool_id)
+            VALUES (0, :tool_id)
+            ON CONFLICT DO NOTHING
+            """
+        ),
+        {"tool_id": tool_id},
+    )
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+
+    result = conn.execute(
+        sa.text("SELECT id FROM tool WHERE name = :name"),
+        {"name": PYTHON_TOOL_NAME},
+    ).fetchone()
+
+    if not result:
+        return
+
+    conn.execute(
+        sa.text(
+            """
+            DELETE FROM persona__tool
+            WHERE persona_id = 0 AND tool_id = :tool_id
+            """
+        ),
+        {"tool_id": result[0]},
+    )

backend/alembic/versions/6b3b4083c5aa_persona_cleanup_and_featured.py

@@ -0,0 +1,112 @@
+"""persona cleanup and featured
+
+Revision ID: 6b3b4083c5aa
+Revises: 57122d037335
+Create Date: 2026-02-26 12:00:00.000000
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "6b3b4083c5aa"
+down_revision = "57122d037335"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Add featured column with nullable=True first
+    op.add_column("persona", sa.Column("featured", sa.Boolean(), nullable=True))
+
+    # Migrate data from is_default_persona to featured
+    op.execute("UPDATE persona SET featured = is_default_persona")
+
+    # Make featured non-nullable with default=False
+    op.alter_column(
+        "persona",
+        "featured",
+        existing_type=sa.Boolean(),
+        nullable=False,
+        server_default=sa.false(),
+    )
+
+    # Drop is_default_persona column
+    op.drop_column("persona", "is_default_persona")
+
+    # Drop unused columns
+    op.drop_column("persona", "num_chunks")
+    op.drop_column("persona", "chunks_above")
+    op.drop_column("persona", "chunks_below")
+    op.drop_column("persona", "llm_relevance_filter")
+    op.drop_column("persona", "llm_filter_extraction")
+    op.drop_column("persona", "recency_bias")
+
+
+def downgrade() -> None:
+    # Add back recency_bias column
+    op.add_column(
+        "persona",
+        sa.Column(
+            "recency_bias",
+            sa.VARCHAR(),
+            nullable=False,
+            server_default="base_decay",
+        ),
+    )
+
+    # Add back llm_filter_extraction column
+    op.add_column(
+        "persona",
+        sa.Column(
+            "llm_filter_extraction",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+
+    # Add back llm_relevance_filter column
+    op.add_column(
+        "persona",
+        sa.Column(
+            "llm_relevance_filter",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+
+    # Add back chunks_below column
+    op.add_column(
+        "persona",
+        sa.Column("chunks_below", sa.Integer(), nullable=False, server_default="0"),
+    )
+
+    # Add back chunks_above column
+    op.add_column(
+        "persona",
+        sa.Column("chunks_above", sa.Integer(), nullable=False, server_default="0"),
+    )
+
+    # Add back num_chunks column
+    op.add_column("persona", sa.Column("num_chunks", sa.Float(), nullable=True))
+
+    # Add back is_default_persona column
+    op.add_column(
+        "persona",
+        sa.Column(
+            "is_default_persona",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+
+    # Migrate data from featured to is_default_persona
+    op.execute("UPDATE persona SET is_default_persona = featured")
+
+    # Drop featured column
+    op.drop_column("persona", "featured")

backend/ee/onyx/main.py

@@ -31,6 +31,7 @@ from ee.onyx.server.query_and_chat.query_backend import (
 from ee.onyx.server.query_and_chat.search_backend import router as search_router
 from ee.onyx.server.query_history.api import router as query_history_router
 from ee.onyx.server.reporting.usage_export_api import router as usage_export_router
+from ee.onyx.server.scim.api import register_scim_exception_handlers
 from ee.onyx.server.scim.api import scim_router
 from ee.onyx.server.seeding import seed_db
 from ee.onyx.server.tenants.api import router as tenants_router
@@ -167,6 +168,7 @@ def get_application() -> FastAPI:
     # they use their own SCIM bearer token auth).
     # Not behind APP_API_PREFIX because IdPs expect /scim/v2/... directly.
     application.include_router(scim_router)
+    register_scim_exception_handlers(application)
 
     # Ensure all routes have auth enabled or are explicitly marked as public
     check_ee_router_auth(application)

backend/ee/onyx/server/scim/api.py

@@ -15,7 +15,9 @@ from uuid import UUID
 
 from fastapi import APIRouter
 from fastapi import Depends
+from fastapi import FastAPI
 from fastapi import Query
+from fastapi import Request
 from fastapi import Response
 from fastapi.responses import JSONResponse
 from fastapi_users.password import PasswordHelper
@@ -24,6 +26,7 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from ee.onyx.db.scim import ScimDAL
+from ee.onyx.server.scim.auth import ScimAuthError
 from ee.onyx.server.scim.auth import verify_scim_token
 from ee.onyx.server.scim.filtering import parse_scim_filter
 from ee.onyx.server.scim.models import SCIM_LIST_RESPONSE_SCHEMA
@@ -77,6 +80,22 @@ scim_router = APIRouter(prefix="/scim/v2", tags=["SCIM"])
 _pw_helper = PasswordHelper()
 
 
+def register_scim_exception_handlers(app: FastAPI) -> None:
+    """Register SCIM-specific exception handlers on the FastAPI app.
+
+    Call this after ``app.include_router(scim_router)`` so that auth
+    failures from ``verify_scim_token`` return RFC 7644 §3.12 error
+    envelopes (with ``schemas`` and ``status`` fields) instead of
+    FastAPI's default ``{"detail": "..."}`` format.
+    """
+
+    @app.exception_handler(ScimAuthError)
+    async def _handle_scim_auth_error(
+        _request: Request, exc: ScimAuthError
+    ) -> ScimJSONResponse:
+        return _scim_error_response(exc.status_code, exc.detail)
+
+
 def _get_provider(
     _token: ScimToken = Depends(verify_scim_token),
 ) -> ScimProvider:
@@ -322,6 +341,7 @@ def list_users(
     """List users with optional SCIM filter and pagination."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
+    dal.commit()
 
     try:
         scim_filter = parse_scim_filter(filter)
@@ -365,6 +385,7 @@ def get_user(
     """Get a single user by ID."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
+    dal.commit()
 
     result = _fetch_user_or_404(user_id, dal)
     if isinstance(result, ScimJSONResponse):
@@ -402,12 +423,6 @@ def create_user(
 
     email = user_resource.userName.strip()
 
-    # externalId is how the IdP correlates this user on subsequent requests.
-    # Without it, the IdP can't find the user and will try to re-create,
-    # hitting a 409 conflict — so we require it up front.
-    if not user_resource.externalId:
-        return _scim_error_response(400, "externalId is required")
-
     # Enforce seat limit
     seat_error = _check_seat_availability(dal)
     if seat_error:
@@ -434,16 +449,19 @@ def create_user(
         dal.rollback()
         return _scim_error_response(409, f"User with email {email} already exists")
 
-    # Create SCIM mapping (externalId is validated above, always present)
+    # Create SCIM mapping when externalId is provided — this is how the IdP
+    # correlates this user on subsequent requests.  Per RFC 7643, externalId
+    # is optional and assigned by the provisioning client.
     external_id = user_resource.externalId
     scim_username = user_resource.userName.strip()
     fields = _fields_from_resource(user_resource)
-    dal.create_user_mapping(
-        external_id=external_id,
-        user_id=user.id,
-        scim_username=scim_username,
-        fields=fields,
-    )
+    if external_id:
+        dal.create_user_mapping(
+            external_id=external_id,
+            user_id=user.id,
+            scim_username=scim_username,
+            fields=fields,
+        )
 
     dal.commit()
 
@@ -721,6 +739,7 @@ def list_groups(
     """List groups with optional SCIM filter and pagination."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
+    dal.commit()
 
     try:
         scim_filter = parse_scim_filter(filter)
@@ -757,6 +776,7 @@ def get_group(
     """Get a single group by ID."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
+    dal.commit()
 
     result = _fetch_group_or_404(group_id, dal)
     if isinstance(result, ScimJSONResponse):

backend/ee/onyx/server/scim/auth.py

@@ -19,7 +19,6 @@ import hashlib
 import secrets
 
 from fastapi import Depends
-from fastapi import HTTPException
 from fastapi import Request
 from sqlalchemy.orm import Session
 
@@ -28,6 +27,21 @@ from onyx.auth.utils import get_hashed_bearer_token_from_request
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.models import ScimToken
 
+
+class ScimAuthError(Exception):
+    """Raised when SCIM bearer token authentication fails.
+
+    Unlike HTTPException, this carries the status and detail so the SCIM
+    exception handler can wrap them in an RFC 7644 §3.12 error envelope
+    with ``schemas`` and ``status`` fields.
+    """
+
+    def __init__(self, status_code: int, detail: str) -> None:
+        self.status_code = status_code
+        self.detail = detail
+        super().__init__(detail)
+
+
 SCIM_TOKEN_PREFIX = "onyx_scim_"
 SCIM_TOKEN_LENGTH = 48
 
@@ -82,23 +96,14 @@ def verify_scim_token(
     """
     hashed = _get_hashed_scim_token_from_request(request)
     if not hashed:
-        raise HTTPException(
-            status_code=401,
-            detail="Missing or invalid SCIM bearer token",
-        )
+        raise ScimAuthError(401, "Missing or invalid SCIM bearer token")
 
     token = dal.get_token_by_hash(hashed)
 
     if not token:
-        raise HTTPException(
-            status_code=401,
-            detail="Invalid SCIM bearer token",
-        )
+        raise ScimAuthError(401, "Invalid SCIM bearer token")
 
     if not token.is_active:
-        raise HTTPException(
-            status_code=401,
-            detail="SCIM token has been revoked",
-        )
+        raise ScimAuthError(401, "SCIM token has been revoked")
 
     return token

backend/ee/onyx/server/scim/providers/base.py

@@ -153,26 +153,28 @@ class ScimProvider(ABC):
         self,
         user: User,
         fields: ScimMappingFields,
-    ) -> ScimName | None:
+    ) -> ScimName:
         """Build SCIM name components for the response.
 
         Round-trips stored ``given_name``/``family_name`` when available (so
         the IdP gets back what it sent). Falls back to splitting
         ``personal_name`` for users provisioned before we stored components.
+        Always returns a ScimName — Okta's spec tests expect ``name``
+        (with ``givenName``/``familyName``) on every user resource.
         Providers may override for custom behavior.
         """
         if fields.given_name is not None or fields.family_name is not None:
             return ScimName(
-                givenName=fields.given_name,
-                familyName=fields.family_name,
-                formatted=user.personal_name,
+                givenName=fields.given_name or "",
+                familyName=fields.family_name or "",
+                formatted=user.personal_name or "",
             )
         if not user.personal_name:
-            return None
+            return ScimName(givenName="", familyName="", formatted="")
         parts = user.personal_name.split(" ", 1)
         return ScimName(
             givenName=parts[0],
-            familyName=parts[1] if len(parts) > 1 else None,
+            familyName=parts[1] if len(parts) > 1 else "",
             formatted=user.personal_name,
         )
 

backend/ee/onyx/server/seeding.py

@@ -18,8 +18,8 @@ from ee.onyx.server.enterprise_settings.store import (
     store_settings as store_ee_settings,
 )
 from ee.onyx.server.enterprise_settings.store import upload_logo
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
+from onyx.db.llm import fetch_existing_llm_provider
 from onyx.db.llm import update_default_provider
 from onyx.db.llm import upsert_llm_provider
 from onyx.db.models import Tool
@@ -117,15 +117,38 @@ def _seed_custom_tools(db_session: Session, tools: List[CustomToolSeed]) -> None
 def _seed_llms(
     db_session: Session, llm_upsert_requests: list[LLMProviderUpsertRequest]
 ) -> None:
-    if llm_upsert_requests:
-        logger.notice("Seeding LLMs")
-        seeded_providers = [
-            upsert_llm_provider(llm_upsert_request, db_session)
-            for llm_upsert_request in llm_upsert_requests
-        ]
-        update_default_provider(
-            provider_id=seeded_providers[0].id, db_session=db_session
-        )
+    if not llm_upsert_requests:
+        return
+
+    logger.notice("Seeding LLMs")
+    for request in llm_upsert_requests:
+        existing = fetch_existing_llm_provider(name=request.name, db_session=db_session)
+        if existing:
+            request.id = existing.id
+    seeded_providers = [
+        upsert_llm_provider(llm_upsert_request, db_session)
+        for llm_upsert_request in llm_upsert_requests
+    ]
+
+    default_provider = next(
+        (p for p in seeded_providers if p.model_configurations), None
+    )
+    if not default_provider:
+        return
+
+    visible_configs = [
+        mc for mc in default_provider.model_configurations if mc.is_visible
+    ]
+    default_config = (
+        visible_configs[0]
+        if visible_configs
+        else default_provider.model_configurations[0]
+    )
+    update_default_provider(
+        provider_id=default_provider.id,
+        model_name=default_config.name,
+        db_session=db_session,
+    )
 
 
 def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) -> None:
@@ -137,12 +160,6 @@ def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) ->
                     user=None,  # Seeding is done as admin
                     name=persona.name,
                     description=persona.description,
-                    num_chunks=(
-                        persona.num_chunks if persona.num_chunks is not None else 0.0
-                    ),
-                    llm_relevance_filter=persona.llm_relevance_filter,
-                    llm_filter_extraction=persona.llm_filter_extraction,
-                    recency_bias=RecencyBiasSetting.AUTO,
                     document_set_ids=persona.document_set_ids,
                     llm_model_provider_override=persona.llm_model_provider_override,
                     llm_model_version_override=persona.llm_model_version_override,
@@ -154,6 +171,7 @@ def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) ->
                     system_prompt=persona.system_prompt,
                     task_prompt=persona.task_prompt,
                     datetime_aware=persona.datetime_aware,
+                    featured=persona.featured,
                     commit=False,
                 )
             db_session.commit()

backend/ee/onyx/server/settings/api.py

@@ -109,6 +109,12 @@ def apply_license_status_to_settings(settings: Settings) -> Settings:
             if metadata.status == _BLOCKING_STATUS:
                 settings.application_status = metadata.status
                 settings.ee_features_enabled = False
+            elif metadata.used_seats > metadata.seats:
+                # License is valid but seat limit exceeded
+                settings.application_status = ApplicationStatus.SEAT_LIMIT_EXCEEDED
+                settings.seat_count = metadata.seats
+                settings.used_seats = metadata.used_seats
+                settings.ee_features_enabled = True
             else:
                 # Has a valid license (GRACE_PERIOD/PAYMENT_REMINDER still allow EE features)
                 settings.ee_features_enabled = True

backend/ee/onyx/server/tenants/provisioning.py

@@ -33,6 +33,7 @@ from onyx.configs.constants import MilestoneRecordType
 from onyx.db.engine.sql_engine import get_session_with_shared_schema
 from onyx.db.engine.sql_engine import get_session_with_tenant
 from onyx.db.image_generation import create_default_image_gen_config_from_api_key
+from onyx.db.llm import fetch_existing_llm_provider
 from onyx.db.llm import update_default_provider
 from onyx.db.llm import upsert_cloud_embedding_provider
 from onyx.db.llm import upsert_llm_provider
@@ -302,12 +303,17 @@ def configure_default_api_keys(db_session: Session) -> None:
 
     has_set_default_provider = False
 
-    def _upsert(request: LLMProviderUpsertRequest) -> None:
+    def _upsert(request: LLMProviderUpsertRequest, default_model: str) -> None:
         nonlocal has_set_default_provider
         try:
+            existing = fetch_existing_llm_provider(
+                name=request.name, db_session=db_session
+            )
+            if existing:
+                request.id = existing.id
             provider = upsert_llm_provider(request, db_session)
             if not has_set_default_provider:
-                update_default_provider(provider.id, db_session)
+                update_default_provider(provider.id, default_model, db_session)
                 has_set_default_provider = True
         except Exception as e:
             logger.error(f"Failed to configure {request.provider} provider: {e}")
@@ -325,14 +331,13 @@ def configure_default_api_keys(db_session: Session) -> None:
             name="OpenAI",
             provider=OPENAI_PROVIDER_NAME,
             api_key=OPENAI_DEFAULT_API_KEY,
-            default_model_name=default_model_name,
             model_configurations=_build_model_configuration_upsert_requests(
                 OPENAI_PROVIDER_NAME, recommendations
             ),
             api_key_changed=True,
             is_auto_mode=True,
         )
-        _upsert(openai_provider)
+        _upsert(openai_provider, default_model_name)
 
         # Create default image generation config using the OpenAI API key
         try:
@@ -361,14 +366,13 @@ def configure_default_api_keys(db_session: Session) -> None:
             name="Anthropic",
             provider=ANTHROPIC_PROVIDER_NAME,
             api_key=ANTHROPIC_DEFAULT_API_KEY,
-            default_model_name=default_model_name,
             model_configurations=_build_model_configuration_upsert_requests(
                 ANTHROPIC_PROVIDER_NAME, recommendations
             ),
             api_key_changed=True,
             is_auto_mode=True,
         )
-        _upsert(anthropic_provider)
+        _upsert(anthropic_provider, default_model_name)
     else:
         logger.info(
             "ANTHROPIC_DEFAULT_API_KEY not set, skipping Anthropic provider configuration"
@@ -393,14 +397,13 @@ def configure_default_api_keys(db_session: Session) -> None:
             name="Google Vertex AI",
             provider=VERTEXAI_PROVIDER_NAME,
             custom_config=custom_config,
-            default_model_name=default_model_name,
             model_configurations=_build_model_configuration_upsert_requests(
                 VERTEXAI_PROVIDER_NAME, recommendations
             ),
             api_key_changed=True,
             is_auto_mode=True,
         )
-        _upsert(vertexai_provider)
+        _upsert(vertexai_provider, default_model_name)
     else:
         logger.info(
             "VERTEXAI_DEFAULT_CREDENTIALS not set, skipping Vertex AI provider configuration"
@@ -432,12 +435,11 @@ def configure_default_api_keys(db_session: Session) -> None:
             name="OpenRouter",
             provider=OPENROUTER_PROVIDER_NAME,
             api_key=OPENROUTER_DEFAULT_API_KEY,
-            default_model_name=default_model_name,
             model_configurations=model_configurations,
             api_key_changed=True,
             is_auto_mode=True,
         )
-        _upsert(openrouter_provider)
+        _upsert(openrouter_provider, default_model_name)
     else:
         logger.info(
             "OPENROUTER_DEFAULT_API_KEY not set, skipping OpenRouter provider configuration"

backend/onyx/auth/users.py

@@ -543,7 +543,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
         result = await db_session.execute(
             select(Persona.id)
             .where(
-                Persona.is_default_persona.is_(True),
+                Persona.featured.is_(True),
                 Persona.is_public.is_(True),
                 Persona.is_visible.is_(True),
                 Persona.deleted.is_(False),
@@ -725,11 +725,19 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
                         if user_by_session:
                             user = user_by_session
 
+                # If the user is inactive, check seat availability before
+                # upgrading role — otherwise they'd become an inactive BASIC
+                # user who still can't log in.
+                if not user.is_active:
+                    with get_session_with_current_tenant() as sync_db:
+                        enforce_seat_limit(sync_db)
+
                 await self.user_db.update(
                     user,
                     {
                         "is_verified": is_verified_by_default,
                         "role": UserRole.BASIC,
+                        **({"is_active": True} if not user.is_active else {}),
                     },
                 )
 

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -241,8 +241,7 @@ _VECTOR_DB_BEAT_TASK_NAMES: set[str] = {
     "check-for-index-attempt-cleanup",
     "check-for-doc-permissions-sync",
     "check-for-external-group-sync",
-    "check-for-documents-for-opensearch-migration",
-    "migrate-documents-from-vespa-to-opensearch",
+    "migrate-chunks-from-vespa-to-opensearch",
 }
 
 if DISABLE_VECTOR_DB:

backend/onyx/background/celery/tasks/user_file_processing/tasks.py

@@ -76,7 +76,7 @@ def _user_file_queued_key(user_file_id: str | UUID) -> str:
     return f"{OnyxRedisLocks.USER_FILE_QUEUED_PREFIX}:{user_file_id}"
 
 
-def _user_file_project_sync_lock_key(user_file_id: str | UUID) -> str:
+def user_file_project_sync_lock_key(user_file_id: str | UUID) -> str:
     return f"{OnyxRedisLocks.USER_FILE_PROJECT_SYNC_LOCK_PREFIX}:{user_file_id}"
 
 
@@ -414,34 +414,31 @@ def _process_user_file_with_indexing(
         raise RuntimeError(f"Indexing pipeline failed for user file {user_file_id}")
 
 
-@shared_task(
-    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
-    bind=True,
-    ignore_result=True,
-)
-def process_single_user_file(
-    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+def process_user_file_impl(
+    *, user_file_id: str, tenant_id: str, redis_locking: bool
 ) -> None:
-    task_logger.info(f"process_single_user_file - Starting id={user_file_id}")
+    """Core implementation for processing a single user file.
+
+    When redis_locking=True, acquires a per-file Redis lock and clears the
+    queued-key guard (Celery path).  When redis_locking=False, skips all Redis
+    operations (BackgroundTask path).
+    """
+    task_logger.info(f"process_user_file_impl - Starting id={user_file_id}")
     start = time.monotonic()
 
-    redis_client = get_redis_client(tenant_id=tenant_id)
-
-    # Clear the "queued" guard set by the beat generator so that the next beat
-    # cycle can re-enqueue this file if it is still in PROCESSING state after
-    # this task completes or fails.
-    redis_client.delete(_user_file_queued_key(user_file_id))
-
-    file_lock: RedisLock = redis_client.lock(
-        _user_file_lock_key(user_file_id),
-        timeout=CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT,
-    )
-
-    if not file_lock.acquire(blocking=False):
-        task_logger.info(
-            f"process_single_user_file - Lock held, skipping user_file_id={user_file_id}"
+    file_lock: RedisLock | None = None
+    if redis_locking:
+        redis_client = get_redis_client(tenant_id=tenant_id)
+        redis_client.delete(_user_file_queued_key(user_file_id))
+        file_lock = redis_client.lock(
+            _user_file_lock_key(user_file_id),
+            timeout=CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT,
         )
-        return None
+        if file_lock is not None and not file_lock.acquire(blocking=False):
+            task_logger.info(
+                f"process_user_file_impl - Lock held, skipping user_file_id={user_file_id}"
+            )
+            return
 
     documents: list[Document] = []
     try:
@@ -449,15 +446,18 @@ def process_single_user_file(
             uf = db_session.get(UserFile, _as_uuid(user_file_id))
             if not uf:
                 task_logger.warning(
-                    f"process_single_user_file - UserFile not found id={user_file_id}"
+                    f"process_user_file_impl - UserFile not found id={user_file_id}"
                 )
-                return None
+                return
 
-            if uf.status != UserFileStatus.PROCESSING:
+            if uf.status not in (
+                UserFileStatus.PROCESSING,
+                UserFileStatus.INDEXING,
+            ):
                 task_logger.info(
-                    f"process_single_user_file - Skipping id={user_file_id} status={uf.status}"
+                    f"process_user_file_impl - Skipping id={user_file_id} status={uf.status}"
                 )
-                return None
+                return
 
             connector = LocalFileConnector(
                 file_locations=[uf.file_id],
@@ -471,7 +471,6 @@ def process_single_user_file(
                         [doc for doc in batch if not isinstance(doc, HierarchyNode)]
                     )
 
-                # update the document id to userfile id in the documents
                 for document in documents:
                     document.id = str(user_file_id)
                     document.source = DocumentSource.USER_FILE
@@ -493,9 +492,8 @@ def process_single_user_file(
 
             except Exception as e:
                 task_logger.exception(
-                    f"process_single_user_file - Error processing file id={user_file_id} - {e.__class__.__name__}"
+                    f"process_user_file_impl - Error processing file id={user_file_id} - {e.__class__.__name__}"
                 )
-                # don't update the status if the user file is being deleted
                 current_user_file = db_session.get(UserFile, _as_uuid(user_file_id))
                 if (
                     current_user_file
@@ -504,33 +502,42 @@ def process_single_user_file(
                     uf.status = UserFileStatus.FAILED
                     db_session.add(uf)
                     db_session.commit()
-                return None
+                return
 
         elapsed = time.monotonic() - start
         task_logger.info(
-            f"process_single_user_file - Finished id={user_file_id} docs={len(documents)} elapsed={elapsed:.2f}s"
+            f"process_user_file_impl - Finished id={user_file_id} docs={len(documents)} elapsed={elapsed:.2f}s"
         )
-        return None
     except Exception as e:
-        # Attempt to mark the file as failed
         with get_session_with_current_tenant() as db_session:
             uf = db_session.get(UserFile, _as_uuid(user_file_id))
             if uf:
-                # don't update the status if the user file is being deleted
                 if uf.status != UserFileStatus.DELETING:
                     uf.status = UserFileStatus.FAILED
                 db_session.add(uf)
                 db_session.commit()
 
         task_logger.exception(
-            f"process_single_user_file - Error processing file id={user_file_id} - {e.__class__.__name__}"
+            f"process_user_file_impl - Error processing file id={user_file_id} - {e.__class__.__name__}"
         )
-        return None
     finally:
-        if file_lock.owned():
+        if file_lock is not None and file_lock.owned():
             file_lock.release()
 
 
+@shared_task(
+    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file(
+    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+) -> None:
+    process_user_file_impl(
+        user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True
+    )
+
+
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_USER_FILE_DELETE,
     soft_time_limit=300,
@@ -581,36 +588,38 @@ def check_for_user_file_delete(self: Task, *, tenant_id: str) -> None:
     return None
 
 
-@shared_task(
-    name=OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
-    bind=True,
-    ignore_result=True,
-)
-def process_single_user_file_delete(
-    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+def delete_user_file_impl(
+    *, user_file_id: str, tenant_id: str, redis_locking: bool
 ) -> None:
-    """Process a single user file delete."""
-    task_logger.info(f"process_single_user_file_delete - Starting id={user_file_id}")
-    redis_client = get_redis_client(tenant_id=tenant_id)
-    file_lock: RedisLock = redis_client.lock(
-        _user_file_delete_lock_key(user_file_id),
-        timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
-    )
-    if not file_lock.acquire(blocking=False):
-        task_logger.info(
-            f"process_single_user_file_delete - Lock held, skipping user_file_id={user_file_id}"
+    """Core implementation for deleting a single user file.
+
+    When redis_locking=True, acquires a per-file Redis lock (Celery path).
+    When redis_locking=False, skips Redis operations (BackgroundTask path).
+    """
+    task_logger.info(f"delete_user_file_impl - Starting id={user_file_id}")
+
+    file_lock: RedisLock | None = None
+    if redis_locking:
+        redis_client = get_redis_client(tenant_id=tenant_id)
+        file_lock = redis_client.lock(
+            _user_file_delete_lock_key(user_file_id),
+            timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
         )
-        return None
+        if file_lock is not None and not file_lock.acquire(blocking=False):
+            task_logger.info(
+                f"delete_user_file_impl - Lock held, skipping user_file_id={user_file_id}"
+            )
+            return
+
     try:
         with get_session_with_current_tenant() as db_session:
             user_file = db_session.get(UserFile, _as_uuid(user_file_id))
             if not user_file:
                 task_logger.info(
-                    f"process_single_user_file_delete - User file not found id={user_file_id}"
+                    f"delete_user_file_impl - User file not found id={user_file_id}"
                 )
-                return None
+                return
 
-            # 1) Delete vector DB chunks (skip when disabled)
             if not DISABLE_VECTOR_DB:
                 if MANAGED_VESPA:
                     httpx_init_vespa_pool(
@@ -648,7 +657,6 @@ def process_single_user_file_delete(
                         chunk_count=chunk_count,
                     )
 
-            # 2) Delete the user-uploaded file content from filestore (blob + metadata)
             file_store = get_default_file_store()
             try:
                 file_store.delete_file(user_file.file_id)
@@ -656,26 +664,33 @@ def process_single_user_file_delete(
                     user_file_id_to_plaintext_file_name(user_file.id)
                 )
             except Exception as e:
-                # This block executed only if the file is not found in the filestore
                 task_logger.exception(
-                    f"process_single_user_file_delete - Error deleting file id={user_file.id} - {e.__class__.__name__}"
+                    f"delete_user_file_impl - Error deleting file id={user_file.id} - {e.__class__.__name__}"
                 )
 
-            # 3) Finally, delete the UserFile row
             db_session.delete(user_file)
             db_session.commit()
-            task_logger.info(
-                f"process_single_user_file_delete - Completed id={user_file_id}"
-            )
+            task_logger.info(f"delete_user_file_impl - Completed id={user_file_id}")
     except Exception as e:
         task_logger.exception(
-            f"process_single_user_file_delete - Error processing file id={user_file_id} - {e.__class__.__name__}"
+            f"delete_user_file_impl - Error processing file id={user_file_id} - {e.__class__.__name__}"
         )
-        return None
     finally:
-        if file_lock.owned():
+        if file_lock is not None and file_lock.owned():
             file_lock.release()
-    return None
+
+
+@shared_task(
+    name=OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file_delete(
+    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+) -> None:
+    delete_user_file_impl(
+        user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True
+    )
 
 
 @shared_task(
@@ -747,32 +762,30 @@ def check_for_user_file_project_sync(self: Task, *, tenant_id: str) -> None:
     return None
 
 
-@shared_task(
-    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE_PROJECT_SYNC,
-    bind=True,
-    ignore_result=True,
-)
-def process_single_user_file_project_sync(
-    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+def project_sync_user_file_impl(
+    *, user_file_id: str, tenant_id: str, redis_locking: bool
 ) -> None:
-    """Process a single user file project sync."""
-    task_logger.info(
-        f"process_single_user_file_project_sync - Starting id={user_file_id}"
-    )
+    """Core implementation for syncing a user file's project/persona metadata.
 
-    redis_client = get_redis_client(tenant_id=tenant_id)
-    redis_client.delete(_user_file_project_sync_queued_key(user_file_id))
+    When redis_locking=True, acquires a per-file Redis lock and clears the
+    queued-key guard (Celery path).  When redis_locking=False, skips Redis
+    operations (BackgroundTask path).
+    """
+    task_logger.info(f"project_sync_user_file_impl - Starting id={user_file_id}")
 
-    file_lock: RedisLock = redis_client.lock(
-        _user_file_project_sync_lock_key(user_file_id),
-        timeout=CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT,
-    )
-
-    if not file_lock.acquire(blocking=False):
-        task_logger.info(
-            f"process_single_user_file_project_sync - Lock held, skipping user_file_id={user_file_id}"
+    file_lock: RedisLock | None = None
+    if redis_locking:
+        redis_client = get_redis_client(tenant_id=tenant_id)
+        redis_client.delete(_user_file_project_sync_queued_key(user_file_id))
+        file_lock = redis_client.lock(
+            user_file_project_sync_lock_key(user_file_id),
+            timeout=CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT,
         )
-        return None
+        if file_lock is not None and not file_lock.acquire(blocking=False):
+            task_logger.info(
+                f"project_sync_user_file_impl - Lock held, skipping user_file_id={user_file_id}"
+            )
+            return
 
     try:
         with get_session_with_current_tenant() as db_session:
@@ -783,11 +796,10 @@ def process_single_user_file_project_sync(
             ).scalar_one_or_none()
             if not user_file:
                 task_logger.info(
-                    f"process_single_user_file_project_sync - User file not found id={user_file_id}"
+                    f"project_sync_user_file_impl - User file not found id={user_file_id}"
                 )
-                return None
+                return
 
-            # Sync project metadata to vector DB (skip when disabled)
             if not DISABLE_VECTOR_DB:
                 if MANAGED_VESPA:
                     httpx_init_vespa_pool(
@@ -822,7 +834,7 @@ def process_single_user_file_project_sync(
                     )
 
             task_logger.info(
-                f"process_single_user_file_project_sync - User file id={user_file_id}"
+                f"project_sync_user_file_impl - User file id={user_file_id}"
             )
 
             user_file.needs_project_sync = False
@@ -835,11 +847,21 @@ def process_single_user_file_project_sync(
 
     except Exception as e:
         task_logger.exception(
-            f"process_single_user_file_project_sync - Error syncing project for file id={user_file_id} - {e.__class__.__name__}"
+            f"project_sync_user_file_impl - Error syncing project for file id={user_file_id} - {e.__class__.__name__}"
         )
-        return None
     finally:
-        if file_lock.owned():
+        if file_lock is not None and file_lock.owned():
             file_lock.release()
 
-    return None
+
+@shared_task(
+    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE_PROJECT_SYNC,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file_project_sync(
+    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
+) -> None:
+    project_sync_user_file_impl(
+        user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True
+    )

backend/onyx/background/periodic_poller.py

@@ -0,0 +1,287 @@
+"""Periodic poller for NO_VECTOR_DB deployments.
+
+Replaces Celery Beat and background workers with a lightweight daemon thread
+that runs from the API server process.  Two responsibilities:
+
+1. Recovery polling (every 30 s): re-processes user files stuck in
+   PROCESSING / DELETING / needs_sync states via the drain loops defined
+   in ``task_utils.py``.
+
+2. Periodic task execution (configurable intervals): runs LLM model updates
+   and scheduled evals at their configured cadences, with Postgres advisory
+   lock deduplication across multiple API server instances.
+"""
+
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from dataclasses import field
+
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+RECOVERY_INTERVAL_SECONDS = 30
+PERIODIC_TASK_LOCK_BASE = 20_000
+PERIODIC_TASK_KV_PREFIX = "periodic_poller:last_claimed:"
+
+
+# ------------------------------------------------------------------
+# Periodic task definitions
+# ------------------------------------------------------------------
+
+
+_NEVER_RAN: float = -1e18
+
+
+@dataclass
+class _PeriodicTaskDef:
+    name: str
+    interval_seconds: float
+    lock_id: int
+    run_fn: Callable[[], None]
+    last_run_at: float = field(default=_NEVER_RAN)
+
+
+def _run_auto_llm_update() -> None:
+    from onyx.configs.app_configs import AUTO_LLM_CONFIG_URL
+
+    if not AUTO_LLM_CONFIG_URL:
+        return
+
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+    from onyx.llm.well_known_providers.auto_update_service import (
+        sync_llm_models_from_github,
+    )
+
+    with get_session_with_current_tenant() as db_session:
+        sync_llm_models_from_github(db_session)
+
+
+def _run_scheduled_eval() -> None:
+    from onyx.configs.app_configs import BRAINTRUST_API_KEY
+    from onyx.configs.app_configs import SCHEDULED_EVAL_DATASET_NAMES
+    from onyx.configs.app_configs import SCHEDULED_EVAL_PERMISSIONS_EMAIL
+    from onyx.configs.app_configs import SCHEDULED_EVAL_PROJECT
+
+    if not all(
+        [
+            BRAINTRUST_API_KEY,
+            SCHEDULED_EVAL_PROJECT,
+            SCHEDULED_EVAL_DATASET_NAMES,
+            SCHEDULED_EVAL_PERMISSIONS_EMAIL,
+        ]
+    ):
+        return
+
+    from datetime import datetime
+    from datetime import timezone
+
+    from onyx.evals.eval import run_eval
+    from onyx.evals.models import EvalConfigurationOptions
+
+    run_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    for dataset_name in SCHEDULED_EVAL_DATASET_NAMES:
+        try:
+            run_eval(
+                configuration=EvalConfigurationOptions(
+                    search_permissions_email=SCHEDULED_EVAL_PERMISSIONS_EMAIL,
+                    dataset_name=dataset_name,
+                    no_send_logs=False,
+                    braintrust_project=SCHEDULED_EVAL_PROJECT,
+                    experiment_name=f"{dataset_name} - {run_timestamp}",
+                ),
+                remote_dataset_name=dataset_name,
+            )
+        except Exception:
+            logger.exception(
+                f"Periodic poller - Failed scheduled eval for dataset {dataset_name}"
+            )
+
+
+def _build_periodic_tasks() -> list[_PeriodicTaskDef]:
+    from onyx.configs.app_configs import AUTO_LLM_CONFIG_URL
+    from onyx.configs.app_configs import AUTO_LLM_UPDATE_INTERVAL_SECONDS
+    from onyx.configs.app_configs import SCHEDULED_EVAL_DATASET_NAMES
+
+    tasks: list[_PeriodicTaskDef] = []
+    if AUTO_LLM_CONFIG_URL:
+        tasks.append(
+            _PeriodicTaskDef(
+                name="auto-llm-update",
+                interval_seconds=AUTO_LLM_UPDATE_INTERVAL_SECONDS,
+                lock_id=PERIODIC_TASK_LOCK_BASE,
+                run_fn=_run_auto_llm_update,
+            )
+        )
+    if SCHEDULED_EVAL_DATASET_NAMES:
+        tasks.append(
+            _PeriodicTaskDef(
+                name="scheduled-eval",
+                interval_seconds=7 * 24 * 3600,
+                lock_id=PERIODIC_TASK_LOCK_BASE + 1,
+                run_fn=_run_scheduled_eval,
+            )
+        )
+    return tasks
+
+
+# ------------------------------------------------------------------
+# Periodic task runner with advisory-lock-guarded claim
+# ------------------------------------------------------------------
+
+
+def _try_claim_task(task_def: _PeriodicTaskDef) -> bool:
+    """Atomically check whether *task_def* should run and record a claim.
+
+    Uses a transaction-scoped advisory lock for atomicity combined with a
+    ``KVStore`` timestamp for cross-instance dedup.  The DB session is held
+    only for this brief claim transaction, not during task execution.
+    """
+    from datetime import datetime
+    from datetime import timezone
+
+    from sqlalchemy import text
+
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+    from onyx.db.models import KVStore
+
+    kv_key = PERIODIC_TASK_KV_PREFIX + task_def.name
+
+    with get_session_with_current_tenant() as db_session:
+        acquired = db_session.execute(
+            text("SELECT pg_try_advisory_xact_lock(:id)"),
+            {"id": task_def.lock_id},
+        ).scalar()
+        if not acquired:
+            return False
+
+        row = db_session.query(KVStore).filter_by(key=kv_key).first()
+        if row and row.value is not None:
+            last_claimed = datetime.fromisoformat(str(row.value))
+            elapsed = (datetime.now(timezone.utc) - last_claimed).total_seconds()
+            if elapsed < task_def.interval_seconds:
+                return False
+
+        now_ts = datetime.now(timezone.utc).isoformat()
+        if row:
+            row.value = now_ts
+        else:
+            db_session.add(KVStore(key=kv_key, value=now_ts))
+        db_session.commit()
+
+    return True
+
+
+def _try_run_periodic_task(task_def: _PeriodicTaskDef) -> None:
+    """Run *task_def* if its interval has elapsed and no peer holds the lock."""
+    now = time.monotonic()
+    if now - task_def.last_run_at < task_def.interval_seconds:
+        return
+
+    if not _try_claim_task(task_def):
+        return
+
+    try:
+        task_def.run_fn()
+        task_def.last_run_at = now
+    except Exception:
+        logger.exception(
+            f"Periodic poller - Error running periodic task {task_def.name}"
+        )
+
+
+# ------------------------------------------------------------------
+# Recovery / drain loop runner
+# ------------------------------------------------------------------
+
+
+def _run_drain_loops(tenant_id: str) -> None:
+    from onyx.background.task_utils import drain_delete_loop
+    from onyx.background.task_utils import drain_processing_loop
+    from onyx.background.task_utils import drain_project_sync_loop
+
+    drain_processing_loop(tenant_id)
+    drain_delete_loop(tenant_id)
+    drain_project_sync_loop(tenant_id)
+
+
+# ------------------------------------------------------------------
+# Startup recovery (10g)
+# ------------------------------------------------------------------
+
+
+def recover_stuck_user_files(tenant_id: str) -> None:
+    """Run all drain loops once to re-process files left in intermediate states.
+
+    Called from ``lifespan()`` on startup when ``DISABLE_VECTOR_DB`` is set.
+    """
+    logger.info("recover_stuck_user_files - Checking for stuck user files")
+    try:
+        _run_drain_loops(tenant_id)
+    except Exception:
+        logger.exception("recover_stuck_user_files - Error during recovery")
+
+
+# ------------------------------------------------------------------
+# Daemon thread (10f)
+# ------------------------------------------------------------------
+
+_shutdown_event = threading.Event()
+_poller_thread: threading.Thread | None = None
+
+
+def _poller_loop(tenant_id: str) -> None:
+    from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
+
+    CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
+
+    periodic_tasks = _build_periodic_tasks()
+    logger.info(
+        f"Periodic poller started with {len(periodic_tasks)} periodic task(s): "
+        f"{[t.name for t in periodic_tasks]}"
+    )
+
+    while not _shutdown_event.is_set():
+        try:
+            _run_drain_loops(tenant_id)
+        except Exception:
+            logger.exception("Periodic poller - Error in recovery polling")
+
+        for task_def in periodic_tasks:
+            try:
+                _try_run_periodic_task(task_def)
+            except Exception:
+                logger.exception(
+                    f"Periodic poller - Unhandled error checking task {task_def.name}"
+                )
+
+        _shutdown_event.wait(RECOVERY_INTERVAL_SECONDS)
+
+
+def start_periodic_poller(tenant_id: str) -> None:
+    """Start the periodic poller daemon thread."""
+    global _poller_thread  # noqa: PLW0603
+    _shutdown_event.clear()
+    _poller_thread = threading.Thread(
+        target=_poller_loop,
+        args=(tenant_id,),
+        daemon=True,
+        name="no-vectordb-periodic-poller",
+    )
+    _poller_thread.start()
+    logger.info("Periodic poller thread started")
+
+
+def stop_periodic_poller() -> None:
+    """Signal the periodic poller to stop and wait for it to exit."""
+    global _poller_thread  # noqa: PLW0603
+    if _poller_thread is None:
+        return
+    _shutdown_event.set()
+    _poller_thread.join(timeout=10)
+    if _poller_thread.is_alive():
+        logger.warning("Periodic poller thread did not stop within timeout")
+    _poller_thread = None
+    logger.info("Periodic poller thread stopped")

backend/onyx/background/task_utils.py

@@ -1,3 +1,33 @@
+"""Background task utilities.
+
+Contains query-history report helpers (used by all deployment modes) and
+in-process background task execution helpers for NO_VECTOR_DB mode:
+
+- Atomic claim-and-mark helpers that prevent duplicate processing
+- Drain loops that process all pending user file work
+
+Each claim function runs a short-lived transaction: SELECT ... FOR UPDATE
+SKIP LOCKED, UPDATE the row to remove it from future queries, COMMIT.
+After the commit the row lock is released, but the row is no longer
+eligible for re-claiming.  No long-lived sessions or advisory locks.
+"""
+
+from uuid import UUID
+
+import sqlalchemy as sa
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from onyx.db.enums import UserFileStatus
+from onyx.db.models import UserFile
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+# ------------------------------------------------------------------
+# Query-history report helpers (pre-existing, used by all modes)
+# ------------------------------------------------------------------
+
 QUERY_REPORT_NAME_PREFIX = "query-history"
 
 
@@ -9,3 +39,142 @@ def construct_query_history_report_name(
 
 def extract_task_id_from_query_history_report_name(name: str) -> str:
     return name.removeprefix(f"{QUERY_REPORT_NAME_PREFIX}-").removesuffix(".csv")
+
+
+# ------------------------------------------------------------------
+# Atomic claim-and-mark helpers
+# ------------------------------------------------------------------
+# Each function runs inside a single short-lived session/transaction:
+#   1. SELECT ... FOR UPDATE SKIP LOCKED  (locks one eligible row)
+#   2. UPDATE the row so it is no longer eligible
+#   3. COMMIT  (releases the row lock)
+# After the commit, no other drain loop can claim the same row.
+
+
+def _claim_next_processing_file(db_session: Session) -> UUID | None:
+    """Claim the next PROCESSING file by transitioning it to INDEXING.
+
+    Returns the file id, or None when no eligible files remain.
+    """
+    file_id = db_session.execute(
+        select(UserFile.id)
+        .where(UserFile.status == UserFileStatus.PROCESSING)
+        .order_by(UserFile.created_at)
+        .limit(1)
+        .with_for_update(skip_locked=True)
+    ).scalar_one_or_none()
+    if file_id is None:
+        return None
+
+    db_session.execute(
+        sa.update(UserFile)
+        .where(UserFile.id == file_id)
+        .values(status=UserFileStatus.INDEXING)
+    )
+    db_session.commit()
+    return file_id
+
+
+def _claim_next_deleting_file(db_session: Session) -> UUID | None:
+    """Claim the next DELETING file.
+
+    No status transition needed — the impl deletes the row on success.
+    The short-lived FOR UPDATE lock prevents concurrent claims.
+    """
+    file_id = db_session.execute(
+        select(UserFile.id)
+        .where(UserFile.status == UserFileStatus.DELETING)
+        .order_by(UserFile.created_at)
+        .limit(1)
+        .with_for_update(skip_locked=True)
+    ).scalar_one_or_none()
+    # Commit to release the row lock promptly.
+    db_session.commit()
+    return file_id
+
+
+def _claim_next_sync_file(db_session: Session) -> UUID | None:
+    """Claim the next file needing project/persona sync.
+
+    No status transition needed — the impl clears the sync flags on
+    success.  The short-lived FOR UPDATE lock prevents concurrent claims.
+    """
+    file_id = db_session.execute(
+        select(UserFile.id)
+        .where(
+            sa.and_(
+                sa.or_(
+                    UserFile.needs_project_sync.is_(True),
+                    UserFile.needs_persona_sync.is_(True),
+                ),
+                UserFile.status == UserFileStatus.COMPLETED,
+            )
+        )
+        .order_by(UserFile.created_at)
+        .limit(1)
+        .with_for_update(skip_locked=True)
+    ).scalar_one_or_none()
+    db_session.commit()
+    return file_id
+
+
+# ------------------------------------------------------------------
+# Drain loops — process *all* pending work of each type
+# ------------------------------------------------------------------
+
+
+def drain_processing_loop(tenant_id: str) -> None:
+    """Process all pending PROCESSING user files."""
+    from onyx.background.celery.tasks.user_file_processing.tasks import (
+        process_user_file_impl,
+    )
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+
+    while True:
+        with get_session_with_current_tenant() as session:
+            file_id = _claim_next_processing_file(session)
+        if file_id is None:
+            break
+        process_user_file_impl(
+            user_file_id=str(file_id),
+            tenant_id=tenant_id,
+            redis_locking=False,
+        )
+
+
+def drain_delete_loop(tenant_id: str) -> None:
+    """Delete all pending DELETING user files."""
+    from onyx.background.celery.tasks.user_file_processing.tasks import (
+        delete_user_file_impl,
+    )
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+
+    while True:
+        with get_session_with_current_tenant() as session:
+            file_id = _claim_next_deleting_file(session)
+        if file_id is None:
+            break
+        delete_user_file_impl(
+            user_file_id=str(file_id),
+            tenant_id=tenant_id,
+            redis_locking=False,
+        )
+
+
+def drain_project_sync_loop(tenant_id: str) -> None:
+    """Sync all pending project/persona metadata for user files."""
+    from onyx.background.celery.tasks.user_file_processing.tasks import (
+        project_sync_user_file_impl,
+    )
+    from onyx.db.engine.sql_engine import get_session_with_current_tenant
+
+    while True:
+        with get_session_with_current_tenant() as session:
+            file_id = _claim_next_sync_file(session)
+        if file_id is None:
+            break
+        project_sync_user_file_impl(
+            user_file_id=str(file_id),
+            tenant_id=tenant_id,
+            redis_locking=False,
+        )

backend/onyx/cache/factory.py

@@ -0,0 +1,45 @@
+from collections.abc import Callable
+
+from onyx.cache.interface import CacheBackend
+from onyx.cache.interface import CacheBackendType
+from onyx.configs.app_configs import CACHE_BACKEND
+
+
+def _build_redis_backend(tenant_id: str) -> CacheBackend:
+    from onyx.cache.redis_backend import RedisCacheBackend
+    from onyx.redis.redis_pool import redis_pool
+
+    return RedisCacheBackend(redis_pool.get_client(tenant_id))
+
+
+_BACKEND_BUILDERS: dict[CacheBackendType, Callable[[str], CacheBackend]] = {
+    CacheBackendType.REDIS: _build_redis_backend,
+    # CacheBackendType.POSTGRES will be added in a follow-up PR.
+}
+
+
+def get_cache_backend(*, tenant_id: str | None = None) -> CacheBackend:
+    """Return a tenant-aware ``CacheBackend``.
+
+    If *tenant_id* is ``None``, the current tenant is read from the
+    thread-local context variable (same behaviour as ``get_redis_client``).
+    """
+    if tenant_id is None:
+        from shared_configs.contextvars import get_current_tenant_id
+
+        tenant_id = get_current_tenant_id()
+
+    builder = _BACKEND_BUILDERS.get(CACHE_BACKEND)
+    if builder is None:
+        raise ValueError(
+            f"Unsupported CACHE_BACKEND={CACHE_BACKEND!r}. "
+            f"Supported values: {[t.value for t in CacheBackendType]}"
+        )
+    return builder(tenant_id)
+
+
+def get_shared_cache_backend() -> CacheBackend:
+    """Return a ``CacheBackend`` in the shared (cross-tenant) namespace."""
+    from shared_configs.configs import DEFAULT_REDIS_PREFIX
+
+    return get_cache_backend(tenant_id=DEFAULT_REDIS_PREFIX)

backend/onyx/cache/interface.py

@@ -0,0 +1,89 @@
+import abc
+from enum import Enum
+
+
+class CacheBackendType(str, Enum):
+    REDIS = "redis"
+    POSTGRES = "postgres"
+
+
+class CacheLock(abc.ABC):
+    """Abstract distributed lock returned by CacheBackend.lock()."""
+
+    @abc.abstractmethod
+    def acquire(
+        self,
+        blocking: bool = True,
+        blocking_timeout: float | None = None,
+    ) -> bool:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def release(self) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def owned(self) -> bool:
+        raise NotImplementedError
+
+
+class CacheBackend(abc.ABC):
+    """Thin abstraction over a key-value cache with TTL, locks, and blocking lists.
+
+    Covers the subset of Redis operations used outside of Celery. When
+    CACHE_BACKEND=postgres, a PostgreSQL-backed implementation is used instead.
+    """
+
+    # -- basic key/value ---------------------------------------------------
+
+    @abc.abstractmethod
+    def get(self, key: str) -> bytes | None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def set(
+        self,
+        key: str,
+        value: str | bytes | int | float,
+        ex: int | None = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def delete(self, key: str) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def exists(self, key: str) -> bool:
+        raise NotImplementedError
+
+    # -- TTL ---------------------------------------------------------------
+
+    @abc.abstractmethod
+    def expire(self, key: str, seconds: int) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def ttl(self, key: str) -> int:
+        """Return remaining TTL in seconds. -1 if no expiry, -2 if key missing."""
+        raise NotImplementedError
+
+    # -- distributed lock --------------------------------------------------
+
+    @abc.abstractmethod
+    def lock(self, name: str, timeout: float | None = None) -> CacheLock:
+        raise NotImplementedError
+
+    # -- blocking list (used by MCP OAuth BLPOP pattern) -------------------
+
+    @abc.abstractmethod
+    def rpush(self, key: str, value: str | bytes) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def blpop(self, keys: list[str], timeout: int = 0) -> tuple[bytes, bytes] | None:
+        """Block until a value is available on one of *keys*, or *timeout* expires.
+
+        Returns ``(key, value)`` or ``None`` on timeout.
+        """
+        raise NotImplementedError

backend/onyx/cache/redis_backend.py

@@ -0,0 +1,92 @@
+from typing import cast
+
+from redis.client import Redis
+from redis.lock import Lock as RedisLock
+
+from onyx.cache.interface import CacheBackend
+from onyx.cache.interface import CacheLock
+
+
+class RedisCacheLock(CacheLock):
+    """Wraps ``redis.lock.Lock`` behind the ``CacheLock`` interface."""
+
+    def __init__(self, lock: RedisLock) -> None:
+        self._lock = lock
+
+    def acquire(
+        self,
+        blocking: bool = True,
+        blocking_timeout: float | None = None,
+    ) -> bool:
+        return bool(
+            self._lock.acquire(
+                blocking=blocking,
+                blocking_timeout=blocking_timeout,
+            )
+        )
+
+    def release(self) -> None:
+        self._lock.release()
+
+    def owned(self) -> bool:
+        return bool(self._lock.owned())
+
+
+class RedisCacheBackend(CacheBackend):
+    """``CacheBackend`` implementation that delegates to a ``redis.Redis`` client.
+
+    This is a thin pass-through — every method maps 1-to-1 to the underlying
+    Redis command.  ``TenantRedis`` key-prefixing is handled by the client
+    itself (provided by ``get_redis_client``).
+    """
+
+    def __init__(self, redis_client: Redis) -> None:
+        self._r = redis_client
+
+    # -- basic key/value ---------------------------------------------------
+
+    def get(self, key: str) -> bytes | None:
+        val = self._r.get(key)
+        if val is None:
+            return None
+        if isinstance(val, bytes):
+            return val
+        return str(val).encode()
+
+    def set(
+        self,
+        key: str,
+        value: str | bytes | int | float,
+        ex: int | None = None,
+    ) -> None:
+        self._r.set(key, value, ex=ex)
+
+    def delete(self, key: str) -> None:
+        self._r.delete(key)
+
+    def exists(self, key: str) -> bool:
+        return bool(self._r.exists(key))
+
+    # -- TTL ---------------------------------------------------------------
+
+    def expire(self, key: str, seconds: int) -> None:
+        self._r.expire(key, seconds)
+
+    def ttl(self, key: str) -> int:
+        return cast(int, self._r.ttl(key))
+
+    # -- distributed lock --------------------------------------------------
+
+    def lock(self, name: str, timeout: float | None = None) -> CacheLock:
+        return RedisCacheLock(self._r.lock(name, timeout=timeout))
+
+    # -- blocking list (MCP OAuth BLPOP pattern) ---------------------------
+
+    def rpush(self, key: str, value: str | bytes) -> None:
+        self._r.rpush(key, value)
+
+    def blpop(self, keys: list[str], timeout: int = 0) -> tuple[bytes, bytes] | None:
+        result = cast(list[bytes] | None, self._r.blpop(keys, timeout=timeout))
+        if result is None:
+            return None
+        return (result[0], result[1])

backend/onyx/chat/chat_state.py

@@ -3,7 +3,6 @@ import time
 from collections.abc import Callable
 from collections.abc import Generator
 from queue import Empty
-from typing import Any
 
 from onyx.chat.citation_processor import CitationMapping
 from onyx.chat.emitter import Emitter
@@ -163,13 +162,11 @@ class ChatStateContainer:
 
 
 def run_chat_loop_with_state_containers(
-    func: Callable[..., None],
+    chat_loop_func: Callable[[Emitter, ChatStateContainer], None],
     completion_callback: Callable[[ChatStateContainer], None],
     is_connected: Callable[[], bool],
     emitter: Emitter,
     state_container: ChatStateContainer,
-    *args: Any,
-    **kwargs: Any,
 ) -> Generator[Packet, None]:
     """
     Explicit wrapper function that runs a function in a background thread
@@ -180,19 +177,18 @@ def run_chat_loop_with_state_containers(
 
     Args:
         func: The function to wrap (should accept emitter and state_container as first and second args)
+        completion_callback: Callback function to call when the function completes
         emitter: Emitter instance for sending packets
         state_container: ChatStateContainer instance for accumulating state
         is_connected: Callable that returns False when stop signal is set
-        *args: Additional positional arguments for func
-        **kwargs: Additional keyword arguments for func
 
     Usage:
         packets = run_chat_loop_with_state_containers(
             my_func,
+            completion_callback=completion_callback,
             emitter=emitter,
             state_container=state_container,
             is_connected=check_func,
-            arg1, arg2, kwarg1=value1
         )
         for packet in packets:
             # Process packets
@@ -201,9 +197,7 @@ def run_chat_loop_with_state_containers(
 
     def run_with_exception_capture() -> None:
         try:
-            # Ensure state_container is passed explicitly, removing it from kwargs if present
-            kwargs_with_state = {**kwargs, "state_container": state_container}
-            func(emitter, *args, **kwargs_with_state)
+            chat_loop_func(emitter, state_container)
         except Exception as e:
             # If execution fails, emit an exception packet
             emitter.emit(

backend/onyx/chat/chat_utils.py

@@ -461,7 +461,7 @@ def _build_tool_call_response_history_message(
 def convert_chat_history(
     chat_history: list[ChatMessage],
     files: list[ChatLoadedFile],
-    project_image_files: list[ChatLoadedFile],
+    context_image_files: list[ChatLoadedFile],
     additional_context: str | None,
     token_counter: Callable[[str], int],
     tool_id_to_name_map: dict[int, str],
@@ -541,11 +541,11 @@ def convert_chat_history(
             )
 
             # Add the user message with image files attached
-            # If this is the last USER message, also include project_image_files
-            # Note: project image file tokens are NOT counted in the token count
+            # If this is the last USER message, also include context_image_files
+            # Note: context image file tokens are NOT counted in the token count
             if idx == last_user_message_idx:
-                if project_image_files:
-                    image_files.extend(project_image_files)
+                if context_image_files:
+                    image_files.extend(context_image_files)
 
                 if additional_context:
                     simple_messages.append(

backend/onyx/chat/llm_loop.py

@@ -1,6 +1,7 @@
 import json
 import time
 from collections.abc import Callable
+from typing import Any
 from typing import Literal
 
 from sqlalchemy.orm import Session
@@ -15,10 +16,10 @@ from onyx.chat.emitter import Emitter
 from onyx.chat.llm_step import extract_tool_calls_from_response_text
 from onyx.chat.llm_step import run_llm_step
 from onyx.chat.models import ChatMessageSimple
-from onyx.chat.models import ExtractedProjectFiles
+from onyx.chat.models import ContextFileMetadata
+from onyx.chat.models import ExtractedContextFiles
 from onyx.chat.models import FileToolMetadata
 from onyx.chat.models import LlmStepResult
-from onyx.chat.models import ProjectFileMetadata
 from onyx.chat.models import ToolCallSimple
 from onyx.chat.prompt_utils import build_reminder_message
 from onyx.chat.prompt_utils import build_system_prompt
@@ -203,17 +204,17 @@ def _try_fallback_tool_extraction(
 MAX_LLM_CYCLES = 6
 
 
-def _build_project_file_citation_mapping(
-    project_file_metadata: list[ProjectFileMetadata],
+def _build_context_file_citation_mapping(
+    file_metadata: list[ContextFileMetadata],
     starting_citation_num: int = 1,
 ) -> CitationMapping:
-    """Build citation mapping for project files.
+    """Build citation mapping for context files.
 
-    Converts project file metadata into SearchDoc objects that can be cited.
+    Converts context file metadata into SearchDoc objects that can be cited.
     Citation numbers start from the provided starting number.
 
     Args:
-        project_file_metadata: List of project file metadata
+        file_metadata: List of context file metadata
         starting_citation_num: Starting citation number (default: 1)
 
     Returns:
@@ -221,8 +222,7 @@ def _build_project_file_citation_mapping(
     """
     citation_mapping: CitationMapping = {}
 
-    for idx, file_meta in enumerate(project_file_metadata, start=starting_citation_num):
-        # Create a SearchDoc for each project file
+    for idx, file_meta in enumerate(file_metadata, start=starting_citation_num):
         search_doc = SearchDoc(
             document_id=file_meta.file_id,
             chunk_ind=0,
@@ -242,29 +242,28 @@ def _build_project_file_citation_mapping(
 
 
 def _build_project_message(
-    project_files: ExtractedProjectFiles | None,
+    context_files: ExtractedContextFiles | None,
     token_counter: Callable[[str], int] | None,
 ) -> list[ChatMessageSimple]:
-    """Build messages for project / tool-backed files.
+    """Build messages for context-injected / tool-backed files.
 
     Returns up to two messages:
-    1. The full-text project files message (if project_file_texts is populated).
+    1. The full-text files message (if file_texts is populated).
     2. A lightweight metadata message for files the LLM should access via the
-       FileReaderTool (e.g. oversized chat-attached files or project files that
-       don't fit in context).
+       FileReaderTool (e.g. oversized files that don't fit in context).
     """
-    if not project_files:
+    if not context_files:
         return []
 
     messages: list[ChatMessageSimple] = []
-    if project_files.project_file_texts:
+    if context_files.file_texts:
         messages.append(
-            _create_project_files_message(project_files, token_counter=None)
+            _create_context_files_message(context_files, token_counter=None)
         )
-    if project_files.file_metadata_for_tool and token_counter:
+    if context_files.file_metadata_for_tool and token_counter:
         messages.append(
             _create_file_tool_metadata_message(
-                project_files.file_metadata_for_tool, token_counter
+                context_files.file_metadata_for_tool, token_counter
             )
         )
     return messages
@@ -275,7 +274,7 @@ def construct_message_history(
     custom_agent_prompt: ChatMessageSimple | None,
     simple_chat_history: list[ChatMessageSimple],
     reminder_message: ChatMessageSimple | None,
-    project_files: ExtractedProjectFiles | None,
+    context_files: ExtractedContextFiles | None,
     available_tokens: int,
     last_n_user_messages: int | None = None,
     token_counter: Callable[[str], int] | None = None,
@@ -289,7 +288,7 @@ def construct_message_history(
 
     # Build the project / file-metadata messages up front so we can use their
     # actual token counts for the budget.
-    project_messages = _build_project_message(project_files, token_counter)
+    project_messages = _build_project_message(context_files, token_counter)
     project_messages_tokens = sum(m.token_count for m in project_messages)
 
     history_token_budget = available_tokens
@@ -445,17 +444,17 @@ def construct_message_history(
                     )
 
     # Attach project images to the last user message
-    if project_files and project_files.project_image_files:
+    if context_files and context_files.image_files:
         existing_images = last_user_message.image_files or []
         last_user_message = ChatMessageSimple(
             message=last_user_message.message,
             token_count=last_user_message.token_count,
             message_type=last_user_message.message_type,
-            image_files=existing_images + project_files.project_image_files,
+            image_files=existing_images + context_files.image_files,
         )
 
     # Build the final message list according to README ordering:
-    # [system], [history_before_last_user], [custom_agent], [project_files],
+    # [system], [history_before_last_user], [custom_agent], [context_files],
     # [forgotten_files], [last_user_message], [messages_after_last_user], [reminder]
     result = [system_prompt] if system_prompt else []
 
@@ -466,14 +465,14 @@ def construct_message_history(
     if custom_agent_prompt:
         result.append(custom_agent_prompt)
 
-    # 3. Add project files / file-metadata messages (inserted before last user message)
+    # 3. Add context files / file-metadata messages (inserted before last user message)
     result.extend(project_messages)
 
     # 4. Add forgotten-files metadata (right before the user's question)
     if forgotten_files_message:
         result.append(forgotten_files_message)
 
-    # 5. Add last user message (with project images attached)
+    # 5. Add last user message (with context images attached)
     result.append(last_user_message)
 
     # 6. Add messages after last user message (tool calls, responses, etc.)
@@ -532,11 +531,13 @@ def _create_file_tool_metadata_message(
     """
     lines = [
         "You have access to the following files. Use the read_file tool to "
-        "read sections of any file:"
+        "read sections of any file. You MUST pass the file_id UUID (not the "
+        "filename) to read_file:"
     ]
     for meta in file_metadata:
         lines.append(
-            f'- {meta.file_id}: "{meta.filename}" (~{meta.approx_char_count:,} chars)'
+            f'- file_id="{meta.file_id}" filename="{meta.filename}" '
+            f"(~{meta.approx_char_count:,} chars)"
         )
 
     message_content = "\n".join(lines)
@@ -547,11 +548,11 @@ def _create_file_tool_metadata_message(
     )
 
 
-def _create_project_files_message(
-    project_files: ExtractedProjectFiles,
+def _create_context_files_message(
+    context_files: ExtractedContextFiles,
     token_counter: Callable[[str], int] | None,  # noqa: ARG001
 ) -> ChatMessageSimple:
-    """Convert project files to a ChatMessageSimple message.
+    """Convert context files to a ChatMessageSimple message.
 
     Format follows the README specification for document representation.
     """
@@ -559,21 +560,25 @@ def _create_project_files_message(
 
     # Format as documents JSON as described in README
     documents_list = []
-    for idx, file_text in enumerate(project_files.project_file_texts, start=1):
-        documents_list.append(
-            {
-                "document": idx,
-                "contents": file_text,
-            }
+    for idx, file_text in enumerate(context_files.file_texts, start=1):
+        title = (
+            context_files.file_metadata[idx - 1].filename
+            if idx - 1 < len(context_files.file_metadata)
+            else None
         )
+        entry: dict[str, Any] = {"document": idx}
+        if title:
+            entry["title"] = title
+        entry["contents"] = file_text
+        documents_list.append(entry)
 
     documents_json = json.dumps({"documents": documents_list}, indent=2)
     message_content = f"Here are some documents provided for context, they may not all be relevant:\n{documents_json}"
 
-    # Use pre-calculated token count from project_files
+    # Use pre-calculated token count from context_files
     return ChatMessageSimple(
         message=message_content,
-        token_count=project_files.total_token_count,
+        token_count=context_files.total_token_count,
         message_type=MessageType.USER,
     )
 
@@ -584,7 +589,7 @@ def run_llm_loop(
     simple_chat_history: list[ChatMessageSimple],
     tools: list[Tool],
     custom_agent_prompt: str | None,
-    project_files: ExtractedProjectFiles,
+    context_files: ExtractedContextFiles,
     persona: Persona | None,
     user_memory_context: UserMemoryContext | None,
     llm: LLM,
@@ -627,9 +632,9 @@ def run_llm_loop(
 
         # Add project file citation mappings if project files are present
         project_citation_mapping: CitationMapping = {}
-        if project_files.project_file_metadata:
-            project_citation_mapping = _build_project_file_citation_mapping(
-                project_files.project_file_metadata
+        if context_files.file_metadata:
+            project_citation_mapping = _build_context_file_citation_mapping(
+                context_files.file_metadata
             )
             citation_processor.update_citation_mapping(project_citation_mapping)
 
@@ -647,7 +652,7 @@ def run_llm_loop(
         # TODO allow citing of images in Projects. Since attached to the last user message, it has no text associated with it.
         # One future workaround is to include the images as separate user messages with citation information and process those.
         always_cite_documents: bool = bool(
-            project_files.project_as_filter or project_files.project_file_texts
+            context_files.use_as_search_filter or context_files.file_texts
         )
         should_cite_documents: bool = False
         ran_image_gen: bool = False
@@ -788,7 +793,7 @@ def run_llm_loop(
                 custom_agent_prompt=custom_agent_prompt_msg,
                 simple_chat_history=simple_chat_history,
                 reminder_message=reminder_msg,
-                project_files=project_files,
+                context_files=context_files,
                 available_tokens=available_tokens,
                 token_counter=token_counter,
                 all_injected_file_metadata=all_injected_file_metadata,

backend/onyx/chat/models.py

@@ -31,13 +31,6 @@ class CustomToolResponse(BaseModel):
     tool_name: str
 
 
-class ProjectSearchConfig(BaseModel):
-    """Configuration for search tool availability in project context."""
-
-    search_usage: SearchToolUsage
-    disable_forced_tool: bool
-
-
 class CreateChatSessionID(BaseModel):
     chat_session_id: UUID
 
@@ -132,8 +125,8 @@ class ChatMessageSimple(BaseModel):
     file_id: str | None = None
 
 
-class ProjectFileMetadata(BaseModel):
-    """Metadata for a project file to enable citation support."""
+class ContextFileMetadata(BaseModel):
+    """Metadata for a context-injected file to enable citation support."""
 
     file_id: str
     filename: str
@@ -167,20 +160,28 @@ class ChatHistoryResult(BaseModel):
     all_injected_file_metadata: dict[str, FileToolMetadata]
 
 
-class ExtractedProjectFiles(BaseModel):
-    project_file_texts: list[str]
-    project_image_files: list[ChatLoadedFile]
-    project_as_filter: bool
+class ExtractedContextFiles(BaseModel):
+    """Result of attempting to load user files (from a project or persona) into context."""
+
+    file_texts: list[str]
+    image_files: list[ChatLoadedFile]
+    use_as_search_filter: bool
     total_token_count: int
-    # Metadata for project files to enable citations
-    project_file_metadata: list[ProjectFileMetadata]
-    # None if not a project
-    project_uncapped_token_count: int | None
     # Lightweight metadata for files exposed via FileReaderTool
-    # (populated when files don't fit in context and vector DB is disabled)
+    # (populated when files don't fit in context and vector DB is disabled).
+    file_metadata: list[ContextFileMetadata]
+    uncapped_token_count: int | None
     file_metadata_for_tool: list[FileToolMetadata] = []
 
 
+class SearchParams(BaseModel):
+    """Resolved search filter IDs and search-tool usage for a chat turn."""
+
+    search_project_id: int | None
+    search_persona_id: int | None
+    search_usage: SearchToolUsage
+
+
 class LlmStepResult(BaseModel):
     reasoning: str | None
     answer: str | None

backend/onyx/chat/process_message.py

@@ -3,6 +3,7 @@ IMPORTANT: familiarize yourself with the design concepts prior to contributing t
 An overview can be found in the README.md file in this directory.
 """
 
+import io
 import re
 import traceback
 from collections.abc import Callable
@@ -33,11 +34,11 @@ from onyx.chat.models import ChatBasicResponse
 from onyx.chat.models import ChatFullResponse
 from onyx.chat.models import ChatLoadedFile
 from onyx.chat.models import ChatMessageSimple
+from onyx.chat.models import ContextFileMetadata
 from onyx.chat.models import CreateChatSessionID
-from onyx.chat.models import ExtractedProjectFiles
+from onyx.chat.models import ExtractedContextFiles
 from onyx.chat.models import FileToolMetadata
-from onyx.chat.models import ProjectFileMetadata
-from onyx.chat.models import ProjectSearchConfig
+from onyx.chat.models import SearchParams
 from onyx.chat.models import StreamingError
 from onyx.chat.models import ToolCallResponse
 from onyx.chat.prompt_utils import calculate_reserved_tokens
@@ -62,11 +63,12 @@ from onyx.db.models import ChatSession
 from onyx.db.models import Persona
 from onyx.db.models import User
 from onyx.db.models import UserFile
-from onyx.db.projects import get_project_token_count
 from onyx.db.projects import get_user_files_from_project
 from onyx.db.tools import get_tools
 from onyx.deep_research.dr_loop import run_deep_research_llm_loop
+from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_store.models import ChatFileType
+from onyx.file_store.models import InMemoryChatFile
 from onyx.file_store.utils import load_in_memory_chat_files
 from onyx.file_store.utils import verify_user_files
 from onyx.llm.factory import get_llm_for_persona
@@ -139,12 +141,12 @@ def _collect_available_file_ids(
                 pass
 
     if project_id:
-        project_files = get_user_files_from_project(
+        user_files = get_user_files_from_project(
             project_id=project_id,
             user_id=user_id,
             db_session=db_session,
         )
-        for uf in project_files:
+        for uf in user_files:
             user_file_ids.add(uf.id)
 
     return _AvailableFiles(
@@ -192,9 +194,67 @@ def _convert_loaded_files_to_chat_files(
     return chat_files
 
 
-def _extract_project_file_texts_and_images(
+def resolve_context_user_files(
+    persona: Persona,
     project_id: int | None,
     user_id: UUID | None,
+    db_session: Session,
+) -> list[UserFile]:
+    """Apply the precedence rule to decide which user files to load.
+
+    A custom persona fully supersedes the project.  When a chat uses a
+    custom persona, the project is purely organisational — its files are
+    never loaded and never made searchable.
+
+    Custom persona → persona's own user_files (may be empty).
+    Default persona inside a project → project files.
+    Otherwise → empty list.
+    """
+    if persona.id != DEFAULT_PERSONA_ID:
+        return list(persona.user_files) if persona.user_files else []
+    if project_id:
+        return get_user_files_from_project(
+            project_id=project_id,
+            user_id=user_id,
+            db_session=db_session,
+        )
+    return []
+
+
+def _empty_extracted_context_files() -> ExtractedContextFiles:
+    return ExtractedContextFiles(
+        file_texts=[],
+        image_files=[],
+        use_as_search_filter=False,
+        total_token_count=0,
+        file_metadata=[],
+        uncapped_token_count=None,
+    )
+
+
+def _extract_text_from_in_memory_file(f: InMemoryChatFile) -> str | None:
+    """Extract text content from an InMemoryChatFile.
+
+    PLAIN_TEXT: the content is pre-extracted UTF-8 plaintext stored during
+    ingestion — decode directly.
+    DOC / CSV / other text types: the content is the original file bytes —
+    use extract_file_text which handles encoding detection and format parsing.
+    """
+    try:
+        if f.file_type == ChatFileType.PLAIN_TEXT:
+            return f.content.decode("utf-8", errors="ignore").replace("\x00", "")
+        return extract_file_text(
+            file=io.BytesIO(f.content),
+            file_name=f.filename or "",
+            break_on_unprocessable=False,
+        )
+    except Exception:
+        logger.warning(f"Failed to extract text from file {f.file_id}", exc_info=True)
+        return None
+
+
+def extract_context_files(
+    user_files: list[UserFile],
     llm_max_context_window: int,
     reserved_token_count: int,
     db_session: Session,
@@ -203,8 +263,12 @@ def _extract_project_file_texts_and_images(
     # 60% of the LLM's max context window. The other benefit is that for projects with
     # more files, this makes it so that we don't throw away the history too quickly every time.
     max_llm_context_percentage: float = 0.6,
-) -> ExtractedProjectFiles:
-    """Extract text content from project files if they fit within the context window.
+) -> ExtractedContextFiles:
+    """Load user files into context if they fit; otherwise flag for search.
+
+    The caller is responsible for deciding *which* user files to pass in
+    (project files, persona files, etc.).  This function only cares about
+    the all-or-nothing fit check and the actual content loading.
 
     Args:
         project_id: The project ID to load files from
@@ -213,160 +277,95 @@ def _extract_project_file_texts_and_images(
         reserved_token_count: Number of tokens to reserve for other content
         db_session: Database session
         max_llm_context_percentage: Maximum percentage of the LLM context window to use.
-
     Returns:
-        ExtractedProjectFiles containing:
-        - List of text content strings from project files (text files only)
-        - List of image files from project (ChatLoadedFile objects)
-        - Project id if the the project should be provided as a filter in search or None if not.
+        ExtractedContextFiles containing:
+        - List of text content strings from context files (text files only)
+        - List of image files from context (ChatLoadedFile objects)
         - Total token count of all extracted files
+        - File metadata for context files
+        - Uncapped token count of all extracted files
+        - File metadata for files that don't fit in context and vector DB is disabled
     """
-    # TODO I believe this is not handling all file types correctly.
-    project_as_filter = False
-    if not project_id:
-        return ExtractedProjectFiles(
-            project_file_texts=[],
-            project_image_files=[],
-            project_as_filter=False,
-            total_token_count=0,
-            project_file_metadata=[],
-            project_uncapped_token_count=None,
-        )
+    # TODO(yuhong): I believe this is not handling all file types correctly.
 
+    if not user_files:
+        return _empty_extracted_context_files()
+
+    aggregate_tokens = sum(uf.token_count or 0 for uf in user_files)
     max_actual_tokens = (
         llm_max_context_window - reserved_token_count
     ) * max_llm_context_percentage
 
-    # Calculate total token count for all user files in the project
-    project_tokens = get_project_token_count(
-        project_id=project_id,
-        user_id=user_id,
+    if aggregate_tokens >= max_actual_tokens:
+        tool_metadata = []
+        use_as_search_filter = not DISABLE_VECTOR_DB
+        if DISABLE_VECTOR_DB:
+            tool_metadata = _build_file_tool_metadata_for_user_files(user_files)
+        return ExtractedContextFiles(
+            file_texts=[],
+            image_files=[],
+            use_as_search_filter=use_as_search_filter,
+            total_token_count=0,
+            file_metadata=[],
+            uncapped_token_count=aggregate_tokens,
+            file_metadata_for_tool=tool_metadata,
+        )
+
+    # Files fit — load them into context
+    user_file_map = {str(uf.id): uf for uf in user_files}
+    in_memory_files = load_in_memory_chat_files(
+        user_file_ids=[uf.id for uf in user_files],
         db_session=db_session,
     )
 
-    project_file_texts: list[str] = []
-    project_image_files: list[ChatLoadedFile] = []
-    project_file_metadata: list[ProjectFileMetadata] = []
+    file_texts: list[str] = []
+    image_files: list[ChatLoadedFile] = []
+    file_metadata: list[ContextFileMetadata] = []
     total_token_count = 0
-    if project_tokens < max_actual_tokens:
-        # Load project files into memory using cached plaintext when available
-        project_user_files = get_user_files_from_project(
-            project_id=project_id,
-            user_id=user_id,
-            db_session=db_session,
-        )
-        if project_user_files:
-            # Create a mapping from file_id to UserFile for token count lookup
-            user_file_map = {str(file.id): file for file in project_user_files}
 
-            project_file_ids = [file.id for file in project_user_files]
-            in_memory_project_files = load_in_memory_chat_files(
-                user_file_ids=project_file_ids,
-                db_session=db_session,
+    for f in in_memory_files:
+        uf = user_file_map.get(str(f.file_id))
+        if f.file_type.is_text_file():
+            text_content = _extract_text_from_in_memory_file(f)
+            if not text_content:
+                continue
+            file_texts.append(text_content)
+            file_metadata.append(
+                ContextFileMetadata(
+                    file_id=str(f.file_id),
+                    filename=f.filename or f"file_{f.file_id}",
+                    file_content=text_content,
+                )
+            )
+            if uf and uf.token_count:
+                total_token_count += uf.token_count
+        elif f.file_type == ChatFileType.IMAGE:
+            token_count = uf.token_count if uf and uf.token_count else 0
+            total_token_count += token_count
+            image_files.append(
+                ChatLoadedFile(
+                    file_id=f.file_id,
+                    content=f.content,
+                    file_type=f.file_type,
+                    filename=f.filename,
+                    content_text=None,
+                    token_count=token_count,
+                )
             )
 
-            # Extract text content from loaded files
-            for file in in_memory_project_files:
-                if file.file_type.is_text_file():
-                    try:
-                        text_content = file.content.decode("utf-8", errors="ignore")
-                        # Strip null bytes
-                        text_content = text_content.replace("\x00", "")
-                        if text_content:
-                            project_file_texts.append(text_content)
-                            # Add metadata for citation support
-                            project_file_metadata.append(
-                                ProjectFileMetadata(
-                                    file_id=str(file.file_id),
-                                    filename=file.filename or f"file_{file.file_id}",
-                                    file_content=text_content,
-                                )
-                            )
-                            # Add token count for text file
-                            user_file = user_file_map.get(str(file.file_id))
-                            if user_file and user_file.token_count:
-                                total_token_count += user_file.token_count
-                    except Exception:
-                        # Skip files that can't be decoded
-                        pass
-                elif file.file_type == ChatFileType.IMAGE:
-                    # Convert InMemoryChatFile to ChatLoadedFile
-                    user_file = user_file_map.get(str(file.file_id))
-                    token_count = (
-                        user_file.token_count
-                        if user_file and user_file.token_count
-                        else 0
-                    )
-                    total_token_count += token_count
-                    chat_loaded_file = ChatLoadedFile(
-                        file_id=file.file_id,
-                        content=file.content,
-                        file_type=file.file_type,
-                        filename=file.filename,
-                        content_text=None,  # Images don't have text content
-                        token_count=token_count,
-                    )
-                    project_image_files.append(chat_loaded_file)
-    else:
-        if DISABLE_VECTOR_DB:
-            # Without a vector DB we can't use project-as-filter search.
-            # Instead, build lightweight metadata so the LLM can call the
-            # FileReaderTool to inspect individual files on demand.
-            file_metadata_for_tool = _build_file_tool_metadata_for_project(
-                project_id=project_id,
-                user_id=user_id,
-                db_session=db_session,
-            )
-            return ExtractedProjectFiles(
-                project_file_texts=[],
-                project_image_files=[],
-                project_as_filter=False,
-                total_token_count=0,
-                project_file_metadata=[],
-                project_uncapped_token_count=project_tokens,
-                file_metadata_for_tool=file_metadata_for_tool,
-            )
-        project_as_filter = True
-
-    return ExtractedProjectFiles(
-        project_file_texts=project_file_texts,
-        project_image_files=project_image_files,
-        project_as_filter=project_as_filter,
+    return ExtractedContextFiles(
+        file_texts=file_texts,
+        image_files=image_files,
+        use_as_search_filter=False,
         total_token_count=total_token_count,
-        project_file_metadata=project_file_metadata,
-        project_uncapped_token_count=project_tokens,
+        file_metadata=file_metadata,
+        uncapped_token_count=aggregate_tokens,
     )
 
 
 APPROX_CHARS_PER_TOKEN = 4
 
 
-def _build_file_tool_metadata_for_project(
-    project_id: int,
-    user_id: UUID | None,
-    db_session: Session,
-) -> list[FileToolMetadata]:
-    """Build lightweight FileToolMetadata for every file in a project.
-
-    Used when files are too large to fit in context and the vector DB is
-    disabled, so the LLM needs to know which files it can read via the
-    FileReaderTool.
-    """
-    project_user_files = get_user_files_from_project(
-        project_id=project_id,
-        user_id=user_id,
-        db_session=db_session,
-    )
-    return [
-        FileToolMetadata(
-            file_id=str(uf.id),
-            filename=uf.name,
-            approx_char_count=(uf.token_count or 0) * APPROX_CHARS_PER_TOKEN,
-        )
-        for uf in project_user_files
-    ]
-
-
 def _build_file_tool_metadata_for_user_files(
     user_files: list[UserFile],
 ) -> list[FileToolMetadata]:
@@ -381,55 +380,46 @@ def _build_file_tool_metadata_for_user_files(
     ]
 
 
-def _get_project_search_availability(
+def determine_search_params(
+    persona_id: int,
     project_id: int | None,
-    persona_id: int | None,
-    loaded_project_files: bool,
-    project_has_files: bool,
-    forced_tool_id: int | None,
-    search_tool_id: int | None,
-) -> ProjectSearchConfig:
-    """Determine search tool availability based on project context.
+    extracted_context_files: ExtractedContextFiles,
+) -> SearchParams:
+    """Decide which search filter IDs and search-tool usage apply for a chat turn.
 
-    Search is disabled when ALL of the following are true:
-    - User is in a project
-    - Using the default persona (not a custom agent)
-    - Project files are already loaded in context
+    A custom persona fully supersedes the project — project files are never
+    searchable and the search tool config is entirely controlled by the
+    persona.  The project_id filter is only set for the default persona.
 
-    When search is disabled and the user tried to force the search tool,
-    that forcing is also disabled.
-
-    Returns AUTO (follow persona config) in all other cases.
+    For the default persona inside a project:
+      - Files overflow  → ENABLED  (vector DB scopes to these files)
+      - Files fit       → DISABLED (content already in prompt)
+      - No files at all → DISABLED (nothing to search)
     """
-    # Not in a project, this should have no impact on search tool availability
-    if not project_id:
-        return ProjectSearchConfig(
-            search_usage=SearchToolUsage.AUTO, disable_forced_tool=False
-        )
+    is_custom_persona = persona_id != DEFAULT_PERSONA_ID
 
-    # Custom persona in project - let persona config decide
-    # Even if there are no files in the project, it's still guided by the persona config.
-    if persona_id != DEFAULT_PERSONA_ID:
-        return ProjectSearchConfig(
-            search_usage=SearchToolUsage.AUTO, disable_forced_tool=False
-        )
+    search_project_id: int | None = None
+    search_persona_id: int | None = None
+    if extracted_context_files.use_as_search_filter:
+        if is_custom_persona:
+            search_persona_id = persona_id
+        else:
+            search_project_id = project_id
 
-    # If in a project with the default persona and the files have been already loaded into the context or
-    # there are no files in the project, disable search as there is nothing to search for.
-    if loaded_project_files or not project_has_files:
-        user_forced_search = (
-            forced_tool_id is not None
-            and search_tool_id is not None
-            and forced_tool_id == search_tool_id
-        )
-        return ProjectSearchConfig(
-            search_usage=SearchToolUsage.DISABLED,
-            disable_forced_tool=user_forced_search,
-        )
+    search_usage = SearchToolUsage.AUTO
+    if not is_custom_persona and project_id:
+        has_context_files = bool(extracted_context_files.uncapped_token_count)
+        files_loaded_in_context = bool(extracted_context_files.file_texts)
 
-    # Default persona in a project with files, but also the files have not been loaded into the context already.
-    return ProjectSearchConfig(
-        search_usage=SearchToolUsage.ENABLED, disable_forced_tool=False
+        if extracted_context_files.use_as_search_filter:
+            search_usage = SearchToolUsage.ENABLED
+        elif files_loaded_in_context or not has_context_files:
+            search_usage = SearchToolUsage.DISABLED
+
+    return SearchParams(
+        search_project_id=search_project_id,
+        search_persona_id=search_persona_id,
+        search_usage=search_usage,
     )
 
 
@@ -661,26 +651,37 @@ def handle_stream_message_objects(
             user_memory_context=prompt_memory_context,
         )
 
-        # Process projects, if all of the files fit in the context, it doesn't need to use RAG
-        extracted_project_files = _extract_project_file_texts_and_images(
+        # Determine which user files to use.  A custom persona fully
+        # supersedes the project — project files are never loaded or
+        # searchable when a custom persona is in play.  Only the default
+        # persona inside a project uses the project's files.
+        context_user_files = resolve_context_user_files(
+            persona=persona,
             project_id=chat_session.project_id,
             user_id=user_id,
+            db_session=db_session,
+        )
+
+        extracted_context_files = extract_context_files(
+            user_files=context_user_files,
             llm_max_context_window=llm.config.max_input_tokens,
             reserved_token_count=reserved_token_count,
             db_session=db_session,
         )
 
-        # When the vector DB is disabled, persona-attached user_files have no
-        # search pipeline path. Inject them as file_metadata_for_tool so the
-        # LLM can read them via the FileReaderTool.
-        if DISABLE_VECTOR_DB and persona.user_files:
-            persona_file_metadata = _build_file_tool_metadata_for_user_files(
-                persona.user_files
-            )
-            # Merge persona file metadata into the extracted project files
-            extracted_project_files.file_metadata_for_tool.extend(persona_file_metadata)
+        search_params = determine_search_params(
+            persona_id=persona.id,
+            project_id=chat_session.project_id,
+            extracted_context_files=extracted_context_files,
+        )
+
+        # Also grant access to persona-attached user files for FileReaderTool
+        if persona.user_files:
+            existing = set(available_files.user_file_ids)
+            for uf in persona.user_files:
+                if uf.id not in existing:
+                    available_files.user_file_ids.append(uf.id)
 
-        # Build a mapping of tool_id to tool_name for history reconstruction
         all_tools = get_tools(db_session)
         tool_id_to_name_map = {tool.id: tool.name for tool in all_tools}
 
@@ -689,30 +690,17 @@ def handle_stream_message_objects(
             None,
         )
 
-        # Determine if search should be disabled for this project context
         forced_tool_id = new_msg_req.forced_tool_id
-        project_search_config = _get_project_search_availability(
-            project_id=chat_session.project_id,
-            persona_id=persona.id,
-            loaded_project_files=bool(extracted_project_files.project_file_texts),
-            project_has_files=bool(
-                extracted_project_files.project_uncapped_token_count
-            ),
-            forced_tool_id=new_msg_req.forced_tool_id,
-            search_tool_id=search_tool_id,
-        )
-        if project_search_config.disable_forced_tool:
+        if (
+            search_params.search_usage == SearchToolUsage.DISABLED
+            and forced_tool_id is not None
+            and search_tool_id is not None
+            and forced_tool_id == search_tool_id
+        ):
             forced_tool_id = None
 
         emitter = get_default_emitter()
 
-        # Also grant access to persona-attached user files
-        if persona.user_files:
-            existing = set(available_files.user_file_ids)
-            for uf in persona.user_files:
-                if uf.id not in existing:
-                    available_files.user_file_ids.append(uf.id)
-
         # Construct tools based on the persona configurations
         tool_dict = construct_tools(
             persona=persona,
@@ -722,11 +710,8 @@ def handle_stream_message_objects(
             llm=llm,
             search_tool_config=SearchToolConfig(
                 user_selected_filters=new_msg_req.internal_search_filters,
-                project_id=(
-                    chat_session.project_id
-                    if extracted_project_files.project_as_filter
-                    else None
-                ),
+                project_id=search_params.search_project_id,
+                persona_id=search_params.search_persona_id,
                 bypass_acl=bypass_acl,
                 slack_context=slack_context,
                 enable_slack_search=_should_enable_slack_search(
@@ -744,7 +729,7 @@ def handle_stream_message_objects(
                 chat_file_ids=available_files.chat_file_ids,
             ),
             allowed_tool_ids=new_msg_req.allowed_tool_ids,
-            search_usage_forcing_setting=project_search_config.search_usage,
+            search_usage_forcing_setting=search_params.search_usage,
         )
         tools: list[Tool] = []
         for tool_list in tool_dict.values():
@@ -783,7 +768,7 @@ def handle_stream_message_objects(
         chat_history_result = convert_chat_history(
             chat_history=chat_history,
             files=files,
-            project_image_files=extracted_project_files.project_image_files,
+            context_image_files=extracted_context_files.image_files,
             additional_context=additional_context,
             token_counter=token_counter,
             tool_id_to_name_map=tool_id_to_name_map,
@@ -879,46 +864,54 @@ def handle_stream_message_objects(
             # (user has already responded to a clarification question)
             skip_clarification = is_last_assistant_message_clarification(chat_history)
 
+            # NOTE: we _could_ pass in a zero argument function since emitter and state_container
+            # are just passed in immediately anyways, but the abstraction is cleaner this way.
             yield from run_chat_loop_with_state_containers(
-                run_deep_research_llm_loop,
+                lambda emitter, state_container: run_deep_research_llm_loop(
+                    emitter=emitter,
+                    state_container=state_container,
+                    simple_chat_history=simple_chat_history,
+                    tools=tools,
+                    custom_agent_prompt=custom_agent_prompt,
+                    llm=llm,
+                    token_counter=token_counter,
+                    db_session=db_session,
+                    skip_clarification=skip_clarification,
+                    user_identity=user_identity,
+                    chat_session_id=str(chat_session.id),
+                    all_injected_file_metadata=all_injected_file_metadata,
+                ),
                 llm_loop_completion_callback,
                 is_connected=check_is_connected,
                 emitter=emitter,
                 state_container=state_container,
-                simple_chat_history=simple_chat_history,
-                tools=tools,
-                custom_agent_prompt=custom_agent_prompt,
-                llm=llm,
-                token_counter=token_counter,
-                db_session=db_session,
-                skip_clarification=skip_clarification,
-                user_identity=user_identity,
-                chat_session_id=str(chat_session.id),
-                all_injected_file_metadata=all_injected_file_metadata,
             )
         else:
             yield from run_chat_loop_with_state_containers(
-                run_llm_loop,
+                lambda emitter, state_container: run_llm_loop(
+                    emitter=emitter,
+                    state_container=state_container,
+                    simple_chat_history=simple_chat_history,
+                    tools=tools,
+                    custom_agent_prompt=custom_agent_prompt,
+                    context_files=extracted_context_files,
+                    persona=persona,
+                    user_memory_context=user_memory_context,
+                    llm=llm,
+                    token_counter=token_counter,
+                    db_session=db_session,
+                    forced_tool_id=forced_tool_id,
+                    user_identity=user_identity,
+                    chat_session_id=str(chat_session.id),
+                    chat_files=chat_files_for_tools,
+                    include_citations=new_msg_req.include_citations,
+                    all_injected_file_metadata=all_injected_file_metadata,
+                    inject_memories_in_prompt=user.use_memories,
+                ),
                 llm_loop_completion_callback,
                 is_connected=check_is_connected,  # Not passed through to run_llm_loop
                 emitter=emitter,
                 state_container=state_container,
-                simple_chat_history=simple_chat_history,
-                tools=tools,
-                custom_agent_prompt=custom_agent_prompt,
-                project_files=extracted_project_files,
-                persona=persona,
-                user_memory_context=user_memory_context,
-                llm=llm,
-                token_counter=token_counter,
-                db_session=db_session,
-                forced_tool_id=forced_tool_id,
-                user_identity=user_identity,
-                chat_session_id=str(chat_session.id),
-                chat_files=chat_files_for_tools,
-                include_citations=new_msg_req.include_citations,
-                all_injected_file_metadata=all_injected_file_metadata,
-                inject_memories_in_prompt=user.use_memories,
             )
 
     except ValueError as e:

backend/onyx/configs/app_configs.py

@@ -6,6 +6,7 @@ from datetime import timezone
 from typing import cast
 
 from onyx.auth.schemas import AuthBackend
+from onyx.cache.interface import CacheBackendType
 from onyx.configs.constants import AuthType
 from onyx.configs.constants import QueryHistoryType
 from onyx.file_processing.enums import HtmlBasedConnectorTransformLinksStrategy
@@ -54,6 +55,12 @@ DISABLE_USER_KNOWLEDGE = os.environ.get("DISABLE_USER_KNOWLEDGE", "").lower() ==
 # are disabled but core chat, tools, user file uploads, and Projects still work.
 DISABLE_VECTOR_DB = os.environ.get("DISABLE_VECTOR_DB", "").lower() == "true"
 
+# Which backend to use for caching, locks, and ephemeral state.
+# "redis" (default) or "postgres" (only valid when DISABLE_VECTOR_DB=true).
+CACHE_BACKEND = CacheBackendType(
+    os.environ.get("CACHE_BACKEND", CacheBackendType.REDIS)
+)
+
 # Maximum token count for a single uploaded file. Files exceeding this are rejected.
 # Defaults to 100k tokens (or 10M when vector DB is disabled).
 _DEFAULT_FILE_TOKEN_LIMIT = 10_000_000 if DISABLE_VECTOR_DB else 100_000

backend/onyx/connectors/gong/connector.py

@@ -32,6 +32,8 @@ class GongConnector(LoadConnector, PollConnector):
     BASE_URL = "https://api.gong.io"
     MAX_CALL_DETAILS_ATTEMPTS = 6
     CALL_DETAILS_DELAY = 30  # in seconds
+    # Gong API limit is 3 calls/sec — stay safely under it
+    MIN_REQUEST_INTERVAL = 0.5  # seconds between requests
 
     def __init__(
         self,
@@ -45,9 +47,13 @@ class GongConnector(LoadConnector, PollConnector):
         self.continue_on_fail = continue_on_fail
         self.auth_token_basic: str | None = None
         self.hide_user_info = hide_user_info
+        self._last_request_time: float = 0.0
 
+        # urllib3 Retry already respects the Retry-After header by default
+        # (respect_retry_after_header=True), so on 429 it will sleep for the
+        # duration Gong specifies before retrying.
         retry_strategy = Retry(
-            total=5,
+            total=10,
             backoff_factor=2,
             status_forcelist=[429, 500, 502, 503, 504],
         )
@@ -61,8 +67,24 @@ class GongConnector(LoadConnector, PollConnector):
         url = f"{GongConnector.BASE_URL}{endpoint}"
         return url
 
+    def _throttled_request(
+        self, method: str, url: str, **kwargs: Any
+    ) -> requests.Response:
+        """Rate-limited request wrapper. Enforces MIN_REQUEST_INTERVAL between
+        calls to stay under Gong's 3 calls/sec limit and avoid triggering 429s."""
+        now = time.monotonic()
+        elapsed = now - self._last_request_time
+        if elapsed < self.MIN_REQUEST_INTERVAL:
+            time.sleep(self.MIN_REQUEST_INTERVAL - elapsed)
+
+        response = self._session.request(method, url, **kwargs)
+        self._last_request_time = time.monotonic()
+        return response
+
     def _get_workspace_id_map(self) -> dict[str, str]:
-        response = self._session.get(GongConnector.make_url("/v2/workspaces"))
+        response = self._throttled_request(
+            "GET", GongConnector.make_url("/v2/workspaces")
+        )
         response.raise_for_status()
 
         workspaces_details = response.json().get("workspaces")
@@ -106,8 +128,8 @@ class GongConnector(LoadConnector, PollConnector):
                     del body["filter"]["workspaceId"]
 
             while True:
-                response = self._session.post(
-                    GongConnector.make_url("/v2/calls/transcript"), json=body
+                response = self._throttled_request(
+                    "POST", GongConnector.make_url("/v2/calls/transcript"), json=body
                 )
                 # If no calls in the range, just break out
                 if response.status_code == 404:
@@ -142,8 +164,8 @@ class GongConnector(LoadConnector, PollConnector):
             "contentSelector": {"exposedFields": {"parties": True}},
         }
 
-        response = self._session.post(
-            GongConnector.make_url("/v2/calls/extensive"), json=body
+        response = self._throttled_request(
+            "POST", GongConnector.make_url("/v2/calls/extensive"), json=body
         )
         response.raise_for_status()
 
@@ -194,7 +216,8 @@ class GongConnector(LoadConnector, PollConnector):
             # There's a likely race condition in the API where a transcript will have a
             # call id but the call to v2/calls/extensive will not return all of the id's
             # retry with exponential backoff has been observed to mitigate this
-            # in ~2 minutes
+            # in ~2 minutes. After max attempts, proceed with whatever we have —
+            # the per-call loop below will skip missing IDs gracefully.
             current_attempt = 0
             while True:
                 current_attempt += 1
@@ -213,11 +236,14 @@ class GongConnector(LoadConnector, PollConnector):
                     f"missing_call_ids={missing_call_ids}"
                 )
                 if current_attempt >= self.MAX_CALL_DETAILS_ATTEMPTS:
-                    raise RuntimeError(
-                        f"Attempt count exceeded for _get_call_details_by_ids: "
-                        f"missing_call_ids={missing_call_ids} "
-                        f"max_attempts={self.MAX_CALL_DETAILS_ATTEMPTS}"
+                    logger.error(
+                        f"Giving up on missing call id's after "
+                        f"{self.MAX_CALL_DETAILS_ATTEMPTS} attempts: "
+                        f"missing_call_ids={missing_call_ids} — "
+                        f"proceeding with {len(call_details_map)} of "
+                        f"{len(transcript_call_ids)} calls"
                     )
+                    break
 
                 wait_seconds = self.CALL_DETAILS_DELAY * pow(2, current_attempt - 1)
                 logger.warning(

backend/onyx/connectors/sharepoint/connector.py

@@ -23,7 +23,6 @@ from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.serialization import pkcs12
 from office365.graph_client import GraphClient  # type: ignore[import-untyped]
-from office365.intune.organizations.organization import Organization  # type: ignore[import-untyped]
 from office365.onedrive.driveitems.driveItem import DriveItem  # type: ignore[import-untyped]
 from office365.onedrive.sites.site import Site  # type: ignore[import-untyped]
 from office365.onedrive.sites.sites_with_root import SitesWithRoot  # type: ignore[import-untyped]
@@ -872,6 +871,56 @@ class SharepointConnector(
                     "Site URLs must be full Sharepoint URLs (e.g. https://your-tenant.sharepoint.com/sites/your-site or https://your-tenant.sharepoint.com/teams/your-team)"
                 )
 
+    def _extract_tenant_domain_from_sites(self) -> str | None:
+        """Extract the tenant domain from configured site URLs.
+
+        Site URLs look like https://{tenant}.sharepoint.com/sites/... so the
+        tenant domain is the first label of the hostname.
+        """
+        for site_url in self.sites:
+            try:
+                hostname = urlsplit(site_url.strip()).hostname
+            except ValueError:
+                continue
+            if not hostname:
+                continue
+            tenant = hostname.split(".")[0]
+            if tenant:
+                return tenant
+        logger.warning(f"No tenant domain found from {len(self.sites)} sites")
+        return None
+
+    def _resolve_tenant_domain_from_root_site(self) -> str:
+        """Resolve tenant domain via GET /v1.0/sites/root which only requires
+        Sites.Read.All (a permission the connector already needs)."""
+        root_site = self.graph_client.sites.root.get().execute_query()
+        hostname = root_site.site_collection.hostname
+        if not hostname:
+            raise ConnectorValidationError(
+                "Could not determine tenant domain from root site"
+            )
+        tenant_domain = hostname.split(".")[0]
+        logger.info(
+            "Resolved tenant domain '%s' from root site hostname '%s'",
+            tenant_domain,
+            hostname,
+        )
+        return tenant_domain
+
+    def _resolve_tenant_domain(self) -> str:
+        """Determine the tenant domain, preferring site URLs over a Graph API
+        call to avoid needing extra permissions."""
+        from_sites = self._extract_tenant_domain_from_sites()
+        if from_sites:
+            logger.info(
+                "Resolved tenant domain '%s' from site URLs",
+                from_sites,
+            )
+            return from_sites
+
+        logger.info("No site URLs available; resolving tenant domain from root site")
+        return self._resolve_tenant_domain_from_root_site()
+
     @property
     def graph_client(self) -> GraphClient:
         if self._graph_client is None:
@@ -1589,6 +1638,11 @@ class SharepointConnector(
         sp_private_key = credentials.get("sp_private_key")
         sp_certificate_password = credentials.get("sp_certificate_password")
 
+        if not sp_client_id:
+            raise ConnectorValidationError("Client ID is required")
+        if not sp_directory_id:
+            raise ConnectorValidationError("Directory (tenant) ID is required")
+
         authority_url = f"{self.authority_host}/{sp_directory_id}"
 
         if auth_method == SharepointAuthMethod.CERTIFICATE.value:
@@ -1641,21 +1695,7 @@ class SharepointConnector(
             _acquire_token_for_graph, environment=self._azure_environment
         )
         if auth_method == SharepointAuthMethod.CERTIFICATE.value:
-            org = self.graph_client.organization.get().execute_query()
-            if not org or len(org) == 0:
-                raise ConnectorValidationError("No organization found")
-
-            tenant_info: Organization = org[
-                0
-            ]  # Access first item directly from collection
-            if not tenant_info.verified_domains:
-                raise ConnectorValidationError("No verified domains found for tenant")
-
-            sp_tenant_domain = tenant_info.verified_domains[0].name
-            if not sp_tenant_domain:
-                raise ConnectorValidationError("No verified domains found for tenant")
-            # remove the .onmicrosoft.com part
-            self.sp_tenant_domain = sp_tenant_domain.split(".")[0]
+            self.sp_tenant_domain = self._resolve_tenant_domain()
         return None
 
     def _get_drive_names_for_site(self, site_url: str) -> list[str]:

backend/onyx/db/chat.py

@@ -98,6 +98,7 @@ def get_chat_sessions_by_user(
     db_session: Session,
     include_onyxbot_flows: bool = False,
     limit: int = 50,
+    before: datetime | None = None,
     project_id: int | None = None,
     only_non_project_chats: bool = False,
     include_failed_chats: bool = False,
@@ -112,6 +113,9 @@ def get_chat_sessions_by_user(
     if deleted is not None:
         stmt = stmt.where(ChatSession.deleted == deleted)
 
+    if before is not None:
+        stmt = stmt.where(ChatSession.time_updated < before)
+
     if limit:
         stmt = stmt.limit(limit)
 

backend/onyx/db/engine/async_sql_engine.py

@@ -21,8 +21,8 @@ from onyx.configs.app_configs import POSTGRES_POOL_RECYCLE
 from onyx.configs.app_configs import POSTGRES_PORT
 from onyx.configs.app_configs import POSTGRES_USE_NULL_POOL
 from onyx.configs.app_configs import POSTGRES_USER
+from onyx.db.engine.iam_auth import create_ssl_context_if_iam
 from onyx.db.engine.iam_auth import get_iam_auth_token
-from onyx.db.engine.iam_auth import ssl_context
 from onyx.db.engine.sql_engine import ASYNC_DB_API
 from onyx.db.engine.sql_engine import build_connection_string
 from onyx.db.engine.sql_engine import is_valid_schema_name
@@ -66,7 +66,7 @@ def get_sqlalchemy_async_engine() -> AsyncEngine:
         if app_name:
             connect_args["server_settings"] = {"application_name": app_name}
 
-        connect_args["ssl"] = ssl_context
+        connect_args["ssl"] = create_ssl_context_if_iam()
 
         engine_kwargs = {
             "connect_args": connect_args,
@@ -97,7 +97,7 @@ def get_sqlalchemy_async_engine() -> AsyncEngine:
                 user = POSTGRES_USER
                 token = get_iam_auth_token(host, port, user, AWS_REGION_NAME)
                 cparams["password"] = token
-                cparams["ssl"] = ssl_context
+                cparams["ssl"] = create_ssl_context_if_iam()
 
     return _ASYNC_ENGINE
 

backend/onyx/db/engine/iam_auth.py

@@ -1,3 +1,4 @@
+import functools
 import os
 import ssl
 from typing import Any
@@ -48,11 +49,9 @@ def provide_iam_token(
         configure_psycopg2_iam_auth(cparams, host, port, user, region)
 
 
+@functools.cache
 def create_ssl_context_if_iam() -> ssl.SSLContext | None:
     """Create an SSL context if IAM authentication is enabled, else return None."""
     if USE_IAM_AUTH:
         return ssl.create_default_context(cafile=SSL_CERT_FILE)
     return None
-
-
-ssl_context = create_ssl_context_if_iam()

backend/onyx/db/enums.py

@@ -186,6 +186,7 @@ class EmbeddingPrecision(str, PyEnum):
 
 class UserFileStatus(str, PyEnum):
     PROCESSING = "PROCESSING"
+    INDEXING = "INDEXING"
     COMPLETED = "COMPLETED"
     FAILED = "FAILED"
     CANCELED = "CANCELED"

backend/onyx/db/image_generation.py

@@ -202,7 +202,6 @@ def create_default_image_gen_config_from_api_key(
             api_key=api_key,
             api_base=None,
             api_version=None,
-            default_model_name=model_name,
             deployment_name=None,
             is_public=True,
         )

backend/onyx/db/llm.py

@@ -109,45 +109,38 @@ def can_user_access_llm_provider(
         is_admin: If True, bypass user group restrictions but still respect persona restrictions
 
     Access logic:
-    1. If is_public=True → everyone has access (public override)
-    2. If is_public=False:
-       - Both groups AND personas set → must satisfy BOTH (AND logic, admins bypass group check)
-       - Only groups set → must be in one of the groups (OR across groups, admins bypass)
-       - Only personas set → must use one of the personas (OR across personas, applies to admins)
-       - Neither set → NOBODY has access unless admin (locked, admin-only)
+    - is_public controls USER access (group bypass): when True, all users can access
+      regardless of group membership. When False, user must be in a whitelisted group
+      (or be admin).
+    - Persona restrictions are ALWAYS enforced when set, regardless of is_public.
+      This allows admins to make a provider available to all users while still
+      restricting which personas (assistants) can use it.
+
+    Decision matrix:
+    1. is_public=True, no personas set → everyone has access
+    2. is_public=True, personas set → all users, but only whitelisted personas
+    3. is_public=False, groups+personas set → must satisfy BOTH (admins bypass groups)
+    4. is_public=False, only groups set → must be in group (admins bypass)
+    5. is_public=False, only personas set → must use whitelisted persona
+    6. is_public=False, neither set → admin-only (locked)
     """
-    # Public override - everyone has access
-    if provider.is_public:
-        return True
-
-    # Extract IDs once to avoid multiple iterations
-    provider_group_ids = (
-        {group.id for group in provider.groups} if provider.groups else set()
-    )
-    provider_persona_ids = (
-        {p.id for p in provider.personas} if provider.personas else set()
-    )
-
+    provider_group_ids = {g.id for g in (provider.groups or [])}
+    provider_persona_ids = {p.id for p in (provider.personas or [])}
     has_groups = bool(provider_group_ids)
     has_personas = bool(provider_persona_ids)
 
-    # Both groups AND personas set → AND logic (must satisfy both)
-    if has_groups and has_personas:
-        # Admins bypass group check but still must satisfy persona restrictions
-        user_in_group = is_admin or bool(user_group_ids & provider_group_ids)
-        persona_allowed = persona.id in provider_persona_ids if persona else False
-        return user_in_group and persona_allowed
+    # Persona restrictions are always enforced when set, regardless of is_public
+    if has_personas and not (persona and persona.id in provider_persona_ids):
+        return False
+
+    if provider.is_public:
+        return True
 
-    # Only groups set → user must be in one of the groups (admins bypass)
     if has_groups:
         return is_admin or bool(user_group_ids & provider_group_ids)
 
-    # Only personas set → persona must be in allowed list (applies to admins too)
-    if has_personas:
-        return persona.id in provider_persona_ids if persona else False
-
-    # Neither groups nor personas set, and not public → admins can access
-    return is_admin
+    # No groups: either persona-whitelisted (already passed) or admin-only if locked
+    return has_personas or is_admin
 
 
 def validate_persona_ids_exist(
@@ -213,11 +206,29 @@ def upsert_llm_provider(
     llm_provider_upsert_request: LLMProviderUpsertRequest,
     db_session: Session,
 ) -> LLMProviderView:
-    existing_llm_provider = fetch_existing_llm_provider(
-        name=llm_provider_upsert_request.name, db_session=db_session
-    )
+    existing_llm_provider: LLMProviderModel | None = None
+    if llm_provider_upsert_request.id:
+        existing_llm_provider = fetch_existing_llm_provider_by_id(
+            id=llm_provider_upsert_request.id, db_session=db_session
+        )
+        if not existing_llm_provider:
+            raise ValueError(
+                f"LLM provider with id {llm_provider_upsert_request.id} not found"
+            )
 
-    if not existing_llm_provider:
+        if existing_llm_provider.name != llm_provider_upsert_request.name:
+            raise ValueError(
+                f"LLM provider with id {llm_provider_upsert_request.id} name change not allowed"
+            )
+    else:
+        existing_llm_provider = fetch_existing_llm_provider(
+            name=llm_provider_upsert_request.name, db_session=db_session
+        )
+        if existing_llm_provider:
+            raise ValueError(
+                f"LLM provider with name '{llm_provider_upsert_request.name}'"
+                " already exists"
+            )
         existing_llm_provider = LLMProviderModel(name=llm_provider_upsert_request.name)
         db_session.add(existing_llm_provider)
 
@@ -238,11 +249,7 @@ def upsert_llm_provider(
     existing_llm_provider.api_base = api_base
     existing_llm_provider.api_version = llm_provider_upsert_request.api_version
     existing_llm_provider.custom_config = custom_config
-    # TODO: Remove default model name on api change
-    # Needed due to /provider/{id}/default endpoint not disclosing the default model name
-    existing_llm_provider.default_model_name = (
-        llm_provider_upsert_request.default_model_name
-    )
+
     existing_llm_provider.is_public = llm_provider_upsert_request.is_public
     existing_llm_provider.is_auto_mode = llm_provider_upsert_request.is_auto_mode
     existing_llm_provider.deployment_name = llm_provider_upsert_request.deployment_name
@@ -306,15 +313,6 @@ def upsert_llm_provider(
                 display_name=model_config.display_name,
             )
 
-    default_model = fetch_default_model(db_session, LLMModelFlowType.CHAT)
-    if default_model and default_model.llm_provider_id == existing_llm_provider.id:
-        _update_default_model(
-            db_session=db_session,
-            provider_id=existing_llm_provider.id,
-            model=existing_llm_provider.default_model_name,
-            flow_type=LLMModelFlowType.CHAT,
-        )
-
     # Make sure the relationship table stays up to date
     update_group_llm_provider_relationships__no_commit(
         llm_provider_id=existing_llm_provider.id,
@@ -488,6 +486,22 @@ def fetch_existing_llm_provider(
     return provider_model
 
 
+def fetch_existing_llm_provider_by_id(
+    id: int, db_session: Session
+) -> LLMProviderModel | None:
+    provider_model = db_session.scalar(
+        select(LLMProviderModel)
+        .where(LLMProviderModel.id == id)
+        .options(
+            selectinload(LLMProviderModel.model_configurations),
+            selectinload(LLMProviderModel.groups),
+            selectinload(LLMProviderModel.personas),
+        )
+    )
+
+    return provider_model
+
+
 def fetch_embedding_provider(
     db_session: Session, provider_type: EmbeddingProvider
 ) -> CloudEmbeddingProviderModel | None:
@@ -604,22 +618,13 @@ def remove_llm_provider__no_commit(db_session: Session, provider_id: int) -> Non
     db_session.flush()
 
 
-def update_default_provider(provider_id: int, db_session: Session) -> None:
-    # Attempt to get the default_model_name from the provider first
-    # TODO: Remove default_model_name check
-    provider = db_session.scalar(
-        select(LLMProviderModel).where(
-            LLMProviderModel.id == provider_id,
-        )
-    )
-
-    if provider is None:
-        raise ValueError(f"LLM Provider with id={provider_id} does not exist")
-
+def update_default_provider(
+    provider_id: int, model_name: str, db_session: Session
+) -> None:
     _update_default_model(
         db_session,
         provider_id,
-        provider.default_model_name,  # type: ignore[arg-type]
+        model_name,
         LLMModelFlowType.CHAT,
     )
 
@@ -805,12 +810,6 @@ def sync_auto_mode_models(
             )
             changes += 1
 
-    # In Auto mode, default model is always set from GitHub config
-    default_model = llm_recommendations.get_default_model(provider.provider)
-    if default_model and provider.default_model_name != default_model.name:
-        provider.default_model_name = default_model.name
-        changes += 1
-
     db_session.commit()
     return changes
 

backend/onyx/db/models.py

@@ -103,7 +103,6 @@ from onyx.utils.encryption import encrypt_string_to_bytes
 from onyx.utils.sensitive import SensitiveValue
 from onyx.utils.headers import HeaderItemDict
 from shared_configs.enums import EmbeddingProvider
-from onyx.context.search.enums import RecencyBiasSetting
 
 # TODO: After anonymous user migration has been deployed, make user_id columns NOT NULL
 # and update Mapped[User | None] relationships to Mapped[User] where needed.
@@ -2823,17 +2822,8 @@ class LLMProvider(Base):
         postgresql.JSONB(), nullable=True
     )
 
-    # Deprecated: use LLMModelFlow with CHAT flow type instead
-    default_model_name: Mapped[str | None] = mapped_column(String, nullable=True)
-
     deployment_name: Mapped[str | None] = mapped_column(String, nullable=True)
 
-    # Deprecated: use LLMModelFlow.is_default with CHAT flow type instead
-    is_default_provider: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
-    # Deprecated: use LLMModelFlow.is_default with VISION flow type instead
-    is_default_vision_provider: Mapped[bool | None] = mapped_column(Boolean)
-    # Deprecated: use LLMModelFlow with VISION flow type instead
-    default_vision_model: Mapped[str | None] = mapped_column(String, nullable=True)
     # EE only
     is_public: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
     # Auto mode: models, visibility, and defaults are managed by GitHub config
@@ -3265,19 +3255,6 @@ class Persona(Base):
     )
     name: Mapped[str] = mapped_column(String)
     description: Mapped[str] = mapped_column(String)
-    # Number of chunks to pass to the LLM for generation.
-    num_chunks: Mapped[float | None] = mapped_column(Float, nullable=True)
-    chunks_above: Mapped[int] = mapped_column(Integer)
-    chunks_below: Mapped[int] = mapped_column(Integer)
-    # Pass every chunk through LLM for evaluation, fairly expensive
-    # Can be turned off globally by admin, in which case, this setting is ignored
-    llm_relevance_filter: Mapped[bool] = mapped_column(Boolean)
-    # Enables using LLM to extract time and source type filters
-    # Can also be admin disabled globally
-    llm_filter_extraction: Mapped[bool] = mapped_column(Boolean)
-    recency_bias: Mapped[RecencyBiasSetting] = mapped_column(
-        Enum(RecencyBiasSetting, native_enum=False)
-    )
 
     # Allows the persona to specify a specific default LLM model
     # NOTE: only is applied on the actual response generation - is not used for things like
@@ -3304,11 +3281,8 @@ class Persona(Base):
     # Treated specially (cannot be user edited etc.)
     builtin_persona: Mapped[bool] = mapped_column(Boolean, default=False)
 
-    # Default personas are personas created by admins and are automatically added
-    # to all users' assistants list.
-    is_default_persona: Mapped[bool] = mapped_column(
-        Boolean, default=False, nullable=False
-    )
+    # Featured personas are highlighted in the UI
+    featured: Mapped[bool] = mapped_column(Boolean, default=False)
     # controls whether the persona is available to be selected by users
     is_visible: Mapped[bool] = mapped_column(Boolean, default=True)
     # controls the ordering of personas in the UI

backend/onyx/db/persona.py

@@ -18,11 +18,8 @@ from sqlalchemy.orm import Session
 from onyx.access.hierarchy_access import get_user_external_group_ids
 from onyx.auth.schemas import UserRole
 from onyx.configs.app_configs import CURATORS_CANNOT_VIEW_OR_EDIT_NON_OWNED_ASSISTANTS
-from onyx.configs.chat_configs import CONTEXT_CHUNKS_ABOVE
-from onyx.configs.chat_configs import CONTEXT_CHUNKS_BELOW
 from onyx.configs.constants import DEFAULT_PERSONA_ID
 from onyx.configs.constants import NotificationType
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.constants import SLACK_BOT_PERSONA_PREFIX
 from onyx.db.document_access import get_accessible_documents_by_ids
 from onyx.db.models import ConnectorCredentialPair
@@ -254,16 +251,15 @@ def create_update_persona(
     # Permission to actually use these is checked later
 
     try:
-        # Default persona validation
-        if create_persona_request.is_default_persona:
-            if not create_persona_request.is_public:
-                raise ValueError("Cannot make a default persona non public")
+        # Featured persona validation
+        if create_persona_request.featured:
 
-            # Curators can edit default personas, but not make them
+            # Curators can edit featured personas, but not make them
+            # TODO this will be reworked soon with RBAC permissions feature
             if user.role == UserRole.CURATOR or user.role == UserRole.GLOBAL_CURATOR:
                 pass
             elif user.role != UserRole.ADMIN:
-                raise ValueError("Only admins can make a default persona")
+                raise ValueError("Only admins can make a featured persona")
 
         # Convert incoming string UUIDs to UUID objects for DB operations
         converted_user_file_ids = None
@@ -284,7 +280,6 @@ def create_update_persona(
             document_set_ids=create_persona_request.document_set_ids,
             tool_ids=create_persona_request.tool_ids,
             is_public=create_persona_request.is_public,
-            recency_bias=create_persona_request.recency_bias,
             llm_model_provider_override=create_persona_request.llm_model_provider_override,
             llm_model_version_override=create_persona_request.llm_model_version_override,
             starter_messages=create_persona_request.starter_messages,
@@ -298,10 +293,7 @@ def create_update_persona(
             remove_image=create_persona_request.remove_image,
             search_start_date=create_persona_request.search_start_date,
             label_ids=create_persona_request.label_ids,
-            num_chunks=create_persona_request.num_chunks,
-            llm_relevance_filter=create_persona_request.llm_relevance_filter,
-            llm_filter_extraction=create_persona_request.llm_filter_extraction,
-            is_default_persona=create_persona_request.is_default_persona,
+            featured=create_persona_request.featured,
             user_file_ids=converted_user_file_ids,
             commit=False,
             hierarchy_node_ids=create_persona_request.hierarchy_node_ids,
@@ -335,6 +327,7 @@ def update_persona_shared(
     db_session: Session,
     group_ids: list[int] | None = None,
     is_public: bool | None = None,
+    label_ids: list[int] | None = None,
 ) -> None:
     """Simplified version of `create_update_persona` which only touches the
     accessibility rather than any of the logic (e.g. prompt, connected data sources,
@@ -344,9 +337,7 @@ def update_persona_shared(
     )
 
     if user and user.role != UserRole.ADMIN and persona.user_id != user.id:
-        raise HTTPException(
-            status_code=403, detail="You don't have permission to modify this persona"
-        )
+        raise PermissionError("You don't have permission to modify this persona")
 
     versioned_update_persona_access = fetch_versioned_implementation(
         "onyx.db.persona", "update_persona_access"
@@ -360,6 +351,15 @@ def update_persona_shared(
         group_ids=group_ids,
     )
 
+    if label_ids is not None:
+        labels = (
+            db_session.query(PersonaLabel).filter(PersonaLabel.id.in_(label_ids)).all()
+        )
+        if len(labels) != len(label_ids):
+            raise ValueError("Some label IDs were not found in the database")
+        persona.labels.clear()
+        persona.labels = labels
+
     db_session.commit()
 
 
@@ -869,10 +869,6 @@ def upsert_persona(
     user: User | None,
     name: str,
     description: str,
-    num_chunks: float,
-    llm_relevance_filter: bool,
-    llm_filter_extraction: bool,
-    recency_bias: RecencyBiasSetting,
     llm_model_provider_override: str | None,
     llm_model_version_override: str | None,
     starter_messages: list[StarterMessage] | None,
@@ -893,13 +889,11 @@ def upsert_persona(
     remove_image: bool | None = None,
     search_start_date: datetime | None = None,
     builtin_persona: bool = False,
-    is_default_persona: bool | None = None,
+    featured: bool | None = None,
     label_ids: list[int] | None = None,
     user_file_ids: list[UUID] | None = None,
     hierarchy_node_ids: list[int] | None = None,
     document_ids: list[str] | None = None,
-    chunks_above: int = CONTEXT_CHUNKS_ABOVE,
-    chunks_below: int = CONTEXT_CHUNKS_BELOW,
     replace_base_system_prompt: bool = False,
 ) -> Persona:
     """
@@ -965,6 +959,8 @@ def upsert_persona(
         labels = (
             db_session.query(PersonaLabel).filter(PersonaLabel.id.in_(label_ids)).all()
         )
+        if len(labels) != len(label_ids):
+            raise ValueError("Some label IDs were not found in the database")
 
     # Fetch and attach hierarchy_nodes by IDs
     hierarchy_nodes = None
@@ -1008,12 +1004,6 @@ def upsert_persona(
         # `default` and `built-in` properties can only be set when creating a persona.
         existing_persona.name = name
         existing_persona.description = description
-        existing_persona.num_chunks = num_chunks
-        existing_persona.chunks_above = chunks_above
-        existing_persona.chunks_below = chunks_below
-        existing_persona.llm_relevance_filter = llm_relevance_filter
-        existing_persona.llm_filter_extraction = llm_filter_extraction
-        existing_persona.recency_bias = recency_bias
         existing_persona.llm_model_provider_override = llm_model_provider_override
         existing_persona.llm_model_version_override = llm_model_version_override
         existing_persona.starter_messages = starter_messages
@@ -1027,10 +1017,8 @@ def upsert_persona(
         if label_ids is not None:
             existing_persona.labels.clear()
             existing_persona.labels = labels or []
-        existing_persona.is_default_persona = (
-            is_default_persona
-            if is_default_persona is not None
-            else existing_persona.is_default_persona
+        existing_persona.featured = (
+            featured if featured is not None else existing_persona.featured
         )
         # Update embedded prompt fields if provided
         if system_prompt is not None:
@@ -1083,12 +1071,6 @@ def upsert_persona(
             is_public=is_public,
             name=name,
             description=description,
-            num_chunks=num_chunks,
-            chunks_above=chunks_above,
-            chunks_below=chunks_below,
-            llm_relevance_filter=llm_relevance_filter,
-            llm_filter_extraction=llm_filter_extraction,
-            recency_bias=recency_bias,
             builtin_persona=builtin_persona,
             system_prompt=system_prompt or "",
             task_prompt=task_prompt or "",
@@ -1104,9 +1086,7 @@ def upsert_persona(
             display_priority=display_priority,
             is_visible=is_visible,
             search_start_date=search_start_date,
-            is_default_persona=(
-                is_default_persona if is_default_persona is not None else False
-            ),
+            featured=(featured if featured is not None else False),
             user_files=user_files or [],
             labels=labels or [],
             hierarchy_nodes=hierarchy_nodes or [],
@@ -1151,9 +1131,9 @@ def delete_old_default_personas(
     db_session.commit()
 
 
-def update_persona_is_default(
+def update_persona_featured(
     persona_id: int,
-    is_default: bool,
+    featured: bool,
     db_session: Session,
     user: User,
 ) -> None:
@@ -1161,10 +1141,7 @@ def update_persona_is_default(
         db_session=db_session, persona_id=persona_id, user=user, get_editable=True
     )
 
-    if not persona.is_public:
-        persona.is_public = True
-
-    persona.is_default_persona = is_default
+    persona.featured = featured
     db_session.commit()
 
 

backend/onyx/db/projects.py

@@ -9,8 +9,9 @@ from pydantic import BaseModel
 from pydantic import ConfigDict
 from sqlalchemy import func
 from sqlalchemy.orm import Session
+from starlette.background import BackgroundTasks
 
-from onyx.background.celery.versioned_apps.client import app as client_app
+from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
@@ -105,8 +106,8 @@ def upload_files_to_user_files_with_indexing(
     user: User,
     temp_id_map: dict[str, str] | None,
     db_session: Session,
+    background_tasks: BackgroundTasks | None = None,
 ) -> CategorizedFilesResult:
-    # Validate project ownership if a project_id is provided
     if project_id is not None and user is not None:
         if not check_project_ownership(project_id, user.id, db_session):
             raise HTTPException(status_code=404, detail="Project not found")
@@ -127,16 +128,27 @@ def upload_files_to_user_files_with_indexing(
         logger.warning(
             f"File {rejected_file.filename} rejected for {rejected_file.reason}"
         )
-    for user_file in user_files:
-        task = client_app.send_task(
-            OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
-            kwargs={"user_file_id": user_file.id, "tenant_id": tenant_id},
-            queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
-            priority=OnyxCeleryPriority.HIGH,
-        )
-        logger.info(
-            f"Triggered indexing for user_file_id={user_file.id} with task_id={task.id}"
-        )
+
+    if DISABLE_VECTOR_DB and background_tasks is not None:
+        from onyx.background.task_utils import drain_processing_loop
+
+        background_tasks.add_task(drain_processing_loop, tenant_id)
+        for user_file in user_files:
+            logger.info(f"Queued in-process processing for user_file_id={user_file.id}")
+    else:
+        from onyx.background.celery.versioned_apps.client import app as client_app
+
+        for user_file in user_files:
+            task = client_app.send_task(
+                OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
+                kwargs={"user_file_id": user_file.id, "tenant_id": tenant_id},
+                queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
+                priority=OnyxCeleryPriority.HIGH,
+            )
+            logger.info(
+                f"Triggered indexing for user_file_id={user_file.id} "
+                f"with task_id={task.id}"
+            )
 
     return CategorizedFilesResult(
         user_files=user_files,

backend/onyx/db/slack_channel_config.py

@@ -5,8 +5,6 @@ from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm import Session
 
-from onyx.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.constants import DEFAULT_PERSONA_SLACK_CHANNEL_NAME
 from onyx.db.constants import SLACK_BOT_PERSONA_PREFIX
 from onyx.db.models import ChannelConfig
@@ -45,8 +43,6 @@ def create_slack_channel_persona(
     channel_name: str | None,
     document_set_ids: list[int],
     existing_persona_id: int | None = None,
-    num_chunks: float = MAX_CHUNKS_FED_TO_CHAT,
-    enable_auto_filters: bool = False,
 ) -> Persona:
     """NOTE: does not commit changes"""
 
@@ -73,17 +69,13 @@ def create_slack_channel_persona(
         system_prompt="",
         task_prompt="",
         datetime_aware=True,
-        num_chunks=num_chunks,
-        llm_relevance_filter=True,
-        llm_filter_extraction=enable_auto_filters,
-        recency_bias=RecencyBiasSetting.AUTO,
         tool_ids=[search_tool.id],
         document_set_ids=document_set_ids,
         llm_model_provider_override=None,
         llm_model_version_override=None,
         starter_messages=None,
         is_public=True,
-        is_default_persona=False,
+        featured=False,
         db_session=db_session,
         commit=False,
     )

backend/onyx/db/user_file.py

@@ -6,6 +6,7 @@ from sqlalchemy import select
 from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
+from onyx.db.models import Project__UserFile
 from onyx.db.models import UserFile
 
 
@@ -57,12 +58,19 @@ def fetch_user_project_ids_for_user_files(
     db_session: Session,
 ) -> dict[str, list[int]]:
     """Fetch user project ids for specified user files"""
-    stmt = select(UserFile).where(UserFile.id.in_(user_file_ids))
-    results = db_session.execute(stmt).scalars().all()
-    return {
-        str(user_file.id): [project.id for project in user_file.projects]
-        for user_file in results
+    user_file_uuid_ids = [UUID(user_file_id) for user_file_id in user_file_ids]
+    stmt = select(Project__UserFile.user_file_id, Project__UserFile.project_id).where(
+        Project__UserFile.user_file_id.in_(user_file_uuid_ids)
+    )
+    rows = db_session.execute(stmt).all()
+
+    user_file_id_to_project_ids: dict[str, list[int]] = {
+        user_file_id: [] for user_file_id in user_file_ids
     }
+    for user_file_id, project_id in rows:
+        user_file_id_to_project_ids[str(user_file_id)].append(project_id)
+
+    return user_file_id_to_project_ids
 
 
 def fetch_persona_ids_for_user_files(

backend/onyx/deep_research/dr_loop.py

@@ -139,7 +139,7 @@ def generate_final_report(
             custom_agent_prompt=None,
             simple_chat_history=history,
             reminder_message=reminder_message,
-            project_files=None,
+            context_files=None,
             available_tokens=llm.config.max_input_tokens,
             all_injected_file_metadata=all_injected_file_metadata,
         )
@@ -257,7 +257,7 @@ def run_deep_research_llm_loop(
                     custom_agent_prompt=None,
                     simple_chat_history=simple_chat_history,
                     reminder_message=None,
-                    project_files=None,
+                    context_files=None,
                     available_tokens=available_tokens,
                     last_n_user_messages=MAX_USER_MESSAGES_FOR_CONTEXT,
                     all_injected_file_metadata=all_injected_file_metadata,
@@ -321,7 +321,7 @@ def run_deep_research_llm_loop(
                 custom_agent_prompt=None,
                 simple_chat_history=simple_chat_history + [reminder_message],
                 reminder_message=None,
-                project_files=None,
+                context_files=None,
                 available_tokens=available_tokens,
                 last_n_user_messages=MAX_USER_MESSAGES_FOR_CONTEXT + 1,
                 all_injected_file_metadata=all_injected_file_metadata,
@@ -485,7 +485,7 @@ def run_deep_research_llm_loop(
                     custom_agent_prompt=None,
                     simple_chat_history=simple_chat_history,
                     reminder_message=first_cycle_reminder_message,
-                    project_files=None,
+                    context_files=None,
                     available_tokens=available_tokens,
                     last_n_user_messages=MAX_USER_MESSAGES_FOR_CONTEXT,
                     all_injected_file_metadata=all_injected_file_metadata,

backend/onyx/document_index/opensearch/opensearch_document_index.py

@@ -6,7 +6,6 @@ import httpx
 from opensearchpy import NotFoundError
 
 from onyx.access.models import DocumentAccess
-from onyx.configs.app_configs import USING_AWS_MANAGED_OPENSEARCH
 from onyx.configs.app_configs import VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT
 from onyx.configs.chat_configs import NUM_RETURNED_HITS
 from onyx.configs.chat_configs import TITLE_CONTENT_RATIO
@@ -563,12 +562,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
         )
 
         if not self._client.index_exists():
-            if USING_AWS_MANAGED_OPENSEARCH:
-                index_settings = (
-                    DocumentSchema.get_index_settings_for_aws_managed_opensearch()
-                )
-            else:
-                index_settings = DocumentSchema.get_index_settings()
+            index_settings = DocumentSchema.get_index_settings_based_on_environment()
             self._client.create_index(
                 mappings=expected_mappings,
                 settings=index_settings,

backend/onyx/document_index/opensearch/schema.py

@@ -12,6 +12,7 @@ from pydantic import model_validator
 from pydantic import SerializerFunctionWrapHandler
 
 from onyx.configs.app_configs import OPENSEARCH_TEXT_ANALYZER
+from onyx.configs.app_configs import USING_AWS_MANAGED_OPENSEARCH
 from onyx.document_index.interfaces_new import TenantState
 from onyx.document_index.opensearch.constants import DEFAULT_MAX_CHUNK_SIZE
 from onyx.document_index.opensearch.constants import EF_CONSTRUCTION
@@ -525,7 +526,7 @@ class DocumentSchema:
         }
 
     @staticmethod
-    def get_index_settings_for_aws_managed_opensearch() -> dict[str, Any]:
+    def get_index_settings_for_aws_managed_opensearch_st_dev() -> dict[str, Any]:
         """
         Settings for AWS-managed OpenSearch.
 
@@ -546,3 +547,41 @@ class DocumentSchema:
                 "knn.algo_param.ef_search": EF_SEARCH,
             }
         }
+
+    @staticmethod
+    def get_index_settings_for_aws_managed_opensearch_mt_cloud() -> dict[str, Any]:
+        """
+        Settings for AWS-managed OpenSearch in multi-tenant cloud.
+
+        324 shards very roughly targets a storage load of ~30Gb per shard, which
+        according to AWS OpenSearch documentation is within a good target range.
+
+        As documented above we need 2 replicas for a total of 3 copies of the
+        data because the cluster is configured with 3-AZ awareness.
+        """
+        return {
+            "index": {
+                "number_of_shards": 324,
+                "number_of_replicas": 2,
+                # Required for vector search.
+                "knn": True,
+                "knn.algo_param.ef_search": EF_SEARCH,
+            }
+        }
+
+    @staticmethod
+    def get_index_settings_based_on_environment() -> dict[str, Any]:
+        """
+        Returns the index settings based on the environment.
+        """
+        if USING_AWS_MANAGED_OPENSEARCH:
+            if MULTI_TENANT:
+                return (
+                    DocumentSchema.get_index_settings_for_aws_managed_opensearch_mt_cloud()
+                )
+            else:
+                return (
+                    DocumentSchema.get_index_settings_for_aws_managed_opensearch_st_dev()
+                )
+        else:
+            return DocumentSchema.get_index_settings()

backend/onyx/main.py

@@ -32,11 +32,14 @@ from onyx.auth.schemas import UserUpdate
 from onyx.auth.users import auth_backend
 from onyx.auth.users import create_onyx_oauth_router
 from onyx.auth.users import fastapi_users
+from onyx.cache.interface import CacheBackendType
 from onyx.configs.app_configs import APP_API_PREFIX
 from onyx.configs.app_configs import APP_HOST
 from onyx.configs.app_configs import APP_PORT
 from onyx.configs.app_configs import AUTH_RATE_LIMITING_ENABLED
 from onyx.configs.app_configs import AUTH_TYPE
+from onyx.configs.app_configs import CACHE_BACKEND
+from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.configs.app_configs import LOG_ENDPOINT_LATENCY
 from onyx.configs.app_configs import OAUTH_CLIENT_ID
 from onyx.configs.app_configs import OAUTH_CLIENT_SECRET
@@ -254,8 +257,53 @@ def include_auth_router_with_prefix(
     )
 
 
+def validate_cache_backend_settings() -> None:
+    """Validate that CACHE_BACKEND=postgres is only used with DISABLE_VECTOR_DB.
+
+    The Postgres cache backend eliminates the Redis dependency, but only works
+    when Celery is not running (which requires DISABLE_VECTOR_DB=true).
+    """
+    if CACHE_BACKEND == CacheBackendType.POSTGRES and not DISABLE_VECTOR_DB:
+        raise RuntimeError(
+            "CACHE_BACKEND=postgres requires DISABLE_VECTOR_DB=true. "
+            "The Postgres cache backend is only supported in no-vector-DB "
+            "deployments where Celery is replaced by the in-process task runner."
+        )
+
+
+def validate_no_vector_db_settings() -> None:
+    """Validate that DISABLE_VECTOR_DB is not combined with incompatible settings.
+
+    Raises RuntimeError if DISABLE_VECTOR_DB is set alongside MULTI_TENANT or ENABLE_CRAFT,
+    since these modes require infrastructure that is removed in no-vector-DB deployments.
+    """
+    if not DISABLE_VECTOR_DB:
+        return
+
+    if MULTI_TENANT:
+        raise RuntimeError(
+            "DISABLE_VECTOR_DB cannot be used with MULTI_TENANT. "
+            "Multi-tenant deployments require the vector database for "
+            "per-tenant document indexing and search. Run in single-tenant "
+            "mode when disabling the vector database."
+        )
+
+    from onyx.server.features.build.configs import ENABLE_CRAFT
+
+    if ENABLE_CRAFT:
+        raise RuntimeError(
+            "DISABLE_VECTOR_DB cannot be used with ENABLE_CRAFT. "
+            "Onyx Craft requires background workers for sandbox lifecycle "
+            "management, which are removed in no-vector-DB deployments. "
+            "Disable Craft (ENABLE_CRAFT=false) when disabling the vector database."
+        )
+
+
 @asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:  # noqa: ARG001
+    validate_no_vector_db_settings()
+    validate_cache_backend_settings()
+
     # Set recursion limit
     if SYSTEM_RECURSION_LIMIT is not None:
         sys.setrecursionlimit(SYSTEM_RECURSION_LIMIT)
@@ -324,8 +372,20 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:  # noqa: ARG001
     if AUTH_RATE_LIMITING_ENABLED:
         await setup_auth_limiter()
 
+    if DISABLE_VECTOR_DB:
+        from onyx.background.periodic_poller import recover_stuck_user_files
+        from onyx.background.periodic_poller import start_periodic_poller
+
+        recover_stuck_user_files(POSTGRES_DEFAULT_SCHEMA)
+        start_periodic_poller(POSTGRES_DEFAULT_SCHEMA)
+
     yield
 
+    if DISABLE_VECTOR_DB:
+        from onyx.background.periodic_poller import stop_periodic_poller
+
+        stop_periodic_poller()
+
     SqlEngine.reset_engine()
 
     if AUTH_RATE_LIMITING_ENABLED:

backend/onyx/onyxbot/slack/handlers/handle_message.py

@@ -3,10 +3,12 @@ import datetime
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
 
+from onyx.auth.schemas import UserRole
 from onyx.configs.onyxbot_configs import ONYX_BOT_FEEDBACK_REMINDER
 from onyx.configs.onyxbot_configs import ONYX_BOT_REACT_EMOJI
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
 from onyx.db.models import SlackChannelConfig
+from onyx.db.user_preferences import activate_user
 from onyx.db.users import add_slack_user_if_not_exists
 from onyx.db.users import get_user_by_email
 from onyx.onyxbot.slack.blocks import get_feedback_reminder_blocks
@@ -243,6 +245,44 @@ def handle_message(
                     )
                     return False
 
+            elif (
+                not existing_user.is_active
+                and existing_user.role == UserRole.SLACK_USER
+            ):
+                check_seat_fn = fetch_ee_implementation_or_noop(
+                    "onyx.db.license",
+                    "check_seat_availability",
+                    None,
+                )
+                seat_result = check_seat_fn(db_session=db_session)
+                if seat_result is not None and not seat_result.available:
+                    logger.info(
+                        f"Blocked inactive Slack user {message_info.email}: "
+                        f"{seat_result.error_message}"
+                    )
+                    respond_in_thread_or_channel(
+                        client=client,
+                        channel=channel,
+                        thread_ts=message_info.msg_to_respond,
+                        text=(
+                            "We weren't able to respond because your organization "
+                            "has reached its user seat limit. Your account is "
+                            "currently deactivated and cannot be reactivated "
+                            "until more seats are available. Please contact "
+                            "your Onyx administrator."
+                        ),
+                    )
+                    return False
+
+                activate_user(existing_user, db_session)
+                invalidate_license_cache_fn = fetch_ee_implementation_or_noop(
+                    "onyx.db.license",
+                    "invalidate_license_cache",
+                    None,
+                )
+                invalidate_license_cache_fn()
+                logger.info(f"Reactivated inactive Slack user {message_info.email}")
+
             add_slack_user_if_not_exists(db_session, message_info.email)
 
         # first check if we need to respond with a standard answer

backend/onyx/server/features/persona/api.py

@@ -32,7 +32,7 @@ from onyx.db.persona import get_persona_snapshots_for_user
 from onyx.db.persona import get_persona_snapshots_paginated
 from onyx.db.persona import mark_persona_as_deleted
 from onyx.db.persona import mark_persona_as_not_deleted
-from onyx.db.persona import update_persona_is_default
+from onyx.db.persona import update_persona_featured
 from onyx.db.persona import update_persona_label
 from onyx.db.persona import update_persona_public_status
 from onyx.db.persona import update_persona_shared
@@ -130,8 +130,8 @@ class IsPublicRequest(BaseModel):
     is_public: bool
 
 
-class IsDefaultRequest(BaseModel):
-    is_default_persona: bool
+class IsFeaturedRequest(BaseModel):
+    featured: bool
 
 
 @admin_router.patch("/{persona_id}/visible")
@@ -168,22 +168,22 @@ def patch_user_persona_public_status(
         raise HTTPException(status_code=403, detail=str(e))
 
 
-@admin_router.patch("/{persona_id}/default")
-def patch_persona_default_status(
+@admin_router.patch("/{persona_id}/featured")
+def patch_persona_featured_status(
     persona_id: int,
-    is_default_request: IsDefaultRequest,
+    is_featured_request: IsFeaturedRequest,
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
 ) -> None:
     try:
-        update_persona_is_default(
+        update_persona_featured(
             persona_id=persona_id,
-            is_default=is_default_request.is_default_persona,
+            featured=is_featured_request.featured,
             db_session=db_session,
             user=user,
         )
     except ValueError as e:
-        logger.exception("Failed to update persona default status")
+        logger.exception("Failed to update persona featured status")
         raise HTTPException(status_code=403, detail=str(e))
 
 
@@ -405,6 +405,7 @@ class PersonaShareRequest(BaseModel):
     user_ids: list[UUID] | None = None
     group_ids: list[int] | None = None
     is_public: bool | None = None
+    label_ids: list[int] | None = None
 
 
 # We notify each user when a user is shared with them
@@ -415,14 +416,22 @@ def share_persona(
     user: User = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> None:
-    update_persona_shared(
-        persona_id=persona_id,
-        user=user,
-        db_session=db_session,
-        user_ids=persona_share_request.user_ids,
-        group_ids=persona_share_request.group_ids,
-        is_public=persona_share_request.is_public,
-    )
+    try:
+        update_persona_shared(
+            persona_id=persona_id,
+            user=user,
+            db_session=db_session,
+            user_ids=persona_share_request.user_ids,
+            group_ids=persona_share_request.group_ids,
+            is_public=persona_share_request.is_public,
+            label_ids=persona_share_request.label_ids,
+        )
+    except PermissionError as e:
+        logger.exception("Failed to share persona")
+        raise HTTPException(status_code=403, detail=str(e))
+    except ValueError as e:
+        logger.exception("Failed to share persona")
+        raise HTTPException(status_code=400, detail=str(e))
 
 
 @basic_router.delete("/{persona_id}", tags=PUBLIC_API_TAGS)

backend/onyx/server/features/persona/models.py

@@ -5,7 +5,6 @@ from pydantic import BaseModel
 from pydantic import Field
 
 from onyx.configs.constants import DocumentSource
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.enums import HierarchyNodeType
 from onyx.db.models import Document
 from onyx.db.models import HierarchyNode
@@ -108,11 +107,7 @@ class PersonaUpsertRequest(BaseModel):
     name: str
     description: str
     document_set_ids: list[int]
-    num_chunks: float
     is_public: bool
-    recency_bias: RecencyBiasSetting
-    llm_filter_extraction: bool
-    llm_relevance_filter: bool
     llm_model_provider_override: str | None = None
     llm_model_version_override: str | None = None
     starter_messages: list[StarterMessage] | None = None
@@ -128,7 +123,7 @@ class PersonaUpsertRequest(BaseModel):
     )
     search_start_date: datetime | None = None
     label_ids: list[int] | None = None
-    is_default_persona: bool = False
+    featured: bool = False
     display_priority: int | None = None
     # Accept string UUIDs from frontend
     user_file_ids: list[str] | None = None
@@ -155,9 +150,6 @@ class MinimalPersonaSnapshot(BaseModel):
     tools: list[ToolSnapshot]
     starter_messages: list[StarterMessage] | None
 
-    llm_relevance_filter: bool
-    llm_filter_extraction: bool
-
     # only show document sets in the UI that the assistant has access to
     document_sets: list[DocumentSetSummary]
     # Counts for knowledge sources (used to determine if search tool should be enabled)
@@ -175,7 +167,7 @@ class MinimalPersonaSnapshot(BaseModel):
     is_public: bool
     is_visible: bool
     display_priority: int | None
-    is_default_persona: bool
+    featured: bool
     builtin_persona: bool
 
     # Used for filtering
@@ -214,8 +206,6 @@ class MinimalPersonaSnapshot(BaseModel):
                 if should_expose_tool_to_fe(tool)
             ],
             starter_messages=persona.starter_messages,
-            llm_relevance_filter=persona.llm_relevance_filter,
-            llm_filter_extraction=persona.llm_filter_extraction,
             document_sets=[
                 DocumentSetSummary.from_model(document_set)
                 for document_set in persona.document_sets
@@ -230,7 +220,7 @@ class MinimalPersonaSnapshot(BaseModel):
             is_public=persona.is_public,
             is_visible=persona.is_visible,
             display_priority=persona.display_priority,
-            is_default_persona=persona.is_default_persona,
+            featured=persona.featured,
             builtin_persona=persona.builtin_persona,
             labels=[PersonaLabelSnapshot.from_model(label) for label in persona.labels],
             owner=(
@@ -252,11 +242,9 @@ class PersonaSnapshot(BaseModel):
     # Return string UUIDs to frontend for consistency
     user_file_ids: list[str]
     display_priority: int | None
-    is_default_persona: bool
+    featured: bool
     builtin_persona: bool
     starter_messages: list[StarterMessage] | None
-    llm_relevance_filter: bool
-    llm_filter_extraction: bool
     tools: list[ToolSnapshot]
     labels: list["PersonaLabelSnapshot"]
     owner: MinimalUserSnapshot | None
@@ -265,7 +253,6 @@ class PersonaSnapshot(BaseModel):
     document_sets: list[DocumentSetSummary]
     llm_model_provider_override: str | None
     llm_model_version_override: str | None
-    num_chunks: float | None
     # Hierarchy nodes attached for scoped search
     hierarchy_nodes: list[HierarchyNodeSnapshot] = Field(default_factory=list)
     # Individual documents attached for scoped search
@@ -289,11 +276,9 @@ class PersonaSnapshot(BaseModel):
             icon_name=persona.icon_name,
             user_file_ids=[str(file.id) for file in persona.user_files],
             display_priority=persona.display_priority,
-            is_default_persona=persona.is_default_persona,
+            featured=persona.featured,
             builtin_persona=persona.builtin_persona,
             starter_messages=persona.starter_messages,
-            llm_relevance_filter=persona.llm_relevance_filter,
-            llm_filter_extraction=persona.llm_filter_extraction,
             tools=[
                 ToolSnapshot.from_model(tool)
                 for tool in persona.tools
@@ -324,7 +309,6 @@ class PersonaSnapshot(BaseModel):
             ],
             llm_model_provider_override=persona.llm_model_provider_override,
             llm_model_version_override=persona.llm_model_version_override,
-            num_chunks=persona.num_chunks,
             system_prompt=persona.system_prompt,
             replace_base_system_prompt=persona.replace_base_system_prompt,
             task_prompt=persona.task_prompt,
@@ -332,12 +316,10 @@ class PersonaSnapshot(BaseModel):
         )
 
 
-# Model with full context on perona's internal settings
+# Model with full context on persona's internal settings
 # This is used for flows which need to know all settings
 class FullPersonaSnapshot(PersonaSnapshot):
     search_start_date: datetime | None = None
-    llm_relevance_filter: bool = False
-    llm_filter_extraction: bool = False
 
     @classmethod
     def from_model(
@@ -360,7 +342,7 @@ class FullPersonaSnapshot(PersonaSnapshot):
             icon_name=persona.icon_name,
             user_file_ids=[str(file.id) for file in persona.user_files],
             display_priority=persona.display_priority,
-            is_default_persona=persona.is_default_persona,
+            featured=persona.featured,
             builtin_persona=persona.builtin_persona,
             starter_messages=persona.starter_messages,
             users=[
@@ -391,10 +373,7 @@ class FullPersonaSnapshot(PersonaSnapshot):
                 DocumentSetSummary.from_model(document_set_model)
                 for document_set_model in persona.document_sets
             ],
-            num_chunks=persona.num_chunks,
             search_start_date=persona.search_start_date,
-            llm_relevance_filter=persona.llm_relevance_filter,
-            llm_filter_extraction=persona.llm_filter_extraction,
             llm_model_provider_override=persona.llm_model_provider_override,
             llm_model_version_override=persona.llm_model_version_override,
             system_prompt=persona.system_prompt,

backend/onyx/server/features/projects/api.py

@@ -2,6 +2,7 @@ import json
 from uuid import UUID
 
 from fastapi import APIRouter
+from fastapi import BackgroundTasks
 from fastapi import Depends
 from fastapi import File
 from fastapi import Form
@@ -12,13 +13,7 @@ from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
 from onyx.auth.users import current_user
-from onyx.background.celery.tasks.user_file_processing.tasks import (
-    enqueue_user_file_project_sync_task,
-)
-from onyx.background.celery.tasks.user_file_processing.tasks import (
-    get_user_file_project_sync_queue_depth,
-)
-from onyx.background.celery.versioned_apps.client import app as client_app
+from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
@@ -34,7 +29,6 @@ from onyx.db.models import UserProject
 from onyx.db.persona import get_personas_by_ids
 from onyx.db.projects import get_project_token_count
 from onyx.db.projects import upload_files_to_user_files_with_indexing
-from onyx.redis.redis_pool import get_redis_client
 from onyx.server.features.projects.models import CategorizedFilesSnapshot
 from onyx.server.features.projects.models import ChatSessionRequest
 from onyx.server.features.projects.models import TokenCountResponse
@@ -55,7 +49,27 @@ class UserFileDeleteResult(BaseModel):
     assistant_names: list[str] = []
 
 
-def _trigger_user_file_project_sync(user_file_id: UUID, tenant_id: str) -> None:
+def _trigger_user_file_project_sync(
+    user_file_id: UUID,
+    tenant_id: str,
+    background_tasks: BackgroundTasks | None = None,
+) -> None:
+    if DISABLE_VECTOR_DB and background_tasks is not None:
+        from onyx.background.task_utils import drain_project_sync_loop
+
+        background_tasks.add_task(drain_project_sync_loop, tenant_id)
+        logger.info(f"Queued in-process project sync for user_file_id={user_file_id}")
+        return
+
+    from onyx.background.celery.tasks.user_file_processing.tasks import (
+        enqueue_user_file_project_sync_task,
+    )
+    from onyx.background.celery.tasks.user_file_processing.tasks import (
+        get_user_file_project_sync_queue_depth,
+    )
+    from onyx.background.celery.versioned_apps.client import app as client_app
+    from onyx.redis.redis_pool import get_redis_client
+
     queue_depth = get_user_file_project_sync_queue_depth(client_app)
     if queue_depth > USER_FILE_PROJECT_SYNC_MAX_QUEUE_DEPTH:
         logger.warning(
@@ -111,6 +125,7 @@ def create_project(
 
 @router.post("/file/upload", tags=PUBLIC_API_TAGS)
 def upload_user_files(
+    bg_tasks: BackgroundTasks,
     files: list[UploadFile] = File(...),
     project_id: int | None = Form(None),
     temp_id_map: str | None = Form(None),  # JSON string mapping hashed key -> temp_id
@@ -137,12 +152,12 @@ def upload_user_files(
             user=user,
             temp_id_map=parsed_temp_id_map,
             db_session=db_session,
+            background_tasks=bg_tasks if DISABLE_VECTOR_DB else None,
         )
 
         return CategorizedFilesSnapshot.from_result(categorized_files_result)
 
     except Exception as e:
-        # Log error with type, message, and stack for easier debugging
         logger.exception(f"Error uploading files - {type(e).__name__}: {str(e)}")
         raise HTTPException(
             status_code=500,
@@ -192,6 +207,7 @@ def get_files_in_project(
 def unlink_user_file_from_project(
     project_id: int,
     file_id: UUID,
+    bg_tasks: BackgroundTasks,
     user: User = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> Response:
@@ -208,7 +224,6 @@ def unlink_user_file_from_project(
     if project is None:
         raise HTTPException(status_code=404, detail="Project not found")
 
-    user_id = user.id
     user_file = (
         db_session.query(UserFile)
         .filter(UserFile.id == file_id, UserFile.user_id == user_id)
@@ -224,7 +239,7 @@ def unlink_user_file_from_project(
         db_session.commit()
 
     tenant_id = get_current_tenant_id()
-    _trigger_user_file_project_sync(user_file.id, tenant_id)
+    _trigger_user_file_project_sync(user_file.id, tenant_id, bg_tasks)
 
     return Response(status_code=204)
 
@@ -237,6 +252,7 @@ def unlink_user_file_from_project(
 def link_user_file_to_project(
     project_id: int,
     file_id: UUID,
+    bg_tasks: BackgroundTasks,
     user: User = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> UserFileSnapshot:
@@ -268,7 +284,7 @@ def link_user_file_to_project(
         db_session.commit()
 
     tenant_id = get_current_tenant_id()
-    _trigger_user_file_project_sync(user_file.id, tenant_id)
+    _trigger_user_file_project_sync(user_file.id, tenant_id, bg_tasks)
 
     return UserFileSnapshot.from_model(user_file)
 
@@ -335,7 +351,7 @@ def upsert_project_instructions(
 class ProjectPayload(BaseModel):
     project: UserProjectSnapshot
     files: list[UserFileSnapshot] | None = None
-    persona_id_to_is_default: dict[int, bool] | None = None
+    persona_id_to_featured: dict[int, bool] | None = None
 
 
 @router.get(
@@ -354,13 +370,11 @@ def get_project_details(
         if session.persona_id is not None
     ]
     personas = get_personas_by_ids(persona_ids, db_session)
-    persona_id_to_is_default = {
-        persona.id: persona.is_default_persona for persona in personas
-    }
+    persona_id_to_featured = {persona.id: persona.featured for persona in personas}
     return ProjectPayload(
         project=project,
         files=files,
-        persona_id_to_is_default=persona_id_to_is_default,
+        persona_id_to_featured=persona_id_to_featured,
     )
 
 
@@ -426,6 +440,7 @@ def delete_project(
 @router.delete("/file/{file_id}", tags=PUBLIC_API_TAGS)
 def delete_user_file(
     file_id: UUID,
+    bg_tasks: BackgroundTasks,
     user: User = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> UserFileDeleteResult:
@@ -458,15 +473,25 @@ def delete_user_file(
     db_session.commit()
 
     tenant_id = get_current_tenant_id()
-    task = client_app.send_task(
-        OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
-        kwargs={"user_file_id": str(user_file.id), "tenant_id": tenant_id},
-        queue=OnyxCeleryQueues.USER_FILE_DELETE,
-        priority=OnyxCeleryPriority.HIGH,
-    )
-    logger.info(
-        f"Triggered delete for user_file_id={user_file.id} with task_id={task.id}"
-    )
+    if DISABLE_VECTOR_DB:
+        from onyx.background.task_utils import drain_delete_loop
+
+        bg_tasks.add_task(drain_delete_loop, tenant_id)
+        logger.info(f"Queued in-process delete for user_file_id={user_file.id}")
+    else:
+        from onyx.background.celery.versioned_apps.client import app as client_app
+
+        task = client_app.send_task(
+            OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
+            kwargs={"user_file_id": str(user_file.id), "tenant_id": tenant_id},
+            queue=OnyxCeleryQueues.USER_FILE_DELETE,
+            priority=OnyxCeleryPriority.HIGH,
+        )
+        logger.info(
+            f"Triggered delete for user_file_id={user_file.id} "
+            f"with task_id={task.id}"
+        )
+
     return UserFileDeleteResult(
         has_associations=False, project_names=[], assistant_names=[]
     )

backend/onyx/server/kg/api.py

@@ -5,7 +5,6 @@ from sqlalchemy.orm import Session
 from onyx.auth.users import current_admin_user
 from onyx.configs.constants import TMP_DRALPHA_PERSONA_NAME
 from onyx.configs.kg_configs import KG_BETA_ASSISTANT_DESCRIPTION
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.entities import get_entity_stats_by_grounded_source_name
 from onyx.db.entity_type import get_configured_entity_types
@@ -134,11 +133,7 @@ def enable_or_disable_kg(
         system_prompt=KG_BETA_ASSISTANT_SYSTEM_PROMPT,
         task_prompt=KG_BETA_ASSISTANT_TASK_PROMPT,
         datetime_aware=False,
-        num_chunks=25,
-        llm_relevance_filter=False,
         is_public=False,
-        llm_filter_extraction=False,
-        recency_bias=RecencyBiasSetting.NO_DECAY,
         document_set_ids=[],
         tool_ids=[search_tool.id, kg_tool.id],
         llm_model_provider_override=None,
@@ -147,7 +142,7 @@ def enable_or_disable_kg(
         users=[user.id],
         groups=[],
         label_ids=[],
-        is_default_persona=False,
+        featured=False,
         display_priority=0,
         user_file_ids=[],
     )

backend/onyx/server/manage/image_generation/api.py

@@ -97,7 +97,6 @@ def _build_llm_provider_request(
             ),  # Only this from source
             api_base=api_base,  # From request
             api_version=api_version,  # From request
-            default_model_name=model_name,
             deployment_name=deployment_name,  # From request
             is_public=True,
             groups=[],
@@ -136,7 +135,6 @@ def _build_llm_provider_request(
         api_key=api_key,
         api_base=api_base,
         api_version=api_version,
-        default_model_name=model_name,
         deployment_name=deployment_name,
         is_public=True,
         groups=[],
@@ -168,7 +166,6 @@ def _create_image_gen_llm_provider__no_commit(
         api_key=provider_request.api_key,
         api_base=provider_request.api_base,
         api_version=provider_request.api_version,
-        default_model_name=provider_request.default_model_name,
         deployment_name=provider_request.deployment_name,
         is_public=provider_request.is_public,
         custom_config=provider_request.custom_config,

backend/onyx/server/manage/llm/api.py

@@ -22,7 +22,10 @@ from onyx.auth.users import current_chat_accessible_user
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.enums import LLMModelFlowType
 from onyx.db.llm import can_user_access_llm_provider
+from onyx.db.llm import fetch_default_llm_model
+from onyx.db.llm import fetch_default_vision_model
 from onyx.db.llm import fetch_existing_llm_provider
+from onyx.db.llm import fetch_existing_llm_provider_by_id
 from onyx.db.llm import fetch_existing_llm_providers
 from onyx.db.llm import fetch_existing_models
 from onyx.db.llm import fetch_persona_with_groups
@@ -52,11 +55,12 @@ from onyx.llm.well_known_providers.llm_provider_options import (
 )
 from onyx.server.manage.llm.models import BedrockFinalModelResponse
 from onyx.server.manage.llm.models import BedrockModelsRequest
+from onyx.server.manage.llm.models import DefaultModel
 from onyx.server.manage.llm.models import LLMCost
 from onyx.server.manage.llm.models import LLMProviderDescriptor
+from onyx.server.manage.llm.models import LLMProviderResponse
 from onyx.server.manage.llm.models import LLMProviderUpsertRequest
 from onyx.server.manage.llm.models import LLMProviderView
-from onyx.server.manage.llm.models import ModelConfigurationUpsertRequest
 from onyx.server.manage.llm.models import OllamaFinalModelResponse
 from onyx.server.manage.llm.models import OllamaModelDetails
 from onyx.server.manage.llm.models import OllamaModelsRequest
@@ -233,12 +237,9 @@ def test_llm_configuration(
 
     test_api_key = test_llm_request.api_key
     test_custom_config = test_llm_request.custom_config
-    if test_llm_request.name:
-        # NOTE: we are querying by name. we probably should be querying by an invariant id, but
-        # as it turns out the name is not editable in the UI and other code also keys off name,
-        # so we won't rock the boat just yet.
-        existing_provider = fetch_existing_llm_provider(
-            name=test_llm_request.name, db_session=db_session
+    if test_llm_request.id:
+        existing_provider = fetch_existing_llm_provider_by_id(
+            id=test_llm_request.id, db_session=db_session
         )
         if existing_provider:
             test_custom_config = _restore_masked_custom_config_values(
@@ -268,7 +269,7 @@ def test_llm_configuration(
 
     llm = get_llm(
         provider=test_llm_request.provider,
-        model=test_llm_request.default_model_name,
+        model=test_llm_request.model,
         api_key=test_api_key,
         api_base=test_llm_request.api_base,
         api_version=test_llm_request.api_version,
@@ -303,7 +304,7 @@ def list_llm_providers(
     include_image_gen: bool = Query(False),
     _: User = Depends(current_admin_user),
     db_session: Session = Depends(get_session),
-) -> list[LLMProviderView]:
+) -> LLMProviderResponse[LLMProviderView]:
     start_time = datetime.now(timezone.utc)
     logger.debug("Starting to fetch LLM providers")
 
@@ -328,7 +329,15 @@ def list_llm_providers(
     duration = (end_time - start_time).total_seconds()
     logger.debug(f"Completed fetching LLM providers in {duration:.2f} seconds")
 
-    return llm_provider_list
+    return LLMProviderResponse[LLMProviderView].from_models(
+        providers=llm_provider_list,
+        default_text=DefaultModel.from_model_config(
+            fetch_default_llm_model(db_session)
+        ),
+        default_vision=DefaultModel.from_model_config(
+            fetch_default_vision_model(db_session)
+        ),
+    )
 
 
 @admin_router.put("/provider")
@@ -344,18 +353,44 @@ def put_llm_provider(
     # validate request (e.g. if we're intending to create but the name already exists we should throw an error)
     # NOTE: may involve duplicate fetching to Postgres, but we're assuming SQLAlchemy is smart enough to cache
     # the result
-    existing_provider = fetch_existing_llm_provider(
+    existing_provider = None
+    if llm_provider_upsert_request.id:
+        existing_provider = fetch_existing_llm_provider_by_id(
+            id=llm_provider_upsert_request.id, db_session=db_session
+        )
+
+    # Check name constraints
+    # TODO: Once port from name to id is complete, unique name will no longer be required
+    if existing_provider and llm_provider_upsert_request.name != existing_provider.name:
+        raise HTTPException(
+            status_code=400,
+            detail="Renaming providers is not currently supported",
+        )
+
+    found_provider = fetch_existing_llm_provider(
         name=llm_provider_upsert_request.name, db_session=db_session
     )
+    if found_provider is not None and found_provider is not existing_provider:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Provider with name={llm_provider_upsert_request.name} already exists",
+        )
+
     if existing_provider and is_creation:
         raise HTTPException(
             status_code=400,
-            detail=f"LLM Provider with name {llm_provider_upsert_request.name} already exists",
+            detail=(
+                f"LLM Provider with name {llm_provider_upsert_request.name} and "
+                f"id={llm_provider_upsert_request.id} already exists"
+            ),
         )
     elif not existing_provider and not is_creation:
         raise HTTPException(
             status_code=400,
-            detail=f"LLM Provider with name {llm_provider_upsert_request.name} does not exist",
+            detail=(
+                f"LLM Provider with name {llm_provider_upsert_request.name} and "
+                f"id={llm_provider_upsert_request.id} does not exist"
+            ),
         )
 
     # SSRF Protection: Validate api_base and custom_config match stored values
@@ -393,22 +428,6 @@ def put_llm_provider(
                 deduplicated_personas.append(persona_id)
         llm_provider_upsert_request.personas = deduplicated_personas
 
-    default_model_found = False
-
-    for model_configuration in llm_provider_upsert_request.model_configurations:
-        if model_configuration.name == llm_provider_upsert_request.default_model_name:
-            model_configuration.is_visible = True
-            default_model_found = True
-
-    # TODO: Remove this logic on api change
-    # Believed to be a dead pathway but we want to be safe for now
-    if not default_model_found:
-        llm_provider_upsert_request.model_configurations.append(
-            ModelConfigurationUpsertRequest(
-                name=llm_provider_upsert_request.default_model_name, is_visible=True
-            )
-        )
-
     # the llm api key is sanitized when returned to clients, so the only time we
     # should get a real key is when it is explicitly changed
     if existing_provider and not llm_provider_upsert_request.api_key_changed:
@@ -438,8 +457,8 @@ def put_llm_provider(
             config = fetch_llm_recommendations_from_github()
             if config and llm_provider_upsert_request.provider in config.providers:
                 # Refetch the provider to get the updated model
-                updated_provider = fetch_existing_llm_provider(
-                    name=llm_provider_upsert_request.name, db_session=db_session
+                updated_provider = fetch_existing_llm_provider_by_id(
+                    id=result.id, db_session=db_session
                 )
                 if updated_provider:
                     sync_auto_mode_models(
@@ -469,28 +488,29 @@ def delete_llm_provider(
         raise HTTPException(status_code=404, detail=str(e))
 
 
-@admin_router.post("/provider/{provider_id}/default")
+@admin_router.post("/default")
 def set_provider_as_default(
-    provider_id: int,
+    default_model_request: DefaultModel,
     _: User = Depends(current_admin_user),
     db_session: Session = Depends(get_session),
 ) -> None:
-    update_default_provider(provider_id=provider_id, db_session=db_session)
+    update_default_provider(
+        provider_id=default_model_request.provider_id,
+        model_name=default_model_request.model_name,
+        db_session=db_session,
+    )
 
 
-@admin_router.post("/provider/{provider_id}/default-vision")
+@admin_router.post("/default-vision")
 def set_provider_as_default_vision(
-    provider_id: int,
-    vision_model: str | None = Query(
-        None, description="The default vision model to use"
-    ),
+    default_model: DefaultModel,
     _: User = Depends(current_admin_user),
     db_session: Session = Depends(get_session),
 ) -> None:
-    if vision_model is None:
-        raise HTTPException(status_code=404, detail="Vision model not provided")
     update_default_vision_provider(
-        provider_id=provider_id, vision_model=vision_model, db_session=db_session
+        provider_id=default_model.provider_id,
+        vision_model=default_model.model_name,
+        db_session=db_session,
     )
 
 
@@ -516,7 +536,7 @@ def get_auto_config(
 def get_vision_capable_providers(
     _: User = Depends(current_admin_user),
     db_session: Session = Depends(get_session),
-) -> list[VisionProviderResponse]:
+) -> LLMProviderResponse[VisionProviderResponse]:
     """Return a list of LLM providers and their models that support image input"""
     vision_models = fetch_existing_models(
         db_session=db_session, flow_types=[LLMModelFlowType.VISION]
@@ -545,7 +565,13 @@ def get_vision_capable_providers(
     ]
 
     logger.debug(f"Found {len(vision_provider_response)} vision-capable providers")
-    return vision_provider_response
+
+    return LLMProviderResponse[VisionProviderResponse].from_models(
+        providers=vision_provider_response,
+        default_vision=DefaultModel.from_model_config(
+            fetch_default_vision_model(db_session)
+        ),
+    )
 
 
 """Endpoints for all"""
@@ -555,7 +581,7 @@ def get_vision_capable_providers(
 def list_llm_provider_basics(
     user: User = Depends(current_chat_accessible_user),
     db_session: Session = Depends(get_session),
-) -> list[LLMProviderDescriptor]:
+) -> LLMProviderResponse[LLMProviderDescriptor]:
     """Get LLM providers accessible to the current user.
 
     Returns:
@@ -577,9 +603,9 @@ def list_llm_provider_basics(
     for provider in all_providers:
         # Use centralized access control logic with persona=None since we're
         # listing providers without a specific persona context. This correctly:
-        # - Includes all public providers
+        # - Includes public providers WITHOUT persona restrictions
         # - Includes providers user can access via group membership
-        # - Excludes persona-only restricted providers (requires specific persona)
+        # - Excludes providers with persona restrictions (requires specific persona)
         # - Excludes non-public providers with no restrictions (admin-only)
         if can_user_access_llm_provider(
             provider, user_group_ids, persona=None, is_admin=is_admin
@@ -592,7 +618,15 @@ def list_llm_provider_basics(
         f"Completed fetching {len(accessible_providers)} user-accessible providers in {duration:.2f} seconds"
     )
 
-    return accessible_providers
+    return LLMProviderResponse[LLMProviderDescriptor].from_models(
+        providers=accessible_providers,
+        default_text=DefaultModel.from_model_config(
+            fetch_default_llm_model(db_session)
+        ),
+        default_vision=DefaultModel.from_model_config(
+            fetch_default_vision_model(db_session)
+        ),
+    )
 
 
 def get_valid_model_names_for_persona(
@@ -604,7 +638,7 @@ def get_valid_model_names_for_persona(
 
     Returns a list of model names (e.g., ["gpt-4o", "claude-3-5-sonnet"]) that are
     available to the user when using this persona, respecting all RBAC restrictions.
-    Public providers are always included.
+    Public providers are included unless they have persona restrictions that exclude this persona.
     """
     persona = fetch_persona_with_groups(db_session, persona_id)
     if not persona:
@@ -618,7 +652,7 @@ def get_valid_model_names_for_persona(
 
     valid_models = []
     for llm_provider_model in all_providers:
-        # Public providers always included, restricted checked via RBAC
+        # Check access with persona context — respects all RBAC restrictions
         if can_user_access_llm_provider(
             llm_provider_model, user_group_ids, persona, is_admin=is_admin
         ):
@@ -635,11 +669,11 @@ def list_llm_providers_for_persona(
     persona_id: int,
     user: User = Depends(current_chat_accessible_user),
     db_session: Session = Depends(get_session),
-) -> list[LLMProviderDescriptor]:
+) -> LLMProviderResponse[LLMProviderDescriptor]:
     """Get LLM providers for a specific persona.
 
     Returns providers that the user can access when using this persona:
-    - All public providers (is_public=True) - ALWAYS included
+    - Public providers (respecting persona restrictions if set)
     - Restricted providers user can access via group/persona restrictions
 
     This endpoint is used for background fetching of restricted providers
@@ -668,7 +702,7 @@ def list_llm_providers_for_persona(
     llm_provider_list: list[LLMProviderDescriptor] = []
 
     for llm_provider_model in all_providers:
-        # Use simplified access check - public providers always included
+        # Check access with persona context — respects persona restrictions
         if can_user_access_llm_provider(
             llm_provider_model, user_group_ids, persona, is_admin=is_admin
         ):
@@ -682,7 +716,51 @@ def list_llm_providers_for_persona(
         f"Completed fetching {len(llm_provider_list)} LLM providers for persona {persona_id} in {duration:.2f} seconds"
     )
 
-    return llm_provider_list
+    # Get the default model and vision model for the persona
+    # TODO: Port persona's over to use ID
+    persona_default_provider = persona.llm_model_provider_override
+    persona_default_model = persona.llm_model_version_override
+
+    default_text_model = fetch_default_llm_model(db_session)
+    default_vision_model = fetch_default_vision_model(db_session)
+
+    # Build default_text and default_vision using persona overrides when available,
+    # falling back to the global defaults.
+    default_text = DefaultModel.from_model_config(default_text_model)
+    default_vision = DefaultModel.from_model_config(default_vision_model)
+
+    if persona_default_provider:
+        provider = fetch_existing_llm_provider(persona_default_provider, db_session)
+        if provider and can_user_access_llm_provider(
+            provider, user_group_ids, persona, is_admin=is_admin
+        ):
+            if persona_default_model:
+                # Persona specifies both provider and model — use them directly
+                default_text = DefaultModel(
+                    provider_id=provider.id,
+                    model_name=persona_default_model,
+                )
+            else:
+                # Persona specifies only the provider — pick a visible (public) model,
+                # falling back to any model on this provider
+                visible_model = next(
+                    (mc for mc in provider.model_configurations if mc.is_visible),
+                    None,
+                )
+                fallback_model = visible_model or next(
+                    iter(provider.model_configurations), None
+                )
+                if fallback_model:
+                    default_text = DefaultModel(
+                        provider_id=provider.id,
+                        model_name=fallback_model.name,
+                    )
+
+    return LLMProviderResponse[LLMProviderDescriptor].from_models(
+        providers=llm_provider_list,
+        default_text=default_text,
+        default_vision=default_vision,
+    )
 
 
 @admin_router.get("/provider-contextual-cost")

backend/onyx/server/manage/llm/models.py

@@ -1,5 +1,9 @@
+from __future__ import annotations
+
 from typing import Any
+from typing import Generic
 from typing import TYPE_CHECKING
+from typing import TypeVar
 
 from pydantic import BaseModel
 from pydantic import Field
@@ -21,50 +25,22 @@ if TYPE_CHECKING:
         ModelConfiguration as ModelConfigurationModel,
     )
 
-
-# TODO: Clear this up on api refactor
-# There is still logic that requires sending each providers default model name
-# There is no logic that requires sending the providers default vision model name
-# We only send for the one that is actually the default
-def get_default_llm_model_name(llm_provider_model: "LLMProviderModel") -> str:
-    """Find the default conversation model name for a provider.
-
-    Returns the model name if found, otherwise returns empty string.
-    """
-    for model_config in llm_provider_model.model_configurations:
-        for flow in model_config.llm_model_flows:
-            if flow.is_default and flow.llm_model_flow_type == LLMModelFlowType.CHAT:
-                return model_config.name
-    return ""
-
-
-def get_default_vision_model_name(llm_provider_model: "LLMProviderModel") -> str | None:
-    """Find the default vision model name for a provider.
-
-    Returns the model name if found, otherwise returns None.
-    """
-    for model_config in llm_provider_model.model_configurations:
-        for flow in model_config.llm_model_flows:
-            if flow.is_default and flow.llm_model_flow_type == LLMModelFlowType.VISION:
-                return model_config.name
-    return None
+T = TypeVar("T", "LLMProviderDescriptor", "LLMProviderView", "VisionProviderResponse")
 
 
 class TestLLMRequest(BaseModel):
     # provider level
-    name: str | None = None
+    id: int | None = None
     provider: str
+    model: str
     api_key: str | None = None
     api_base: str | None = None
     api_version: str | None = None
     custom_config: dict[str, str] | None = None
 
     # model level
-    default_model_name: str
     deployment_name: str | None = None
 
-    model_configurations: list["ModelConfigurationUpsertRequest"]
-
     # if try and use the existing API/custom config key
     api_key_changed: bool
     custom_config_changed: bool
@@ -80,13 +56,10 @@ class LLMProviderDescriptor(BaseModel):
     """A descriptor for an LLM provider that can be safely viewed by
     non-admin users. Used when giving a list of available LLMs."""
 
+    id: int
     name: str
     provider: str
     provider_display_name: str  # Human-friendly name like "Claude (Anthropic)"
-    default_model_name: str
-    is_default_provider: bool | None
-    is_default_vision_provider: bool | None
-    default_vision_model: str | None
     model_configurations: list["ModelConfigurationView"]
 
     @classmethod
@@ -99,24 +72,12 @@ class LLMProviderDescriptor(BaseModel):
         )
 
         provider = llm_provider_model.provider
-        default_model_name = get_default_llm_model_name(llm_provider_model)
-        default_vision_model = get_default_vision_model_name(llm_provider_model)
-
-        is_default_provider = bool(default_model_name)
-        is_default_vision_provider = default_vision_model is not None
-
-        default_model_name = (
-            default_model_name or llm_provider_model.default_model_name or ""
-        )
 
         return cls(
+            id=llm_provider_model.id,
             name=llm_provider_model.name,
             provider=provider,
             provider_display_name=get_provider_display_name(provider),
-            default_model_name=default_model_name,
-            is_default_provider=is_default_provider,
-            is_default_vision_provider=is_default_vision_provider,
-            default_vision_model=default_vision_model,
             model_configurations=filter_model_configurations(
                 llm_provider_model.model_configurations, provider
             ),
@@ -130,18 +91,17 @@ class LLMProvider(BaseModel):
     api_base: str | None = None
     api_version: str | None = None
     custom_config: dict[str, str] | None = None
-    default_model_name: str
     is_public: bool = True
     is_auto_mode: bool = False
     groups: list[int] = Field(default_factory=list)
     personas: list[int] = Field(default_factory=list)
     deployment_name: str | None = None
-    default_vision_model: str | None = None
 
 
 class LLMProviderUpsertRequest(LLMProvider):
     # should only be used for a "custom" provider
     # for default providers, the built-in model names are used
+    id: int | None = None
     api_key_changed: bool = False
     custom_config_changed: bool = False
     model_configurations: list["ModelConfigurationUpsertRequest"] = []
@@ -157,8 +117,6 @@ class LLMProviderView(LLMProvider):
     """Stripped down representation of LLMProvider for display / limited access info only"""
 
     id: int
-    is_default_provider: bool | None = None
-    is_default_vision_provider: bool | None = None
     model_configurations: list["ModelConfigurationView"]
 
     @classmethod
@@ -180,16 +138,6 @@ class LLMProviderView(LLMProvider):
 
         provider = llm_provider_model.provider
 
-        default_model_name = get_default_llm_model_name(llm_provider_model)
-        default_vision_model = get_default_vision_model_name(llm_provider_model)
-
-        is_default_provider = bool(default_model_name)
-        is_default_vision_provider = default_vision_model is not None
-
-        default_model_name = (
-            default_model_name or llm_provider_model.default_model_name or ""
-        )
-
         return cls(
             id=llm_provider_model.id,
             name=llm_provider_model.name,
@@ -202,10 +150,6 @@ class LLMProviderView(LLMProvider):
             api_base=llm_provider_model.api_base,
             api_version=llm_provider_model.api_version,
             custom_config=llm_provider_model.custom_config,
-            default_model_name=default_model_name,
-            is_default_provider=is_default_provider,
-            is_default_vision_provider=is_default_vision_provider,
-            default_vision_model=default_vision_model,
             is_public=llm_provider_model.is_public,
             is_auto_mode=llm_provider_model.is_auto_mode,
             groups=groups,
@@ -425,3 +369,38 @@ class OpenRouterFinalModelResponse(BaseModel):
         int | None
     )  # From OpenRouter API context_length (may be missing for some models)
     supports_image_input: bool
+
+
+class DefaultModel(BaseModel):
+    provider_id: int
+    model_name: str
+
+    @classmethod
+    def from_model_config(
+        cls, model_config: ModelConfigurationModel | None
+    ) -> DefaultModel | None:
+        if not model_config:
+            return None
+        return cls(
+            provider_id=model_config.llm_provider_id,
+            model_name=model_config.name,
+        )
+
+
+class LLMProviderResponse(BaseModel, Generic[T]):
+    providers: list[T]
+    default_text: DefaultModel | None = None
+    default_vision: DefaultModel | None = None
+
+    @classmethod
+    def from_models(
+        cls,
+        providers: list[T],
+        default_text: DefaultModel | None = None,
+        default_vision: DefaultModel | None = None,
+    ) -> LLMProviderResponse[T]:
+        return cls(
+            providers=providers,
+            default_text=default_text,
+            default_vision=default_vision,
+        )

backend/onyx/server/manage/slack_bot.py

@@ -198,7 +198,6 @@ def patch_slack_channel_config(
             channel_name=channel_config["channel_name"],
             document_set_ids=slack_channel_config_creation_request.document_sets,
             existing_persona_id=existing_persona_id,
-            enable_auto_filters=slack_channel_config_creation_request.enable_auto_filters,
         ).id
 
     slack_channel_config_model = update_slack_channel_config(

backend/onyx/server/metrics/per_tenant.py

@@ -0,0 +1,27 @@
+"""Per-tenant request counter metric.
+
+Increments a counter on every request, labelled by tenant, so Grafana can
+answer "which tenant is generating the most traffic?"
+"""
+
+from prometheus_client import Counter
+from prometheus_fastapi_instrumentator.metrics import Info
+
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
+
+_requests_by_tenant = Counter(
+    "onyx_api_requests_by_tenant_total",
+    "Total API requests by tenant",
+    ["tenant_id", "method", "handler", "status"],
+)
+
+
+def per_tenant_request_callback(info: Info) -> None:
+    """Increment per-tenant request counter for every request."""
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get() or "unknown"
+    _requests_by_tenant.labels(
+        tenant_id=tenant_id,
+        method=info.method,
+        handler=info.modified_handler,
+        status=info.modified_status,
+    ).inc()

backend/onyx/server/metrics/postgres_connection_pool.py

@@ -32,6 +32,7 @@ from sqlalchemy.pool import QueuePool
 
 from onyx.utils.logger import setup_logger
 from shared_configs.contextvars import CURRENT_ENDPOINT_CONTEXTVAR
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
@@ -72,7 +73,7 @@ _checkout_timeout_total = Counter(
 _connections_held = Gauge(
     "onyx_db_connections_held_by_endpoint",
     "Number of DB connections currently held, by endpoint and engine",
-    ["handler", "engine"],
+    ["handler", "engine", "tenant_id"],
 )
 
 _hold_seconds = Histogram(
@@ -163,10 +164,14 @@ def _register_pool_events(engine: Engine, label: str) -> None:
         conn_proxy: PoolProxiedConnection,  # noqa: ARG001
     ) -> None:
         handler = CURRENT_ENDPOINT_CONTEXTVAR.get() or "unknown"
+        tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get() or "unknown"
         conn_record.info["_metrics_endpoint"] = handler
+        conn_record.info["_metrics_tenant_id"] = tenant_id
         conn_record.info["_metrics_checkout_time"] = time.monotonic()
         _checkout_total.labels(engine=label).inc()
-        _connections_held.labels(handler=handler, engine=label).inc()
+        _connections_held.labels(
+            handler=handler, engine=label, tenant_id=tenant_id
+        ).inc()
 
     @event.listens_for(engine, "checkin")
     def on_checkin(
@@ -174,9 +179,12 @@ def _register_pool_events(engine: Engine, label: str) -> None:
         conn_record: ConnectionPoolEntry,
     ) -> None:
         handler = conn_record.info.pop("_metrics_endpoint", "unknown")
+        tenant_id = conn_record.info.pop("_metrics_tenant_id", "unknown")
         start = conn_record.info.pop("_metrics_checkout_time", None)
         _checkin_total.labels(engine=label).inc()
-        _connections_held.labels(handler=handler, engine=label).dec()
+        _connections_held.labels(
+            handler=handler, engine=label, tenant_id=tenant_id
+        ).dec()
         if start is not None:
             _hold_seconds.labels(handler=handler, engine=label).observe(
                 time.monotonic() - start
@@ -199,9 +207,12 @@ def _register_pool_events(engine: Engine, label: str) -> None:
         # Defensively clean up the held-connections gauge in case checkin
         # doesn't fire after invalidation (e.g. hard pool shutdown).
         handler = conn_record.info.pop("_metrics_endpoint", None)
+        tenant_id = conn_record.info.pop("_metrics_tenant_id", "unknown")
         start = conn_record.info.pop("_metrics_checkout_time", None)
         if handler:
-            _connections_held.labels(handler=handler, engine=label).dec()
+            _connections_held.labels(
+                handler=handler, engine=label, tenant_id=tenant_id
+            ).dec()
         if start is not None:
             _hold_seconds.labels(handler=handler or "unknown", engine=label).observe(
                 time.monotonic() - start

backend/onyx/server/metrics/prometheus_setup.py

@@ -11,9 +11,11 @@ SQLAlchemy connection pool metrics are registered separately via
 """
 
 from prometheus_fastapi_instrumentator import Instrumentator
+from prometheus_fastapi_instrumentator.metrics import default as default_metrics
 from sqlalchemy.exc import TimeoutError as SATimeoutError
 from starlette.applications import Starlette
 
+from onyx.server.metrics.per_tenant import per_tenant_request_callback
 from onyx.server.metrics.postgres_connection_pool import pool_timeout_handler
 from onyx.server.metrics.slow_requests import slow_request_callback
 
@@ -59,6 +61,15 @@ def setup_prometheus_metrics(app: Starlette) -> None:
         excluded_handlers=_EXCLUDED_HANDLERS,
     )
 
+    # Explicitly create the default metrics (http_requests_total,
+    # http_request_duration_seconds, etc.) and add them first.  The library
+    # skips creating defaults when ANY custom instrumentations are registered
+    # via .add(), so we must include them ourselves.
+    default_callback = default_metrics(latency_lowr_buckets=_LATENCY_BUCKETS)
+    if default_callback:
+        instrumentator.add(default_callback)
+
     instrumentator.add(slow_request_callback)
+    instrumentator.add(per_tenant_request_callback)
 
     instrumentator.instrument(app, latency_lowr_buckets=_LATENCY_BUCKETS).expose(app)

backend/onyx/server/query_and_chat/chat_backend.py

@@ -152,10 +152,20 @@ def get_user_chat_sessions(
     project_id: int | None = None,
     only_non_project_chats: bool = True,
     include_failed_chats: bool = False,
+    page_size: int = Query(default=50, ge=1, le=100),
+    before: str | None = Query(default=None),
 ) -> ChatSessionsResponse:
     user_id = user.id
 
     try:
+        before_dt = (
+            datetime.datetime.fromisoformat(before) if before is not None else None
+        )
+    except ValueError:
+        raise HTTPException(status_code=422, detail="Invalid 'before' timestamp format")
+
+    try:
+        # Fetch one extra to determine if there are more results
         chat_sessions = get_chat_sessions_by_user(
             user_id=user_id,
             deleted=False,
@@ -163,11 +173,16 @@ def get_user_chat_sessions(
             project_id=project_id,
             only_non_project_chats=only_non_project_chats,
             include_failed_chats=include_failed_chats,
+            limit=page_size + 1,
+            before=before_dt,
         )
 
     except ValueError:
         raise ValueError("Chat session does not exist or has been deleted")
 
+    has_more = len(chat_sessions) > page_size
+    chat_sessions = chat_sessions[:page_size]
+
     return ChatSessionsResponse(
         sessions=[
             ChatSessionDetails(
@@ -181,7 +196,8 @@ def get_user_chat_sessions(
                 current_temperature_override=chat.temperature_override,
             )
             for chat in chat_sessions
-        ]
+        ],
+        has_more=has_more,
     )
 
 

backend/onyx/server/query_and_chat/models.py

@@ -192,6 +192,7 @@ class ChatSessionDetails(BaseModel):
 
 class ChatSessionsResponse(BaseModel):
     sessions: list[ChatSessionDetails]
+    has_more: bool = False
 
 
 class ChatMessageDetail(BaseModel):

backend/onyx/server/settings/models.py

@@ -19,6 +19,7 @@ class ApplicationStatus(str, Enum):
     PAYMENT_REMINDER = "payment_reminder"
     GRACE_PERIOD = "grace_period"
     GATED_ACCESS = "gated_access"
+    SEAT_LIMIT_EXCEEDED = "seat_limit_exceeded"
 
 
 class Notification(BaseModel):
@@ -82,6 +83,10 @@ class Settings(BaseModel):
     # Default Assistant settings
     disable_default_assistant: bool | None = False
 
+    # Seat usage - populated by license enforcement when seat limit is exceeded
+    seat_count: int | None = None
+    used_seats: int | None = None
+
     # OpenSearch migration
     opensearch_indexing_enabled: bool = False
 

backend/onyx/setup.py

@@ -25,6 +25,7 @@ from onyx.db.enums import EmbeddingPrecision
 from onyx.db.index_attempt import cancel_indexing_attempts_past_model
 from onyx.db.index_attempt import expire_index_attempts
 from onyx.db.llm import fetch_default_llm_model
+from onyx.db.llm import fetch_existing_llm_provider
 from onyx.db.llm import update_default_provider
 from onyx.db.llm import upsert_llm_provider
 from onyx.db.search_settings import get_active_search_settings
@@ -254,14 +255,18 @@ def setup_postgres(db_session: Session) -> None:
         logger.notice("Setting up default OpenAI LLM for dev.")
 
         llm_model = GEN_AI_MODEL_VERSION or "gpt-4o-mini"
+        provider_name = "DevEnvPresetOpenAI"
+        existing = fetch_existing_llm_provider(
+            name=provider_name, db_session=db_session
+        )
         model_req = LLMProviderUpsertRequest(
-            name="DevEnvPresetOpenAI",
+            id=existing.id if existing else None,
+            name=provider_name,
             provider=LlmProviderNames.OPENAI,
             api_key=GEN_AI_API_KEY,
             api_base=None,
             api_version=None,
             custom_config=None,
-            default_model_name=llm_model,
             is_public=True,
             groups=[],
             model_configurations=[
@@ -273,7 +278,9 @@ def setup_postgres(db_session: Session) -> None:
         new_llm_provider = upsert_llm_provider(
             llm_provider_upsert_request=model_req, db_session=db_session
         )
-        update_default_provider(provider_id=new_llm_provider.id, db_session=db_session)
+        update_default_provider(
+            provider_id=new_llm_provider.id, model_name=llm_model, db_session=db_session
+        )
 
 
 def update_default_multipass_indexing(db_session: Session) -> None:

backend/onyx/tools/fake_tools/research_agent.py

@@ -120,7 +120,7 @@ def generate_intermediate_report(
             custom_agent_prompt=None,
             simple_chat_history=history,
             reminder_message=reminder_message,
-            project_files=None,
+            context_files=None,
             available_tokens=llm.config.max_input_tokens,
         )
 
@@ -325,7 +325,7 @@ def run_research_agent_call(
                     custom_agent_prompt=None,
                     simple_chat_history=msg_history,
                     reminder_message=reminder_message,
-                    project_files=None,
+                    context_files=None,
                     available_tokens=llm.config.max_input_tokens,
                 )
 

backend/pyproject.toml

@@ -8,37 +8,3 @@ dependencies = [
 
 [tool.uv.sources]
 onyx = { workspace = true }
-
-[tool.mypy]
-plugins = "sqlalchemy.ext.mypy.plugin"
-mypy_path = "backend"
-explicit_package_bases = true
-disallow_untyped_defs = true
-warn_unused_ignores = true
-enable_error_code = ["possibly-undefined"]
-strict_equality = true
-# Patterns match paths whether mypy is run from backend/ (CI) or repo root (e.g. VS Code extension with target ./backend)
-exclude = [
-  "(?:^|/)generated/",
-  "(?:^|/)\\.venv/",
-  "(?:^|/)onyx/server/features/build/sandbox/kubernetes/docker/skills/",
-  "(?:^|/)onyx/server/features/build/sandbox/kubernetes/docker/templates/",
-]
-
-[[tool.mypy.overrides]]
-module = "alembic.versions.*"
-disable_error_code = ["var-annotated"]
-
-[[tool.mypy.overrides]]
-module = "alembic_tenants.versions.*"
-disable_error_code = ["var-annotated"]
-
-[[tool.mypy.overrides]]
-module = "generated.*"
-follow_imports = "silent"
-ignore_errors = true
-
-[[tool.mypy.overrides]]
-module = "transformers.*"
-follow_imports = "skip"
-ignore_errors = true

backend/requirements/default.txt

@@ -257,7 +257,7 @@ exceptiongroup==1.3.0
     # via
     #   braintrust
     #   fastmcp
-fastapi==0.128.0
+fastapi==0.133.1
     # via
     #   fastapi-limiter
     #   fastapi-users
@@ -528,7 +528,7 @@ lxml==5.3.0
     #   unstructured
     #   xmlsec
     #   zeep
-lxml-html-clean==0.4.3
+lxml-html-clean==0.4.4
     # via lxml
 magika==0.6.3
     # via markitdown
@@ -809,7 +809,7 @@ pypandoc-binary==1.16.2
     # via onyx
 pyparsing==3.2.5
     # via httplib2
-pypdf==6.6.2
+pypdf==6.7.5
     # via
     #   onyx
     #   unstructured-client
@@ -1155,6 +1155,7 @@ typing-inspect==0.9.0
     # via dataclasses-json
 typing-inspection==0.4.2
     # via
+    #   fastapi
     #   mcp
     #   pydantic
     #   pydantic-settings
@@ -1216,7 +1217,7 @@ websockets==15.0.1
     # via
     #   fastmcp
     #   google-genai
-werkzeug==3.1.5
+werkzeug==3.1.6
     # via sendgrid
 wrapt==1.17.3
     # via

backend/requirements/dev.txt

@@ -125,7 +125,7 @@ executing==2.2.1
     # via stack-data
 faker==40.1.2
     # via onyx
-fastapi==0.128.0
+fastapi==0.133.1
     # via
     #   onyx
     #   onyx-devtools
@@ -619,6 +619,7 @@ typing-extensions==4.15.0
     #   typing-inspection
 typing-inspection==0.4.2
     # via
+    #   fastapi
     #   mcp
     #   pydantic
     #   pydantic-settings

backend/requirements/ee.txt

@@ -90,7 +90,7 @@ docstring-parser==0.17.0
     # via google-cloud-aiplatform
 durationpy==0.10
     # via kubernetes
-fastapi==0.128.0
+fastapi==0.133.1
     # via onyx
 fastavro==1.12.1
     # via cohere
@@ -398,6 +398,7 @@ typing-extensions==4.15.0
     #   typing-inspection
 typing-inspection==0.4.2
     # via
+    #   fastapi
     #   mcp
     #   pydantic
     #   pydantic-settings

backend/requirements/model_server.txt

@@ -108,7 +108,7 @@ durationpy==0.10
     # via kubernetes
 einops==0.8.1
     # via onyx
-fastapi==0.128.0
+fastapi==0.133.1
     # via
     #   onyx
     #   sentry-sdk
@@ -525,6 +525,7 @@ typing-extensions==4.15.0
     #   typing-inspection
 typing-inspection==0.4.2
     # via
+    #   fastapi
     #   mcp
     #   pydantic
     #   pydantic-settings

backend/tests/daily/connectors/sharepoint/test_sharepoint_connector.py

@@ -12,6 +12,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.models import Document
 from onyx.connectors.models import HierarchyNode
 from onyx.connectors.models import ImageSection
+from onyx.connectors.sharepoint.connector import SharepointAuthMethod
 from onyx.connectors.sharepoint.connector import SharepointConnector
 from onyx.db.enums import HierarchyNodeType
 from tests.daily.connectors.utils import load_all_from_connector
@@ -521,3 +522,46 @@ def test_sharepoint_connector_hierarchy_nodes(
                 f"Document {doc.semantic_identifier} should have "
                 "parent_hierarchy_raw_node_id set"
             )
+
+
+@pytest.fixture
+def sharepoint_cert_credentials() -> dict[str, str]:
+    return {
+        "authentication_method": SharepointAuthMethod.CERTIFICATE.value,
+        "sp_client_id": os.environ["PERM_SYNC_SHAREPOINT_CLIENT_ID"],
+        "sp_private_key": os.environ["PERM_SYNC_SHAREPOINT_PRIVATE_KEY"],
+        "sp_certificate_password": os.environ[
+            "PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD"
+        ],
+        "sp_directory_id": os.environ["PERM_SYNC_SHAREPOINT_DIRECTORY_ID"],
+    }
+
+
+def test_resolve_tenant_domain_from_site_urls(
+    sharepoint_cert_credentials: dict[str, str],
+) -> None:
+    """Verify that certificate auth resolves the tenant domain from site URLs
+    without calling the /organization endpoint."""
+    site_url = os.environ["SHAREPOINT_SITE"]
+    connector = SharepointConnector(sites=[site_url])
+    connector.load_credentials(sharepoint_cert_credentials)
+
+    assert connector.sp_tenant_domain is not None
+    assert len(connector.sp_tenant_domain) > 0
+    # The tenant domain should match the first label of the site URL hostname
+    from urllib.parse import urlsplit
+
+    expected = urlsplit(site_url).hostname.split(".")[0]  # type: ignore
+    assert connector.sp_tenant_domain == expected
+
+
+def test_resolve_tenant_domain_from_root_site(
+    sharepoint_cert_credentials: dict[str, str],
+) -> None:
+    """Verify that certificate auth resolves the tenant domain via the root
+    site endpoint when no site URLs are configured."""
+    connector = SharepointConnector(sites=[])
+    connector.load_credentials(sharepoint_cert_credentials)
+
+    assert connector.sp_tenant_domain is not None
+    assert len(connector.sp_tenant_domain) > 0

backend/tests/daily/llm/test_bedrock.py

@@ -17,7 +17,7 @@ def test_bedrock_llm_configuration(client: TestClient) -> None:
     # Prepare the test request payload
     test_request: dict[str, Any] = {
         "provider": LlmProviderNames.BEDROCK,
-        "default_model_name": _DEFAULT_BEDROCK_MODEL,
+        "model": _DEFAULT_BEDROCK_MODEL,
         "api_key": None,
         "api_base": None,
         "api_version": None,
@@ -44,7 +44,7 @@ def test_bedrock_llm_configuration_invalid_key(client: TestClient) -> None:
     # Prepare the test request payload with invalid credentials
     test_request: dict[str, Any] = {
         "provider": LlmProviderNames.BEDROCK,
-        "default_model_name": _DEFAULT_BEDROCK_MODEL,
+        "model": _DEFAULT_BEDROCK_MODEL,
         "api_key": None,
         "api_base": None,
         "api_version": None,

backend/tests/external_dependency_unit/answer/conftest.py

@@ -28,7 +28,6 @@ def ensure_default_llm_provider(db_session: Session) -> None:
             provider=LlmProviderNames.OPENAI,
             api_key=os.environ.get("OPENAI_API_KEY", "test"),
             is_public=True,
-            default_model_name="gpt-4o-mini",
             model_configurations=[
                 ModelConfigurationUpsertRequest(
                     name="gpt-4o-mini",
@@ -41,7 +40,7 @@ def ensure_default_llm_provider(db_session: Session) -> None:
             llm_provider_upsert_request=llm_provider_request,
             db_session=db_session,
         )
-        update_default_provider(provider.id, db_session)
+        update_default_provider(provider.id, "gpt-4o-mini", db_session)
     except Exception as exc:  # pragma: no cover - only hits on duplicate setup issues
         # Rollback to clear the pending transaction state
         db_session.rollback()

backend/tests/external_dependency_unit/answer/test_answer_without_openai.py

@@ -47,7 +47,6 @@ def test_answer_with_only_anthropic_provider(
             name=provider_name,
             provider=LlmProviderNames.ANTHROPIC,
             api_key=anthropic_api_key,
-            default_model_name=anthropic_model,
             is_public=True,
             groups=[],
             model_configurations=[
@@ -59,7 +58,7 @@ def test_answer_with_only_anthropic_provider(
     )
 
     try:
-        update_default_provider(anthropic_provider.id, db_session)
+        update_default_provider(anthropic_provider.id, anthropic_model, db_session)
 
         test_user = create_test_user(db_session, email_prefix="anthropic_only")
         chat_session = create_chat_session(

backend/tests/external_dependency_unit/answer/test_stream_chat_message_objects.py

@@ -9,7 +9,6 @@ from onyx.chat.models import AnswerStreamPart
 from onyx.chat.models import StreamingError
 from onyx.chat.process_message import handle_stream_message_objects
 from onyx.db.chat import create_chat_session
-from onyx.db.models import RecencyBiasSetting
 from onyx.db.models import User
 from onyx.db.persona import upsert_persona
 from onyx.server.query_and_chat.models import MessageResponseIDInfo
@@ -74,10 +73,6 @@ def test_stream_chat_message_objects_without_web_search(
         user=None,  # System persona
         name=f"Test Persona {uuid.uuid4()}",
         description="Test persona with no tools for web search test",
-        num_chunks=10.0,
-        llm_relevance_filter=True,
-        llm_filter_extraction=True,
-        recency_bias=RecencyBiasSetting.BASE_DECAY,
         llm_model_provider_override=None,
         llm_model_version_override=None,
         starter_messages=None,

backend/tests/external_dependency_unit/background/test_periodic_task_claim.py

@@ -0,0 +1,257 @@
+"""External dependency unit tests for periodic task claiming.
+
+Tests ``_try_claim_task`` and ``_try_run_periodic_task`` against real
+PostgreSQL, verifying happy-path behavior and concurrent-access safety.
+
+The claim mechanism uses a transaction-scoped advisory lock + a KVStore
+timestamp for cross-instance dedup.  The DB session is released before
+the task runs, so long-running tasks don't hold connections.
+"""
+
+import time
+from collections.abc import Generator
+from concurrent.futures import as_completed
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+from unittest.mock import MagicMock
+from uuid import uuid4
+
+import pytest
+
+from onyx.background.periodic_poller import _PeriodicTaskDef
+from onyx.background.periodic_poller import _try_claim_task
+from onyx.background.periodic_poller import _try_run_periodic_task
+from onyx.background.periodic_poller import PERIODIC_TASK_KV_PREFIX
+from onyx.db.engine.sql_engine import get_session_with_current_tenant
+from onyx.db.engine.sql_engine import SqlEngine
+from onyx.db.models import KVStore
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
+from tests.external_dependency_unit.constants import TEST_TENANT_ID
+
+_TEST_LOCK_BASE = 90_000
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _init_engine() -> None:
+    SqlEngine.init_engine(pool_size=10, max_overflow=5)
+
+
+def _make_task(
+    *,
+    name: str | None = None,
+    interval: float = 3600,
+    lock_id: int | None = None,
+    run_fn: MagicMock | None = None,
+) -> _PeriodicTaskDef:
+    return _PeriodicTaskDef(
+        name=name if name is not None else f"test-{uuid4().hex[:8]}",
+        interval_seconds=interval,
+        lock_id=lock_id if lock_id is not None else _TEST_LOCK_BASE,
+        run_fn=run_fn if run_fn is not None else MagicMock(),
+    )
+
+
+@pytest.fixture(autouse=True)
+def _cleanup_kv(
+    tenant_context: None,  # noqa: ARG001
+) -> Generator[None, None, None]:
+    yield
+    with get_session_with_current_tenant() as db_session:
+        db_session.query(KVStore).filter(
+            KVStore.key.like(f"{PERIODIC_TASK_KV_PREFIX}test-%")
+        ).delete(synchronize_session=False)
+        db_session.commit()
+
+
+# ------------------------------------------------------------------
+# Happy-path: _try_claim_task
+# ------------------------------------------------------------------
+
+
+class TestClaimHappyPath:
+    def test_first_claim_succeeds(self) -> None:
+        assert _try_claim_task(_make_task()) is True
+
+    def test_first_claim_creates_kv_row(self) -> None:
+        task = _make_task()
+        _try_claim_task(task)
+
+        with get_session_with_current_tenant() as db_session:
+            row = (
+                db_session.query(KVStore)
+                .filter_by(key=PERIODIC_TASK_KV_PREFIX + task.name)
+                .first()
+            )
+        assert row is not None
+        assert row.value is not None
+
+    def test_second_claim_within_interval_fails(self) -> None:
+        task = _make_task(interval=3600)
+        assert _try_claim_task(task) is True
+        assert _try_claim_task(task) is False
+
+    def test_claim_after_interval_succeeds(self) -> None:
+        task = _make_task(interval=1)
+        assert _try_claim_task(task) is True
+
+        kv_key = PERIODIC_TASK_KV_PREFIX + task.name
+        with get_session_with_current_tenant() as db_session:
+            row = db_session.query(KVStore).filter_by(key=kv_key).first()
+            assert row is not None
+            row.value = (datetime.now(timezone.utc) - timedelta(seconds=10)).isoformat()
+            db_session.commit()
+
+        assert _try_claim_task(task) is True
+
+
+# ------------------------------------------------------------------
+# Happy-path: _try_run_periodic_task
+# ------------------------------------------------------------------
+
+
+class TestRunHappyPath:
+    def test_runs_task_and_updates_last_run_at(self) -> None:
+        mock_fn = MagicMock()
+        task = _make_task(run_fn=mock_fn)
+
+        _try_run_periodic_task(task)
+
+        mock_fn.assert_called_once()
+        assert task.last_run_at > 0
+
+    def test_skips_when_in_memory_interval_not_elapsed(self) -> None:
+        mock_fn = MagicMock()
+        task = _make_task(run_fn=mock_fn, interval=3600)
+        task.last_run_at = time.monotonic()
+
+        _try_run_periodic_task(task)
+
+        mock_fn.assert_not_called()
+
+    def test_skips_when_db_claim_blocked(self) -> None:
+        name = f"test-{uuid4().hex[:8]}"
+        lock_id = _TEST_LOCK_BASE + 10
+
+        _try_claim_task(_make_task(name=name, lock_id=lock_id, interval=3600))
+
+        mock_fn = MagicMock()
+        task = _make_task(name=name, lock_id=lock_id, interval=3600, run_fn=mock_fn)
+        _try_run_periodic_task(task)
+
+        mock_fn.assert_not_called()
+
+    def test_task_exception_does_not_propagate(self) -> None:
+        task = _make_task(run_fn=MagicMock(side_effect=RuntimeError("boom")))
+        _try_run_periodic_task(task)
+
+    def test_claim_committed_before_task_runs(self) -> None:
+        """The KV claim must be visible in the DB when run_fn executes."""
+        task_name = f"test-order-{uuid4().hex[:8]}"
+        kv_key = PERIODIC_TASK_KV_PREFIX + task_name
+        claim_visible: list[bool] = []
+
+        def check_claim() -> None:
+            with get_session_with_current_tenant() as db_session:
+                row = db_session.query(KVStore).filter_by(key=kv_key).first()
+                claim_visible.append(row is not None and row.value is not None)
+
+        task = _PeriodicTaskDef(
+            name=task_name,
+            interval_seconds=3600,
+            lock_id=_TEST_LOCK_BASE + 11,
+            run_fn=check_claim,
+        )
+
+        _try_run_periodic_task(task)
+
+        assert claim_visible == [True]
+
+
+# ------------------------------------------------------------------
+# Concurrency: only one claimer should win
+# ------------------------------------------------------------------
+
+
+class TestClaimConcurrency:
+    def test_concurrent_claims_single_winner(self) -> None:
+        """Many threads claim the same task — exactly one should succeed."""
+        num_threads = 20
+        task_name = f"test-race-{uuid4().hex[:8]}"
+        lock_id = _TEST_LOCK_BASE + 20
+
+        def claim() -> bool:
+            CURRENT_TENANT_ID_CONTEXTVAR.set(TEST_TENANT_ID)
+            return _try_claim_task(
+                _PeriodicTaskDef(
+                    name=task_name,
+                    interval_seconds=3600,
+                    lock_id=lock_id,
+                    run_fn=lambda: None,
+                )
+            )
+
+        results: list[bool] = []
+        with ThreadPoolExecutor(max_workers=num_threads) as executor:
+            futures = [executor.submit(claim) for _ in range(num_threads)]
+            for future in as_completed(futures):
+                results.append(future.result())
+
+        winners = sum(1 for r in results if r)
+        assert winners == 1, f"Expected 1 winner, got {winners}"
+
+    def test_concurrent_run_single_execution(self) -> None:
+        """Many threads run the same task — run_fn fires exactly once."""
+        num_threads = 20
+        task_name = f"test-run-race-{uuid4().hex[:8]}"
+        lock_id = _TEST_LOCK_BASE + 21
+        counter = MagicMock()
+
+        def run() -> None:
+            CURRENT_TENANT_ID_CONTEXTVAR.set(TEST_TENANT_ID)
+            _try_run_periodic_task(
+                _PeriodicTaskDef(
+                    name=task_name,
+                    interval_seconds=3600,
+                    lock_id=lock_id,
+                    run_fn=counter,
+                )
+            )
+
+        with ThreadPoolExecutor(max_workers=num_threads) as executor:
+            futures = [executor.submit(run) for _ in range(num_threads)]
+            for future in as_completed(futures):
+                future.result()
+
+        assert (
+            counter.call_count == 1
+        ), f"Expected run_fn called once, got {counter.call_count}"
+
+    def test_no_errors_under_contention(self) -> None:
+        """All threads complete without exceptions under high contention."""
+        num_threads = 30
+        task_name = f"test-err-{uuid4().hex[:8]}"
+        lock_id = _TEST_LOCK_BASE + 22
+        errors: list[Exception] = []
+
+        def claim() -> bool:
+            CURRENT_TENANT_ID_CONTEXTVAR.set(TEST_TENANT_ID)
+            return _try_claim_task(
+                _PeriodicTaskDef(
+                    name=task_name,
+                    interval_seconds=3600,
+                    lock_id=lock_id,
+                    run_fn=lambda: None,
+                )
+            )
+
+        with ThreadPoolExecutor(max_workers=num_threads) as executor:
+            futures = [executor.submit(claim) for _ in range(num_threads)]
+            for future in as_completed(futures):
+                try:
+                    future.result()
+                except Exception as e:
+                    errors.append(e)
+
+        assert errors == [], f"Got {len(errors)} errors: {errors}"

backend/tests/external_dependency_unit/background/test_startup_recovery.py

@@ -0,0 +1,219 @@
+"""External dependency unit tests for startup recovery (Step 10g).
+
+Seeds ``UserFile`` records in stuck states (PROCESSING, DELETING,
+needs_project_sync) then calls ``recover_stuck_user_files`` and verifies
+the drain loops pick them up via ``FOR UPDATE SKIP LOCKED``.
+
+Uses real PostgreSQL (via ``db_session`` / ``tenant_context`` fixtures).
+The per-file ``*_impl`` functions are mocked so no real file store or
+connector is needed — we only verify that recovery finds and dispatches
+the correct files.
+"""
+
+from collections.abc import Generator
+from unittest.mock import MagicMock
+from unittest.mock import patch
+from uuid import uuid4
+
+import pytest
+from sqlalchemy.orm import Session
+
+from onyx.background.periodic_poller import recover_stuck_user_files
+from onyx.db.enums import UserFileStatus
+from onyx.db.models import UserFile
+from tests.external_dependency_unit.conftest import create_test_user
+from tests.external_dependency_unit.constants import TEST_TENANT_ID
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_IMPL_MODULE = "onyx.background.celery.tasks.user_file_processing.tasks"
+
+
+def _create_user_file(
+    db_session: Session,
+    user_id: object,
+    *,
+    status: UserFileStatus = UserFileStatus.PROCESSING,
+    needs_project_sync: bool = False,
+    needs_persona_sync: bool = False,
+) -> UserFile:
+    uf = UserFile(
+        id=uuid4(),
+        user_id=user_id,
+        file_id=f"test_file_{uuid4().hex[:8]}",
+        name=f"test_{uuid4().hex[:8]}.txt",
+        file_type="text/plain",
+        status=status,
+        needs_project_sync=needs_project_sync,
+        needs_persona_sync=needs_persona_sync,
+    )
+    db_session.add(uf)
+    db_session.commit()
+    db_session.refresh(uf)
+    return uf
+
+
+@pytest.fixture()
+def _cleanup_user_files(db_session: Session) -> Generator[list[UserFile], None, None]:
+    """Track created UserFile rows and delete them after each test."""
+    created: list[UserFile] = []
+    yield created
+    for uf in created:
+        existing = db_session.get(UserFile, uf.id)
+        if existing:
+            db_session.delete(existing)
+    db_session.commit()
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestRecoverProcessingFiles:
+    """Files in PROCESSING status are re-processed via the processing drain loop."""
+
+    def test_processing_files_recovered(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+        _cleanup_user_files: list[UserFile],
+    ) -> None:
+        user = create_test_user(db_session, "recovery_proc")
+        uf = _create_user_file(db_session, user.id, status=UserFileStatus.PROCESSING)
+        _cleanup_user_files.append(uf)
+
+        mock_impl = MagicMock()
+        with patch(f"{_IMPL_MODULE}.process_user_file_impl", mock_impl):
+            recover_stuck_user_files(TEST_TENANT_ID)
+
+        called_ids = [call.kwargs["user_file_id"] for call in mock_impl.call_args_list]
+        assert (
+            str(uf.id) in called_ids
+        ), f"Expected file {uf.id} to be recovered but got: {called_ids}"
+
+    def test_completed_files_not_recovered(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+        _cleanup_user_files: list[UserFile],
+    ) -> None:
+        user = create_test_user(db_session, "recovery_comp")
+        uf = _create_user_file(db_session, user.id, status=UserFileStatus.COMPLETED)
+        _cleanup_user_files.append(uf)
+
+        mock_impl = MagicMock()
+        with patch(f"{_IMPL_MODULE}.process_user_file_impl", mock_impl):
+            recover_stuck_user_files(TEST_TENANT_ID)
+
+        called_ids = [call.kwargs["user_file_id"] for call in mock_impl.call_args_list]
+        assert (
+            str(uf.id) not in called_ids
+        ), f"COMPLETED file {uf.id} should not have been recovered"
+
+
+class TestRecoverDeletingFiles:
+    """Files in DELETING status are recovered via the delete drain loop."""
+
+    def test_deleting_files_recovered(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+        _cleanup_user_files: list[UserFile],
+    ) -> None:
+        user = create_test_user(db_session, "recovery_del")
+        uf = _create_user_file(db_session, user.id, status=UserFileStatus.DELETING)
+        _cleanup_user_files.append(uf)
+
+        mock_impl = MagicMock()
+        with patch(f"{_IMPL_MODULE}.delete_user_file_impl", mock_impl):
+            recover_stuck_user_files(TEST_TENANT_ID)
+
+        called_ids = [call.kwargs["user_file_id"] for call in mock_impl.call_args_list]
+        assert (
+            str(uf.id) in called_ids
+        ), f"Expected file {uf.id} to be recovered for deletion but got: {called_ids}"
+
+
+class TestRecoverSyncFiles:
+    """Files needing project/persona sync are recovered via the sync drain loop."""
+
+    def test_needs_project_sync_recovered(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+        _cleanup_user_files: list[UserFile],
+    ) -> None:
+        user = create_test_user(db_session, "recovery_sync")
+        uf = _create_user_file(
+            db_session,
+            user.id,
+            status=UserFileStatus.COMPLETED,
+            needs_project_sync=True,
+        )
+        _cleanup_user_files.append(uf)
+
+        mock_impl = MagicMock()
+        with patch(f"{_IMPL_MODULE}.project_sync_user_file_impl", mock_impl):
+            recover_stuck_user_files(TEST_TENANT_ID)
+
+        called_ids = [call.kwargs["user_file_id"] for call in mock_impl.call_args_list]
+        assert (
+            str(uf.id) in called_ids
+        ), f"Expected file {uf.id} to be recovered for sync but got: {called_ids}"
+
+    def test_needs_persona_sync_recovered(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+        _cleanup_user_files: list[UserFile],
+    ) -> None:
+        user = create_test_user(db_session, "recovery_psync")
+        uf = _create_user_file(
+            db_session,
+            user.id,
+            status=UserFileStatus.COMPLETED,
+            needs_persona_sync=True,
+        )
+        _cleanup_user_files.append(uf)
+
+        mock_impl = MagicMock()
+        with patch(f"{_IMPL_MODULE}.project_sync_user_file_impl", mock_impl):
+            recover_stuck_user_files(TEST_TENANT_ID)
+
+        called_ids = [call.kwargs["user_file_id"] for call in mock_impl.call_args_list]
+        assert (
+            str(uf.id) in called_ids
+        ), f"Expected file {uf.id} to be recovered for persona sync but got: {called_ids}"
+
+
+class TestRecoveryMultipleFiles:
+    """Recovery processes all stuck files in one pass, not just the first."""
+
+    def test_multiple_processing_files(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+        _cleanup_user_files: list[UserFile],
+    ) -> None:
+        user = create_test_user(db_session, "recovery_multi")
+        files = []
+        for _ in range(3):
+            uf = _create_user_file(
+                db_session, user.id, status=UserFileStatus.PROCESSING
+            )
+            _cleanup_user_files.append(uf)
+            files.append(uf)
+
+        mock_impl = MagicMock()
+        with patch(f"{_IMPL_MODULE}.process_user_file_impl", mock_impl):
+            recover_stuck_user_files(TEST_TENANT_ID)
+
+        called_ids = {call.kwargs["user_file_id"] for call in mock_impl.call_args_list}
+        expected_ids = {str(uf.id) for uf in files}
+        assert expected_ids.issubset(called_ids), (
+            f"Expected all {len(files)} files to be recovered. "
+            f"Missing: {expected_ids - called_ids}"
+        )

backend/tests/external_dependency_unit/celery/test_persona_file_sync.py

@@ -0,0 +1,515 @@
+"""
+External dependency unit tests for persona file sync.
+
+Validates that:
+
+1. The check_for_user_file_project_sync beat task picks up UserFiles with
+   needs_persona_sync=True (not just needs_project_sync).
+
+2. The process_single_user_file_project_sync worker task reads persona
+   associations from the DB, passes persona_ids to the document index via
+   VespaDocumentUserFields, and clears needs_persona_sync afterwards.
+
+3. upsert_persona correctly marks affected UserFiles with
+   needs_persona_sync=True when file associations change.
+
+Uses real Redis and PostgreSQL.  Document index (Vespa) calls are mocked
+since we only need to verify the arguments passed to update_single.
+"""
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from typing import Any
+from unittest.mock import MagicMock
+from unittest.mock import patch
+from unittest.mock import PropertyMock
+from uuid import uuid4
+
+from sqlalchemy.orm import Session
+
+from onyx.background.celery.tasks.user_file_processing.tasks import (
+    check_for_user_file_project_sync,
+)
+from onyx.background.celery.tasks.user_file_processing.tasks import (
+    process_single_user_file_project_sync,
+)
+from onyx.background.celery.tasks.user_file_processing.tasks import (
+    user_file_project_sync_lock_key,
+)
+from onyx.db.enums import UserFileStatus
+from onyx.db.models import Persona
+from onyx.db.models import Persona__UserFile
+from onyx.db.models import User
+from onyx.db.models import UserFile
+from onyx.db.persona import upsert_persona
+from onyx.document_index.interfaces import VespaDocumentUserFields
+from onyx.redis.redis_pool import get_redis_client
+from tests.external_dependency_unit.conftest import create_test_user
+from tests.external_dependency_unit.constants import TEST_TENANT_ID
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _create_completed_user_file(
+    db_session: Session,
+    user: User,
+    needs_persona_sync: bool = False,
+    needs_project_sync: bool = False,
+) -> UserFile:
+    """Insert a UserFile in COMPLETED status."""
+    uf = UserFile(
+        id=uuid4(),
+        user_id=user.id,
+        file_id=f"test_file_{uuid4().hex[:8]}",
+        name=f"test_{uuid4().hex[:8]}.txt",
+        file_type="text/plain",
+        status=UserFileStatus.COMPLETED,
+        needs_persona_sync=needs_persona_sync,
+        needs_project_sync=needs_project_sync,
+        chunk_count=5,
+    )
+    db_session.add(uf)
+    db_session.commit()
+    db_session.refresh(uf)
+    return uf
+
+
+def _create_test_persona(
+    db_session: Session,
+    user: User,
+    user_files: list[UserFile] | None = None,
+) -> Persona:
+    """Create a minimal Persona via direct model insert."""
+    persona = Persona(
+        name=f"Test Persona {uuid4().hex[:8]}",
+        description="Test persona",
+        system_prompt="You are a test assistant",
+        task_prompt="Answer the question",
+        tools=[],
+        document_sets=[],
+        users=[user],
+        groups=[],
+        is_visible=True,
+        is_public=True,
+        display_priority=None,
+        starter_messages=None,
+        deleted=False,
+        user_files=user_files or [],
+        user_id=user.id,
+    )
+    db_session.add(persona)
+    db_session.commit()
+    db_session.refresh(persona)
+    return persona
+
+
+def _link_file_to_persona(
+    db_session: Session, persona: Persona, user_file: UserFile
+) -> None:
+    """Create the join table row between a persona and a user file."""
+    link = Persona__UserFile(persona_id=persona.id, user_file_id=user_file.id)
+    db_session.add(link)
+    db_session.commit()
+
+
+_PATCH_QUEUE_DEPTH = (
+    "onyx.background.celery.tasks.user_file_processing.tasks"
+    ".get_user_file_project_sync_queue_depth"
+)
+
+
+@contextmanager
+def _patch_task_app(task: Any, mock_app: MagicMock) -> Generator[None, None, None]:
+    """Patch the ``app`` property on a bound Celery task."""
+    task_instance = task.run.__self__
+    with (
+        patch.object(
+            type(task_instance),
+            "app",
+            new_callable=PropertyMock,
+            return_value=mock_app,
+        ),
+        patch(_PATCH_QUEUE_DEPTH, return_value=0),
+    ):
+        yield
+
+
+# ---------------------------------------------------------------------------
+# Test: check_for_user_file_project_sync picks up persona sync
+# ---------------------------------------------------------------------------
+
+
+class TestCheckSweepIncludesPersonaSync:
+    """The beat task must pick up files needing persona sync, not just project sync."""
+
+    def test_persona_sync_flag_enqueues_task(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """A file with needs_persona_sync=True (and COMPLETED) gets enqueued."""
+        user = create_test_user(db_session, "persona_sweep")
+        uf = _create_completed_user_file(db_session, user, needs_persona_sync=True)
+
+        mock_app = MagicMock()
+
+        with _patch_task_app(check_for_user_file_project_sync, mock_app):
+            check_for_user_file_project_sync.run(tenant_id=TEST_TENANT_ID)
+
+        enqueued_ids = {
+            call.kwargs["kwargs"]["user_file_id"]
+            for call in mock_app.send_task.call_args_list
+        }
+        assert str(uf.id) in enqueued_ids
+
+    def test_neither_flag_does_not_enqueue(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """A file with both flags False is not enqueued."""
+        user = create_test_user(db_session, "no_sync")
+        uf = _create_completed_user_file(db_session, user)
+
+        mock_app = MagicMock()
+
+        with _patch_task_app(check_for_user_file_project_sync, mock_app):
+            check_for_user_file_project_sync.run(tenant_id=TEST_TENANT_ID)
+
+        enqueued_ids = {
+            call.kwargs["kwargs"]["user_file_id"]
+            for call in mock_app.send_task.call_args_list
+        }
+        assert str(uf.id) not in enqueued_ids
+
+    def test_both_flags_enqueues_once(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """A file with BOTH flags True is enqueued exactly once."""
+        user = create_test_user(db_session, "both_flags")
+        uf = _create_completed_user_file(
+            db_session, user, needs_persona_sync=True, needs_project_sync=True
+        )
+
+        mock_app = MagicMock()
+
+        with _patch_task_app(check_for_user_file_project_sync, mock_app):
+            check_for_user_file_project_sync.run(tenant_id=TEST_TENANT_ID)
+
+        matching_calls = [
+            call
+            for call in mock_app.send_task.call_args_list
+            if call.kwargs["kwargs"]["user_file_id"] == str(uf.id)
+        ]
+        assert len(matching_calls) == 1
+
+
+# ---------------------------------------------------------------------------
+# Test: process_single_user_file_project_sync passes persona_ids to index
+# ---------------------------------------------------------------------------
+
+_PATCH_GET_SETTINGS = (
+    "onyx.background.celery.tasks.user_file_processing.tasks.get_active_search_settings"
+)
+_PATCH_GET_INDICES = (
+    "onyx.background.celery.tasks.user_file_processing.tasks.get_all_document_indices"
+)
+_PATCH_HTTPX_INIT = (
+    "onyx.background.celery.tasks.user_file_processing.tasks.httpx_init_vespa_pool"
+)
+_PATCH_DISABLE_VDB = (
+    "onyx.background.celery.tasks.user_file_processing.tasks.DISABLE_VECTOR_DB"
+)
+
+
+class TestSyncTaskWritesPersonaIds:
+    """The sync task reads persona associations and sends them to the index."""
+
+    def test_passes_persona_ids_to_update_single(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """After linking a file to a persona, sync sends the persona ID."""
+        user = create_test_user(db_session, "sync_persona")
+        uf = _create_completed_user_file(db_session, user, needs_persona_sync=True)
+        persona = _create_test_persona(db_session, user)
+        _link_file_to_persona(db_session, persona, uf)
+
+        mock_doc_index = MagicMock()
+        mock_search_settings = MagicMock()
+        mock_search_settings.primary = MagicMock()
+        mock_search_settings.secondary = None
+
+        redis_client = get_redis_client(tenant_id=TEST_TENANT_ID)
+        lock_key = user_file_project_sync_lock_key(str(uf.id))
+        redis_client.delete(lock_key)
+
+        with (
+            patch(_PATCH_DISABLE_VDB, False),
+            patch(_PATCH_HTTPX_INIT),
+            patch(_PATCH_GET_SETTINGS, return_value=mock_search_settings),
+            patch(_PATCH_GET_INDICES, return_value=[mock_doc_index]),
+        ):
+            process_single_user_file_project_sync.run(
+                user_file_id=str(uf.id), tenant_id=TEST_TENANT_ID
+            )
+
+        mock_doc_index.update_single.assert_called_once()
+        call_args = mock_doc_index.update_single.call_args
+        user_fields: VespaDocumentUserFields = call_args.kwargs["user_fields"]
+        assert user_fields.personas is not None
+        assert persona.id in user_fields.personas
+        assert call_args.args[0] == str(uf.id)
+
+    def test_clears_persona_sync_flag(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """After a successful sync the needs_persona_sync flag is cleared."""
+        user = create_test_user(db_session, "sync_clear")
+        uf = _create_completed_user_file(db_session, user, needs_persona_sync=True)
+
+        redis_client = get_redis_client(tenant_id=TEST_TENANT_ID)
+        lock_key = user_file_project_sync_lock_key(str(uf.id))
+        redis_client.delete(lock_key)
+
+        with patch(_PATCH_DISABLE_VDB, True):
+            process_single_user_file_project_sync.run(
+                user_file_id=str(uf.id), tenant_id=TEST_TENANT_ID
+            )
+
+        db_session.refresh(uf)
+        assert uf.needs_persona_sync is False
+
+    def test_passes_both_project_and_persona_ids(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """A file linked to both a project and a persona gets both IDs."""
+        from onyx.db.models import Project__UserFile
+        from onyx.db.models import UserProject
+
+        user = create_test_user(db_session, "sync_both")
+        uf = _create_completed_user_file(
+            db_session, user, needs_persona_sync=True, needs_project_sync=True
+        )
+        persona = _create_test_persona(db_session, user)
+        _link_file_to_persona(db_session, persona, uf)
+
+        project = UserProject(user_id=user.id, name="test-project", instructions="")
+        db_session.add(project)
+        db_session.commit()
+        db_session.refresh(project)
+
+        link = Project__UserFile(project_id=project.id, user_file_id=uf.id)
+        db_session.add(link)
+        db_session.commit()
+
+        mock_doc_index = MagicMock()
+        mock_search_settings = MagicMock()
+        mock_search_settings.primary = MagicMock()
+        mock_search_settings.secondary = None
+
+        redis_client = get_redis_client(tenant_id=TEST_TENANT_ID)
+        lock_key = user_file_project_sync_lock_key(str(uf.id))
+        redis_client.delete(lock_key)
+
+        with (
+            patch(_PATCH_DISABLE_VDB, False),
+            patch(_PATCH_HTTPX_INIT),
+            patch(_PATCH_GET_SETTINGS, return_value=mock_search_settings),
+            patch(_PATCH_GET_INDICES, return_value=[mock_doc_index]),
+        ):
+            process_single_user_file_project_sync.run(
+                user_file_id=str(uf.id), tenant_id=TEST_TENANT_ID
+            )
+
+        call_kwargs = mock_doc_index.update_single.call_args.kwargs
+        user_fields: VespaDocumentUserFields = call_kwargs["user_fields"]
+        assert user_fields.personas is not None
+        assert user_fields.user_projects is not None
+        assert persona.id in user_fields.personas
+        assert project.id in user_fields.user_projects
+
+        # Both flags should be cleared
+        db_session.refresh(uf)
+        assert uf.needs_persona_sync is False
+        assert uf.needs_project_sync is False
+
+    def test_deleted_persona_excluded_from_ids(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """A soft-deleted persona should NOT appear in the persona_ids sent to Vespa."""
+        user = create_test_user(db_session, "sync_deleted")
+        uf = _create_completed_user_file(db_session, user, needs_persona_sync=True)
+        persona = _create_test_persona(db_session, user)
+        _link_file_to_persona(db_session, persona, uf)
+
+        persona.deleted = True
+        db_session.commit()
+
+        mock_doc_index = MagicMock()
+        mock_search_settings = MagicMock()
+        mock_search_settings.primary = MagicMock()
+        mock_search_settings.secondary = None
+
+        redis_client = get_redis_client(tenant_id=TEST_TENANT_ID)
+        lock_key = user_file_project_sync_lock_key(str(uf.id))
+        redis_client.delete(lock_key)
+
+        with (
+            patch(_PATCH_DISABLE_VDB, False),
+            patch(_PATCH_HTTPX_INIT),
+            patch(_PATCH_GET_SETTINGS, return_value=mock_search_settings),
+            patch(_PATCH_GET_INDICES, return_value=[mock_doc_index]),
+        ):
+            process_single_user_file_project_sync.run(
+                user_file_id=str(uf.id), tenant_id=TEST_TENANT_ID
+            )
+
+        call_kwargs = mock_doc_index.update_single.call_args.kwargs
+        user_fields: VespaDocumentUserFields = call_kwargs["user_fields"]
+        assert user_fields.personas is not None
+        assert persona.id not in user_fields.personas
+
+
+# ---------------------------------------------------------------------------
+# Test: upsert_persona marks files for persona sync
+# ---------------------------------------------------------------------------
+
+
+class TestUpsertPersonaMarksSyncFlag:
+    """upsert_persona must set needs_persona_sync on affected UserFiles."""
+
+    def test_creating_persona_with_files_marks_sync(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        user = create_test_user(db_session, "upsert_create")
+        uf = _create_completed_user_file(db_session, user)
+        assert uf.needs_persona_sync is False
+
+        upsert_persona(
+            user=user,
+            name=f"persona-{uuid4().hex[:8]}",
+            description="test",
+            llm_model_provider_override=None,
+            llm_model_version_override=None,
+            starter_messages=None,
+            system_prompt="test",
+            task_prompt="test",
+            datetime_aware=None,
+            is_public=True,
+            db_session=db_session,
+            user_file_ids=[uf.id],
+        )
+
+        db_session.refresh(uf)
+        assert uf.needs_persona_sync is True
+
+    def test_updating_persona_files_marks_both_old_and_new(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """When file associations change, both the removed and added files are flagged."""
+        user = create_test_user(db_session, "upsert_update")
+        uf_old = _create_completed_user_file(db_session, user)
+        uf_new = _create_completed_user_file(db_session, user)
+
+        persona = upsert_persona(
+            user=user,
+            name=f"persona-{uuid4().hex[:8]}",
+            description="test",
+            llm_model_provider_override=None,
+            llm_model_version_override=None,
+            starter_messages=None,
+            system_prompt="test",
+            task_prompt="test",
+            datetime_aware=None,
+            is_public=True,
+            db_session=db_session,
+            user_file_ids=[uf_old.id],
+        )
+
+        # Clear the flag from creation so we can observe the update
+        uf_old.needs_persona_sync = False
+        db_session.commit()
+
+        # Now update the persona to swap files
+        upsert_persona(
+            user=user,
+            name=persona.name,
+            description=persona.description,
+            llm_model_provider_override=None,
+            llm_model_version_override=None,
+            starter_messages=None,
+            system_prompt=persona.system_prompt,
+            task_prompt=persona.task_prompt,
+            datetime_aware=None,
+            is_public=persona.is_public,
+            db_session=db_session,
+            persona_id=persona.id,
+            user_file_ids=[uf_new.id],
+        )
+
+        db_session.refresh(uf_old)
+        db_session.refresh(uf_new)
+        assert uf_old.needs_persona_sync is True, "Removed file should be flagged"
+        assert uf_new.needs_persona_sync is True, "Added file should be flagged"
+
+    def test_removing_all_files_marks_old_files(
+        self,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """Removing all files from a persona flags the previously associated files."""
+        user = create_test_user(db_session, "upsert_remove")
+        uf = _create_completed_user_file(db_session, user)
+
+        persona = upsert_persona(
+            user=user,
+            name=f"persona-{uuid4().hex[:8]}",
+            description="test",
+            llm_model_provider_override=None,
+            llm_model_version_override=None,
+            starter_messages=None,
+            system_prompt="test",
+            task_prompt="test",
+            datetime_aware=None,
+            is_public=True,
+            db_session=db_session,
+            user_file_ids=[uf.id],
+        )
+
+        uf.needs_persona_sync = False
+        db_session.commit()
+
+        upsert_persona(
+            user=user,
+            name=persona.name,
+            description=persona.description,
+            llm_model_provider_override=None,
+            llm_model_version_override=None,
+            starter_messages=None,
+            system_prompt=persona.system_prompt,
+            task_prompt=persona.task_prompt,
+            datetime_aware=None,
+            is_public=persona.is_public,
+            db_session=db_session,
+            persona_id=persona.id,
+            user_file_ids=[],
+        )
+
+        db_session.refresh(uf)
+        assert uf.needs_persona_sync is True

backend/tests/external_dependency_unit/celery/test_user_file_indexing_adapter.py

@@ -0,0 +1,311 @@
+"""
+External dependency unit tests for UserFileIndexingAdapter metadata writing.
+
+Validates that build_metadata_aware_chunks produces DocMetadataAwareIndexChunk
+objects with both `user_project` and `personas` fields populated correctly
+based on actual DB associations.
+
+Uses real PostgreSQL for UserFile/Persona/UserProject rows.
+Mocks the LLM tokenizer and file store since they are not relevant here.
+"""
+
+from unittest.mock import MagicMock
+from unittest.mock import patch
+from uuid import uuid4
+
+from sqlalchemy.orm import Session
+
+from onyx.configs.constants import DocumentSource
+from onyx.connectors.models import Document
+from onyx.connectors.models import TextSection
+from onyx.db.enums import UserFileStatus
+from onyx.db.models import Persona
+from onyx.db.models import Persona__UserFile
+from onyx.db.models import Project__UserFile
+from onyx.db.models import User
+from onyx.db.models import UserFile
+from onyx.db.models import UserProject
+from onyx.indexing.adapters.user_file_indexing_adapter import UserFileIndexingAdapter
+from onyx.indexing.indexing_pipeline import DocumentBatchPrepareContext
+from onyx.indexing.models import ChunkEmbedding
+from onyx.indexing.models import IndexChunk
+from tests.external_dependency_unit.conftest import create_test_user
+from tests.external_dependency_unit.constants import TEST_TENANT_ID
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _create_user_file(db_session: Session, user: User) -> UserFile:
+    uf = UserFile(
+        id=uuid4(),
+        user_id=user.id,
+        file_id=f"test_file_{uuid4().hex[:8]}",
+        name=f"test_{uuid4().hex[:8]}.txt",
+        file_type="text/plain",
+        status=UserFileStatus.COMPLETED,
+        chunk_count=1,
+    )
+    db_session.add(uf)
+    db_session.commit()
+    db_session.refresh(uf)
+    return uf
+
+
+def _create_persona(db_session: Session, user: User) -> Persona:
+    persona = Persona(
+        name=f"Test Persona {uuid4().hex[:8]}",
+        description="Test persona",
+        system_prompt="test",
+        task_prompt="test",
+        tools=[],
+        document_sets=[],
+        users=[user],
+        groups=[],
+        is_visible=True,
+        is_public=True,
+        display_priority=None,
+        starter_messages=None,
+        deleted=False,
+        user_id=user.id,
+    )
+    db_session.add(persona)
+    db_session.commit()
+    db_session.refresh(persona)
+    return persona
+
+
+def _create_project(db_session: Session, user: User) -> UserProject:
+    project = UserProject(
+        user_id=user.id,
+        name=f"project-{uuid4().hex[:8]}",
+        instructions="",
+    )
+    db_session.add(project)
+    db_session.commit()
+    db_session.refresh(project)
+    return project
+
+
+def _make_index_chunk(user_file: UserFile) -> IndexChunk:
+    """Build a minimal IndexChunk whose source document ID matches the UserFile."""
+    doc = Document(
+        id=str(user_file.id),
+        source=DocumentSource.USER_FILE,
+        semantic_identifier=user_file.name,
+        sections=[TextSection(text="test chunk content", link=None)],
+        metadata={},
+    )
+    return IndexChunk(
+        source_document=doc,
+        chunk_id=0,
+        blurb="test chunk",
+        content="test chunk content",
+        source_links={0: ""},
+        image_file_id=None,
+        section_continuation=False,
+        title_prefix="",
+        metadata_suffix_semantic="",
+        metadata_suffix_keyword="",
+        contextual_rag_reserved_tokens=0,
+        doc_summary="",
+        chunk_context="",
+        mini_chunk_texts=None,
+        large_chunk_id=None,
+        embeddings=ChunkEmbedding(
+            full_embedding=[0.0] * 768,
+            mini_chunk_embeddings=[],
+        ),
+        title_embedding=None,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestAdapterWritesBothMetadataFields:
+    """build_metadata_aware_chunks must populate user_project AND personas."""
+
+    @patch(
+        "onyx.indexing.adapters.user_file_indexing_adapter.get_default_llm",
+        side_effect=Exception("no LLM in test"),
+    )
+    def test_file_linked_to_persona_gets_persona_id(
+        self,
+        _mock_llm: MagicMock,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        user = create_test_user(db_session, "adapter_persona")
+        uf = _create_user_file(db_session, user)
+        persona = _create_persona(db_session, user)
+
+        db_session.add(Persona__UserFile(persona_id=persona.id, user_file_id=uf.id))
+        db_session.commit()
+
+        adapter = UserFileIndexingAdapter(
+            tenant_id=TEST_TENANT_ID, db_session=db_session
+        )
+        chunk = _make_index_chunk(uf)
+        doc = chunk.source_document
+        context = DocumentBatchPrepareContext(updatable_docs=[doc], id_to_boost_map={})
+
+        result = adapter.build_metadata_aware_chunks(
+            chunks_with_embeddings=[chunk],
+            chunk_content_scores=[1.0],
+            tenant_id=TEST_TENANT_ID,
+            context=context,
+        )
+
+        assert len(result.chunks) == 1
+        aware_chunk = result.chunks[0]
+        assert persona.id in aware_chunk.personas
+        assert aware_chunk.user_project == []
+
+    @patch(
+        "onyx.indexing.adapters.user_file_indexing_adapter.get_default_llm",
+        side_effect=Exception("no LLM in test"),
+    )
+    def test_file_linked_to_project_gets_project_id(
+        self,
+        _mock_llm: MagicMock,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        user = create_test_user(db_session, "adapter_project")
+        uf = _create_user_file(db_session, user)
+        project = _create_project(db_session, user)
+
+        db_session.add(Project__UserFile(project_id=project.id, user_file_id=uf.id))
+        db_session.commit()
+
+        adapter = UserFileIndexingAdapter(
+            tenant_id=TEST_TENANT_ID, db_session=db_session
+        )
+        chunk = _make_index_chunk(uf)
+        context = DocumentBatchPrepareContext(
+            updatable_docs=[chunk.source_document], id_to_boost_map={}
+        )
+
+        result = adapter.build_metadata_aware_chunks(
+            chunks_with_embeddings=[chunk],
+            chunk_content_scores=[1.0],
+            tenant_id=TEST_TENANT_ID,
+            context=context,
+        )
+
+        assert len(result.chunks) == 1
+        aware_chunk = result.chunks[0]
+        assert project.id in aware_chunk.user_project
+        assert aware_chunk.personas == []
+
+    @patch(
+        "onyx.indexing.adapters.user_file_indexing_adapter.get_default_llm",
+        side_effect=Exception("no LLM in test"),
+    )
+    def test_file_linked_to_both_gets_both_ids(
+        self,
+        _mock_llm: MagicMock,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        user = create_test_user(db_session, "adapter_both")
+        uf = _create_user_file(db_session, user)
+        persona = _create_persona(db_session, user)
+        project = _create_project(db_session, user)
+
+        db_session.add(Persona__UserFile(persona_id=persona.id, user_file_id=uf.id))
+        db_session.add(Project__UserFile(project_id=project.id, user_file_id=uf.id))
+        db_session.commit()
+
+        adapter = UserFileIndexingAdapter(
+            tenant_id=TEST_TENANT_ID, db_session=db_session
+        )
+        chunk = _make_index_chunk(uf)
+        context = DocumentBatchPrepareContext(
+            updatable_docs=[chunk.source_document], id_to_boost_map={}
+        )
+
+        result = adapter.build_metadata_aware_chunks(
+            chunks_with_embeddings=[chunk],
+            chunk_content_scores=[1.0],
+            tenant_id=TEST_TENANT_ID,
+            context=context,
+        )
+
+        aware_chunk = result.chunks[0]
+        assert persona.id in aware_chunk.personas
+        assert project.id in aware_chunk.user_project
+
+    @patch(
+        "onyx.indexing.adapters.user_file_indexing_adapter.get_default_llm",
+        side_effect=Exception("no LLM in test"),
+    )
+    def test_file_with_no_associations_gets_empty_lists(
+        self,
+        _mock_llm: MagicMock,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        user = create_test_user(db_session, "adapter_empty")
+        uf = _create_user_file(db_session, user)
+
+        adapter = UserFileIndexingAdapter(
+            tenant_id=TEST_TENANT_ID, db_session=db_session
+        )
+        chunk = _make_index_chunk(uf)
+        context = DocumentBatchPrepareContext(
+            updatable_docs=[chunk.source_document], id_to_boost_map={}
+        )
+
+        result = adapter.build_metadata_aware_chunks(
+            chunks_with_embeddings=[chunk],
+            chunk_content_scores=[1.0],
+            tenant_id=TEST_TENANT_ID,
+            context=context,
+        )
+
+        aware_chunk = result.chunks[0]
+        assert aware_chunk.personas == []
+        assert aware_chunk.user_project == []
+
+    @patch(
+        "onyx.indexing.adapters.user_file_indexing_adapter.get_default_llm",
+        side_effect=Exception("no LLM in test"),
+    )
+    def test_multiple_personas_all_appear(
+        self,
+        _mock_llm: MagicMock,
+        db_session: Session,
+        tenant_context: None,  # noqa: ARG002
+    ) -> None:
+        """A file linked to multiple personas should have all their IDs."""
+        user = create_test_user(db_session, "adapter_multi")
+        uf = _create_user_file(db_session, user)
+        persona_a = _create_persona(db_session, user)
+        persona_b = _create_persona(db_session, user)
+
+        db_session.add(Persona__UserFile(persona_id=persona_a.id, user_file_id=uf.id))
+        db_session.add(Persona__UserFile(persona_id=persona_b.id, user_file_id=uf.id))
+        db_session.commit()
+
+        adapter = UserFileIndexingAdapter(
+            tenant_id=TEST_TENANT_ID, db_session=db_session
+        )
+        chunk = _make_index_chunk(uf)
+        context = DocumentBatchPrepareContext(
+            updatable_docs=[chunk.source_document], id_to_boost_map={}
+        )
+
+        result = adapter.build_metadata_aware_chunks(
+            chunks_with_embeddings=[chunk],
+            chunk_content_scores=[1.0],
+            tenant_id=TEST_TENANT_ID,
+            context=context,
+        )
+
+        aware_chunk = result.chunks[0]
+        assert set(aware_chunk.personas) == {persona_a.id, persona_b.id}

backend/tests/external_dependency_unit/llm/test_llm_provider.py

@@ -29,6 +29,7 @@ from onyx.server.manage.llm.api import (
     test_llm_configuration as run_test_llm_configuration,
 )
 from onyx.server.manage.llm.models import LLMProviderUpsertRequest
+from onyx.server.manage.llm.models import LLMProviderView
 from onyx.server.manage.llm.models import ModelConfigurationUpsertRequest
 from onyx.server.manage.llm.models import TestLLMRequest as LLMTestRequest
 
@@ -44,15 +45,14 @@ def _create_test_provider(
     db_session: Session,
     name: str,
     api_key: str = "sk-test-key-00000000000000000000000000000000000",
-) -> None:
+) -> LLMProviderView:
     """Helper to create a test LLM provider in the database."""
-    upsert_llm_provider(
+    return upsert_llm_provider(
         LLMProviderUpsertRequest(
             name=name,
             provider=LlmProviderNames.OPENAI,
             api_key=api_key,
             api_key_changed=True,
-            default_model_name="gpt-4o-mini",
             model_configurations=[
                 ModelConfigurationUpsertRequest(name="gpt-4o-mini", is_visible=True)
             ],
@@ -102,17 +102,11 @@ class TestLLMConfigurationEndpoint:
                 # This should complete without exception
                 run_test_llm_configuration(
                     test_llm_request=LLMTestRequest(
-                        name=None,  # New provider (not in DB)
                         provider=LlmProviderNames.OPENAI,
                         api_key="sk-new-test-key-0000000000000000000000000000",
                         api_key_changed=True,
                         custom_config_changed=False,
-                        default_model_name="gpt-4o-mini",
-                        model_configurations=[
-                            ModelConfigurationUpsertRequest(
-                                name="gpt-4o-mini", is_visible=True
-                            )
-                        ],
+                        model="gpt-4o-mini",
                     ),
                     _=_create_mock_admin(),
                     db_session=db_session,
@@ -152,17 +146,11 @@ class TestLLMConfigurationEndpoint:
                 with pytest.raises(HTTPException) as exc_info:
                     run_test_llm_configuration(
                         test_llm_request=LLMTestRequest(
-                            name=None,
                             provider=LlmProviderNames.OPENAI,
                             api_key="sk-invalid-key-00000000000000000000000000",
                             api_key_changed=True,
                             custom_config_changed=False,
-                            default_model_name="gpt-4o-mini",
-                            model_configurations=[
-                                ModelConfigurationUpsertRequest(
-                                    name="gpt-4o-mini", is_visible=True
-                                )
-                            ],
+                            model="gpt-4o-mini",
                         ),
                         _=_create_mock_admin(),
                         db_session=db_session,
@@ -194,7 +182,9 @@ class TestLLMConfigurationEndpoint:
 
         try:
             # First, create the provider in the database
-            _create_test_provider(db_session, provider_name, api_key=original_api_key)
+            provider = _create_test_provider(
+                db_session, provider_name, api_key=original_api_key
+            )
 
             with patch(
                 "onyx.server.manage.llm.api.test_llm", side_effect=mock_test_llm_capture
@@ -202,17 +192,12 @@ class TestLLMConfigurationEndpoint:
                 # Test with api_key_changed=False - should use stored key
                 run_test_llm_configuration(
                     test_llm_request=LLMTestRequest(
-                        name=provider_name,  # Existing provider
+                        id=provider.id,
                         provider=LlmProviderNames.OPENAI,
                         api_key=None,  # Not providing a new key
                         api_key_changed=False,  # Using existing key
                         custom_config_changed=False,
-                        default_model_name="gpt-4o-mini",
-                        model_configurations=[
-                            ModelConfigurationUpsertRequest(
-                                name="gpt-4o-mini", is_visible=True
-                            )
-                        ],
+                        model="gpt-4o-mini",
                     ),
                     _=_create_mock_admin(),
                     db_session=db_session,
@@ -246,7 +231,9 @@ class TestLLMConfigurationEndpoint:
 
         try:
             # First, create the provider in the database
-            _create_test_provider(db_session, provider_name, api_key=original_api_key)
+            provider = _create_test_provider(
+                db_session, provider_name, api_key=original_api_key
+            )
 
             with patch(
                 "onyx.server.manage.llm.api.test_llm", side_effect=mock_test_llm_capture
@@ -254,17 +241,12 @@ class TestLLMConfigurationEndpoint:
                 # Test with api_key_changed=True - should use new key
                 run_test_llm_configuration(
                     test_llm_request=LLMTestRequest(
-                        name=provider_name,  # Existing provider
+                        id=provider.id,
                         provider=LlmProviderNames.OPENAI,
                         api_key=new_api_key,  # Providing a new key
                         api_key_changed=True,  # Key is being changed
                         custom_config_changed=False,
-                        default_model_name="gpt-4o-mini",
-                        model_configurations=[
-                            ModelConfigurationUpsertRequest(
-                                name="gpt-4o-mini", is_visible=True
-                            )
-                        ],
+                        model="gpt-4o-mini",
                     ),
                     _=_create_mock_admin(),
                     db_session=db_session,
@@ -297,7 +279,7 @@ class TestLLMConfigurationEndpoint:
 
         try:
             # First, create the provider in the database with custom_config
-            upsert_llm_provider(
+            provider = upsert_llm_provider(
                 LLMProviderUpsertRequest(
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
@@ -305,7 +287,6 @@ class TestLLMConfigurationEndpoint:
                     api_key_changed=True,
                     custom_config=original_custom_config,
                     custom_config_changed=True,
-                    default_model_name="gpt-4o-mini",
                     model_configurations=[
                         ModelConfigurationUpsertRequest(
                             name="gpt-4o-mini", is_visible=True
@@ -321,18 +302,13 @@ class TestLLMConfigurationEndpoint:
                 # Test with custom_config_changed=False - should use stored config
                 run_test_llm_configuration(
                     test_llm_request=LLMTestRequest(
-                        name=provider_name,
+                        id=provider.id,
                         provider=LlmProviderNames.OPENAI,
                         api_key=None,
                         api_key_changed=False,
                         custom_config=None,  # Not providing new config
                         custom_config_changed=False,  # Using existing config
-                        default_model_name="gpt-4o-mini",
-                        model_configurations=[
-                            ModelConfigurationUpsertRequest(
-                                name="gpt-4o-mini", is_visible=True
-                            )
-                        ],
+                        model="gpt-4o-mini",
                     ),
                     _=_create_mock_admin(),
                     db_session=db_session,
@@ -368,17 +344,11 @@ class TestLLMConfigurationEndpoint:
                 for model_name in test_models:
                     run_test_llm_configuration(
                         test_llm_request=LLMTestRequest(
-                            name=None,
                             provider=LlmProviderNames.OPENAI,
                             api_key="sk-test-key-00000000000000000000000000000000000",
                             api_key_changed=True,
                             custom_config_changed=False,
-                            default_model_name=model_name,
-                            model_configurations=[
-                                ModelConfigurationUpsertRequest(
-                                    name=model_name, is_visible=True
-                                )
-                            ],
+                            model=model_name,
                         ),
                         _=_create_mock_admin(),
                         db_session=db_session,
@@ -442,7 +412,6 @@ class TestDefaultProviderEndpoint:
                     provider=LlmProviderNames.OPENAI,
                     api_key=provider_1_api_key,
                     api_key_changed=True,
-                    default_model_name=provider_1_initial_model,
                     model_configurations=[
                         ModelConfigurationUpsertRequest(name="gpt-4", is_visible=True),
                         ModelConfigurationUpsertRequest(name="gpt-4o", is_visible=True),
@@ -452,7 +421,7 @@ class TestDefaultProviderEndpoint:
             )
 
             # Set provider 1 as the default provider explicitly
-            update_default_provider(provider_1.id, db_session)
+            update_default_provider(provider_1.id, provider_1_initial_model, db_session)
 
             # Step 2: Call run_test_default_provider - should use provider 1's default model
             with patch(
@@ -472,7 +441,6 @@ class TestDefaultProviderEndpoint:
                     provider=LlmProviderNames.OPENAI,
                     api_key=provider_2_api_key,
                     api_key_changed=True,
-                    default_model_name=provider_2_default_model,
                     model_configurations=[
                         ModelConfigurationUpsertRequest(
                             name="gpt-4o-mini", is_visible=True
@@ -499,11 +467,11 @@ class TestDefaultProviderEndpoint:
             # Step 5: Update provider 1's default model
             upsert_llm_provider(
                 LLMProviderUpsertRequest(
+                    id=provider_1.id,
                     name=provider_1_name,
                     provider=LlmProviderNames.OPENAI,
                     api_key=provider_1_api_key,
                     api_key_changed=True,
-                    default_model_name=provider_1_updated_model,  # Changed
                     model_configurations=[
                         ModelConfigurationUpsertRequest(name="gpt-4", is_visible=True),
                         ModelConfigurationUpsertRequest(name="gpt-4o", is_visible=True),
@@ -512,6 +480,9 @@ class TestDefaultProviderEndpoint:
                 db_session=db_session,
             )
 
+            # Set provider 1's default model to the updated model
+            update_default_provider(provider_1.id, provider_1_updated_model, db_session)
+
             # Step 6: Call run_test_default_provider - should use new model on provider 1
             with patch(
                 "onyx.server.manage.llm.api.test_llm", side_effect=mock_test_llm_capture
@@ -524,7 +495,7 @@ class TestDefaultProviderEndpoint:
             captured_llms.clear()
 
             # Step 7: Change the default provider to provider 2
-            update_default_provider(provider_2.id, db_session)
+            update_default_provider(provider_2.id, provider_2_default_model, db_session)
 
             # Step 8: Call run_test_default_provider - should use provider 2
             with patch(
@@ -596,7 +567,6 @@ class TestDefaultProviderEndpoint:
                     provider=LlmProviderNames.OPENAI,
                     api_key="sk-test-key-00000000000000000000000000000000000",
                     api_key_changed=True,
-                    default_model_name="gpt-4o-mini",
                     model_configurations=[
                         ModelConfigurationUpsertRequest(
                             name="gpt-4o-mini", is_visible=True
@@ -605,7 +575,7 @@ class TestDefaultProviderEndpoint:
                 ),
                 db_session=db_session,
             )
-            update_default_provider(provider.id, db_session)
+            update_default_provider(provider.id, "gpt-4o-mini", db_session)
 
             # Test should fail
             with patch(

backend/tests/external_dependency_unit/llm/test_llm_provider_api_base.py

@@ -49,7 +49,6 @@ def _create_test_provider(
             api_key_changed=True,
             api_base=api_base,
             custom_config=custom_config,
-            default_model_name="gpt-4o-mini",
             model_configurations=[
                 ModelConfigurationUpsertRequest(name="gpt-4o-mini", is_visible=True)
             ],
@@ -91,14 +90,14 @@ class TestLLMProviderChanges:
         the API key should be blocked.
         """
         try:
-            _create_test_provider(db_session, provider_name)
+            provider = _create_test_provider(db_session, provider_name)
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", True):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     api_base="https://attacker.example.com",
-                    default_model_name="gpt-4o-mini",
                 )
 
                 with pytest.raises(HTTPException) as exc_info:
@@ -125,16 +124,16 @@ class TestLLMProviderChanges:
         Changing api_base IS allowed when the API key is also being changed.
         """
         try:
-            _create_test_provider(db_session, provider_name)
+            provider = _create_test_provider(db_session, provider_name)
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", True):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     api_key="sk-new-key-00000000000000000000000000000000000",
                     api_key_changed=True,
                     api_base="https://custom-endpoint.example.com/v1",
-                    default_model_name="gpt-4o-mini",
                 )
 
                 result = put_llm_provider(
@@ -159,14 +158,16 @@ class TestLLMProviderChanges:
         original_api_base = "https://original.example.com/v1"
 
         try:
-            _create_test_provider(db_session, provider_name, api_base=original_api_base)
+            provider = _create_test_provider(
+                db_session, provider_name, api_base=original_api_base
+            )
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", True):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     api_base=original_api_base,
-                    default_model_name="gpt-4o-mini",
                 )
 
                 result = put_llm_provider(
@@ -190,14 +191,14 @@ class TestLLMProviderChanges:
         changes. This allows model-only updates when provider has no custom base URL.
         """
         try:
-            _create_test_provider(db_session, provider_name, api_base=None)
+            view = _create_test_provider(db_session, provider_name, api_base=None)
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", True):
                 update_request = LLMProviderUpsertRequest(
+                    id=view.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     api_base="",
-                    default_model_name="gpt-4o-mini",
                 )
 
                 result = put_llm_provider(
@@ -223,14 +224,16 @@ class TestLLMProviderChanges:
         original_api_base = "https://original.example.com/v1"
 
         try:
-            _create_test_provider(db_session, provider_name, api_base=original_api_base)
+            provider = _create_test_provider(
+                db_session, provider_name, api_base=original_api_base
+            )
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", True):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     api_base=None,
-                    default_model_name="gpt-4o-mini",
                 )
 
                 with pytest.raises(HTTPException) as exc_info:
@@ -259,14 +262,14 @@ class TestLLMProviderChanges:
         users have full control over their deployment.
         """
         try:
-            _create_test_provider(db_session, provider_name)
+            provider = _create_test_provider(db_session, provider_name)
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", False):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     api_base="https://custom.example.com/v1",
-                    default_model_name="gpt-4o-mini",
                 )
 
                 result = put_llm_provider(
@@ -297,7 +300,6 @@ class TestLLMProviderChanges:
                     api_key="sk-new-key-00000000000000000000000000000000000",
                     api_key_changed=True,
                     api_base="https://custom.example.com/v1",
-                    default_model_name="gpt-4o-mini",
                 )
 
                 result = put_llm_provider(
@@ -322,7 +324,7 @@ class TestLLMProviderChanges:
         redirect LLM API requests).
         """
         try:
-            _create_test_provider(
+            provider = _create_test_provider(
                 db_session,
                 provider_name,
                 custom_config={"SOME_CONFIG": "original_value"},
@@ -330,11 +332,11 @@ class TestLLMProviderChanges:
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", True):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     custom_config={"OPENAI_API_BASE": "https://attacker.example.com"},
                     custom_config_changed=True,
-                    default_model_name="gpt-4o-mini",
                 )
 
                 with pytest.raises(HTTPException) as exc_info:
@@ -362,15 +364,15 @@ class TestLLMProviderChanges:
         without changing the API key.
         """
         try:
-            _create_test_provider(db_session, provider_name)
+            provider = _create_test_provider(db_session, provider_name)
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", True):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     custom_config={"OPENAI_API_BASE": "https://attacker.example.com"},
                     custom_config_changed=True,
-                    default_model_name="gpt-4o-mini",
                 )
 
                 with pytest.raises(HTTPException) as exc_info:
@@ -399,7 +401,7 @@ class TestLLMProviderChanges:
         new_config = {"AWS_REGION_NAME": "us-west-2"}
 
         try:
-            _create_test_provider(
+            provider = _create_test_provider(
                 db_session,
                 provider_name,
                 custom_config={"AWS_REGION_NAME": "us-east-1"},
@@ -407,13 +409,13 @@ class TestLLMProviderChanges:
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", True):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     api_key="sk-new-key-00000000000000000000000000000000000",
                     api_key_changed=True,
                     custom_config_changed=True,
                     custom_config=new_config,
-                    default_model_name="gpt-4o-mini",
                 )
 
                 result = put_llm_provider(
@@ -438,17 +440,17 @@ class TestLLMProviderChanges:
         original_config = {"AWS_REGION_NAME": "us-east-1"}
 
         try:
-            _create_test_provider(
+            provider = _create_test_provider(
                 db_session, provider_name, custom_config=original_config
             )
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", True):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     custom_config=original_config,
                     custom_config_changed=True,
-                    default_model_name="gpt-4o-mini",
                 )
 
                 result = put_llm_provider(
@@ -474,7 +476,7 @@ class TestLLMProviderChanges:
         new_config = {"AWS_REGION_NAME": "eu-west-1"}
 
         try:
-            _create_test_provider(
+            provider = _create_test_provider(
                 db_session,
                 provider_name,
                 custom_config={"AWS_REGION_NAME": "us-east-1"},
@@ -482,10 +484,10 @@ class TestLLMProviderChanges:
 
             with patch("onyx.server.manage.llm.api.MULTI_TENANT", False):
                 update_request = LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=provider_name,
                     provider=LlmProviderNames.OPENAI,
                     custom_config=new_config,
-                    default_model_name="gpt-4o-mini",
                     custom_config_changed=True,
                 )
 
@@ -530,14 +532,8 @@ def test_upload_with_custom_config_then_change(
         with patch("onyx.server.manage.llm.api.test_llm", side_effect=capture_test_llm):
             run_llm_config_test(
                 LLMTestRequest(
-                    name=name,
                     provider=provider_name,
-                    default_model_name=default_model_name,
-                    model_configurations=[
-                        ModelConfigurationUpsertRequest(
-                            name=default_model_name, is_visible=True
-                        )
-                    ],
+                    model=default_model_name,
                     api_key_changed=False,
                     custom_config_changed=True,
                     custom_config=custom_config,
@@ -546,11 +542,10 @@ def test_upload_with_custom_config_then_change(
                 db_session=db_session,
             )
 
-            put_llm_provider(
+            provider = put_llm_provider(
                 llm_provider_upsert_request=LLMProviderUpsertRequest(
                     name=name,
                     provider=provider_name,
-                    default_model_name=default_model_name,
                     custom_config=custom_config,
                     model_configurations=[
                         ModelConfigurationUpsertRequest(
@@ -569,14 +564,9 @@ def test_upload_with_custom_config_then_change(
             # Turn auto mode off
             run_llm_config_test(
                 LLMTestRequest(
-                    name=name,
+                    id=provider.id,
                     provider=provider_name,
-                    default_model_name=default_model_name,
-                    model_configurations=[
-                        ModelConfigurationUpsertRequest(
-                            name=default_model_name, is_visible=True
-                        )
-                    ],
+                    model=default_model_name,
                     api_key_changed=False,
                     custom_config_changed=False,
                 ),
@@ -586,9 +576,9 @@ def test_upload_with_custom_config_then_change(
 
             put_llm_provider(
                 llm_provider_upsert_request=LLMProviderUpsertRequest(
+                    id=provider.id,
                     name=name,
                     provider=provider_name,
-                    default_model_name=default_model_name,
                     model_configurations=[
                         ModelConfigurationUpsertRequest(
                             name=default_model_name, is_visible=True
@@ -616,13 +606,13 @@ def test_upload_with_custom_config_then_change(
                 )
 
             # Check inside the database and check that custom_config is the same as the original
-            provider = fetch_existing_llm_provider(name=name, db_session=db_session)
-            if not provider:
+            db_provider = fetch_existing_llm_provider(name=name, db_session=db_session)
+            if not db_provider:
                 assert False, "Provider not found in the database"
 
-            assert provider.custom_config == custom_config, (
+            assert db_provider.custom_config == custom_config, (
                 f"Expected custom_config {custom_config}, "
-                f"but got {provider.custom_config}"
+                f"but got {db_provider.custom_config}"
             )
     finally:
         db_session.rollback()
@@ -642,11 +632,10 @@ def test_preserves_masked_sensitive_custom_config_on_provider_update(
     }
 
     try:
-        put_llm_provider(
+        view = put_llm_provider(
             llm_provider_upsert_request=LLMProviderUpsertRequest(
                 name=name,
                 provider=provider,
-                default_model_name=default_model_name,
                 custom_config=original_custom_config,
                 model_configurations=[
                     ModelConfigurationUpsertRequest(
@@ -665,9 +654,9 @@ def test_preserves_masked_sensitive_custom_config_on_provider_update(
         with patch("onyx.server.manage.llm.api.MULTI_TENANT", False):
             put_llm_provider(
                 llm_provider_upsert_request=LLMProviderUpsertRequest(
+                    id=view.id,
                     name=name,
                     provider=provider,
-                    default_model_name=default_model_name,
                     custom_config={
                         "vertex_credentials": _mask_string(
                             original_custom_config["vertex_credentials"]
@@ -706,7 +695,7 @@ def test_preserves_masked_sensitive_custom_config_on_test_request(
 ) -> None:
     """LLM test should restore masked sensitive custom config values before invocation."""
     name = f"test-provider-vertex-test-{uuid4().hex[:8]}"
-    provider = LlmProviderNames.VERTEX_AI.value
+    provider_name = LlmProviderNames.VERTEX_AI.value
     default_model_name = "gemini-2.5-pro"
     original_custom_config = {
         "vertex_credentials": '{"type":"service_account","private_key":"REAL_PRIVATE_KEY"}',
@@ -719,11 +708,10 @@ def test_preserves_masked_sensitive_custom_config_on_test_request(
         return ""
 
     try:
-        put_llm_provider(
+        provider = put_llm_provider(
             llm_provider_upsert_request=LLMProviderUpsertRequest(
                 name=name,
-                provider=provider,
-                default_model_name=default_model_name,
+                provider=provider_name,
                 custom_config=original_custom_config,
                 model_configurations=[
                     ModelConfigurationUpsertRequest(
@@ -742,14 +730,9 @@ def test_preserves_masked_sensitive_custom_config_on_test_request(
         with patch("onyx.server.manage.llm.api.test_llm", side_effect=capture_test_llm):
             run_llm_config_test(
                 LLMTestRequest(
-                    name=name,
-                    provider=provider,
-                    default_model_name=default_model_name,
-                    model_configurations=[
-                        ModelConfigurationUpsertRequest(
-                            name=default_model_name, is_visible=True
-                        )
-                    ],
+                    id=provider.id,
+                    provider=provider_name,
+                    model=default_model_name,
                     api_key_changed=False,
                     custom_config_changed=True,
                     custom_config={

backend/tests/external_dependency_unit/llm/test_llm_provider_auto_mode.py

@@ -15,9 +15,11 @@ import pytest
 from sqlalchemy.orm import Session
 
 from onyx.db.enums import LLMModelFlowType
+from onyx.db.llm import fetch_auto_mode_providers
 from onyx.db.llm import fetch_default_llm_model
 from onyx.db.llm import fetch_existing_llm_provider
 from onyx.db.llm import fetch_existing_llm_providers
+from onyx.db.llm import fetch_llm_provider_view
 from onyx.db.llm import remove_llm_provider
 from onyx.db.llm import sync_auto_mode_models
 from onyx.db.llm import update_default_provider
@@ -135,7 +137,6 @@ class TestAutoModeSyncFeature:
                         api_key="sk-test-key-00000000000000000000000000000000000",
                         api_key_changed=True,
                         is_auto_mode=True,
-                        default_model_name=expected_default_model,
                         model_configurations=[],  # No model configs provided
                     ),
                     is_creation=True,
@@ -163,13 +164,8 @@ class TestAutoModeSyncFeature:
                 if mc.name in all_expected_models:
                     assert mc.is_visible is True, f"Model '{mc.name}' should be visible"
 
-            # Verify the default model was set correctly
-            assert (
-                provider.default_model_name == expected_default_model
-            ), f"Default model should be '{expected_default_model}'"
-
             # Step 4: Set the provider as default
-            update_default_provider(provider.id, db_session)
+            update_default_provider(provider.id, expected_default_model, db_session)
 
             # Step 5: Fetch the default provider and verify
             default_model = fetch_default_llm_model(db_session)
@@ -238,7 +234,6 @@ class TestAutoModeSyncFeature:
                         api_key="sk-test-key-00000000000000000000000000000000000",
                         api_key_changed=True,
                         is_auto_mode=True,
-                        default_model_name="gpt-4o",
                         model_configurations=[],
                     ),
                     is_creation=True,
@@ -317,7 +312,6 @@ class TestAutoModeSyncFeature:
                     api_key="sk-test-key-00000000000000000000000000000000000",
                     api_key_changed=True,
                     is_auto_mode=False,  # Not in auto mode initially
-                    default_model_name="gpt-4",
                     model_configurations=initial_models,
                 ),
                 is_creation=True,
@@ -326,13 +320,13 @@ class TestAutoModeSyncFeature:
             )
 
             # Verify initial state: all models are visible
-            provider = fetch_existing_llm_provider(
+            initial_provider = fetch_existing_llm_provider(
                 name=provider_name, db_session=db_session
             )
-            assert provider is not None
-            assert provider.is_auto_mode is False
+            assert initial_provider is not None
+            assert initial_provider.is_auto_mode is False
 
-            for mc in provider.model_configurations:
+            for mc in initial_provider.model_configurations:
                 assert (
                     mc.is_visible is True
                 ), f"Initial model '{mc.name}' should be visible"
@@ -344,12 +338,12 @@ class TestAutoModeSyncFeature:
             ):
                 put_llm_provider(
                     llm_provider_upsert_request=LLMProviderUpsertRequest(
+                        id=initial_provider.id,
                         name=provider_name,
                         provider=LlmProviderNames.OPENAI,
                         api_key=None,  # Not changing API key
                         api_key_changed=False,
                         is_auto_mode=True,  # Now enabling auto mode
-                        default_model_name=auto_mode_default,
                         model_configurations=[],  # Auto mode will sync from config
                     ),
                     is_creation=False,  # This is an update
@@ -360,15 +354,15 @@ class TestAutoModeSyncFeature:
             # Step 3: Verify model visibility after auto mode transition
             # Expire session cache to force fresh fetch after sync_auto_mode_models committed
             db_session.expire_all()
-            provider = fetch_existing_llm_provider(
-                name=provider_name, db_session=db_session
+            provider_view = fetch_llm_provider_view(
+                provider_name=provider_name, db_session=db_session
             )
-            assert provider is not None
-            assert provider.is_auto_mode is True
+            assert provider_view is not None
+            assert provider_view.is_auto_mode is True
 
             # Build a map of model name -> visibility
             model_visibility = {
-                mc.name: mc.is_visible for mc in provider.model_configurations
+                mc.name: mc.is_visible for mc in provider_view.model_configurations
             }
 
             # Models in auto mode config should be visible
@@ -388,9 +382,6 @@ class TestAutoModeSyncFeature:
                         model_visibility[model_name] is False
                     ), f"Model '{model_name}' not in auto config should NOT be visible"
 
-            # Verify the default model was updated
-            assert provider.default_model_name == auto_mode_default
-
         finally:
             db_session.rollback()
             _cleanup_provider(db_session, provider_name)
@@ -432,8 +423,12 @@ class TestAutoModeSyncFeature:
                         api_key="sk-test-key-00000000000000000000000000000000000",
                         api_key_changed=True,
                         is_auto_mode=True,
-                        default_model_name="gpt-4o",
-                        model_configurations=[],
+                        model_configurations=[
+                            ModelConfigurationUpsertRequest(
+                                name="gpt-4o",
+                                is_visible=True,
+                            )
+                        ],
                     ),
                     is_creation=True,
                     _=_create_mock_admin(),
@@ -535,7 +530,6 @@ class TestAutoModeSyncFeature:
                         api_key=provider_1_api_key,
                         api_key_changed=True,
                         is_auto_mode=True,
-                        default_model_name=provider_1_default_model,
                         model_configurations=[],
                     ),
                     is_creation=True,
@@ -549,7 +543,7 @@ class TestAutoModeSyncFeature:
                 name=provider_1_name, db_session=db_session
             )
             assert provider_1 is not None
-            update_default_provider(provider_1.id, db_session)
+            update_default_provider(provider_1.id, provider_1_default_model, db_session)
 
             with patch(
                 "onyx.server.manage.llm.api.fetch_llm_recommendations_from_github",
@@ -563,7 +557,6 @@ class TestAutoModeSyncFeature:
                         api_key=provider_2_api_key,
                         api_key_changed=True,
                         is_auto_mode=True,
-                        default_model_name=provider_2_default_model,
                         model_configurations=[],
                     ),
                     is_creation=True,
@@ -584,7 +577,7 @@ class TestAutoModeSyncFeature:
                 name=provider_2_name, db_session=db_session
             )
             assert provider_2 is not None
-            update_default_provider(provider_2.id, db_session)
+            update_default_provider(provider_2.id, provider_2_default_model, db_session)
 
             # Step 5: Verify provider 2 is now the default
             db_session.expire_all()
@@ -644,7 +637,6 @@ class TestAutoModeMissingFlows:
                     api_key="sk-test-key-00000000000000000000000000000000000",
                     api_key_changed=True,
                     is_auto_mode=True,
-                    default_model_name="gpt-4o",
                     model_configurations=[],
                 ),
                 is_creation=True,
@@ -701,3 +693,364 @@ class TestAutoModeMissingFlows:
         finally:
             db_session.rollback()
             _cleanup_provider(db_session, provider_name)
+
+
+class TestAutoModeTransitionsAndResync:
+    """Tests for auto/manual transitions, config evolution, and sync idempotency."""
+
+    def test_auto_to_manual_mode_preserves_models_and_stops_syncing(
+        self,
+        db_session: Session,
+        provider_name: str,
+    ) -> None:
+        """Disabling auto mode should preserve the current model list and
+        prevent future syncs from altering visibility.
+
+        Steps:
+        1. Create provider in auto mode — models synced from config.
+        2. Update provider to manual mode (is_auto_mode=False).
+        3. Verify all models remain with unchanged visibility.
+        4. Call sync_auto_mode_models with a *different* config.
+        5. Verify fetch_auto_mode_providers excludes this provider, so the
+           periodic task would never call sync on it.
+        """
+        initial_config = _create_mock_llm_recommendations(
+            provider=LlmProviderNames.OPENAI,
+            default_model_name="gpt-4o",
+            additional_models=["gpt-4o-mini"],
+        )
+
+        try:
+            # Step 1: Create in auto mode
+            with patch(
+                "onyx.server.manage.llm.api.fetch_llm_recommendations_from_github",
+                return_value=initial_config,
+            ):
+                put_llm_provider(
+                    llm_provider_upsert_request=LLMProviderUpsertRequest(
+                        name=provider_name,
+                        provider=LlmProviderNames.OPENAI,
+                        api_key="sk-test-key-00000000000000000000000000000000000",
+                        api_key_changed=True,
+                        is_auto_mode=True,
+                        model_configurations=[],
+                    ),
+                    is_creation=True,
+                    _=_create_mock_admin(),
+                    db_session=db_session,
+                )
+
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+            visibility_before = {
+                mc.name: mc.is_visible for mc in provider.model_configurations
+            }
+            assert visibility_before == {"gpt-4o": True, "gpt-4o-mini": True}
+
+            # Step 2: Switch to manual mode
+            put_llm_provider(
+                llm_provider_upsert_request=LLMProviderUpsertRequest(
+                    id=provider.id,
+                    name=provider_name,
+                    provider=LlmProviderNames.OPENAI,
+                    api_key=None,
+                    api_key_changed=False,
+                    is_auto_mode=False,
+                    model_configurations=[
+                        ModelConfigurationUpsertRequest(name="gpt-4o", is_visible=True),
+                        ModelConfigurationUpsertRequest(
+                            name="gpt-4o-mini", is_visible=True
+                        ),
+                    ],
+                ),
+                is_creation=False,
+                _=_create_mock_admin(),
+                db_session=db_session,
+            )
+
+            # Step 3: Models unchanged
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+            assert provider.is_auto_mode is False
+            visibility_after = {
+                mc.name: mc.is_visible for mc in provider.model_configurations
+            }
+            assert visibility_after == visibility_before
+
+            # Step 4-5: Provider excluded from auto mode queries
+            auto_providers = fetch_auto_mode_providers(db_session)
+            auto_provider_ids = {p.id for p in auto_providers}
+            assert provider.id not in auto_provider_ids
+
+        finally:
+            db_session.rollback()
+            _cleanup_provider(db_session, provider_name)
+
+    def test_resync_adds_new_and_hides_removed_models(
+        self,
+        db_session: Session,
+        provider_name: str,
+    ) -> None:
+        """When the GitHub config changes between syncs, a subsequent sync
+        should add newly listed models and hide models that were removed.
+
+        Steps:
+        1. Create provider in auto mode with config v1: [gpt-4o, gpt-4o-mini].
+        2. Sync with config v2: [gpt-4o, gpt-4-turbo] (gpt-4o-mini removed,
+           gpt-4-turbo added).
+        3. Verify gpt-4o still visible, gpt-4o-mini hidden, gpt-4-turbo added
+           and visible.
+        """
+        config_v1 = _create_mock_llm_recommendations(
+            provider=LlmProviderNames.OPENAI,
+            default_model_name="gpt-4o",
+            additional_models=["gpt-4o-mini"],
+        )
+        config_v2 = _create_mock_llm_recommendations(
+            provider=LlmProviderNames.OPENAI,
+            default_model_name="gpt-4o",
+            additional_models=["gpt-4-turbo"],
+        )
+
+        try:
+            # Step 1: Create with config v1
+            with patch(
+                "onyx.server.manage.llm.api.fetch_llm_recommendations_from_github",
+                return_value=config_v1,
+            ):
+                put_llm_provider(
+                    llm_provider_upsert_request=LLMProviderUpsertRequest(
+                        name=provider_name,
+                        provider=LlmProviderNames.OPENAI,
+                        api_key="sk-test-key-00000000000000000000000000000000000",
+                        api_key_changed=True,
+                        is_auto_mode=True,
+                        model_configurations=[],
+                    ),
+                    is_creation=True,
+                    _=_create_mock_admin(),
+                    db_session=db_session,
+                )
+
+            # Step 2: Re-sync with config v2
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+
+            changes = sync_auto_mode_models(
+                db_session=db_session,
+                provider=provider,
+                llm_recommendations=config_v2,
+            )
+            assert changes > 0
+
+            # Step 3: Verify
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+
+            visibility = {
+                mc.name: mc.is_visible for mc in provider.model_configurations
+            }
+
+            # gpt-4o: still in config -> visible
+            assert visibility["gpt-4o"] is True
+            # gpt-4o-mini: removed from config -> hidden (not deleted)
+            assert "gpt-4o-mini" in visibility, "Removed model should still exist in DB"
+            assert visibility["gpt-4o-mini"] is False
+            # gpt-4-turbo: newly added -> visible
+            assert visibility["gpt-4-turbo"] is True
+
+        finally:
+            db_session.rollback()
+            _cleanup_provider(db_session, provider_name)
+
+    def test_sync_is_idempotent(
+        self,
+        db_session: Session,
+        provider_name: str,
+    ) -> None:
+        """Running sync twice with the same config should produce zero
+        changes on the second call."""
+        config = _create_mock_llm_recommendations(
+            provider=LlmProviderNames.OPENAI,
+            default_model_name="gpt-4o",
+            additional_models=["gpt-4o-mini", "gpt-4-turbo"],
+        )
+
+        try:
+            with patch(
+                "onyx.server.manage.llm.api.fetch_llm_recommendations_from_github",
+                return_value=config,
+            ):
+                put_llm_provider(
+                    llm_provider_upsert_request=LLMProviderUpsertRequest(
+                        name=provider_name,
+                        provider=LlmProviderNames.OPENAI,
+                        api_key="sk-test-key-00000000000000000000000000000000000",
+                        api_key_changed=True,
+                        is_auto_mode=True,
+                        model_configurations=[],
+                    ),
+                    is_creation=True,
+                    _=_create_mock_admin(),
+                    db_session=db_session,
+                )
+
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+
+            # First explicit sync (may report changes if creation already synced)
+            sync_auto_mode_models(
+                db_session=db_session,
+                provider=provider,
+                llm_recommendations=config,
+            )
+
+            # Snapshot state after first sync
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+            snapshot = {
+                mc.name: (mc.is_visible, mc.display_name)
+                for mc in provider.model_configurations
+            }
+
+            # Second sync — should be a no-op
+            changes = sync_auto_mode_models(
+                db_session=db_session,
+                provider=provider,
+                llm_recommendations=config,
+            )
+            assert (
+                changes == 0
+            ), f"Expected 0 changes on idempotent re-sync, got {changes}"
+
+            # State should be identical
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+            current = {
+                mc.name: (mc.is_visible, mc.display_name)
+                for mc in provider.model_configurations
+            }
+            assert current == snapshot
+
+        finally:
+            db_session.rollback()
+            _cleanup_provider(db_session, provider_name)
+
+    def test_default_model_hidden_when_removed_from_config(
+        self,
+        db_session: Session,
+        provider_name: str,
+    ) -> None:
+        """When the current default model is removed from the config, sync
+        should hide it. The default model flow row should still exist (it
+        points at the ModelConfiguration), but the model is no longer visible.
+
+        Steps:
+        1. Create provider with config: default=gpt-4o, additional=[gpt-4o-mini].
+        2. Set gpt-4o as the global default.
+        3. Re-sync with config: default=gpt-4o-mini (gpt-4o removed entirely).
+        4. Verify gpt-4o is hidden, gpt-4o-mini is visible, and
+           fetch_default_llm_model still returns a result (the flow row persists).
+        """
+        config_v1 = _create_mock_llm_recommendations(
+            provider=LlmProviderNames.OPENAI,
+            default_model_name="gpt-4o",
+            additional_models=["gpt-4o-mini"],
+        )
+        config_v2 = _create_mock_llm_recommendations(
+            provider=LlmProviderNames.OPENAI,
+            default_model_name="gpt-4o-mini",
+            additional_models=[],
+        )
+
+        try:
+            with patch(
+                "onyx.server.manage.llm.api.fetch_llm_recommendations_from_github",
+                return_value=config_v1,
+            ):
+                put_llm_provider(
+                    llm_provider_upsert_request=LLMProviderUpsertRequest(
+                        name=provider_name,
+                        provider=LlmProviderNames.OPENAI,
+                        api_key="sk-test-key-00000000000000000000000000000000000",
+                        api_key_changed=True,
+                        is_auto_mode=True,
+                        model_configurations=[],
+                    ),
+                    is_creation=True,
+                    _=_create_mock_admin(),
+                    db_session=db_session,
+                )
+
+            # Step 2: Set gpt-4o as global default
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+            update_default_provider(provider.id, "gpt-4o", db_session)
+
+            default_before = fetch_default_llm_model(db_session)
+            assert default_before is not None
+            assert default_before.name == "gpt-4o"
+
+            # Step 3: Re-sync with config v2 (gpt-4o removed)
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+
+            changes = sync_auto_mode_models(
+                db_session=db_session,
+                provider=provider,
+                llm_recommendations=config_v2,
+            )
+            assert changes > 0
+
+            # Step 4: Verify visibility
+            db_session.expire_all()
+            provider = fetch_existing_llm_provider(
+                name=provider_name, db_session=db_session
+            )
+            assert provider is not None
+
+            visibility = {
+                mc.name: mc.is_visible for mc in provider.model_configurations
+            }
+            assert visibility["gpt-4o"] is False, "Removed default should be hidden"
+            assert visibility["gpt-4o-mini"] is True, "New default should be visible"
+
+            # The LLMModelFlow row for gpt-4o still exists (is_default=True),
+            # but the model is hidden. fetch_default_llm_model filters on
+            # is_visible=True, so it should NOT return gpt-4o.
+            db_session.expire_all()
+            default_after = fetch_default_llm_model(db_session)
+            assert (
+                default_after is None or default_after.name != "gpt-4o"
+            ), "Hidden model should not be returned as the default"
+
+        finally:
+            db_session.rollback()
+            _cleanup_provider(db_session, provider_name)

backend/tests/external_dependency_unit/llm/test_llm_provider_called.py

@@ -64,7 +64,6 @@ def _create_provider(
             name=name,
             provider=provider,
             api_key="sk-ant-api03-...",
-            default_model_name="claude-3-5-sonnet-20240620",
             is_public=is_public,
             model_configurations=[
                 ModelConfigurationUpsertRequest(
@@ -154,7 +153,9 @@ def test_user_sends_message_to_private_provider(
     )
     _create_provider(db_session, LlmProviderNames.GOOGLE, "private-provider", False)
 
-    update_default_provider(public_provider_id, db_session)
+    update_default_provider(
+        public_provider_id, "claude-3-5-sonnet-20240620", db_session
+    )
 
     try:
         # Create chat session

backend/tests/external_dependency_unit/search_settings/test_search_settings.py

@@ -42,7 +42,6 @@ def _create_llm_provider_and_model(
             name=provider_name,
             provider="openai",
             api_key="test-api-key",
-            default_model_name=model_name,
             model_configurations=[
                 ModelConfigurationUpsertRequest(
                     name=model_name,

backend/tests/external_dependency_unit/slack_bot/test_slack_bot_federated_search.py

@@ -22,7 +22,6 @@ from sqlalchemy.orm import Session
 from slack_sdk.errors import SlackApiError
 
 from onyx.configs.constants import FederatedConnectorSource
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.context.search.federated.slack_search import fetch_and_cache_channel_metadata
 from onyx.db.models import DocumentSet
 from onyx.db.models import FederatedConnector
@@ -55,11 +54,6 @@ def _create_test_persona_with_slack_config(db_session: Session) -> Persona | Non
     persona = Persona(
         name=f"test_slack_persona_{unique_id}",
         description="Test persona for Slack federated search",
-        chunks_above=0,
-        chunks_below=0,
-        llm_relevance_filter=True,
-        llm_filter_extraction=True,
-        recency_bias=RecencyBiasSetting.AUTO,
         system_prompt="You are a helpful assistant.",
         task_prompt="Answer the user's question based on the provided context.",
     )
@@ -434,7 +428,6 @@ class TestSlackBotFederatedSearch:
                 name=f"test-llm-provider-{uuid4().hex[:8]}",
                 provider=LlmProviderNames.OPENAI,
                 api_key=api_key,
-                default_model_name="gpt-4o",
                 is_public=True,
                 model_configurations=[
                     ModelConfigurationUpsertRequest(
@@ -448,7 +441,7 @@ class TestSlackBotFederatedSearch:
             db_session=db_session,
         )
 
-        update_default_provider(provider_view.id, db_session)
+        update_default_provider(provider_view.id, "gpt-4o", db_session)
 
     def _teardown_common_mocks(self, patches: list) -> None:
         """Stop all patches"""
@@ -819,11 +812,6 @@ def test_slack_channel_config_eager_loads_persona(db_session: Session) -> None:
     persona = Persona(
         name=f"test_eager_load_persona_{unique_id}",
         description="Test persona for eager loading test",
-        chunks_above=0,
-        chunks_below=0,
-        llm_relevance_filter=True,
-        llm_filter_extraction=True,
-        recency_bias=RecencyBiasSetting.AUTO,
         system_prompt="You are a helpful assistant.",
         task_prompt="Answer the user's question.",
     )

backend/tests/external_dependency_unit/tools/test_mcp_passthrough_oauth.py

@@ -21,7 +21,6 @@ import pytest
 from sqlalchemy.orm import Session
 
 from onyx.chat.emitter import get_default_emitter
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.enums import MCPAuthenticationPerformer
 from onyx.db.enums import MCPAuthenticationType
 from onyx.db.enums import MCPTransport
@@ -47,12 +46,6 @@ def _create_test_persona_with_mcp_tool(
     persona = Persona(
         name=f"Test MCP Persona {uuid4().hex[:8]}",
         description="Test persona with MCP tools",
-        num_chunks=10.0,
-        chunks_above=0,
-        chunks_below=0,
-        llm_relevance_filter=False,
-        llm_filter_extraction=False,
-        recency_bias=RecencyBiasSetting.NO_DECAY,
         system_prompt="You are a helpful assistant",
         task_prompt="Answer the user's question",
         tools=tools,

backend/tests/external_dependency_unit/tools/test_oauth_tool_integration.py

@@ -17,7 +17,6 @@ import pytest
 from sqlalchemy.orm import Session
 
 from onyx.chat.emitter import get_default_emitter
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.models import OAuthAccount
 from onyx.db.models import OAuthConfig
 from onyx.db.models import Persona
@@ -57,12 +56,6 @@ def _create_test_persona(db_session: Session, user: User, tools: list[Tool]) ->
     persona = Persona(
         name=f"Test Persona {uuid4().hex[:8]}",
         description="Test persona",
-        num_chunks=10.0,
-        chunks_above=0,
-        chunks_below=0,
-        llm_relevance_filter=False,
-        llm_filter_extraction=False,
-        recency_bias=RecencyBiasSetting.NO_DECAY,
         system_prompt="You are a helpful assistant",
         task_prompt="Answer the user's question",
         tools=tools,

backend/tests/external_dependency_unit/tools/test_python_tool.py

@@ -933,6 +933,7 @@ from unittest.mock import patch
 
 import pytest
 from fastapi import UploadFile
+from fastapi.background import BackgroundTasks
 from sqlalchemy.orm import Session
 from starlette.datastructures import Headers
 
@@ -1139,6 +1140,7 @@ def test_code_interpreter_receives_chat_files(
     # Upload a test CSV
     csv_content = b"name,age,city\nAlice,30,NYC\nBob,25,SF\n"
     result = upload_user_files(
+        bg_tasks=BackgroundTasks(),
         files=[
             UploadFile(
                 file=io.BytesIO(csv_content),

backend/tests/integration/common_utils/managers/chat.py

@@ -76,9 +76,12 @@ class ChatSessionManager:
         user_performing_action: DATestUser,
         persona_id: int = 0,
         description: str = "Test chat session",
+        project_id: int | None = None,
     ) -> DATestChatSession:
         chat_session_creation_req = ChatSessionCreationRequest(
-            persona_id=persona_id, description=description
+            persona_id=persona_id,
+            description=description,
+            project_id=project_id,
         )
         response = requests.post(
             f"{API_SERVER_URL}/chat/create-chat-session",

backend/tests/integration/common_utils/managers/llm_provider.py

@@ -4,10 +4,12 @@ from uuid import uuid4
 import requests
 
 from onyx.llm.constants import LlmProviderNames
+from onyx.server.manage.llm.models import DefaultModel
 from onyx.server.manage.llm.models import LLMProviderUpsertRequest
 from onyx.server.manage.llm.models import LLMProviderView
 from onyx.server.manage.llm.models import ModelConfigurationUpsertRequest
 from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.constants import GENERAL_HEADERS
 from tests.integration.common_utils.test_models import DATestLLMProvider
 from tests.integration.common_utils.test_models import DATestUser
 
@@ -32,7 +34,6 @@ class LLMProviderManager:
         llm_provider = LLMProviderUpsertRequest(
             name=name or f"test-provider-{uuid4()}",
             provider=provider or LlmProviderNames.OPENAI,
-            default_model_name=default_model_name or "gpt-4o-mini",
             api_key=api_key or os.environ["OPENAI_API_KEY"],
             api_base=api_base,
             api_version=api_version,
@@ -65,7 +66,7 @@ class LLMProviderManager:
             name=response_data["name"],
             provider=response_data["provider"],
             api_key=response_data["api_key"],
-            default_model_name=response_data["default_model_name"],
+            default_model_name=default_model_name or "gpt-4o-mini",
             is_public=response_data["is_public"],
             is_auto_mode=response_data.get("is_auto_mode", False),
             groups=response_data["groups"],
@@ -75,9 +76,19 @@ class LLMProviderManager:
         )
 
         if set_as_default:
+            if default_model_name is None:
+                default_model_name = "gpt-4o-mini"
             set_default_response = requests.post(
-                f"{API_SERVER_URL}/admin/llm/provider/{llm_response.json()['id']}/default",
-                headers=user_performing_action.headers,
+                f"{API_SERVER_URL}/admin/llm/default",
+                json={
+                    "provider_id": response_data["id"],
+                    "model_name": default_model_name,
+                },
+                headers=(
+                    user_performing_action.headers
+                    if user_performing_action
+                    else GENERAL_HEADERS
+                ),
             )
             set_default_response.raise_for_status()
 
@@ -104,7 +115,7 @@ class LLMProviderManager:
             headers=user_performing_action.headers,
         )
         response.raise_for_status()
-        return [LLMProviderView(**ug) for ug in response.json()]
+        return [LLMProviderView(**p) for p in response.json()["providers"]]
 
     @staticmethod
     def verify(
@@ -113,7 +124,11 @@ class LLMProviderManager:
         verify_deleted: bool = False,
     ) -> None:
         all_llm_providers = LLMProviderManager.get_all(user_performing_action)
+        default_model = LLMProviderManager.get_default_model(user_performing_action)
         for fetched_llm_provider in all_llm_providers:
+            model_names = [
+                model.name for model in fetched_llm_provider.model_configurations
+            ]
             if llm_provider.id == fetched_llm_provider.id:
                 if verify_deleted:
                     raise ValueError(
@@ -126,11 +141,30 @@ class LLMProviderManager:
                 if (
                     fetched_llm_groups == llm_provider_groups
                     and llm_provider.provider == fetched_llm_provider.provider
-                    and llm_provider.default_model_name
-                    == fetched_llm_provider.default_model_name
+                    and (
+                        default_model is None or default_model.model_name in model_names
+                    )
                     and llm_provider.is_public == fetched_llm_provider.is_public
                     and set(fetched_llm_provider.personas) == set(llm_provider.personas)
                 ):
                     return
         if not verify_deleted:
             raise ValueError(f"LLM Provider {llm_provider.id} not found")
+
+    @staticmethod
+    def get_default_model(
+        user_performing_action: DATestUser | None = None,
+    ) -> DefaultModel | None:
+        response = requests.get(
+            f"{API_SERVER_URL}/admin/llm/provider",
+            headers=(
+                user_performing_action.headers
+                if user_performing_action
+                else GENERAL_HEADERS
+            ),
+        )
+        response.raise_for_status()
+        default_text = response.json().get("default_text")
+        if default_text is None:
+            return None
+        return DefaultModel(**default_text)

backend/tests/integration/common_utils/managers/persona.py

@@ -3,7 +3,6 @@ from uuid import uuid4
 
 import requests
 
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.server.features.persona.models import FullPersonaSnapshot
 from onyx.server.features.persona.models import PersonaUpsertRequest
 from tests.integration.common_utils.constants import API_SERVER_URL
@@ -20,11 +19,7 @@ class PersonaManager:
         description: str | None = None,
         system_prompt: str | None = None,
         task_prompt: str | None = None,
-        num_chunks: float = 5,
-        llm_relevance_filter: bool = True,
         is_public: bool = True,
-        llm_filter_extraction: bool = True,
-        recency_bias: RecencyBiasSetting = RecencyBiasSetting.AUTO,
         datetime_aware: bool = False,
         document_set_ids: list[int] | None = None,
         tool_ids: list[int] | None = None,
@@ -35,6 +30,7 @@ class PersonaManager:
         label_ids: list[int] | None = None,
         user_file_ids: list[str] | None = None,
         display_priority: int | None = None,
+        featured: bool = False,
     ) -> DATestPersona:
         name = name or f"test-persona-{uuid4()}"
         description = description or f"Description for {name}"
@@ -47,11 +43,7 @@ class PersonaManager:
             system_prompt=system_prompt,
             task_prompt=task_prompt,
             datetime_aware=datetime_aware,
-            num_chunks=num_chunks,
-            llm_relevance_filter=llm_relevance_filter,
             is_public=is_public,
-            llm_filter_extraction=llm_filter_extraction,
-            recency_bias=recency_bias,
             document_set_ids=document_set_ids or [],
             tool_ids=tool_ids or [],
             llm_model_provider_override=llm_model_provider_override,
@@ -61,6 +53,7 @@ class PersonaManager:
             label_ids=label_ids or [],
             user_file_ids=user_file_ids or [],
             display_priority=display_priority,
+            featured=featured,
         )
 
         response = requests.post(
@@ -75,11 +68,7 @@ class PersonaManager:
             id=persona_data["id"],
             name=name,
             description=description,
-            num_chunks=num_chunks,
-            llm_relevance_filter=llm_relevance_filter,
             is_public=is_public,
-            llm_filter_extraction=llm_filter_extraction,
-            recency_bias=recency_bias,
             system_prompt=system_prompt,
             task_prompt=task_prompt,
             datetime_aware=datetime_aware,
@@ -90,6 +79,7 @@ class PersonaManager:
             users=users or [],
             groups=groups or [],
             label_ids=label_ids or [],
+            featured=featured,
         )
 
     @staticmethod
@@ -100,11 +90,7 @@ class PersonaManager:
         description: str | None = None,
         system_prompt: str | None = None,
         task_prompt: str | None = None,
-        num_chunks: float | None = None,
-        llm_relevance_filter: bool | None = None,
         is_public: bool | None = None,
-        llm_filter_extraction: bool | None = None,
-        recency_bias: RecencyBiasSetting | None = None,
         datetime_aware: bool = False,
         document_set_ids: list[int] | None = None,
         tool_ids: list[int] | None = None,
@@ -113,6 +99,7 @@ class PersonaManager:
         users: list[str] | None = None,
         groups: list[int] | None = None,
         label_ids: list[int] | None = None,
+        featured: bool | None = None,
     ) -> DATestPersona:
         system_prompt = system_prompt or f"System prompt for {persona.name}"
         task_prompt = task_prompt or f"Task prompt for {persona.name}"
@@ -123,13 +110,7 @@ class PersonaManager:
             system_prompt=system_prompt,
             task_prompt=task_prompt,
             datetime_aware=datetime_aware,
-            num_chunks=num_chunks or persona.num_chunks,
-            llm_relevance_filter=llm_relevance_filter or persona.llm_relevance_filter,
             is_public=persona.is_public if is_public is None else is_public,
-            llm_filter_extraction=(
-                llm_filter_extraction or persona.llm_filter_extraction
-            ),
-            recency_bias=recency_bias or persona.recency_bias,
             document_set_ids=document_set_ids or persona.document_set_ids,
             tool_ids=tool_ids or persona.tool_ids,
             llm_model_provider_override=(
@@ -141,6 +122,7 @@ class PersonaManager:
             users=[UUID(user) for user in (users or persona.users)],
             groups=groups or persona.groups,
             label_ids=label_ids or persona.label_ids,
+            featured=featured if featured is not None else persona.featured,
         )
 
         response = requests.patch(
@@ -155,16 +137,12 @@ class PersonaManager:
             id=updated_persona_data["id"],
             name=updated_persona_data["name"],
             description=updated_persona_data["description"],
-            num_chunks=updated_persona_data["num_chunks"],
-            llm_relevance_filter=updated_persona_data["llm_relevance_filter"],
             is_public=updated_persona_data["is_public"],
-            llm_filter_extraction=updated_persona_data["llm_filter_extraction"],
-            recency_bias=recency_bias or persona.recency_bias,
             system_prompt=system_prompt,
             task_prompt=task_prompt,
             datetime_aware=datetime_aware,
-            document_set_ids=updated_persona_data["document_sets"],
-            tool_ids=updated_persona_data["tools"],
+            document_set_ids=[ds["id"] for ds in updated_persona_data["document_sets"]],
+            tool_ids=[t["id"] for t in updated_persona_data["tools"]],
             llm_model_provider_override=updated_persona_data[
                 "llm_model_provider_override"
             ],
@@ -173,7 +151,8 @@ class PersonaManager:
             ],
             users=[user["email"] for user in updated_persona_data["users"]],
             groups=updated_persona_data["groups"],
-            label_ids=updated_persona_data["labels"],
+            label_ids=[label["id"] for label in updated_persona_data["labels"]],
+            featured=updated_persona_data["featured"],
         )
 
     @staticmethod
@@ -222,32 +201,13 @@ class PersonaManager:
                             fetched_persona.description,
                         )
                     )
-                if fetched_persona.num_chunks != persona.num_chunks:
-                    mismatches.append(
-                        ("num_chunks", persona.num_chunks, fetched_persona.num_chunks)
-                    )
-                if fetched_persona.llm_relevance_filter != persona.llm_relevance_filter:
-                    mismatches.append(
-                        (
-                            "llm_relevance_filter",
-                            persona.llm_relevance_filter,
-                            fetched_persona.llm_relevance_filter,
-                        )
-                    )
                 if fetched_persona.is_public != persona.is_public:
                     mismatches.append(
                         ("is_public", persona.is_public, fetched_persona.is_public)
                     )
-                if (
-                    fetched_persona.llm_filter_extraction
-                    != persona.llm_filter_extraction
-                ):
+                if fetched_persona.featured != persona.featured:
                     mismatches.append(
-                        (
-                            "llm_filter_extraction",
-                            persona.llm_filter_extraction,
-                            fetched_persona.llm_filter_extraction,
-                        )
+                        ("featured", persona.featured, fetched_persona.featured)
                     )
                 if (
                     fetched_persona.llm_model_provider_override

backend/tests/integration/common_utils/managers/scim_client.py

@@ -0,0 +1,66 @@
+import requests
+
+from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.constants import GENERAL_HEADERS
+
+
+class ScimClient:
+    """HTTP client for making authenticated SCIM v2 requests."""
+
+    @staticmethod
+    def _headers(raw_token: str) -> dict[str, str]:
+        return {
+            **GENERAL_HEADERS,
+            "Authorization": f"Bearer {raw_token}",
+        }
+
+    @staticmethod
+    def get(path: str, raw_token: str) -> requests.Response:
+        return requests.get(
+            f"{API_SERVER_URL}/scim/v2{path}",
+            headers=ScimClient._headers(raw_token),
+            timeout=60,
+        )
+
+    @staticmethod
+    def post(path: str, raw_token: str, json: dict) -> requests.Response:
+        return requests.post(
+            f"{API_SERVER_URL}/scim/v2{path}",
+            json=json,
+            headers=ScimClient._headers(raw_token),
+            timeout=60,
+        )
+
+    @staticmethod
+    def put(path: str, raw_token: str, json: dict) -> requests.Response:
+        return requests.put(
+            f"{API_SERVER_URL}/scim/v2{path}",
+            json=json,
+            headers=ScimClient._headers(raw_token),
+            timeout=60,
+        )
+
+    @staticmethod
+    def patch(path: str, raw_token: str, json: dict) -> requests.Response:
+        return requests.patch(
+            f"{API_SERVER_URL}/scim/v2{path}",
+            json=json,
+            headers=ScimClient._headers(raw_token),
+            timeout=60,
+        )
+
+    @staticmethod
+    def delete(path: str, raw_token: str) -> requests.Response:
+        return requests.delete(
+            f"{API_SERVER_URL}/scim/v2{path}",
+            headers=ScimClient._headers(raw_token),
+            timeout=60,
+        )
+
+    @staticmethod
+    def get_no_auth(path: str) -> requests.Response:
+        return requests.get(
+            f"{API_SERVER_URL}/scim/v2{path}",
+            headers=GENERAL_HEADERS,
+            timeout=60,
+        )

backend/tests/integration/common_utils/managers/scim_token.py

@@ -0,0 +1,52 @@
+import requests
+
+from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.test_models import DATestScimToken
+from tests.integration.common_utils.test_models import DATestUser
+
+
+class ScimTokenManager:
+    @staticmethod
+    def create(
+        name: str,
+        user_performing_action: DATestUser,
+    ) -> DATestScimToken:
+        response = requests.post(
+            f"{API_SERVER_URL}/admin/enterprise-settings/scim/token",
+            json={"name": name},
+            headers=user_performing_action.headers,
+            timeout=60,
+        )
+        response.raise_for_status()
+        data = response.json()
+        return DATestScimToken(
+            id=data["id"],
+            name=data["name"],
+            token_display=data["token_display"],
+            is_active=data["is_active"],
+            created_at=data["created_at"],
+            last_used_at=data.get("last_used_at"),
+            raw_token=data["raw_token"],
+        )
+
+    @staticmethod
+    def get_active(
+        user_performing_action: DATestUser,
+    ) -> DATestScimToken | None:
+        response = requests.get(
+            f"{API_SERVER_URL}/admin/enterprise-settings/scim/token",
+            headers=user_performing_action.headers,
+            timeout=60,
+        )
+        if response.status_code == 404:
+            return None
+        response.raise_for_status()
+        data = response.json()
+        return DATestScimToken(
+            id=data["id"],
+            name=data["name"],
+            token_display=data["token_display"],
+            is_active=data["is_active"],
+            created_at=data["created_at"],
+            last_used_at=data.get("last_used_at"),
+        )

backend/tests/integration/common_utils/test_models.py

@@ -10,7 +10,6 @@ from pydantic import Field
 from onyx.auth.schemas import UserRole
 from onyx.configs.constants import MessageType
 from onyx.configs.constants import QAFeedbackType
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.context.search.models import SavedSearchDoc
 from onyx.context.search.models import SearchDoc
 from onyx.db.enums import AccessType
@@ -42,6 +41,18 @@ class DATestPAT(BaseModel):
     last_used_at: str | None = None
 
 
+class DATestScimToken(BaseModel):
+    """SCIM bearer token model for testing."""
+
+    id: int
+    name: str
+    raw_token: str | None = None  # Only present on initial creation
+    token_display: str
+    is_active: bool
+    created_at: str
+    last_used_at: str | None = None
+
+
 class DATestAPIKey(BaseModel):
     api_key_id: int
     api_key_display: str
@@ -116,7 +127,7 @@ class DATestLLMProvider(BaseModel):
     name: str
     provider: str
     api_key: str
-    default_model_name: str
+    default_model_name: str | None = None
     is_public: bool
     is_auto_mode: bool = False
     groups: list[int]
@@ -150,11 +161,7 @@ class DATestPersona(BaseModel):
     id: int
     name: str
     description: str
-    num_chunks: float
-    llm_relevance_filter: bool
     is_public: bool
-    llm_filter_extraction: bool
-    recency_bias: RecencyBiasSetting
     document_set_ids: list[int]
     tool_ids: list[int]
     llm_model_provider_override: str | None
@@ -162,6 +169,7 @@ class DATestPersona(BaseModel):
     users: list[str]
     groups: list[int]
     label_ids: list[int]
+    featured: bool = False
 
     # Embedded prompt fields (no longer separate prompt_ids)
     system_prompt: str | None = None

backend/tests/integration/tests/discord_bot/test_discord_bot_db.py

@@ -8,7 +8,6 @@ from collections.abc import Generator
 import pytest
 from sqlalchemy.orm import Session
 
-from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.discord_bot import bulk_create_channel_configs
 from onyx.db.discord_bot import create_discord_bot_config
 from onyx.db.discord_bot import create_guild_config
@@ -36,14 +35,8 @@ def _create_test_persona(db_session: Session, persona_id: int, name: str) -> Per
         id=persona_id,
         name=name,
         description="Test persona for Discord bot tests",
-        num_chunks=5.0,
-        chunks_above=1,
-        chunks_below=1,
-        llm_relevance_filter=False,
-        llm_filter_extraction=False,
-        recency_bias=RecencyBiasSetting.FAVOR_RECENT,
         is_visible=True,
-        is_default_persona=False,
+        featured=False,
         deleted=False,
         builtin_persona=False,
     )

backend/tests/integration/tests/indexing/test_checkpointing.py

@@ -414,6 +414,24 @@ def test_mock_connector_checkpoint_recovery(
     )
     assert finished_index_attempt.status == IndexingStatus.FAILED
 
+    # Pause the connector immediately to prevent check_for_indexing from
+    # creating automatic retry attempts while we reset the mock server.
+    # Without this, the INITIAL_INDEXING status causes immediate retries
+    # that would consume (or fail against) the mock server before we can
+    # set up the recovery behavior.
+    CCPairManager.pause_cc_pair(cc_pair, user_performing_action=admin_user)
+
+    # Collect all index attempt IDs created so far (the initial one plus
+    # any automatic retries that may have started before the pause took effect).
+    all_prior_attempt_ids: list[int] = []
+    index_attempts_page = IndexAttemptManager.get_index_attempt_page(
+        cc_pair_id=cc_pair.id,
+        page=0,
+        page_size=100,
+        user_performing_action=admin_user,
+    )
+    all_prior_attempt_ids = [ia.id for ia in index_attempts_page.items]
+
     # Verify initial state: both docs should be indexed
     with get_session_with_current_tenant() as db_session:
         documents = DocumentManager.fetch_documents_for_cc_pair(
@@ -465,17 +483,14 @@ def test_mock_connector_checkpoint_recovery(
     )
     assert response.status_code == 200
 
-    # After the failure, the connector is in repeated error state and paused.
-    # Set the manual indexing trigger first (while paused), then unpause.
-    # This ensures the trigger is set before CHECK_FOR_INDEXING runs, which will
-    # prevent the connector from being re-paused when repeated error state is detected.
+    # Set the manual indexing trigger, then unpause to allow the recovery run.
     CCPairManager.run_once(
         cc_pair, from_beginning=False, user_performing_action=admin_user
     )
     CCPairManager.unpause_cc_pair(cc_pair, user_performing_action=admin_user)
     recovery_index_attempt = IndexAttemptManager.wait_for_index_attempt_start(
         cc_pair_id=cc_pair.id,
-        index_attempts_to_ignore=[initial_index_attempt.id],
+        index_attempts_to_ignore=all_prior_attempt_ids,
         user_performing_action=admin_user,
     )
     IndexAttemptManager.wait_for_index_attempt_completion(