chore(helm): remove broken code-interpreter dependency

The code-interpreter Helm chart repo at
https://onyx-dot-app.github.io/code-interpreter/ returns 404,
causing ct lint to fail in CI. Remove it from Chart.yaml
dependencies, Chart.lock, ct.yaml chart-repos, and the CI
workflow's helm repo add step.

.github/actions/build-backend-image/action.yml

@@ -1,73 +0,0 @@
-name: "Build Backend Image"
-description: "Builds and pushes the backend Docker image with cache reuse"
-inputs:
-  runs-on-ecr-cache:
-    description: "ECR cache registry from runs-on/action"
-    required: true
-  ref-name:
-    description: "Git ref name used for cache suffix fallback"
-    required: true
-  pr-number:
-    description: "Optional PR number for cache suffix"
-    required: false
-    default: ""
-  github-sha:
-    description: "Commit SHA used for cache keys"
-    required: true
-  run-id:
-    description: "GitHub run ID used in output image tag"
-    required: true
-  docker-username:
-    description: "Docker Hub username"
-    required: true
-  docker-token:
-    description: "Docker Hub token"
-    required: true
-  docker-no-cache:
-    description: "Set to 'true' to disable docker build cache"
-    required: false
-    default: "false"
-runs:
-  using: "composite"
-  steps:
-    - name: Format branch name for cache
-      id: format-branch
-      shell: bash
-      env:
-        PR_NUMBER: ${{ inputs.pr-number }}
-        REF_NAME: ${{ inputs.ref-name }}
-      run: |
-        if [ -n "${PR_NUMBER}" ]; then
-          CACHE_SUFFIX="${PR_NUMBER}"
-        else
-          # shellcheck disable=SC2001
-          CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-        fi
-        echo "cache-suffix=${CACHE_SUFFIX}" >> "$GITHUB_OUTPUT"
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # ratchet:docker/setup-buildx-action@v3
-
-    - name: Login to Docker Hub
-      uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # ratchet:docker/login-action@v3
-      with:
-        username: ${{ inputs.docker-username }}
-        password: ${{ inputs.docker-token }}
-
-    - name: Build and push Backend Docker image
-      uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
-      with:
-        context: ./backend
-        file: ./backend/Dockerfile
-        push: true
-        tags: ${{ inputs.runs-on-ecr-cache }}:nightly-llm-it-backend-${{ inputs.run-id }}
-        cache-from: |
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:backend-cache-${{ inputs.github-sha }}
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }}
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:backend-cache
-          type=registry,ref=onyxdotapp/onyx-backend:latest
-        cache-to: |
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:backend-cache-${{ inputs.github-sha }},mode=max
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:backend-cache,mode=max
-        no-cache: ${{ inputs.docker-no-cache == 'true' }}

.github/actions/build-integration-image/action.yml

@@ -1,76 +0,0 @@
-name: "Build Integration Image"
-description: "Builds and pushes the integration test image with docker bake"
-inputs:
-  runs-on-ecr-cache:
-    description: "ECR cache registry from runs-on/action"
-    required: true
-  ref-name:
-    description: "Git ref name used for cache suffix fallback"
-    required: true
-  pr-number:
-    description: "Optional PR number for cache suffix"
-    required: false
-    default: ""
-  github-sha:
-    description: "Commit SHA used for cache keys"
-    required: true
-  run-id:
-    description: "GitHub run ID used in output image tag"
-    required: true
-  docker-username:
-    description: "Docker Hub username"
-    required: true
-  docker-token:
-    description: "Docker Hub token"
-    required: true
-runs:
-  using: "composite"
-  steps:
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # ratchet:docker/setup-buildx-action@v3
-
-    - name: Login to Docker Hub
-      uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # ratchet:docker/login-action@v3
-      with:
-        username: ${{ inputs.docker-username }}
-        password: ${{ inputs.docker-token }}
-
-    - name: Format branch name for cache
-      id: format-branch
-      shell: bash
-      env:
-        PR_NUMBER: ${{ inputs.pr-number }}
-        REF_NAME: ${{ inputs.ref-name }}
-      run: |
-        if [ -n "${PR_NUMBER}" ]; then
-          CACHE_SUFFIX="${PR_NUMBER}"
-        else
-          # shellcheck disable=SC2001
-          CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-        fi
-        echo "cache-suffix=${CACHE_SUFFIX}" >> "$GITHUB_OUTPUT"
-
-    - name: Build and push integration test image with Docker Bake
-      shell: bash
-      env:
-        RUNS_ON_ECR_CACHE: ${{ inputs.runs-on-ecr-cache }}
-        INTEGRATION_REPOSITORY: ${{ inputs.runs-on-ecr-cache }}
-        TAG: nightly-llm-it-${{ inputs.run-id }}
-        CACHE_SUFFIX: ${{ steps.format-branch.outputs.cache-suffix }}
-        HEAD_SHA: ${{ inputs.github-sha }}
-      run: |
-        docker buildx bake --push \
-          --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${HEAD_SHA} \
-          --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${CACHE_SUFFIX} \
-          --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache \
-          --set backend.cache-from=type=registry,ref=onyxdotapp/onyx-backend:latest \
-          --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${HEAD_SHA},mode=max \
-          --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${CACHE_SUFFIX},mode=max \
-          --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache,mode=max \
-          --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${HEAD_SHA} \
-          --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${CACHE_SUFFIX} \
-          --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache \
-          --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${HEAD_SHA},mode=max \
-          --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${CACHE_SUFFIX},mode=max \
-          --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache,mode=max \
-          integration

.github/actions/build-model-server-image/action.yml

@@ -1,68 +0,0 @@
-name: "Build Model Server Image"
-description: "Builds and pushes the model server Docker image with cache reuse"
-inputs:
-  runs-on-ecr-cache:
-    description: "ECR cache registry from runs-on/action"
-    required: true
-  ref-name:
-    description: "Git ref name used for cache suffix fallback"
-    required: true
-  pr-number:
-    description: "Optional PR number for cache suffix"
-    required: false
-    default: ""
-  github-sha:
-    description: "Commit SHA used for cache keys"
-    required: true
-  run-id:
-    description: "GitHub run ID used in output image tag"
-    required: true
-  docker-username:
-    description: "Docker Hub username"
-    required: true
-  docker-token:
-    description: "Docker Hub token"
-    required: true
-runs:
-  using: "composite"
-  steps:
-    - name: Format branch name for cache
-      id: format-branch
-      shell: bash
-      env:
-        PR_NUMBER: ${{ inputs.pr-number }}
-        REF_NAME: ${{ inputs.ref-name }}
-      run: |
-        if [ -n "${PR_NUMBER}" ]; then
-          CACHE_SUFFIX="${PR_NUMBER}"
-        else
-          # shellcheck disable=SC2001
-          CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-        fi
-        echo "cache-suffix=${CACHE_SUFFIX}" >> "$GITHUB_OUTPUT"
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # ratchet:docker/setup-buildx-action@v3
-
-    - name: Login to Docker Hub
-      uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # ratchet:docker/login-action@v3
-      with:
-        username: ${{ inputs.docker-username }}
-        password: ${{ inputs.docker-token }}
-
-    - name: Build and push Model Server Docker image
-      uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
-      with:
-        context: ./backend
-        file: ./backend/Dockerfile.model_server
-        push: true
-        tags: ${{ inputs.runs-on-ecr-cache }}:nightly-llm-it-model-server-${{ inputs.run-id }}
-        cache-from: |
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:model-server-cache-${{ inputs.github-sha }}
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }}
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:model-server-cache
-          type=registry,ref=onyxdotapp/onyx-model-server:latest
-        cache-to: |
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:model-server-cache-${{ inputs.github-sha }},mode=max
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
-          type=registry,ref=${{ inputs.runs-on-ecr-cache }}:model-server-cache,mode=max

.github/actions/run-nightly-provider-chat-test/action.yml

@@ -1,130 +0,0 @@
-name: "Run Nightly Provider Chat Test"
-description: "Starts required compose services and runs nightly provider integration test"
-inputs:
-  provider:
-    description: "Provider slug for NIGHTLY_LLM_PROVIDER"
-    required: true
-  models:
-    description: "Comma-separated model list for NIGHTLY_LLM_MODELS"
-    required: true
-  provider-api-key:
-    description: "API key for NIGHTLY_LLM_API_KEY"
-    required: false
-    default: ""
-  strict:
-    description: "String true/false for NIGHTLY_LLM_STRICT"
-    required: true
-  api-base:
-    description: "Optional NIGHTLY_LLM_API_BASE"
-    required: false
-    default: ""
-  api-version:
-    description: "Optional NIGHTLY_LLM_API_VERSION"
-    required: false
-    default: ""
-  deployment-name:
-    description: "Optional NIGHTLY_LLM_DEPLOYMENT_NAME"
-    required: false
-    default: ""
-  custom-config-json:
-    description: "Optional NIGHTLY_LLM_CUSTOM_CONFIG_JSON"
-    required: false
-    default: ""
-  runs-on-ecr-cache:
-    description: "ECR cache registry from runs-on/action"
-    required: true
-  run-id:
-    description: "GitHub run ID used in image tags"
-    required: true
-  docker-username:
-    description: "Docker Hub username"
-    required: true
-  docker-token:
-    description: "Docker Hub token"
-    required: true
-runs:
-  using: "composite"
-  steps:
-    - name: Login to Docker Hub
-      uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # ratchet:docker/login-action@v3
-      with:
-        username: ${{ inputs.docker-username }}
-        password: ${{ inputs.docker-token }}
-
-    - name: Create .env file for Docker Compose
-      shell: bash
-      env:
-        ECR_CACHE: ${{ inputs.runs-on-ecr-cache }}
-        RUN_ID: ${{ inputs.run-id }}
-      run: |
-        cat <<EOF2 > deployment/docker_compose/.env
-        COMPOSE_PROFILES=s3-filestore
-        ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
-        LICENSE_ENFORCEMENT_ENABLED=false
-        AUTH_TYPE=basic
-        POSTGRES_POOL_PRE_PING=true
-        POSTGRES_USE_NULL_POOL=true
-        REQUIRE_EMAIL_VERIFICATION=false
-        DISABLE_TELEMETRY=true
-        INTEGRATION_TESTS_MODE=true
-        AUTO_LLM_UPDATE_INTERVAL_SECONDS=10
-        AWS_REGION_NAME=us-west-2
-        ONYX_BACKEND_IMAGE=${ECR_CACHE}:nightly-llm-it-backend-${RUN_ID}
-        ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:nightly-llm-it-model-server-${RUN_ID}
-        EOF2
-
-    - name: Start Docker containers
-      shell: bash
-      run: |
-        cd deployment/docker_compose
-        docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait \
-          relational_db \
-          index \
-          cache \
-          minio \
-          api_server \
-          inference_model_server
-
-    - name: Run nightly provider integration test
-      uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
-      env:
-        MODELS: ${{ inputs.models }}
-        NIGHTLY_LLM_PROVIDER: ${{ inputs.provider }}
-        NIGHTLY_LLM_API_KEY: ${{ inputs.provider-api-key }}
-        NIGHTLY_LLM_API_BASE: ${{ inputs.api-base }}
-        NIGHTLY_LLM_API_VERSION: ${{ inputs.api-version }}
-        NIGHTLY_LLM_DEPLOYMENT_NAME: ${{ inputs.deployment-name }}
-        NIGHTLY_LLM_CUSTOM_CONFIG_JSON: ${{ inputs.custom-config-json }}
-        NIGHTLY_LLM_STRICT: ${{ inputs.strict }}
-        RUNS_ON_ECR_CACHE: ${{ inputs.runs-on-ecr-cache }}
-        RUN_ID: ${{ inputs.run-id }}
-      with:
-        timeout_minutes: 20
-        max_attempts: 2
-        retry_wait_seconds: 10
-        command: |
-          docker run --rm --network onyx_default \
-            --name test-runner \
-            -e POSTGRES_HOST=relational_db \
-            -e POSTGRES_USER=postgres \
-            -e POSTGRES_PASSWORD=password \
-            -e POSTGRES_DB=postgres \
-            -e DB_READONLY_USER=db_readonly_user \
-            -e DB_READONLY_PASSWORD=password \
-            -e POSTGRES_POOL_PRE_PING=true \
-            -e POSTGRES_USE_NULL_POOL=true \
-            -e VESPA_HOST=index \
-            -e REDIS_HOST=cache \
-            -e API_SERVER_HOST=api_server \
-            -e TEST_WEB_HOSTNAME=test-runner \
-            -e AWS_REGION_NAME=us-west-2 \
-            -e NIGHTLY_LLM_PROVIDER="${NIGHTLY_LLM_PROVIDER}" \
-            -e NIGHTLY_LLM_MODELS="${MODELS}" \
-            -e NIGHTLY_LLM_API_KEY="${NIGHTLY_LLM_API_KEY}" \
-            -e NIGHTLY_LLM_API_BASE="${NIGHTLY_LLM_API_BASE}" \
-            -e NIGHTLY_LLM_API_VERSION="${NIGHTLY_LLM_API_VERSION}" \
-            -e NIGHTLY_LLM_DEPLOYMENT_NAME="${NIGHTLY_LLM_DEPLOYMENT_NAME}" \
-            -e NIGHTLY_LLM_CUSTOM_CONFIG_JSON="${NIGHTLY_LLM_CUSTOM_CONFIG_JSON}" \
-            -e NIGHTLY_LLM_STRICT="${NIGHTLY_LLM_STRICT}" \
-            ${RUNS_ON_ECR_CACHE}:nightly-llm-it-${RUN_ID} \
-            /app/tests/integration/tests/llm_workflows/test_nightly_provider_chat_workflow.py

.github/pull_request_template.md

@@ -8,5 +8,5 @@
 
 ## Additional Options
 
-- [ ] [Optional] Please cherry-pick this PR to the latest release version.
+- [ ] [Required] I have considered whether this PR needs to be cherry-picked to the latest beta branch.
 - [ ] [Optional] Override Linear Check

.github/workflows/deployment.yml

@@ -426,9 +426,8 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-amd64
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-amd64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-amd64,mode=max
@@ -500,9 +499,8 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-arm64
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-arm64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-arm64,mode=max
@@ -648,8 +646,8 @@ jobs:
             NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-amd64
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-amd64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-amd64,mode=max
@@ -730,8 +728,8 @@ jobs:
             NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
             NODE_OPTIONS=--max-old-space-size=8192
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-arm64
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-arm64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-arm64,mode=max
@@ -864,9 +862,8 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-amd64
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-amd64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-amd64,mode=max
@@ -937,9 +934,8 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-arm64
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-arm64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-arm64,mode=max
@@ -1076,8 +1072,8 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             ENABLE_CRAFT=true
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-amd64
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-amd64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-amd64,mode=max
@@ -1149,8 +1145,8 @@ jobs:
             ONYX_VERSION=${{ github.ref_name }}
             ENABLE_CRAFT=true
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-arm64
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-arm64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-craft-cache-arm64,mode=max
@@ -1291,9 +1287,8 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-amd64
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-amd64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-amd64,mode=max
@@ -1371,9 +1366,8 @@ jobs:
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
           cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-arm64
-            type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
+            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-arm64
           cache-to: |
             type=inline
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-arm64,mode=max

.github/workflows/helm-chart-releases.yml

@@ -33,7 +33,7 @@ jobs:
           helm repo add cloudnative-pg https://cloudnative-pg.github.io/charts
           helm repo add ot-container-kit https://ot-container-kit.github.io/helm-charts
           helm repo add minio https://charts.min.io/
-          helm repo add code-interpreter https://onyx-dot-app.github.io/python-sandbox/
+          helm repo add code-interpreter https://onyx-dot-app.github.io/code-interpreter/
           helm repo update
 
       - name: Build chart dependencies

.github/workflows/nightly-llm-provider-chat.yml

@@ -1,49 +0,0 @@
-name: Nightly LLM Provider Chat Tests
-concurrency:
-  group: Nightly-LLM-Provider-Chat-${{ github.workflow }}-${{ github.ref_name }}
-  cancel-in-progress: true
-
-on:
-  schedule:
-    # Runs daily at 10:30 UTC (2:30 AM PST / 3:30 AM PDT)
-    - cron: "30 10 * * *"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  provider-chat-test:
-    uses: ./.github/workflows/reusable-nightly-llm-provider-chat.yml
-    permissions:
-      contents: read
-      id-token: write
-    with:
-      openai_models: ${{ vars.NIGHTLY_LLM_OPENAI_MODELS }}
-      anthropic_models: ${{ vars.NIGHTLY_LLM_ANTHROPIC_MODELS }}
-      bedrock_models: ${{ vars.NIGHTLY_LLM_BEDROCK_MODELS }}
-      vertex_ai_models: ${{ vars.NIGHTLY_LLM_VERTEX_AI_MODELS }}
-      azure_models: ${{ vars.NIGHTLY_LLM_AZURE_MODELS }}
-      azure_api_base: ${{ vars.NIGHTLY_LLM_AZURE_API_BASE }}
-      ollama_models: ${{ vars.NIGHTLY_LLM_OLLAMA_MODELS }}
-      openrouter_models: ${{ vars.NIGHTLY_LLM_OPENROUTER_MODELS }}
-      strict: true
-
-  notify-slack-on-failure:
-    needs: [provider-chat-test]
-    if: failure() && github.event_name == 'schedule'
-    runs-on: ubuntu-slim
-    timeout-minutes: 5
-    steps:
-      - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Send Slack notification
-        uses: ./.github/actions/slack-notify
-        with:
-          webhook-url: ${{ secrets.SLACK_WEBHOOK }}
-          failed-jobs: provider-chat-test
-          title: "🚨 Scheduled LLM Provider Chat Tests failed!"
-          ref-name: ${{ github.ref_name }}

.github/workflows/post-merge-beta-cherry-pick.yml

@@ -1,163 +0,0 @@
-name: Post-Merge Beta Cherry-Pick
-
-on:
-  push:
-    branches:
-      - main
-
-permissions:
-  contents: write
-  pull-requests: write
-
-jobs:
-  cherry-pick-to-latest-release:
-    outputs:
-      should_cherrypick: ${{ steps.gate.outputs.should_cherrypick }}
-      pr_number: ${{ steps.gate.outputs.pr_number }}
-      cherry_pick_reason: ${{ steps.run_cherry_pick.outputs.reason }}
-      cherry_pick_details: ${{ steps.run_cherry_pick.outputs.details }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    steps:
-      - name: Resolve merged PR and checkbox state
-        id: gate
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          # For the commit that triggered this workflow (HEAD on main), fetch all
-          # associated PRs and keep only the PR that was actually merged into main
-          # with this exact merge commit SHA.
-          pr_numbers="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/pulls" | jq -r --arg sha "${GITHUB_SHA}" '.[] | select(.merged_at != null and .base.ref == "main" and .merge_commit_sha == $sha) | .number')"
-          match_count="$(printf '%s\n' "$pr_numbers" | sed '/^[[:space:]]*$/d' | wc -l | tr -d ' ')"
-          pr_number="$(printf '%s\n' "$pr_numbers" | sed '/^[[:space:]]*$/d' | head -n 1)"
-
-          if [ "${match_count}" -gt 1 ]; then
-            echo "::warning::Multiple merged PRs matched commit ${GITHUB_SHA}. Using PR #${pr_number}."
-          fi
-
-          if [ -z "$pr_number" ]; then
-            echo "No merged PR associated with commit ${GITHUB_SHA}; skipping."
-            echo "should_cherrypick=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          # Read the PR once so we can gate behavior and infer preferred actor.
-          pr_json="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${pr_number}")"
-          pr_body="$(printf '%s' "$pr_json" | jq -r '.body // ""')"
-          merged_by="$(printf '%s' "$pr_json" | jq -r '.merged_by.login // ""')"
-
-          echo "pr_number=$pr_number" >> "$GITHUB_OUTPUT"
-          echo "merged_by=$merged_by" >> "$GITHUB_OUTPUT"
-
-          if echo "$pr_body" | grep -qiE "\\[x\\][[:space:]]*(\\[[^]]+\\][[:space:]]*)?Please cherry-pick this PR to the latest release version"; then
-            echo "should_cherrypick=true" >> "$GITHUB_OUTPUT"
-            echo "Cherry-pick checkbox checked for PR #${pr_number}."
-            exit 0
-          fi
-
-          echo "should_cherrypick=false" >> "$GITHUB_OUTPUT"
-          echo "Cherry-pick checkbox not checked for PR #${pr_number}. Skipping."
-
-      - name: Checkout repository
-        if: steps.gate.outputs.should_cherrypick == 'true'
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
-        with:
-          fetch-depth: 0
-          persist-credentials: true
-          ref: main
-
-      - name: Install the latest version of uv
-        if: steps.gate.outputs.should_cherrypick == 'true'
-        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # ratchet:astral-sh/setup-uv@v7
-        with:
-          enable-cache: false
-          version: "0.9.9"
-
-      - name: Configure git identity
-        if: steps.gate.outputs.should_cherrypick == 'true'
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-      - name: Create cherry-pick PR to latest release
-        id: run_cherry_pick
-        if: steps.gate.outputs.should_cherrypick == 'true'
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ github.token }}
-          GITHUB_TOKEN: ${{ github.token }}
-          CHERRY_PICK_ASSIGNEE: ${{ steps.gate.outputs.merged_by }}
-        run: |
-          set -o pipefail
-          output_file="$(mktemp)"
-          uv run --no-sync --with onyx-devtools ods cherry-pick "${GITHUB_SHA}" --yes --no-verify 2>&1 | tee "$output_file"
-          exit_code="${PIPESTATUS[0]}"
-
-          if [ "${exit_code}" -eq 0 ]; then
-            echo "status=success" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          echo "status=failure" >> "$GITHUB_OUTPUT"
-
-          reason="command-failed"
-          if grep -qiE "merge conflict during cherry-pick|CONFLICT|could not apply|cherry-pick in progress with staged changes" "$output_file"; then
-            reason="merge-conflict"
-          fi
-          echo "reason=${reason}" >> "$GITHUB_OUTPUT"
-
-          {
-            echo "details<<EOF"
-            tail -n 40 "$output_file"
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Mark workflow as failed if cherry-pick failed
-        if: steps.gate.outputs.should_cherrypick == 'true' && steps.run_cherry_pick.outputs.status == 'failure'
-        env:
-          CHERRY_PICK_REASON: ${{ steps.run_cherry_pick.outputs.reason }}
-        run: |
-          echo "::error::Automated cherry-pick failed (${CHERRY_PICK_REASON})."
-          exit 1
-
-  notify-slack-on-cherry-pick-failure:
-    needs:
-      - cherry-pick-to-latest-release
-    if: always() && needs.cherry-pick-to-latest-release.outputs.should_cherrypick == 'true' && needs.cherry-pick-to-latest-release.result != 'success'
-    runs-on: ubuntu-slim
-    timeout-minutes: 10
-    steps:
-      - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Build cherry-pick failure summary
-        id: failure-summary
-        env:
-          SOURCE_PR_NUMBER: ${{ needs.cherry-pick-to-latest-release.outputs.pr_number }}
-          CHERRY_PICK_REASON: ${{ needs.cherry-pick-to-latest-release.outputs.cherry_pick_reason }}
-          CHERRY_PICK_DETAILS: ${{ needs.cherry-pick-to-latest-release.outputs.cherry_pick_details }}
-        run: |
-          source_pr_url="https://github.com/${GITHUB_REPOSITORY}/pull/${SOURCE_PR_NUMBER}"
-
-          reason_text="cherry-pick command failed"
-          if [ "${CHERRY_PICK_REASON}" = "merge-conflict" ]; then
-            reason_text="merge conflict during cherry-pick"
-          fi
-
-          details_excerpt="$(printf '%s' "${CHERRY_PICK_DETAILS}" | tail -n 8 | tr '\n' ' ' | sed "s/[[:space:]]\\+/ /g" | sed "s/\"/'/g" | cut -c1-350)"
-          failed_jobs="• cherry-pick-to-latest-release\\n• source PR: ${source_pr_url}\\n• reason: ${reason_text}"
-          if [ -n "${details_excerpt}" ]; then
-            failed_jobs="${failed_jobs}\\n• excerpt: ${details_excerpt}"
-          fi
-
-          echo "jobs=${failed_jobs}" >> "$GITHUB_OUTPUT"
-
-      - name: Notify #cherry-pick-prs about cherry-pick failure
-        uses: ./.github/actions/slack-notify
-        with:
-          webhook-url: ${{ secrets.CHERRY_PICK_PRS_WEBHOOK }}
-          failed-jobs: ${{ steps.failure-summary.outputs.jobs }}
-          title: "🚨 Automated Cherry-Pick Failed"
-          ref-name: ${{ github.ref_name }}

.github/workflows/pr-beta-cherrypick-check.yml

@@ -0,0 +1,28 @@
+name: Require beta cherry-pick consideration
+concurrency:
+  group: Require-Beta-Cherrypick-Consideration-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    types: [opened, edited, reopened, synchronize]
+
+permissions:
+  contents: read
+
+jobs:
+  beta-cherrypick-check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Check PR body for beta cherry-pick consideration
+        env:
+          PR_BODY: ${{ github.event.pull_request.body }}
+        run: |
+          if echo "$PR_BODY" | grep -qiE "\\[x\\][[:space:]]*\\[Required\\][[:space:]]*I have considered whether this PR needs to be cherry[- ]picked to the latest beta branch"; then
+            echo "Cherry-pick consideration box is checked. Check passed."
+            exit 0
+          fi
+
+          echo "::error::Please check the 'I have considered whether this PR needs to be cherry-picked to the latest beta branch' box in the PR description."
+          exit 1

.github/workflows/pr-external-dependency-unit-tests.yml

@@ -45,6 +45,9 @@ env:
   # TODO: debug why this is failing and enable
   CODE_INTERPRETER_BASE_URL: http://localhost:8000
 
+  # OpenSearch
+  OPENSEARCH_ADMIN_PASSWORD: "StrongPassword123!"
+
 jobs:
   discover-test-dirs:
     # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
@@ -115,9 +118,9 @@ jobs:
       - name: Create .env file for Docker Compose
         run: |
           cat <<EOF > deployment/docker_compose/.env
-          COMPOSE_PROFILES=s3-filestore,opensearch-enabled
+          COMPOSE_PROFILES=s3-filestore
+          CODE_INTERPRETER_BETA_ENABLED=true
           DISABLE_TELEMETRY=true
-          OPENSEARCH_FOR_ONYX_ENABLED=true
           EOF
 
       - name: Set up Standard Dependencies
@@ -126,6 +129,7 @@ jobs:
           docker compose \
             -f docker-compose.yml \
             -f docker-compose.dev.yml \
+            -f docker-compose.opensearch.yml \
             up -d \
             minio \
             relational_db \
@@ -160,7 +164,7 @@ jobs:
           cd deployment/docker_compose
 
           # Get list of running containers
-          containers=$(docker compose -f docker-compose.yml -f docker-compose.dev.yml ps -q)
+          containers=$(docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.opensearch.yml ps -q)
 
           # Collect logs from each container
           for container in $containers; do

.github/workflows/pr-helm-chart-testing.yml

@@ -91,7 +91,6 @@ jobs:
           helm repo add cloudnative-pg https://cloudnative-pg.github.io/charts
           helm repo add ot-container-kit https://ot-container-kit.github.io/helm-charts
           helm repo add minio https://charts.min.io/
-          helm repo add code-interpreter https://onyx-dot-app.github.io/python-sandbox/
           helm repo update
 
       - name: Install Redis operator

.github/workflows/pr-integration-tests.yml

@@ -20,7 +20,6 @@ env:
   # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-  SLACK_BOT_TOKEN_TEST_SPACE: ${{ secrets.SLACK_BOT_TOKEN_TEST_SPACE }}
   CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
@@ -424,7 +423,6 @@ jobs:
               -e OPENAI_API_KEY=${OPENAI_API_KEY} \
               -e EXA_API_KEY=${EXA_API_KEY} \
               -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
-              -e SLACK_BOT_TOKEN_TEST_SPACE=${SLACK_BOT_TOKEN_TEST_SPACE} \
               -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
               -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
               -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
@@ -445,7 +443,6 @@ jobs:
               -e TEST_WEB_HOSTNAME=test-runner \
               -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
               -e MOCK_CONNECTOR_SERVER_PORT=8001 \
-              -e ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=${{ matrix.edition == 'ee' && 'true' || 'false' }} \
               ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-${{ github.run_id }} \
               /app/tests/integration/${{ matrix.test-dir.path }}
 
@@ -704,7 +701,6 @@ jobs:
             -e OPENAI_API_KEY=${OPENAI_API_KEY} \
             -e EXA_API_KEY=${EXA_API_KEY} \
             -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
-            -e SLACK_BOT_TOKEN_TEST_SPACE=${SLACK_BOT_TOKEN_TEST_SPACE} \
             -e TEST_WEB_HOSTNAME=test-runner \
             -e AUTH_TYPE=cloud \
             -e MULTI_TENANT=true \

.github/workflows/pr-playwright-tests.yml

@@ -603,7 +603,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Download visual diff summaries
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
+        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # ratchet:actions/download-artifact@v4
         with:
           pattern: screenshot-diff-summary-*
           path: summaries/

.github/workflows/pr-python-checks.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
     branches:
       - main
-      - "release/**"
+      - 'release/**'
   push:
     tags:
       - "v*.*.*"
@@ -21,13 +21,7 @@ jobs:
     # See https://runs-on.com/runners/linux/
     # Note: Mypy seems quite optimized for x64 compared to arm64.
     # Similarly, mypy is single-threaded and incremental, so 2cpu is sufficient.
-    runs-on:
-      [
-        runs-on,
-        runner=2cpu-linux-x64,
-        "run-id=${{ github.run_id }}-mypy-check",
-        "extras=s3-cache",
-      ]
+    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}-mypy-check", "extras=s3-cache"]
     timeout-minutes: 45
 
     steps:
@@ -58,14 +52,21 @@ jobs:
         if: ${{ vars.DISABLE_MYPY_CACHE != 'true' }}
         uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
         with:
-          path: .mypy_cache
-          key: mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-${{ hashFiles('**/*.py', '**/*.pyi', 'pyproject.toml') }}
+          path: backend/.mypy_cache
+          key: mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-${{ hashFiles('**/*.py', '**/*.pyi', 'backend/pyproject.toml') }}
           restore-keys: |
             mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-
             mypy-${{ runner.os }}-
 
       - name: Run MyPy
+        working-directory: ./backend
         env:
           MYPY_FORCE_COLOR: 1
           TERM: xterm-256color
         run: mypy .
+
+      - name: Run MyPy (tools/)
+        env:
+          MYPY_FORCE_COLOR: 1
+          TERM: xterm-256color
+        run: mypy tools/

.github/workflows/pr-python-connector-tests.yml

@@ -89,10 +89,6 @@ env:
   SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
   SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ vars.SHAREPOINT_CLIENT_DIRECTORY_ID }}
   SHAREPOINT_SITE: ${{ vars.SHAREPOINT_SITE }}
-  PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
-  PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
-  PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
-  PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
 
   # Github
   ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}

.github/workflows/reusable-nightly-llm-provider-chat.yml

@@ -1,329 +0,0 @@
-name: Reusable Nightly LLM Provider Chat Tests
-
-on:
-  workflow_call:
-    inputs:
-      openai_models:
-        description: "Comma-separated models for openai"
-        required: false
-        default: ""
-        type: string
-      anthropic_models:
-        description: "Comma-separated models for anthropic"
-        required: false
-        default: ""
-        type: string
-      bedrock_models:
-        description: "Comma-separated models for bedrock"
-        required: false
-        default: ""
-        type: string
-      vertex_ai_models:
-        description: "Comma-separated models for vertex_ai"
-        required: false
-        default: ""
-        type: string
-      azure_models:
-        description: "Comma-separated models for azure"
-        required: false
-        default: ""
-        type: string
-      ollama_models:
-        description: "Comma-separated models for ollama_chat"
-        required: false
-        default: ""
-        type: string
-      openrouter_models:
-        description: "Comma-separated models for openrouter"
-        required: false
-        default: ""
-        type: string
-      azure_api_base:
-        description: "API base for azure provider"
-        required: false
-        default: ""
-        type: string
-      strict:
-        description: "Default NIGHTLY_LLM_STRICT passed to tests"
-        required: false
-        default: true
-        type: boolean
-
-permissions:
-  contents: read
-  id-token: write
-
-jobs:
-  build-backend-image:
-    runs-on:
-      [
-        runs-on,
-        runner=1cpu-linux-arm64,
-        "run-id=${{ github.run_id }}-build-backend-image",
-        "extras=ecr-cache",
-      ]
-    timeout-minutes: 45
-    environment: ci-protected
-    steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
-        with:
-          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
-          aws-region: us-east-2
-
-      - name: Get AWS Secrets
-        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
-        with:
-          secret-ids: |
-            DOCKER_USERNAME, test/docker-username
-            DOCKER_TOKEN, test/docker-token
-
-      - name: Build backend image
-        uses: ./.github/actions/build-backend-image
-        with:
-          runs-on-ecr-cache: ${{ env.RUNS_ON_ECR_CACHE }}
-          ref-name: ${{ github.ref_name }}
-          pr-number: ${{ github.event.pull_request.number }}
-          github-sha: ${{ github.sha }}
-          run-id: ${{ github.run_id }}
-          docker-username: ${{ env.DOCKER_USERNAME }}
-          docker-token: ${{ env.DOCKER_TOKEN }}
-          docker-no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' && 'true' || 'false' }}
-
-  build-model-server-image:
-    runs-on:
-      [
-        runs-on,
-        runner=1cpu-linux-arm64,
-        "run-id=${{ github.run_id }}-build-model-server-image",
-        "extras=ecr-cache",
-      ]
-    timeout-minutes: 45
-    environment: ci-protected
-    steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
-        with:
-          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
-          aws-region: us-east-2
-
-      - name: Get AWS Secrets
-        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
-        with:
-          secret-ids: |
-            DOCKER_USERNAME, test/docker-username
-            DOCKER_TOKEN, test/docker-token
-
-      - name: Build model server image
-        uses: ./.github/actions/build-model-server-image
-        with:
-          runs-on-ecr-cache: ${{ env.RUNS_ON_ECR_CACHE }}
-          ref-name: ${{ github.ref_name }}
-          pr-number: ${{ github.event.pull_request.number }}
-          github-sha: ${{ github.sha }}
-          run-id: ${{ github.run_id }}
-          docker-username: ${{ env.DOCKER_USERNAME }}
-          docker-token: ${{ env.DOCKER_TOKEN }}
-
-  build-integration-image:
-    runs-on:
-      [
-        runs-on,
-        runner=2cpu-linux-arm64,
-        "run-id=${{ github.run_id }}-build-integration-image",
-        "extras=ecr-cache",
-      ]
-    timeout-minutes: 45
-    environment: ci-protected
-    steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
-        with:
-          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
-          aws-region: us-east-2
-
-      - name: Get AWS Secrets
-        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
-        with:
-          secret-ids: |
-            DOCKER_USERNAME, test/docker-username
-            DOCKER_TOKEN, test/docker-token
-
-      - name: Build integration image
-        uses: ./.github/actions/build-integration-image
-        with:
-          runs-on-ecr-cache: ${{ env.RUNS_ON_ECR_CACHE }}
-          ref-name: ${{ github.ref_name }}
-          pr-number: ${{ github.event.pull_request.number }}
-          github-sha: ${{ github.sha }}
-          run-id: ${{ github.run_id }}
-          docker-username: ${{ env.DOCKER_USERNAME }}
-          docker-token: ${{ env.DOCKER_TOKEN }}
-
-  provider-chat-test:
-    needs:
-      [
-        build-backend-image,
-        build-model-server-image,
-        build-integration-image,
-      ]
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - provider: openai
-            models: ${{ inputs.openai_models }}
-            api_key_env: OPENAI_API_KEY
-            custom_config_env: ""
-            api_base: ""
-            api_version: ""
-            deployment_name: ""
-            required: true
-          - provider: anthropic
-            models: ${{ inputs.anthropic_models }}
-            api_key_env: ANTHROPIC_API_KEY
-            custom_config_env: ""
-            api_base: ""
-            api_version: ""
-            deployment_name: ""
-            required: true
-          - provider: bedrock
-            models: ${{ inputs.bedrock_models }}
-            api_key_env: BEDROCK_API_KEY
-            custom_config_env: ""
-            api_base: ""
-            api_version: ""
-            deployment_name: ""
-            required: false
-          - provider: vertex_ai
-            models: ${{ inputs.vertex_ai_models }}
-            api_key_env: ""
-            custom_config_env: NIGHTLY_LLM_VERTEX_AI_CUSTOM_CONFIG_JSON
-            api_base: ""
-            api_version: ""
-            deployment_name: ""
-            required: false
-          - provider: azure
-            models: ${{ inputs.azure_models }}
-            api_key_env: AZURE_API_KEY
-            custom_config_env: ""
-            api_base: ${{ inputs.azure_api_base }}
-            api_version: "2025-04-01-preview"
-            deployment_name: ""
-            required: false
-          - provider: ollama_chat
-            models: ${{ inputs.ollama_models }}
-            api_key_env: OLLAMA_API_KEY
-            custom_config_env: ""
-            api_base: "https://ollama.com"
-            api_version: ""
-            deployment_name: ""
-            required: false
-          - provider: openrouter
-            models: ${{ inputs.openrouter_models }}
-            api_key_env: OPENROUTER_API_KEY
-            custom_config_env: ""
-            api_base: "https://openrouter.ai/api/v1"
-            api_version: ""
-            deployment_name: ""
-            required: false
-    runs-on:
-      - runs-on
-      - runner=4cpu-linux-arm64
-      - "run-id=${{ github.run_id }}-nightly-${{ matrix.provider }}-provider-chat-test"
-      - extras=ecr-cache
-    timeout-minutes: 45
-    environment: ci-protected
-    steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
-        with:
-          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
-          aws-region: us-east-2
-
-      - name: Get AWS Secrets
-        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
-        with:
-          # Keep JSON values unparsed so vertex custom config is passed as raw JSON.
-          parse-json-secrets: false
-          secret-ids: |
-            DOCKER_USERNAME, test/docker-username
-            DOCKER_TOKEN, test/docker-token
-            OPENAI_API_KEY, test/openai-api-key
-            ANTHROPIC_API_KEY, test/anthropic-api-key
-            BEDROCK_API_KEY, test/bedrock-api-key
-            NIGHTLY_LLM_VERTEX_AI_CUSTOM_CONFIG_JSON, test/nightly-llm-vertex-ai-custom-config-json
-            AZURE_API_KEY, test/azure-api-key
-            OLLAMA_API_KEY, test/ollama-api-key
-            OPENROUTER_API_KEY, test/openrouter-api-key
-
-      - name: Run nightly provider chat test
-        uses: ./.github/actions/run-nightly-provider-chat-test
-        with:
-          provider: ${{ matrix.provider }}
-          models: ${{ matrix.models }}
-          provider-api-key: ${{ matrix.api_key_env && env[matrix.api_key_env] || '' }}
-          strict: ${{ inputs.strict && 'true' || 'false' }}
-          api-base: ${{ matrix.api_base }}
-          api-version: ${{ matrix.api_version }}
-          deployment-name: ${{ matrix.deployment_name }}
-          custom-config-json: ${{ matrix.custom_config_env && env[matrix.custom_config_env] || '' }}
-          runs-on-ecr-cache: ${{ env.RUNS_ON_ECR_CACHE }}
-          run-id: ${{ github.run_id }}
-          docker-username: ${{ env.DOCKER_USERNAME }}
-          docker-token: ${{ env.DOCKER_TOKEN }}
-
-      - name: Dump API server logs
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose logs --no-color api_server > $GITHUB_WORKSPACE/api_server.log || true
-
-      - name: Dump all-container logs
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true
-
-      - name: Upload logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
-        with:
-          name: docker-all-logs-nightly-${{ matrix.provider }}-llm-provider
-          path: |
-            ${{ github.workspace }}/api_server.log
-            ${{ github.workspace }}/docker-compose.log
-
-      - name: Stop Docker containers
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose down -v

AGENTS.md

@@ -548,7 +548,7 @@ class in the utils over directly calling the APIs with a library like `requests`
 calling the utilities directly (e.g. do NOT create admin users with
 `admin_user = UserManager.create(name="admin_user")`, instead use the `admin_user` fixture).
 
-A great example of this type of test is `backend/tests/integration/tests/streaming_endpoints/test_chat_stream.py`.
+A great example of this type of test is `backend/tests/integration/dev_apis/test_simple_chat_api.py`.
 
 To run them:
 
@@ -616,9 +616,3 @@ This is a minimal list - feel free to include more. Do NOT write code as part of
 Keep it high level. You can reference certain files or functions though.
 
 Before writing your plan, make sure to do research. Explore the relevant sections in the codebase.
-
-## Best Practices
-
-In addition to the other content in this file, best practices for contributing
-to the codebase can be found at `contributing_guides/best_practices.md`.
-Understand its contents and follow them.

backend/alembic/run_multitenant_migrations.py

@@ -21,14 +21,15 @@ import sys
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import NamedTuple
+from typing import List, NamedTuple
 
 from alembic.config import Config
 from alembic.script import ScriptDirectory
+from sqlalchemy import text
 
+from onyx.db.engine.sql_engine import is_valid_schema_name
 from onyx.db.engine.sql_engine import SqlEngine
 from onyx.db.engine.tenant_utils import get_all_tenant_ids
-from onyx.db.engine.tenant_utils import get_schemas_needing_migration
 from shared_configs.configs import TENANT_ID_PREFIX
 
 
@@ -104,6 +105,56 @@ def get_head_revision() -> str | None:
     return script.get_current_head()
 
 
+def get_schemas_needing_migration(
+    tenant_schemas: List[str], head_rev: str
+) -> List[str]:
+    """Return only schemas whose current alembic version is not at head."""
+    if not tenant_schemas:
+        return []
+
+    engine = SqlEngine.get_engine()
+
+    with engine.connect() as conn:
+        # Find which schemas actually have an alembic_version table
+        rows = conn.execute(
+            text(
+                "SELECT table_schema FROM information_schema.tables "
+                "WHERE table_name = 'alembic_version' "
+                "AND table_schema = ANY(:schemas)"
+            ),
+            {"schemas": tenant_schemas},
+        )
+        schemas_with_table = set(row[0] for row in rows)
+
+        # Schemas without the table definitely need migration
+        needs_migration = [s for s in tenant_schemas if s not in schemas_with_table]
+
+        if not schemas_with_table:
+            return needs_migration
+
+        # Validate schema names before interpolating into SQL
+        for schema in schemas_with_table:
+            if not is_valid_schema_name(schema):
+                raise ValueError(f"Invalid schema name: {schema}")
+
+        # Single query to get every schema's current revision at once.
+        # Use integer tags instead of interpolating schema names into
+        # string literals to avoid quoting issues.
+        schema_list = list(schemas_with_table)
+        union_parts = [
+            f'SELECT {i} AS idx, version_num FROM "{schema}".alembic_version'
+            for i, schema in enumerate(schema_list)
+        ]
+        rows = conn.execute(text(" UNION ALL ".join(union_parts)))
+        version_by_schema = {schema_list[row[0]]: row[1] for row in rows}
+
+        needs_migration.extend(
+            s for s in schemas_with_table if version_by_schema.get(s) != head_rev
+        )
+
+    return needs_migration
+
+
 def run_migrations_parallel(
     schemas: list[str],
     max_workers: int,

backend/alembic/versions/07b98176f1de_code_interpreter_seed.py

@@ -1,29 +0,0 @@
-"""code interpreter seed
-
-Revision ID: 07b98176f1de
-Revises: 7cb492013621
-Create Date: 2026-02-23 15:55:07.606784
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "07b98176f1de"
-down_revision = "7cb492013621"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Seed the single instance of code_interpreter_server
-    # NOTE: There should only exist at most and at minimum 1 code_interpreter_server row
-    op.execute(
-        sa.text("INSERT INTO code_interpreter_server (server_enabled) VALUES (true)")
-    )
-
-
-def downgrade() -> None:
-    op.execute(sa.text("DELETE FROM code_interpreter_server"))

backend/alembic/versions/0bb4558f35df_add_scim_username_to_scim_user_mapping.py

@@ -1,28 +0,0 @@
-"""add scim_username to scim_user_mapping
-
-Revision ID: 0bb4558f35df
-Revises: 631fd2504136
-Create Date: 2026-02-20 10:45:30.340188
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "0bb4558f35df"
-down_revision = "631fd2504136"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "scim_user_mapping",
-        sa.Column("scim_username", sa.String(), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("scim_user_mapping", "scim_username")

backend/alembic/versions/4a1e4b1c89d2_add_indexing_to_userfilestatus.py

@@ -1,51 +0,0 @@
-"""Add INDEXING to UserFileStatus
-
-Revision ID: 4a1e4b1c89d2
-Revises: 6b3b4083c5aa
-Create Date: 2026-02-28 00:00:00.000000
-
-"""
-
-import sqlalchemy as sa
-from alembic import op
-
-revision = "4a1e4b1c89d2"
-down_revision = "6b3b4083c5aa"
-branch_labels = None
-depends_on = None
-
-TABLE = "user_file"
-COLUMN = "status"
-CONSTRAINT_NAME = "ck_user_file_status"
-
-OLD_VALUES = ("PROCESSING", "COMPLETED", "FAILED", "CANCELED", "DELETING")
-NEW_VALUES = ("PROCESSING", "INDEXING", "COMPLETED", "FAILED", "CANCELED", "DELETING")
-
-
-def _drop_status_check_constraint() -> None:
-    """Drop the existing CHECK constraint on user_file.status.
-
-    The constraint name is auto-generated by SQLAlchemy and unknown,
-    so we look it up via the inspector.
-    """
-    inspector = sa.inspect(op.get_bind())
-    for constraint in inspector.get_check_constraints(TABLE):
-        if COLUMN in constraint.get("sqltext", ""):
-            constraint_name = constraint["name"]
-            if constraint_name is not None:
-                op.drop_constraint(constraint_name, TABLE, type_="check")
-
-
-def upgrade() -> None:
-    _drop_status_check_constraint()
-    in_clause = ", ".join(f"'{v}'" for v in NEW_VALUES)
-    op.create_check_constraint(CONSTRAINT_NAME, TABLE, f"{COLUMN} IN ({in_clause})")
-
-
-def downgrade() -> None:
-    op.execute(
-        f"UPDATE {TABLE} SET {COLUMN} = 'PROCESSING' WHERE {COLUMN} = 'INDEXING'"
-    )
-    op.drop_constraint(CONSTRAINT_NAME, TABLE, type_="check")
-    in_clause = ", ".join(f"'{v}'" for v in OLD_VALUES)
-    op.create_check_constraint(CONSTRAINT_NAME, TABLE, f"{COLUMN} IN ({in_clause})")

backend/alembic/versions/57122d037335_add_python_tool_on_default.py

@@ -1,69 +0,0 @@
-"""add python tool on default
-
-Revision ID: 57122d037335
-Revises: c0c937d5c9e5
-Create Date: 2026-02-27 10:10:40.124925
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "57122d037335"
-down_revision = "c0c937d5c9e5"
-branch_labels = None
-depends_on = None
-
-
-PYTHON_TOOL_NAME = "python"
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # Look up the PythonTool id
-    result = conn.execute(
-        sa.text("SELECT id FROM tool WHERE name = :name"),
-        {"name": PYTHON_TOOL_NAME},
-    ).fetchone()
-
-    if not result:
-        return
-
-    tool_id = result[0]
-
-    # Attach to the default persona (id=0) if not already attached
-    conn.execute(
-        sa.text(
-            """
-            INSERT INTO persona__tool (persona_id, tool_id)
-            VALUES (0, :tool_id)
-            ON CONFLICT DO NOTHING
-            """
-        ),
-        {"tool_id": tool_id},
-    )
-
-
-def downgrade() -> None:
-    conn = op.get_bind()
-
-    result = conn.execute(
-        sa.text("SELECT id FROM tool WHERE name = :name"),
-        {"name": PYTHON_TOOL_NAME},
-    ).fetchone()
-
-    if not result:
-        return
-
-    conn.execute(
-        sa.text(
-            """
-            DELETE FROM persona__tool
-            WHERE persona_id = 0 AND tool_id = :tool_id
-            """
-        ),
-        {"tool_id": result[0]},
-    )

backend/alembic/versions/6b3b4083c5aa_persona_cleanup_and_featured.py

@@ -1,112 +0,0 @@
-"""persona cleanup and featured
-
-Revision ID: 6b3b4083c5aa
-Revises: 57122d037335
-Create Date: 2026-02-26 12:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "6b3b4083c5aa"
-down_revision = "57122d037335"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add featured column with nullable=True first
-    op.add_column("persona", sa.Column("featured", sa.Boolean(), nullable=True))
-
-    # Migrate data from is_default_persona to featured
-    op.execute("UPDATE persona SET featured = is_default_persona")
-
-    # Make featured non-nullable with default=False
-    op.alter_column(
-        "persona",
-        "featured",
-        existing_type=sa.Boolean(),
-        nullable=False,
-        server_default=sa.false(),
-    )
-
-    # Drop is_default_persona column
-    op.drop_column("persona", "is_default_persona")
-
-    # Drop unused columns
-    op.drop_column("persona", "num_chunks")
-    op.drop_column("persona", "chunks_above")
-    op.drop_column("persona", "chunks_below")
-    op.drop_column("persona", "llm_relevance_filter")
-    op.drop_column("persona", "llm_filter_extraction")
-    op.drop_column("persona", "recency_bias")
-
-
-def downgrade() -> None:
-    # Add back recency_bias column
-    op.add_column(
-        "persona",
-        sa.Column(
-            "recency_bias",
-            sa.VARCHAR(),
-            nullable=False,
-            server_default="base_decay",
-        ),
-    )
-
-    # Add back llm_filter_extraction column
-    op.add_column(
-        "persona",
-        sa.Column(
-            "llm_filter_extraction",
-            sa.Boolean(),
-            nullable=False,
-            server_default=sa.false(),
-        ),
-    )
-
-    # Add back llm_relevance_filter column
-    op.add_column(
-        "persona",
-        sa.Column(
-            "llm_relevance_filter",
-            sa.Boolean(),
-            nullable=False,
-            server_default=sa.false(),
-        ),
-    )
-
-    # Add back chunks_below column
-    op.add_column(
-        "persona",
-        sa.Column("chunks_below", sa.Integer(), nullable=False, server_default="0"),
-    )
-
-    # Add back chunks_above column
-    op.add_column(
-        "persona",
-        sa.Column("chunks_above", sa.Integer(), nullable=False, server_default="0"),
-    )
-
-    # Add back num_chunks column
-    op.add_column("persona", sa.Column("num_chunks", sa.Float(), nullable=True))
-
-    # Add back is_default_persona column
-    op.add_column(
-        "persona",
-        sa.Column(
-            "is_default_persona",
-            sa.Boolean(),
-            nullable=False,
-            server_default=sa.false(),
-        ),
-    )
-
-    # Migrate data from featured to is_default_persona
-    op.execute("UPDATE persona SET is_default_persona = featured")
-
-    # Drop featured column
-    op.drop_column("persona", "featured")

backend/alembic/versions/7616121f6e97_add_enterprise_fields_to_scim_user_mapping.py

@@ -1,48 +0,0 @@
-"""add enterprise and name fields to scim_user_mapping
-
-Revision ID: 7616121f6e97
-Revises: 07b98176f1de
-Create Date: 2026-02-23 12:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "7616121f6e97"
-down_revision = "07b98176f1de"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "scim_user_mapping",
-        sa.Column("department", sa.String(), nullable=True),
-    )
-    op.add_column(
-        "scim_user_mapping",
-        sa.Column("manager", sa.String(), nullable=True),
-    )
-    op.add_column(
-        "scim_user_mapping",
-        sa.Column("given_name", sa.String(), nullable=True),
-    )
-    op.add_column(
-        "scim_user_mapping",
-        sa.Column("family_name", sa.String(), nullable=True),
-    )
-    op.add_column(
-        "scim_user_mapping",
-        sa.Column("scim_emails_json", sa.Text(), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("scim_user_mapping", "scim_emails_json")
-    op.drop_column("scim_user_mapping", "family_name")
-    op.drop_column("scim_user_mapping", "given_name")
-    op.drop_column("scim_user_mapping", "manager")
-    op.drop_column("scim_user_mapping", "department")

backend/alembic/versions/7cb492013621_code_interpreter_server_model.py

@@ -1,31 +0,0 @@
-"""code interpreter server model
-
-Revision ID: 7cb492013621
-Revises: 0bb4558f35df
-Create Date: 2026-02-22 18:54:54.007265
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "7cb492013621"
-down_revision = "0bb4558f35df"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "code_interpreter_server",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "server_enabled", sa.Boolean, nullable=False, server_default=sa.true()
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("code_interpreter_server")

backend/alembic/versions/8ffcc2bcfc11_add_needs_persona_sync_to_user_file.py

@@ -1,33 +0,0 @@
-"""add needs_persona_sync to user_file
-
-Revision ID: 8ffcc2bcfc11
-Revises: 7616121f6e97
-Create Date: 2026-02-23 10:48:48.343826
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "8ffcc2bcfc11"
-down_revision = "7616121f6e97"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "user_file",
-        sa.Column(
-            "needs_persona_sync",
-            sa.Boolean(),
-            nullable=False,
-            server_default=sa.text("false"),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("user_file", "needs_persona_sync")

backend/alembic/versions/c0c937d5c9e5_llm_provider_deprecate_fields.py

@@ -1,70 +0,0 @@
-"""llm provider deprecate fields
-
-Revision ID: c0c937d5c9e5
-Revises: 8ffcc2bcfc11
-Create Date: 2026-02-25 17:35:46.125102
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "c0c937d5c9e5"
-down_revision = "8ffcc2bcfc11"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Make default_model_name nullable (was NOT NULL)
-    op.alter_column(
-        "llm_provider",
-        "default_model_name",
-        existing_type=sa.String(),
-        nullable=True,
-    )
-
-    # Drop unique constraint on is_default_provider (defaults now tracked via LLMModelFlow)
-    op.drop_constraint(
-        "llm_provider_is_default_provider_key",
-        "llm_provider",
-        type_="unique",
-    )
-
-    # Remove server_default from is_default_vision_provider (was server_default=false())
-    op.alter_column(
-        "llm_provider",
-        "is_default_vision_provider",
-        existing_type=sa.Boolean(),
-        server_default=None,
-    )
-
-
-def downgrade() -> None:
-    # Restore default_model_name to NOT NULL (set empty string for any NULLs first)
-    op.execute(
-        "UPDATE llm_provider SET default_model_name = '' WHERE default_model_name IS NULL"
-    )
-    op.alter_column(
-        "llm_provider",
-        "default_model_name",
-        existing_type=sa.String(),
-        nullable=False,
-    )
-
-    # Restore unique constraint on is_default_provider
-    op.create_unique_constraint(
-        "llm_provider_is_default_provider_key",
-        "llm_provider",
-        ["is_default_provider"],
-    )
-
-    # Restore server_default for is_default_vision_provider
-    op.alter_column(
-        "llm_provider",
-        "is_default_vision_provider",
-        existing_type=sa.Boolean(),
-        server_default=sa.false(),
-    )

backend/ee/onyx/db/scim.py

@@ -34,7 +34,6 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
 
 from ee.onyx.server.scim.filtering import ScimFilter
 from ee.onyx.server.scim.filtering import ScimFilterOperator
-from ee.onyx.server.scim.models import ScimMappingFields
 from onyx.db.dal import DAL
 from onyx.db.models import ScimGroupMapping
 from onyx.db.models import ScimToken
@@ -128,21 +127,9 @@ class ScimDAL(DAL):
         self,
         external_id: str,
         user_id: UUID,
-        scim_username: str | None = None,
-        fields: ScimMappingFields | None = None,
     ) -> ScimUserMapping:
         """Create a mapping between a SCIM externalId and an Onyx user."""
-        f = fields or ScimMappingFields()
-        mapping = ScimUserMapping(
-            external_id=external_id,
-            user_id=user_id,
-            scim_username=scim_username,
-            department=f.department,
-            manager=f.manager,
-            given_name=f.given_name,
-            family_name=f.family_name,
-            scim_emails_json=f.scim_emails_json,
-        )
+        mapping = ScimUserMapping(external_id=external_id, user_id=user_id)
         self._session.add(mapping)
         self._session.flush()
         return mapping
@@ -261,11 +248,11 @@ class ScimDAL(DAL):
         scim_filter: ScimFilter | None,
         start_index: int = 1,
         count: int = 100,
-    ) -> tuple[list[tuple[User, ScimUserMapping | None]], int]:
+    ) -> tuple[list[tuple[User, str | None]], int]:
         """Query users with optional SCIM filter and pagination.
 
         Returns:
-            A tuple of (list of (user, mapping) pairs, total_count).
+            A tuple of (list of (user, external_id) pairs, total_count).
 
         Raises:
             ValueError: If the filter uses an unsupported attribute.
@@ -305,117 +292,33 @@ class ScimDAL(DAL):
         users = list(
             self._session.scalars(
                 query.order_by(User.id).offset(offset).limit(count)  # type: ignore[arg-type]
-            )
-            .unique()
-            .all()
+            ).all()
         )
 
-        # Batch-fetch SCIM mappings to avoid N+1 queries
-        mapping_map = self._get_user_mappings_batch([u.id for u in users])
-        return [(u, mapping_map.get(u.id)) for u in users], total
+        # Batch-fetch external IDs to avoid N+1 queries
+        ext_id_map = self._get_user_external_ids([u.id for u in users])
+        return [(u, ext_id_map.get(u.id)) for u in users], total
 
-    def sync_user_external_id(
-        self,
-        user_id: UUID,
-        new_external_id: str | None,
-        scim_username: str | None = None,
-        fields: ScimMappingFields | None = None,
-    ) -> None:
-        """Create, update, or delete the external ID mapping for a user.
-
-        When *fields* is provided, all mapping fields are written
-        unconditionally — including ``None`` values — so that a caller can
-        clear a previously-set field (e.g. removing a department).
-        """
+    def sync_user_external_id(self, user_id: UUID, new_external_id: str | None) -> None:
+        """Create, update, or delete the external ID mapping for a user."""
         mapping = self.get_user_mapping_by_user_id(user_id)
         if new_external_id:
             if mapping:
                 if mapping.external_id != new_external_id:
                     mapping.external_id = new_external_id
-                if scim_username is not None:
-                    mapping.scim_username = scim_username
-                if fields is not None:
-                    mapping.department = fields.department
-                    mapping.manager = fields.manager
-                    mapping.given_name = fields.given_name
-                    mapping.family_name = fields.family_name
-                    mapping.scim_emails_json = fields.scim_emails_json
             else:
-                self.create_user_mapping(
-                    external_id=new_external_id,
-                    user_id=user_id,
-                    scim_username=scim_username,
-                    fields=fields,
-                )
+                self.create_user_mapping(external_id=new_external_id, user_id=user_id)
         elif mapping:
             self.delete_user_mapping(mapping.id)
 
-    def _get_user_mappings_batch(
-        self, user_ids: list[UUID]
-    ) -> dict[UUID, ScimUserMapping]:
-        """Batch-fetch SCIM user mappings keyed by user ID."""
+    def _get_user_external_ids(self, user_ids: list[UUID]) -> dict[UUID, str]:
+        """Batch-fetch external IDs for a list of user IDs."""
         if not user_ids:
             return {}
         mappings = self._session.scalars(
             select(ScimUserMapping).where(ScimUserMapping.user_id.in_(user_ids))
         ).all()
-        return {m.user_id: m for m in mappings}
-
-    def get_user_groups(self, user_id: UUID) -> list[tuple[int, str]]:
-        """Get groups a user belongs to as ``(group_id, group_name)`` pairs.
-
-        Excludes groups marked for deletion.
-        """
-        rels = self._session.scalars(
-            select(User__UserGroup).where(User__UserGroup.user_id == user_id)
-        ).all()
-
-        group_ids = [r.user_group_id for r in rels]
-        if not group_ids:
-            return []
-
-        groups = self._session.scalars(
-            select(UserGroup).where(
-                UserGroup.id.in_(group_ids),
-                UserGroup.is_up_for_deletion.is_(False),
-            )
-        ).all()
-        return [(g.id, g.name) for g in groups]
-
-    def get_users_groups_batch(
-        self, user_ids: list[UUID]
-    ) -> dict[UUID, list[tuple[int, str]]]:
-        """Batch-fetch group memberships for multiple users.
-
-        Returns a mapping of ``user_id → [(group_id, group_name), ...]``.
-        Avoids N+1 queries when building user list responses.
-        """
-        if not user_ids:
-            return {}
-
-        rels = self._session.scalars(
-            select(User__UserGroup).where(User__UserGroup.user_id.in_(user_ids))
-        ).all()
-
-        group_ids = list({r.user_group_id for r in rels})
-        if not group_ids:
-            return {}
-
-        groups = self._session.scalars(
-            select(UserGroup).where(
-                UserGroup.id.in_(group_ids),
-                UserGroup.is_up_for_deletion.is_(False),
-            )
-        ).all()
-        groups_by_id = {g.id: g.name for g in groups}
-
-        result: dict[UUID, list[tuple[int, str]]] = {}
-        for r in rels:
-            if r.user_id and r.user_group_id in groups_by_id:
-                result.setdefault(r.user_id, []).append(
-                    (r.user_group_id, groups_by_id[r.user_group_id])
-                )
-        return result
+        return {m.user_id: m.external_id for m in mappings}
 
     # ------------------------------------------------------------------
     # Group mapping operations
@@ -580,13 +483,9 @@ class ScimDAL(DAL):
         if not user_ids:
             return []
 
-        users = (
-            self._session.scalars(
-                select(User).where(User.id.in_(user_ids))  # type: ignore[attr-defined]
-            )
-            .unique()
-            .all()
-        )
+        users = self._session.scalars(
+            select(User).where(User.id.in_(user_ids))  # type: ignore[attr-defined]
+        ).all()
         users_by_id = {u.id: u for u in users}
 
         return [
@@ -605,13 +504,9 @@ class ScimDAL(DAL):
         """
         if not uuids:
             return []
-        existing_users = (
-            self._session.scalars(
-                select(User).where(User.id.in_(uuids))  # type: ignore[attr-defined]
-            )
-            .unique()
-            .all()
-        )
+        existing_users = self._session.scalars(
+            select(User).where(User.id.in_(uuids))  # type: ignore[attr-defined]
+        ).all()
         existing_ids = {u.id for u in existing_users}
         return [uid for uid in uuids if uid not in existing_ids]
 

backend/ee/onyx/db/user_group.py

@@ -9,7 +9,6 @@ from sqlalchemy import Select
 from sqlalchemy import select
 from sqlalchemy import update
 from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
 from ee.onyx.server.user_group.models import SetCuratorRequest
@@ -19,15 +18,11 @@ from onyx.db.connector_credential_pair import get_connector_credential_pair_from
 from onyx.db.enums import AccessType
 from onyx.db.enums import ConnectorCredentialPairStatus
 from onyx.db.models import ConnectorCredentialPair
-from onyx.db.models import Credential
 from onyx.db.models import Credential__UserGroup
 from onyx.db.models import Document
 from onyx.db.models import DocumentByConnectorCredentialPair
-from onyx.db.models import DocumentSet
 from onyx.db.models import DocumentSet__UserGroup
-from onyx.db.models import FederatedConnector__DocumentSet
 from onyx.db.models import LLMProvider__UserGroup
-from onyx.db.models import Persona
 from onyx.db.models import Persona__UserGroup
 from onyx.db.models import TokenRateLimit__UserGroup
 from onyx.db.models import User
@@ -200,60 +195,8 @@ def fetch_user_group(db_session: Session, user_group_id: int) -> UserGroup | Non
     return db_session.scalar(stmt)
 
 
-def _add_user_group_snapshot_eager_loads(
-    stmt: Select,
-) -> Select:
-    """Add eager loading options needed by UserGroup.from_model snapshot creation."""
-    return stmt.options(
-        selectinload(UserGroup.users),
-        selectinload(UserGroup.user_group_relationships),
-        selectinload(UserGroup.cc_pair_relationships)
-        .selectinload(UserGroup__ConnectorCredentialPair.cc_pair)
-        .options(
-            selectinload(ConnectorCredentialPair.connector),
-            selectinload(ConnectorCredentialPair.credential).selectinload(
-                Credential.user
-            ),
-        ),
-        selectinload(UserGroup.document_sets).options(
-            selectinload(DocumentSet.connector_credential_pairs).selectinload(
-                ConnectorCredentialPair.connector
-            ),
-            selectinload(DocumentSet.users),
-            selectinload(DocumentSet.groups),
-            selectinload(DocumentSet.federated_connectors).selectinload(
-                FederatedConnector__DocumentSet.federated_connector
-            ),
-        ),
-        selectinload(UserGroup.personas).options(
-            selectinload(Persona.tools),
-            selectinload(Persona.hierarchy_nodes),
-            selectinload(Persona.attached_documents).selectinload(
-                Document.parent_hierarchy_node
-            ),
-            selectinload(Persona.labels),
-            selectinload(Persona.document_sets).options(
-                selectinload(DocumentSet.connector_credential_pairs).selectinload(
-                    ConnectorCredentialPair.connector
-                ),
-                selectinload(DocumentSet.users),
-                selectinload(DocumentSet.groups),
-                selectinload(DocumentSet.federated_connectors).selectinload(
-                    FederatedConnector__DocumentSet.federated_connector
-                ),
-            ),
-            selectinload(Persona.user),
-            selectinload(Persona.user_files),
-            selectinload(Persona.users),
-            selectinload(Persona.groups),
-        ),
-    )
-
-
 def fetch_user_groups(
-    db_session: Session,
-    only_up_to_date: bool = True,
-    eager_load_for_snapshot: bool = False,
+    db_session: Session, only_up_to_date: bool = True
 ) -> Sequence[UserGroup]:
     """
     Fetches user groups from the database.
@@ -266,8 +209,6 @@ def fetch_user_groups(
         db_session (Session): The SQLAlchemy session used to query the database.
         only_up_to_date (bool, optional): Flag to determine whether to filter the results
             to include only up to date user groups. Defaults to `True`.
-        eager_load_for_snapshot: If True, adds eager loading for all relationships
-            needed by UserGroup.from_model snapshot creation.
 
     Returns:
         Sequence[UserGroup]: A sequence of `UserGroup` objects matching the query criteria.
@@ -275,16 +216,11 @@ def fetch_user_groups(
     stmt = select(UserGroup)
     if only_up_to_date:
         stmt = stmt.where(UserGroup.is_up_to_date == True)  # noqa: E712
-    if eager_load_for_snapshot:
-        stmt = _add_user_group_snapshot_eager_loads(stmt)
-    return db_session.scalars(stmt).unique().all()
+    return db_session.scalars(stmt).all()
 
 
 def fetch_user_groups_for_user(
-    db_session: Session,
-    user_id: UUID,
-    only_curator_groups: bool = False,
-    eager_load_for_snapshot: bool = False,
+    db_session: Session, user_id: UUID, only_curator_groups: bool = False
 ) -> Sequence[UserGroup]:
     stmt = (
         select(UserGroup)
@@ -294,9 +230,7 @@ def fetch_user_groups_for_user(
     )
     if only_curator_groups:
         stmt = stmt.where(User__UserGroup.is_curator == True)  # noqa: E712
-    if eager_load_for_snapshot:
-        stmt = _add_user_group_snapshot_eager_loads(stmt)
-    return db_session.scalars(stmt).unique().all()
+    return db_session.scalars(stmt).all()
 
 
 def construct_document_id_select_by_usergroup(

backend/ee/onyx/external_permissions/sharepoint/group_sync.py

@@ -1,13 +1,9 @@
 from collections.abc import Generator
 
-from office365.sharepoint.client_context import ClientContext  # type: ignore[import-untyped]
-
 from ee.onyx.db.external_perm import ExternalUserGroup
 from ee.onyx.external_permissions.sharepoint.permission_utils import (
     get_sharepoint_external_groups,
 )
-from onyx.configs.app_configs import SHAREPOINT_EXHAUSTIVE_AD_ENUMERATION
-from onyx.connectors.sharepoint.connector import acquire_token_for_rest
 from onyx.connectors.sharepoint.connector import SharepointConnector
 from onyx.db.models import ConnectorCredentialPair
 from onyx.utils.logger import setup_logger
@@ -47,27 +43,14 @@ def sharepoint_group_sync(
 
     logger.info(f"Processing {len(site_descriptors)} sites for group sync")
 
-    enumerate_all = connector_config.get(
-        "exhaustive_ad_enumeration", SHAREPOINT_EXHAUSTIVE_AD_ENUMERATION
-    )
-
-    msal_app = connector.msal_app
-    sp_tenant_domain = connector.sp_tenant_domain
-    sp_domain_suffix = connector.sharepoint_domain_suffix
+    # Process each site
     for site_descriptor in site_descriptors:
         logger.debug(f"Processing site: {site_descriptor.url}")
 
-        ctx = ClientContext(site_descriptor.url).with_access_token(
-            lambda: acquire_token_for_rest(msal_app, sp_tenant_domain, sp_domain_suffix)
-        )
+        ctx = connector._create_rest_client_context(site_descriptor.url)
 
-        external_groups = get_sharepoint_external_groups(
-            ctx,
-            connector.graph_client,
-            graph_api_base=connector.graph_api_base,
-            get_access_token=connector._get_graph_access_token,
-            enumerate_all_ad_groups=enumerate_all,
-        )
+        # Get external groups for this site
+        external_groups = get_sharepoint_external_groups(ctx, connector.graph_client)
 
         # Yield each group
         for group in external_groups:

backend/ee/onyx/external_permissions/sharepoint/permission_utils.py

@@ -1,12 +1,9 @@
 import re
-import time
 from collections import deque
-from collections.abc import Callable
-from collections.abc import Generator
 from typing import Any
+from urllib.parse import unquote
 from urllib.parse import urlparse
 
-import requests as _requests
 from office365.graph_client import GraphClient  # type: ignore[import-untyped]
 from office365.onedrive.driveitems.driveItem import DriveItem  # type: ignore[import-untyped]
 from office365.runtime.client_request import ClientRequestException  # type: ignore
@@ -17,10 +14,7 @@ from pydantic import BaseModel
 from ee.onyx.db.external_perm import ExternalUserGroup
 from onyx.access.models import ExternalAccess
 from onyx.access.utils import build_ext_group_name_for_onyx
-from onyx.configs.app_configs import REQUEST_TIMEOUT_SECONDS
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.sharepoint.connector import GRAPH_API_MAX_RETRIES
-from onyx.connectors.sharepoint.connector import GRAPH_API_RETRYABLE_STATUSES
 from onyx.connectors.sharepoint.connector import SHARED_DOCUMENTS_MAP_REVERSE
 from onyx.connectors.sharepoint.connector import sleep_and_retry
 from onyx.utils.logger import setup_logger
@@ -39,70 +33,6 @@ LIMITED_ACCESS_ROLE_TYPES = [1, 9]
 LIMITED_ACCESS_ROLE_NAMES = ["Limited Access", "Web-Only Limited Access"]
 
 
-AD_GROUP_ENUMERATION_THRESHOLD = 100_000
-
-
-def _graph_api_get(
-    url: str,
-    get_access_token: Callable[[], str],
-    params: dict[str, str] | None = None,
-) -> dict[str, Any]:
-    """Authenticated Graph API GET with retry on transient errors."""
-    for attempt in range(GRAPH_API_MAX_RETRIES + 1):
-        access_token = get_access_token()
-        headers = {"Authorization": f"Bearer {access_token}"}
-        try:
-            resp = _requests.get(
-                url, headers=headers, params=params, timeout=REQUEST_TIMEOUT_SECONDS
-            )
-            if (
-                resp.status_code in GRAPH_API_RETRYABLE_STATUSES
-                and attempt < GRAPH_API_MAX_RETRIES
-            ):
-                wait = min(int(resp.headers.get("Retry-After", str(2**attempt))), 60)
-                logger.warning(
-                    f"Graph API {resp.status_code} on attempt {attempt + 1}, "
-                    f"retrying in {wait}s: {url}"
-                )
-                time.sleep(wait)
-                continue
-            resp.raise_for_status()
-            return resp.json()
-        except (_requests.ConnectionError, _requests.Timeout, _requests.HTTPError):
-            if attempt < GRAPH_API_MAX_RETRIES:
-                wait = min(2**attempt, 60)
-                logger.warning(
-                    f"Graph API connection error on attempt {attempt + 1}, "
-                    f"retrying in {wait}s: {url}"
-                )
-                time.sleep(wait)
-                continue
-            raise
-    raise RuntimeError(
-        f"Graph API request failed after {GRAPH_API_MAX_RETRIES + 1} attempts: {url}"
-    )
-
-
-def _iter_graph_collection(
-    initial_url: str,
-    get_access_token: Callable[[], str],
-    params: dict[str, str] | None = None,
-) -> Generator[dict[str, Any], None, None]:
-    """Paginate through a Graph API collection, yielding items one at a time."""
-    url: str | None = initial_url
-    while url:
-        data = _graph_api_get(url, get_access_token, params)
-        params = None
-        yield from data.get("value", [])
-        url = data.get("@odata.nextLink")
-
-
-def _normalize_email(email: str) -> str:
-    if MICROSOFT_DOMAIN in email:
-        return email.replace(MICROSOFT_DOMAIN, "")
-    return email
-
-
 class SharepointGroup(BaseModel):
     model_config = {"frozen": True}
 
@@ -597,12 +527,8 @@ def get_external_access_from_sharepoint(
         )
     elif site_page:
         site_url = site_page.get("webUrl")
-        # Keep percent-encoding intact so the path matches the encoding
-        # used by the Office365 library's SPResPath.create_relative(),
-        # which compares against urlparse(context.base_url).path.
-        # Decoding (e.g. %27 → ') causes a mismatch that duplicates
-        # the site prefix in the constructed URL.
-        server_relative_url = urlparse(site_url).path
+        # Prefer server-relative URL to avoid OData filters that break on apostrophes
+        server_relative_url = unquote(urlparse(site_url).path)
         file_obj = client_context.web.get_file_by_server_relative_url(
             server_relative_url
         )
@@ -646,65 +572,8 @@ def get_external_access_from_sharepoint(
     )
 
 
-def _enumerate_ad_groups_paginated(
-    get_access_token: Callable[[], str],
-    already_resolved: set[str],
-    graph_api_base: str,
-) -> Generator[ExternalUserGroup, None, None]:
-    """Paginate through all Azure AD groups and yield ExternalUserGroup for each.
-
-    Skips groups whose suffixed name is already in *already_resolved*.
-    Stops early if the number of groups exceeds AD_GROUP_ENUMERATION_THRESHOLD.
-    """
-    groups_url = f"{graph_api_base}/groups"
-    groups_params: dict[str, str] = {"$select": "id,displayName", "$top": "999"}
-    total_groups = 0
-
-    for group_json in _iter_graph_collection(
-        groups_url, get_access_token, groups_params
-    ):
-        group_id: str = group_json.get("id", "")
-        display_name: str = group_json.get("displayName", "")
-        if not group_id or not display_name:
-            continue
-
-        total_groups += 1
-        if total_groups > AD_GROUP_ENUMERATION_THRESHOLD:
-            logger.warning(
-                f"Azure AD group enumeration exceeded {AD_GROUP_ENUMERATION_THRESHOLD} "
-                "groups — stopping to avoid excessive memory/API usage. "
-                "Remaining groups will be resolved from role assignments only."
-            )
-            return
-
-        name = f"{display_name}_{group_id}"
-        if name in already_resolved:
-            continue
-
-        member_emails: list[str] = []
-        members_url = f"{graph_api_base}/groups/{group_id}/members"
-        members_params: dict[str, str] = {
-            "$select": "userPrincipalName,mail",
-            "$top": "999",
-        }
-        for member_json in _iter_graph_collection(
-            members_url, get_access_token, members_params
-        ):
-            email = member_json.get("userPrincipalName") or member_json.get("mail")
-            if email:
-                member_emails.append(_normalize_email(email))
-
-        yield ExternalUserGroup(id=name, user_emails=member_emails)
-
-    logger.info(f"Enumerated {total_groups} Azure AD groups via paginated Graph API")
-
-
 def get_sharepoint_external_groups(
-    client_context: ClientContext,
-    graph_client: GraphClient,
-    graph_api_base: str,
-    get_access_token: Callable[[], str] | None = None,
-    enumerate_all_ad_groups: bool = False,
+    client_context: ClientContext, graph_client: GraphClient
 ) -> list[ExternalUserGroup]:
 
     groups: set[SharepointGroup] = set()
@@ -760,22 +629,57 @@ def get_sharepoint_external_groups(
         client_context, graph_client, groups, is_group_sync=True
     )
 
-    external_user_groups: list[ExternalUserGroup] = [
-        ExternalUserGroup(id=group_name, user_emails=list(emails))
-        for group_name, emails in groups_and_members.groups_to_emails.items()
-    ]
+    # get all Azure AD groups because if any group is assigned to the drive item, we don't want to miss them
+    # We can't assign sharepoint groups to drive items or drives, so we don't need to get all sharepoint groups
+    azure_ad_groups = sleep_and_retry(
+        graph_client.groups.get_all(page_loaded=lambda _: None),
+        "get_sharepoint_external_groups:get_azure_ad_groups",
+    )
+    logger.info(f"Azure AD Groups: {len(azure_ad_groups)}")
+    identified_groups: set[str] = set(groups_and_members.groups_to_emails.keys())
+    ad_groups_to_emails: dict[str, set[str]] = {}
+    for group in azure_ad_groups:
+        # If the group is already identified, we don't need to get the members
+        if group.display_name in identified_groups:
+            continue
+        # AD groups allows same display name for multiple groups, so we need to add the GUID to the name
+        name = group.display_name
+        name = _get_group_name_with_suffix(group.id, name, graph_client)
 
-    if not enumerate_all_ad_groups or get_access_token is None:
-        logger.info(
-            "Skipping exhaustive Azure AD group enumeration. "
-            "Only groups found in site role assignments are included."
+        members = sleep_and_retry(
+            group.members.get_all(page_loaded=lambda _: None),
+            "get_sharepoint_external_groups:get_azure_ad_groups:get_members",
         )
-        return external_user_groups
+        for member in members:
+            member_data = member.to_json()
+            user_principal_name = member_data.get("userPrincipalName")
+            mail = member_data.get("mail")
+            if not ad_groups_to_emails.get(name):
+                ad_groups_to_emails[name] = set()
+            if user_principal_name:
+                if MICROSOFT_DOMAIN in user_principal_name:
+                    user_principal_name = user_principal_name.replace(
+                        MICROSOFT_DOMAIN, ""
+                    )
+                ad_groups_to_emails[name].add(user_principal_name)
+            elif mail:
+                if MICROSOFT_DOMAIN in mail:
+                    mail = mail.replace(MICROSOFT_DOMAIN, "")
+                ad_groups_to_emails[name].add(mail)
 
-    already_resolved = set(groups_and_members.groups_to_emails.keys())
-    for group in _enumerate_ad_groups_paginated(
-        get_access_token, already_resolved, graph_api_base
-    ):
-        external_user_groups.append(group)
+    external_user_groups: list[ExternalUserGroup] = []
+    for group_name, emails in groups_and_members.groups_to_emails.items():
+        external_user_group = ExternalUserGroup(
+            id=group_name,
+            user_emails=list(emails),
+        )
+        external_user_groups.append(external_user_group)
+
+    for group_name, emails in ad_groups_to_emails.items():
+        external_user_group = ExternalUserGroup(
+            id=group_name,
+            user_emails=list(emails),
+        )
+        external_user_groups.append(external_user_group)
 
     return external_user_groups

backend/ee/onyx/main.py

@@ -31,7 +31,6 @@ from ee.onyx.server.query_and_chat.query_backend import (
 from ee.onyx.server.query_and_chat.search_backend import router as search_router
 from ee.onyx.server.query_history.api import router as query_history_router
 from ee.onyx.server.reporting.usage_export_api import router as usage_export_router
-from ee.onyx.server.scim.api import register_scim_exception_handlers
 from ee.onyx.server.scim.api import scim_router
 from ee.onyx.server.seeding import seed_db
 from ee.onyx.server.tenants.api import router as tenants_router
@@ -168,7 +167,6 @@ def get_application() -> FastAPI:
     # they use their own SCIM bearer token auth).
     # Not behind APP_API_PREFIX because IdPs expect /scim/v2/... directly.
     application.include_router(scim_router)
-    register_scim_exception_handlers(application)
 
     # Ensure all routes have auth enabled or are explicitly marked as public
     check_ee_router_auth(application)

backend/ee/onyx/server/query_and_chat/models.py

@@ -34,7 +34,7 @@ class SendSearchQueryRequest(BaseModel):
     filters: BaseFilters | None = None
     num_docs_fed_to_llm_selection: int | None = None
     run_query_expansion: bool = False
-    num_hits: int = 30
+    num_hits: int = 50
 
     include_content: bool = False
     stream: bool = False

backend/ee/onyx/server/scim/api.py

@@ -15,9 +15,7 @@ from uuid import UUID
 
 from fastapi import APIRouter
 from fastapi import Depends
-from fastapi import FastAPI
 from fastapi import Query
-from fastapi import Request
 from fastapi import Response
 from fastapi.responses import JSONResponse
 from fastapi_users.password import PasswordHelper
@@ -26,26 +24,23 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from ee.onyx.db.scim import ScimDAL
-from ee.onyx.server.scim.auth import ScimAuthError
 from ee.onyx.server.scim.auth import verify_scim_token
 from ee.onyx.server.scim.filtering import parse_scim_filter
-from ee.onyx.server.scim.models import SCIM_LIST_RESPONSE_SCHEMA
+from ee.onyx.server.scim.models import ScimEmail
 from ee.onyx.server.scim.models import ScimError
 from ee.onyx.server.scim.models import ScimGroupMember
 from ee.onyx.server.scim.models import ScimGroupResource
 from ee.onyx.server.scim.models import ScimListResponse
-from ee.onyx.server.scim.models import ScimMappingFields
+from ee.onyx.server.scim.models import ScimMeta
 from ee.onyx.server.scim.models import ScimName
 from ee.onyx.server.scim.models import ScimPatchRequest
+from ee.onyx.server.scim.models import ScimResourceType
+from ee.onyx.server.scim.models import ScimSchemaDefinition
 from ee.onyx.server.scim.models import ScimServiceProviderConfig
 from ee.onyx.server.scim.models import ScimUserResource
 from ee.onyx.server.scim.patch import apply_group_patch
 from ee.onyx.server.scim.patch import apply_user_patch
 from ee.onyx.server.scim.patch import ScimPatchError
-from ee.onyx.server.scim.providers.base import get_default_provider
-from ee.onyx.server.scim.providers.base import ScimProvider
-from ee.onyx.server.scim.providers.base import serialize_emails
-from ee.onyx.server.scim.schema_definitions import ENTERPRISE_USER_SCHEMA_DEF
 from ee.onyx.server.scim.schema_definitions import GROUP_RESOURCE_TYPE
 from ee.onyx.server.scim.schema_definitions import GROUP_SCHEMA_DEF
 from ee.onyx.server.scim.schema_definitions import SERVICE_PROVIDER_CONFIG
@@ -53,61 +48,21 @@ from ee.onyx.server.scim.schema_definitions import USER_RESOURCE_TYPE
 from ee.onyx.server.scim.schema_definitions import USER_SCHEMA_DEF
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.models import ScimToken
-from onyx.db.models import ScimUserMapping
 from onyx.db.models import User
 from onyx.db.models import UserGroup
 from onyx.db.models import UserRole
-from onyx.utils.logger import setup_logger
 from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
 
-logger = setup_logger()
-
-
-class ScimJSONResponse(JSONResponse):
-    """JSONResponse with Content-Type: application/scim+json (RFC 7644 §3.1)."""
-
-    media_type = "application/scim+json"
-
 
 # NOTE: All URL paths in this router (/ServiceProviderConfig, /ResourceTypes,
 # /Schemas, /Users, /Groups) are mandated by the SCIM spec (RFC 7643/7644).
 # IdPs like Okta and Azure AD hardcode these exact paths, so they cannot be
 # changed to kebab-case.
-
-
 scim_router = APIRouter(prefix="/scim/v2", tags=["SCIM"])
 
 _pw_helper = PasswordHelper()
 
 
-def register_scim_exception_handlers(app: FastAPI) -> None:
-    """Register SCIM-specific exception handlers on the FastAPI app.
-
-    Call this after ``app.include_router(scim_router)`` so that auth
-    failures from ``verify_scim_token`` return RFC 7644 §3.12 error
-    envelopes (with ``schemas`` and ``status`` fields) instead of
-    FastAPI's default ``{"detail": "..."}`` format.
-    """
-
-    @app.exception_handler(ScimAuthError)
-    async def _handle_scim_auth_error(
-        _request: Request, exc: ScimAuthError
-    ) -> ScimJSONResponse:
-        return _scim_error_response(exc.status_code, exc.detail)
-
-
-def _get_provider(
-    _token: ScimToken = Depends(verify_scim_token),
-) -> ScimProvider:
-    """Resolve the SCIM provider for the current request.
-
-    Currently returns OktaProvider for all requests. When multi-provider
-    support is added (ENG-3652), this will resolve based on token metadata
-    or tenant configuration — no endpoint changes required.
-    """
-    return get_default_provider()
-
-
 # ---------------------------------------------------------------------------
 # Service Discovery Endpoints (unauthenticated)
 # ---------------------------------------------------------------------------
@@ -120,39 +75,15 @@ def get_service_provider_config() -> ScimServiceProviderConfig:
 
 
 @scim_router.get("/ResourceTypes")
-def get_resource_types() -> ScimJSONResponse:
-    """List available SCIM resource types (RFC 7643 §6).
-
-    Wrapped in a ListResponse envelope (RFC 7644 §3.4.2) because IdPs
-    like Entra ID expect a JSON object, not a bare array.
-    """
-    resources = [USER_RESOURCE_TYPE, GROUP_RESOURCE_TYPE]
-    return ScimJSONResponse(
-        content={
-            "schemas": [SCIM_LIST_RESPONSE_SCHEMA],
-            "totalResults": len(resources),
-            "Resources": [
-                r.model_dump(exclude_none=True, by_alias=True) for r in resources
-            ],
-        }
-    )
+def get_resource_types() -> list[ScimResourceType]:
+    """List available SCIM resource types (RFC 7643 §6)."""
+    return [USER_RESOURCE_TYPE, GROUP_RESOURCE_TYPE]
 
 
 @scim_router.get("/Schemas")
-def get_schemas() -> ScimJSONResponse:
-    """Return SCIM schema definitions (RFC 7643 §7).
-
-    Wrapped in a ListResponse envelope (RFC 7644 §3.4.2) because IdPs
-    like Entra ID expect a JSON object, not a bare array.
-    """
-    schemas = [USER_SCHEMA_DEF, GROUP_SCHEMA_DEF, ENTERPRISE_USER_SCHEMA_DEF]
-    return ScimJSONResponse(
-        content={
-            "schemas": [SCIM_LIST_RESPONSE_SCHEMA],
-            "totalResults": len(schemas),
-            "Resources": [s.model_dump(exclude_none=True) for s in schemas],
-        }
-    )
+def get_schemas() -> list[ScimSchemaDefinition]:
+    """Return SCIM schema definitions (RFC 7643 §7)."""
+    return [USER_SCHEMA_DEF, GROUP_SCHEMA_DEF]
 
 
 # ---------------------------------------------------------------------------
@@ -160,43 +91,35 @@ def get_schemas() -> ScimJSONResponse:
 # ---------------------------------------------------------------------------
 
 
-def _scim_error_response(status: int, detail: str) -> ScimJSONResponse:
+def _scim_error_response(status: int, detail: str) -> JSONResponse:
     """Build a SCIM-compliant error response (RFC 7644 §3.12)."""
-    logger.warning("SCIM error response: status=%s detail=%s", status, detail)
     body = ScimError(status=str(status), detail=detail)
-    return ScimJSONResponse(
+    return JSONResponse(
         status_code=status,
         content=body.model_dump(exclude_none=True),
     )
 
 
-def _parse_excluded_attributes(raw: str | None) -> set[str]:
-    """Parse the ``excludedAttributes`` query parameter (RFC 7644 §3.4.2.5).
+def _user_to_scim(user: User, external_id: str | None = None) -> ScimUserResource:
+    """Convert an Onyx User to a SCIM User resource representation."""
+    name = None
+    if user.personal_name:
+        parts = user.personal_name.split(" ", 1)
+        name = ScimName(
+            givenName=parts[0],
+            familyName=parts[1] if len(parts) > 1 else None,
+            formatted=user.personal_name,
+        )
 
-    Returns a set of lowercased attribute names to omit from responses.
-    """
-    if not raw:
-        return set()
-    return {attr.strip().lower() for attr in raw.split(",") if attr.strip()}
-
-
-def _apply_exclusions(
-    resource: ScimUserResource | ScimGroupResource,
-    excluded: set[str],
-) -> dict:
-    """Serialize a SCIM resource, omitting attributes the IdP excluded.
-
-    RFC 7644 §3.4.2.5 lets the IdP pass ``?excludedAttributes=groups,emails``
-    to reduce response payload size. We strip those fields after serialization
-    so the rest of the pipeline doesn't need to know about them.
-    """
-    data = resource.model_dump(exclude_none=True, by_alias=True)
-    for attr in excluded:
-        # Match case-insensitively against the camelCase field names
-        keys_to_remove = [k for k in data if k.lower() == attr]
-        for k in keys_to_remove:
-            del data[k]
-    return data
+    return ScimUserResource(
+        id=str(user.id),
+        externalId=external_id,
+        userName=user.email,
+        name=name,
+        emails=[ScimEmail(value=user.email, type="work", primary=True)],
+        active=user.is_active,
+        meta=ScimMeta(resourceType="User"),
+    )
 
 
 def _check_seat_availability(dal: ScimDAL) -> str | None:
@@ -212,7 +135,7 @@ def _check_seat_availability(dal: ScimDAL) -> str | None:
     return None
 
 
-def _fetch_user_or_404(user_id: str, dal: ScimDAL) -> User | ScimJSONResponse:
+def _fetch_user_or_404(user_id: str, dal: ScimDAL) -> User | JSONResponse:
     """Parse *user_id* as UUID, look up the user, or return a 404 error."""
     try:
         uid = UUID(user_id)
@@ -232,94 +155,8 @@ def _scim_name_to_str(name: ScimName | None) -> str | None:
     """
     if not name:
         return None
-    # If the client explicitly provides ``formatted``, prefer it — the client
-    # knows what display string it wants. Otherwise build from components.
-    if name.formatted:
-        return name.formatted
-    parts = " ".join(part for part in [name.givenName, name.familyName] if part)
-    return parts or None
-
-
-def _scim_resource_response(
-    resource: ScimUserResource | ScimGroupResource | ScimListResponse,
-    status_code: int = 200,
-) -> ScimJSONResponse:
-    """Serialize a SCIM resource as ``application/scim+json``."""
-    content = resource.model_dump(exclude_none=True, by_alias=True)
-    return ScimJSONResponse(
-        status_code=status_code,
-        content=content,
-    )
-
-
-def _build_list_response(
-    resources: list[ScimUserResource | ScimGroupResource],
-    total: int,
-    start_index: int,
-    count: int,
-    excluded: set[str] | None = None,
-) -> ScimListResponse | ScimJSONResponse:
-    """Build a SCIM list response, optionally applying attribute exclusions.
-
-    RFC 7644 §3.4.2.5 — IdPs may request certain attributes be omitted via
-    the ``excludedAttributes`` query parameter.
-    """
-    if excluded:
-        envelope = ScimListResponse(
-            totalResults=total,
-            startIndex=start_index,
-            itemsPerPage=count,
-        )
-        data = envelope.model_dump(exclude_none=True)
-        data["Resources"] = [_apply_exclusions(r, excluded) for r in resources]
-        return ScimJSONResponse(content=data)
-
-    return _scim_resource_response(
-        ScimListResponse(
-            totalResults=total,
-            startIndex=start_index,
-            itemsPerPage=count,
-            Resources=resources,
-        )
-    )
-
-
-def _extract_enterprise_fields(
-    resource: ScimUserResource,
-) -> tuple[str | None, str | None]:
-    """Extract department and manager from enterprise extension."""
-    ext = resource.enterprise_extension
-    if not ext:
-        return None, None
-    department = ext.department
-    manager = ext.manager.value if ext.manager else None
-    return department, manager
-
-
-def _mapping_to_fields(
-    mapping: ScimUserMapping | None,
-) -> ScimMappingFields | None:
-    """Extract round-trip fields from a SCIM user mapping."""
-    if not mapping:
-        return None
-    return ScimMappingFields(
-        department=mapping.department,
-        manager=mapping.manager,
-        given_name=mapping.given_name,
-        family_name=mapping.family_name,
-        scim_emails_json=mapping.scim_emails_json,
-    )
-
-
-def _fields_from_resource(resource: ScimUserResource) -> ScimMappingFields:
-    """Build mapping fields from an incoming SCIM user resource."""
-    department, manager = _extract_enterprise_fields(resource)
-    return ScimMappingFields(
-        department=department,
-        manager=manager,
-        given_name=resource.name.givenName if resource.name else None,
-        family_name=resource.name.familyName if resource.name else None,
-        scim_emails_json=serialize_emails(resource.emails),
+    return name.formatted or " ".join(
+        part for part in [name.givenName, name.familyName] if part
     )
 
 
@@ -331,17 +168,14 @@ def _fields_from_resource(resource: ScimUserResource) -> ScimMappingFields:
 @scim_router.get("/Users", response_model=None)
 def list_users(
     filter: str | None = Query(None),
-    excludedAttributes: str | None = None,
     startIndex: int = Query(1, ge=1),
     count: int = Query(100, ge=0, le=500),
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimListResponse | ScimJSONResponse:
+) -> ScimListResponse | JSONResponse:
     """List users with optional SCIM filter and pagination."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
-    dal.commit()
 
     try:
         scim_filter = parse_scim_filter(filter)
@@ -349,79 +183,58 @@ def list_users(
         return _scim_error_response(400, str(e))
 
     try:
-        users_with_mappings, total = dal.list_users(scim_filter, startIndex, count)
+        users_with_ext_ids, total = dal.list_users(scim_filter, startIndex, count)
     except ValueError as e:
         return _scim_error_response(400, str(e))
 
-    user_groups_map = dal.get_users_groups_batch([u.id for u, _ in users_with_mappings])
     resources: list[ScimUserResource | ScimGroupResource] = [
-        provider.build_user_resource(
-            user,
-            mapping.external_id if mapping else None,
-            groups=user_groups_map.get(user.id, []),
-            scim_username=mapping.scim_username if mapping else None,
-            fields=_mapping_to_fields(mapping),
-        )
-        for user, mapping in users_with_mappings
+        _user_to_scim(user, ext_id) for user, ext_id in users_with_ext_ids
     ]
 
-    return _build_list_response(
-        resources,
-        total,
-        startIndex,
-        count,
-        excluded=_parse_excluded_attributes(excludedAttributes),
+    return ScimListResponse(
+        totalResults=total,
+        startIndex=startIndex,
+        itemsPerPage=count,
+        Resources=resources,
     )
 
 
 @scim_router.get("/Users/{user_id}", response_model=None)
 def get_user(
     user_id: str,
-    excludedAttributes: str | None = None,
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimUserResource | ScimJSONResponse:
+) -> ScimUserResource | JSONResponse:
     """Get a single user by ID."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
-    dal.commit()
 
     result = _fetch_user_or_404(user_id, dal)
-    if isinstance(result, ScimJSONResponse):
+    if isinstance(result, JSONResponse):
         return result
     user = result
 
     mapping = dal.get_user_mapping_by_user_id(user.id)
-
-    resource = provider.build_user_resource(
-        user,
-        mapping.external_id if mapping else None,
-        groups=dal.get_user_groups(user.id),
-        scim_username=mapping.scim_username if mapping else None,
-        fields=_mapping_to_fields(mapping),
-    )
-
-    # RFC 7644 §3.4.2.5 — IdP may request certain attributes be omitted
-    excluded = _parse_excluded_attributes(excludedAttributes)
-    if excluded:
-        return ScimJSONResponse(content=_apply_exclusions(resource, excluded))
-
-    return _scim_resource_response(resource)
+    return _user_to_scim(user, mapping.external_id if mapping else None)
 
 
 @scim_router.post("/Users", status_code=201, response_model=None)
 def create_user(
     user_resource: ScimUserResource,
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimUserResource | ScimJSONResponse:
+) -> ScimUserResource | JSONResponse:
     """Create a new user from a SCIM provisioning request."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
 
-    email = user_resource.userName.strip()
+    email = user_resource.userName.strip().lower()
+
+    # externalId is how the IdP correlates this user on subsequent requests.
+    # Without it, the IdP can't find the user and will try to re-create,
+    # hitting a 409 conflict — so we require it up front.
+    if not user_resource.externalId:
+        return _scim_error_response(400, "externalId is required")
 
     # Enforce seat limit
     seat_error = _check_seat_availability(dal)
@@ -449,31 +262,13 @@ def create_user(
         dal.rollback()
         return _scim_error_response(409, f"User with email {email} already exists")
 
-    # Create SCIM mapping when externalId is provided — this is how the IdP
-    # correlates this user on subsequent requests.  Per RFC 7643, externalId
-    # is optional and assigned by the provisioning client.
+    # Create SCIM mapping (externalId is validated above, always present)
     external_id = user_resource.externalId
-    scim_username = user_resource.userName.strip()
-    fields = _fields_from_resource(user_resource)
-    if external_id:
-        dal.create_user_mapping(
-            external_id=external_id,
-            user_id=user.id,
-            scim_username=scim_username,
-            fields=fields,
-        )
+    dal.create_user_mapping(external_id=external_id, user_id=user.id)
 
     dal.commit()
 
-    return _scim_resource_response(
-        provider.build_user_resource(
-            user,
-            external_id,
-            scim_username=scim_username,
-            fields=fields,
-        ),
-        status_code=201,
-    )
+    return _user_to_scim(user, external_id)
 
 
 @scim_router.put("/Users/{user_id}", response_model=None)
@@ -481,15 +276,14 @@ def replace_user(
     user_id: str,
     user_resource: ScimUserResource,
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimUserResource | ScimJSONResponse:
+) -> ScimUserResource | JSONResponse:
     """Replace a user entirely (RFC 7644 §3.5.1)."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
 
     result = _fetch_user_or_404(user_id, dal)
-    if isinstance(result, ScimJSONResponse):
+    if isinstance(result, JSONResponse):
         return result
     user = result
 
@@ -499,36 +293,19 @@ def replace_user(
         if seat_error:
             return _scim_error_response(403, seat_error)
 
-    personal_name = _scim_name_to_str(user_resource.name)
-
     dal.update_user(
         user,
-        email=user_resource.userName.strip(),
+        email=user_resource.userName.strip().lower(),
         is_active=user_resource.active,
-        personal_name=personal_name,
+        personal_name=_scim_name_to_str(user_resource.name),
     )
 
     new_external_id = user_resource.externalId
-    scim_username = user_resource.userName.strip()
-    fields = _fields_from_resource(user_resource)
-    dal.sync_user_external_id(
-        user.id,
-        new_external_id,
-        scim_username=scim_username,
-        fields=fields,
-    )
+    dal.sync_user_external_id(user.id, new_external_id)
 
     dal.commit()
 
-    return _scim_resource_response(
-        provider.build_user_resource(
-            user,
-            new_external_id,
-            groups=dal.get_user_groups(user.id),
-            scim_username=scim_username,
-            fields=fields,
-        )
-    )
+    return _user_to_scim(user, new_external_id)
 
 
 @scim_router.patch("/Users/{user_id}", response_model=None)
@@ -536,9 +313,8 @@ def patch_user(
     user_id: str,
     patch_request: ScimPatchRequest,
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimUserResource | ScimJSONResponse:
+) -> ScimUserResource | JSONResponse:
     """Partially update a user (RFC 7644 §3.5.2).
 
     This is the primary endpoint for user deprovisioning — Okta sends
@@ -548,27 +324,17 @@ def patch_user(
     dal.update_token_last_used(_token.id)
 
     result = _fetch_user_or_404(user_id, dal)
-    if isinstance(result, ScimJSONResponse):
+    if isinstance(result, JSONResponse):
         return result
     user = result
 
     mapping = dal.get_user_mapping_by_user_id(user.id)
     external_id = mapping.external_id if mapping else None
-    current_scim_username = mapping.scim_username if mapping else None
-    current_fields = _mapping_to_fields(mapping)
 
-    current = provider.build_user_resource(
-        user,
-        external_id,
-        groups=dal.get_user_groups(user.id),
-        scim_username=current_scim_username,
-        fields=current_fields,
-    )
+    current = _user_to_scim(user, external_id)
 
     try:
-        patched, ent_data = apply_user_patch(
-            patch_request.Operations, current, provider.ignored_patch_paths
-        )
+        patched = apply_user_patch(patch_request.Operations, current)
     except ScimPatchError as e:
         return _scim_error_response(e.status, e.detail)
 
@@ -579,60 +345,22 @@ def patch_user(
             if seat_error:
                 return _scim_error_response(403, seat_error)
 
-    # Track the scim_username — if userName was patched, update it
-    new_scim_username = patched.userName.strip() if patched.userName else None
-
-    # If displayName was explicitly patched (different from the original), use
-    # it as personal_name directly.  Otherwise, derive from name components.
-    personal_name: str | None
-    if patched.displayName and patched.displayName != current.displayName:
-        personal_name = patched.displayName
-    else:
-        personal_name = _scim_name_to_str(patched.name)
-
     dal.update_user(
         user,
         email=(
-            patched.userName.strip()
-            if patched.userName.strip().lower() != user.email.lower()
+            patched.userName.strip().lower()
+            if patched.userName.lower() != user.email
             else None
         ),
         is_active=patched.active if patched.active != user.is_active else None,
-        personal_name=personal_name,
+        personal_name=_scim_name_to_str(patched.name),
     )
 
-    # Build updated fields by merging PATCH enterprise data with current values
-    cf = current_fields or ScimMappingFields()
-    fields = ScimMappingFields(
-        department=ent_data.get("department", cf.department),
-        manager=ent_data.get("manager", cf.manager),
-        given_name=patched.name.givenName if patched.name else cf.given_name,
-        family_name=patched.name.familyName if patched.name else cf.family_name,
-        scim_emails_json=(
-            serialize_emails(patched.emails)
-            if patched.emails is not None
-            else cf.scim_emails_json
-        ),
-    )
-
-    dal.sync_user_external_id(
-        user.id,
-        patched.externalId,
-        scim_username=new_scim_username,
-        fields=fields,
-    )
+    dal.sync_user_external_id(user.id, patched.externalId)
 
     dal.commit()
 
-    return _scim_resource_response(
-        provider.build_user_resource(
-            user,
-            patched.externalId,
-            groups=dal.get_user_groups(user.id),
-            scim_username=new_scim_username,
-            fields=fields,
-        )
-    )
+    return _user_to_scim(user, patched.externalId)
 
 
 @scim_router.delete("/Users/{user_id}", status_code=204, response_model=None)
@@ -640,29 +368,25 @@ def delete_user(
     user_id: str,
     _token: ScimToken = Depends(verify_scim_token),
     db_session: Session = Depends(get_session),
-) -> Response | ScimJSONResponse:
+) -> Response | JSONResponse:
     """Delete a user (RFC 7644 §3.6).
 
     Deactivates the user and removes the SCIM mapping. Note that Okta
     typically uses PATCH active=false instead of DELETE.
-    A second DELETE returns 404 per RFC 7644 §3.6.
     """
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
 
     result = _fetch_user_or_404(user_id, dal)
-    if isinstance(result, ScimJSONResponse):
+    if isinstance(result, JSONResponse):
         return result
     user = result
 
-    # If no SCIM mapping exists, the user was already deleted from
-    # SCIM's perspective — return 404 per RFC 7644 §3.6.
-    mapping = dal.get_user_mapping_by_user_id(user.id)
-    if not mapping:
-        return _scim_error_response(404, f"User {user_id} not found")
-
     dal.deactivate_user(user)
-    dal.delete_user_mapping(mapping.id)
+
+    mapping = dal.get_user_mapping_by_user_id(user.id)
+    if mapping:
+        dal.delete_user_mapping(mapping.id)
 
     dal.commit()
 
@@ -674,7 +398,25 @@ def delete_user(
 # ---------------------------------------------------------------------------
 
 
-def _fetch_group_or_404(group_id: str, dal: ScimDAL) -> UserGroup | ScimJSONResponse:
+def _group_to_scim(
+    group: UserGroup,
+    members: list[tuple[UUID, str | None]],
+    external_id: str | None = None,
+) -> ScimGroupResource:
+    """Convert an Onyx UserGroup to a SCIM Group resource."""
+    scim_members = [
+        ScimGroupMember(value=str(uid), display=email) for uid, email in members
+    ]
+    return ScimGroupResource(
+        id=str(group.id),
+        externalId=external_id,
+        displayName=group.name,
+        members=scim_members,
+        meta=ScimMeta(resourceType="Group"),
+    )
+
+
+def _fetch_group_or_404(group_id: str, dal: ScimDAL) -> UserGroup | JSONResponse:
     """Parse *group_id* as int, look up the group, or return a 404 error."""
     try:
         gid = int(group_id)
@@ -729,17 +471,14 @@ def _validate_and_parse_members(
 @scim_router.get("/Groups", response_model=None)
 def list_groups(
     filter: str | None = Query(None),
-    excludedAttributes: str | None = None,
     startIndex: int = Query(1, ge=1),
     count: int = Query(100, ge=0, le=500),
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimListResponse | ScimJSONResponse:
+) -> ScimListResponse | JSONResponse:
     """List groups with optional SCIM filter and pagination."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
-    dal.commit()
 
     try:
         scim_filter = parse_scim_filter(filter)
@@ -752,59 +491,45 @@ def list_groups(
         return _scim_error_response(400, str(e))
 
     resources: list[ScimUserResource | ScimGroupResource] = [
-        provider.build_group_resource(group, dal.get_group_members(group.id), ext_id)
+        _group_to_scim(group, dal.get_group_members(group.id), ext_id)
         for group, ext_id in groups_with_ext_ids
     ]
 
-    return _build_list_response(
-        resources,
-        total,
-        startIndex,
-        count,
-        excluded=_parse_excluded_attributes(excludedAttributes),
+    return ScimListResponse(
+        totalResults=total,
+        startIndex=startIndex,
+        itemsPerPage=count,
+        Resources=resources,
     )
 
 
 @scim_router.get("/Groups/{group_id}", response_model=None)
 def get_group(
     group_id: str,
-    excludedAttributes: str | None = None,
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimGroupResource | ScimJSONResponse:
+) -> ScimGroupResource | JSONResponse:
     """Get a single group by ID."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
-    dal.commit()
 
     result = _fetch_group_or_404(group_id, dal)
-    if isinstance(result, ScimJSONResponse):
+    if isinstance(result, JSONResponse):
         return result
     group = result
 
     mapping = dal.get_group_mapping_by_group_id(group.id)
     members = dal.get_group_members(group.id)
 
-    resource = provider.build_group_resource(
-        group, members, mapping.external_id if mapping else None
-    )
-
-    # RFC 7644 §3.4.2.5 — IdP may request certain attributes be omitted
-    excluded = _parse_excluded_attributes(excludedAttributes)
-    if excluded:
-        return ScimJSONResponse(content=_apply_exclusions(resource, excluded))
-
-    return _scim_resource_response(resource)
+    return _group_to_scim(group, members, mapping.external_id if mapping else None)
 
 
 @scim_router.post("/Groups", status_code=201, response_model=None)
 def create_group(
     group_resource: ScimGroupResource,
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimGroupResource | ScimJSONResponse:
+) -> ScimGroupResource | JSONResponse:
     """Create a new group from a SCIM provisioning request."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
@@ -840,10 +565,7 @@ def create_group(
     dal.commit()
 
     members = dal.get_group_members(db_group.id)
-    return _scim_resource_response(
-        provider.build_group_resource(db_group, members, external_id),
-        status_code=201,
-    )
+    return _group_to_scim(db_group, members, external_id)
 
 
 @scim_router.put("/Groups/{group_id}", response_model=None)
@@ -851,15 +573,14 @@ def replace_group(
     group_id: str,
     group_resource: ScimGroupResource,
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimGroupResource | ScimJSONResponse:
+) -> ScimGroupResource | JSONResponse:
     """Replace a group entirely (RFC 7644 §3.5.1)."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
 
     result = _fetch_group_or_404(group_id, dal)
-    if isinstance(result, ScimJSONResponse):
+    if isinstance(result, JSONResponse):
         return result
     group = result
 
@@ -874,9 +595,7 @@ def replace_group(
     dal.commit()
 
     members = dal.get_group_members(group.id)
-    return _scim_resource_response(
-        provider.build_group_resource(group, members, group_resource.externalId)
-    )
+    return _group_to_scim(group, members, group_resource.externalId)
 
 
 @scim_router.patch("/Groups/{group_id}", response_model=None)
@@ -884,9 +603,8 @@ def patch_group(
     group_id: str,
     patch_request: ScimPatchRequest,
     _token: ScimToken = Depends(verify_scim_token),
-    provider: ScimProvider = Depends(_get_provider),
     db_session: Session = Depends(get_session),
-) -> ScimGroupResource | ScimJSONResponse:
+) -> ScimGroupResource | JSONResponse:
     """Partially update a group (RFC 7644 §3.5.2).
 
     Handles member add/remove operations from Okta and Azure AD.
@@ -895,7 +613,7 @@ def patch_group(
     dal.update_token_last_used(_token.id)
 
     result = _fetch_group_or_404(group_id, dal)
-    if isinstance(result, ScimJSONResponse):
+    if isinstance(result, JSONResponse):
         return result
     group = result
 
@@ -903,11 +621,11 @@ def patch_group(
     external_id = mapping.external_id if mapping else None
 
     current_members = dal.get_group_members(group.id)
-    current = provider.build_group_resource(group, current_members, external_id)
+    current = _group_to_scim(group, current_members, external_id)
 
     try:
         patched, added_ids, removed_ids = apply_group_patch(
-            patch_request.Operations, current, provider.ignored_patch_paths
+            patch_request.Operations, current
         )
     except ScimPatchError as e:
         return _scim_error_response(e.status, e.detail)
@@ -934,9 +652,7 @@ def patch_group(
     dal.commit()
 
     members = dal.get_group_members(group.id)
-    return _scim_resource_response(
-        provider.build_group_resource(group, members, patched.externalId)
-    )
+    return _group_to_scim(group, members, patched.externalId)
 
 
 @scim_router.delete("/Groups/{group_id}", status_code=204, response_model=None)
@@ -944,13 +660,13 @@ def delete_group(
     group_id: str,
     _token: ScimToken = Depends(verify_scim_token),
     db_session: Session = Depends(get_session),
-) -> Response | ScimJSONResponse:
+) -> Response | JSONResponse:
     """Delete a group (RFC 7644 §3.6)."""
     dal = ScimDAL(db_session)
     dal.update_token_last_used(_token.id)
 
     result = _fetch_group_or_404(group_id, dal)
-    if isinstance(result, ScimJSONResponse):
+    if isinstance(result, JSONResponse):
         return result
     group = result
 

backend/ee/onyx/server/scim/auth.py

@@ -19,6 +19,7 @@ import hashlib
 import secrets
 
 from fastapi import Depends
+from fastapi import HTTPException
 from fastapi import Request
 from sqlalchemy.orm import Session
 
@@ -27,21 +28,6 @@ from onyx.auth.utils import get_hashed_bearer_token_from_request
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.models import ScimToken
 
-
-class ScimAuthError(Exception):
-    """Raised when SCIM bearer token authentication fails.
-
-    Unlike HTTPException, this carries the status and detail so the SCIM
-    exception handler can wrap them in an RFC 7644 §3.12 error envelope
-    with ``schemas`` and ``status`` fields.
-    """
-
-    def __init__(self, status_code: int, detail: str) -> None:
-        self.status_code = status_code
-        self.detail = detail
-        super().__init__(detail)
-
-
 SCIM_TOKEN_PREFIX = "onyx_scim_"
 SCIM_TOKEN_LENGTH = 48
 
@@ -96,14 +82,23 @@ def verify_scim_token(
     """
     hashed = _get_hashed_scim_token_from_request(request)
     if not hashed:
-        raise ScimAuthError(401, "Missing or invalid SCIM bearer token")
+        raise HTTPException(
+            status_code=401,
+            detail="Missing or invalid SCIM bearer token",
+        )
 
     token = dal.get_token_by_hash(hashed)
 
     if not token:
-        raise ScimAuthError(401, "Invalid SCIM bearer token")
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid SCIM bearer token",
+        )
 
     if not token.is_active:
-        raise ScimAuthError(401, "SCIM token has been revoked")
+        raise HTTPException(
+            status_code=401,
+            detail="SCIM token has been revoked",
+        )
 
     return token

backend/ee/onyx/server/scim/models.py

@@ -7,14 +7,12 @@ SCIM protocol schemas follow the wire format defined in:
 Admin API schemas are internal to Onyx and used for SCIM token management.
 """
 
-from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
 from pydantic import Field
-from pydantic import field_validator
 
 
 # ---------------------------------------------------------------------------
@@ -33,9 +31,6 @@ SCIM_SERVICE_PROVIDER_CONFIG_SCHEMA = (
 )
 SCIM_RESOURCE_TYPE_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:ResourceType"
 SCIM_SCHEMA_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:Schema"
-SCIM_ENTERPRISE_USER_SCHEMA = (
-    "urn:ietf:params:scim:schemas:extension:enterprise:2.0:User"
-)
 
 
 # ---------------------------------------------------------------------------
@@ -68,43 +63,6 @@ class ScimMeta(BaseModel):
     location: str | None = None
 
 
-class ScimUserGroupRef(BaseModel):
-    """Group reference within a User resource (RFC 7643 §4.1.2, read-only)."""
-
-    value: str
-    display: str | None = None
-
-
-class ScimManagerRef(BaseModel):
-    """Manager sub-attribute for the enterprise extension (RFC 7643 §4.3)."""
-
-    value: str | None = None
-
-
-class ScimEnterpriseExtension(BaseModel):
-    """Enterprise User extension attributes (RFC 7643 §4.3)."""
-
-    department: str | None = None
-    manager: ScimManagerRef | None = None
-
-
-@dataclass
-class ScimMappingFields:
-    """Stored SCIM mapping fields that need to round-trip through the IdP.
-
-    Entra ID sends structured name components, email metadata, and enterprise
-    extension attributes that must be returned verbatim in subsequent GET
-    responses. These fields are persisted on ScimUserMapping and threaded
-    through the DAL, provider, and endpoint layers.
-    """
-
-    department: str | None = None
-    manager: str | None = None
-    given_name: str | None = None
-    family_name: str | None = None
-    scim_emails_json: str | None = None
-
-
 class ScimUserResource(BaseModel):
     """SCIM User resource representation (RFC 7643 §4.1).
 
@@ -113,22 +71,14 @@ class ScimUserResource(BaseModel):
     to match the SCIM wire format (not Python convention).
     """
 
-    model_config = ConfigDict(populate_by_name=True)
-
     schemas: list[str] = Field(default_factory=lambda: [SCIM_USER_SCHEMA])
     id: str | None = None  # Onyx's internal user ID, set on responses
     externalId: str | None = None  # IdP's identifier for this user
     userName: str  # Typically the user's email address
     name: ScimName | None = None
-    displayName: str | None = None
     emails: list[ScimEmail] = Field(default_factory=list)
     active: bool = True
-    groups: list[ScimUserGroupRef] = Field(default_factory=list)
     meta: ScimMeta | None = None
-    enterprise_extension: ScimEnterpriseExtension | None = Field(
-        default=None,
-        alias="urn:ietf:params:scim:schemas:extension:enterprise:2.0:User",
-    )
 
 
 class ScimGroupMember(BaseModel):
@@ -171,53 +121,12 @@ class ScimPatchOperationType(str, Enum):
     REMOVE = "remove"
 
 
-class ScimPatchResourceValue(BaseModel):
-    """Partial resource dict for path-less PATCH replace operations.
-
-    When an IdP sends a PATCH without a ``path``, the ``value`` is a dict
-    of resource attributes to set.  IdPs may include read-only fields
-    (``id``, ``schemas``, ``meta``) alongside actual changes — these are
-    stripped by the provider's ``ignored_patch_paths`` before processing.
-
-    ``extra="allow"`` lets unknown attributes pass through so the patch
-    handler can decide what to do with them (ignore or reject).
-    """
-
-    model_config = ConfigDict(extra="allow")
-
-    active: bool | None = None
-    userName: str | None = None
-    displayName: str | None = None
-    externalId: str | None = None
-    name: ScimName | None = None
-    members: list[ScimGroupMember] | None = None
-    id: str | None = None
-    schemas: list[str] | None = None
-    meta: ScimMeta | None = None
-
-
-ScimPatchValue = str | bool | list[ScimGroupMember] | ScimPatchResourceValue | None
-
-
 class ScimPatchOperation(BaseModel):
     """Single PATCH operation (RFC 7644 §3.5.2)."""
 
     op: ScimPatchOperationType
     path: str | None = None
-    value: ScimPatchValue = None
-
-    @field_validator("op", mode="before")
-    @classmethod
-    def normalize_operation(cls, v: object) -> object:
-        """Normalize op to lowercase for case-insensitive matching.
-
-        Some IdPs (e.g. Entra ID) send capitalized ops like ``"Replace"``
-        instead of ``"replace"``. This is safe for all providers since the
-        enum values are lowercase. If a future provider requires other
-        pre-processing quirks, move patch deserialization into the provider
-        subclass instead of adding more special cases here.
-        """
-        return v.lower() if isinstance(v, str) else v
+    value: str | list[dict[str, str]] | dict[str, str | bool] | bool | None = None
 
 
 class ScimPatchRequest(BaseModel):

backend/ee/onyx/server/scim/patch.py

@@ -14,70 +14,13 @@ responsible for persisting changes.
 
 from __future__ import annotations
 
-import logging
 import re
-from dataclasses import dataclass
-from dataclasses import field
-from typing import Any
 
-from ee.onyx.server.scim.models import SCIM_ENTERPRISE_USER_SCHEMA
-from ee.onyx.server.scim.models import ScimGroupMember
 from ee.onyx.server.scim.models import ScimGroupResource
 from ee.onyx.server.scim.models import ScimPatchOperation
 from ee.onyx.server.scim.models import ScimPatchOperationType
-from ee.onyx.server.scim.models import ScimPatchResourceValue
-from ee.onyx.server.scim.models import ScimPatchValue
 from ee.onyx.server.scim.models import ScimUserResource
 
-logger = logging.getLogger(__name__)
-
-# Lowercased enterprise extension URN for case-insensitive matching
-_ENTERPRISE_URN_LOWER = SCIM_ENTERPRISE_USER_SCHEMA.lower()
-
-# Pattern for email filter paths, e.g.:
-#   emails[primary eq true].value  (Okta)
-#   emails[type eq "work"].value   (Azure AD / Entra ID)
-_EMAIL_FILTER_RE = re.compile(
-    r"^emails\[.+\]\.value$",
-    re.IGNORECASE,
-)
-
-# Pattern for member removal path: members[value eq "user-id"]
-_MEMBER_FILTER_RE = re.compile(
-    r'^members\[value\s+eq\s+"([^"]+)"\]$',
-    re.IGNORECASE,
-)
-
-# ---------------------------------------------------------------------------
-# Dispatch tables for user PATCH paths
-#
-# Maps lowercased SCIM path → (camelCase key, target dict name).
-# "data" writes to the top-level resource dict, "name" writes to the
-# name sub-object dict. This replaces the elif chains for simple fields.
-# ---------------------------------------------------------------------------
-
-_USER_REPLACE_PATHS: dict[str, tuple[str, str]] = {
-    "active": ("active", "data"),
-    "username": ("userName", "data"),
-    "externalid": ("externalId", "data"),
-    "name.givenname": ("givenName", "name"),
-    "name.familyname": ("familyName", "name"),
-    "name.formatted": ("formatted", "name"),
-}
-
-_USER_REMOVE_PATHS: dict[str, tuple[str, str]] = {
-    "externalid": ("externalId", "data"),
-    "name.givenname": ("givenName", "name"),
-    "name.familyname": ("familyName", "name"),
-    "name.formatted": ("formatted", "name"),
-    "displayname": ("displayName", "data"),
-}
-
-_GROUP_REPLACE_PATHS: dict[str, tuple[str, str]] = {
-    "displayname": ("displayName", "data"),
-    "externalid": ("externalId", "data"),
-}
-
 
 class ScimPatchError(Exception):
     """Raised when a PATCH operation cannot be applied."""
@@ -88,223 +31,94 @@ class ScimPatchError(Exception):
         super().__init__(detail)
 
 
-@dataclass
-class _UserPatchCtx:
-    """Bundles the mutable state for user PATCH operations."""
-
-    data: dict[str, Any]
-    name_data: dict[str, Any]
-    ent_data: dict[str, str | None] = field(default_factory=dict)
-
-
-# ---------------------------------------------------------------------------
-# User PATCH
-# ---------------------------------------------------------------------------
+# Pattern for member removal path: members[value eq "user-id"]
+_MEMBER_FILTER_RE = re.compile(
+    r'^members\[value\s+eq\s+"([^"]+)"\]$',
+    re.IGNORECASE,
+)
 
 
 def apply_user_patch(
     operations: list[ScimPatchOperation],
     current: ScimUserResource,
-    ignored_paths: frozenset[str] = frozenset(),
-) -> tuple[ScimUserResource, dict[str, str | None]]:
+) -> ScimUserResource:
     """Apply SCIM PATCH operations to a user resource.
 
-    Args:
-        operations: The PATCH operations to apply.
-        current: The current user resource state.
-        ignored_paths: SCIM attribute paths to silently skip (from provider).
-
-    Returns:
-        A tuple of (modified user resource, enterprise extension data dict).
-        The enterprise dict has keys ``"department"`` and ``"manager"``
-        with values set only when a PATCH operation touched them.
+    Returns a new ``ScimUserResource`` with the modifications applied.
+    The original object is not mutated.
 
     Raises:
         ScimPatchError: If an operation targets an unsupported path.
     """
     data = current.model_dump()
-    ctx = _UserPatchCtx(data=data, name_data=data.get("name") or {})
+    name_data = data.get("name") or {}
 
     for op in operations:
-        if op.op in (ScimPatchOperationType.REPLACE, ScimPatchOperationType.ADD):
-            _apply_user_replace(op, ctx, ignored_paths)
-        elif op.op == ScimPatchOperationType.REMOVE:
-            _apply_user_remove(op, ctx, ignored_paths)
+        if op.op == ScimPatchOperationType.REPLACE:
+            _apply_user_replace(op, data, name_data)
+        elif op.op == ScimPatchOperationType.ADD:
+            _apply_user_replace(op, data, name_data)
         else:
             raise ScimPatchError(
                 f"Unsupported operation '{op.op.value}' on User resource"
             )
 
-    ctx.data["name"] = ctx.name_data
-    return ScimUserResource.model_validate(ctx.data), ctx.ent_data
+    data["name"] = name_data
+    return ScimUserResource.model_validate(data)
 
 
 def _apply_user_replace(
     op: ScimPatchOperation,
-    ctx: _UserPatchCtx,
-    ignored_paths: frozenset[str],
+    data: dict,
+    name_data: dict,
 ) -> None:
     """Apply a replace/add operation to user data."""
     path = (op.path or "").lower()
 
     if not path:
-        # No path — value is a resource dict of top-level attributes to set.
-        if isinstance(op.value, ScimPatchResourceValue):
-            for key, val in op.value.model_dump(exclude_unset=True).items():
-                _set_user_field(key.lower(), val, ctx, ignored_paths, strict=False)
+        # No path — value is a dict of top-level attributes to set
+        if isinstance(op.value, dict):
+            for key, val in op.value.items():
+                _set_user_field(key.lower(), val, data, name_data)
         else:
             raise ScimPatchError("Replace without path requires a dict value")
         return
 
-    _set_user_field(path, op.value, ctx, ignored_paths)
-
-
-def _apply_user_remove(
-    op: ScimPatchOperation,
-    ctx: _UserPatchCtx,
-    ignored_paths: frozenset[str],
-) -> None:
-    """Apply a remove operation to user data — clears the target field."""
-    path = (op.path or "").lower()
-    if not path:
-        raise ScimPatchError("Remove operation requires a path")
-
-    if path in ignored_paths:
-        return
-
-    entry = _USER_REMOVE_PATHS.get(path)
-    if entry:
-        key, target = entry
-        target_dict = ctx.data if target == "data" else ctx.name_data
-        target_dict[key] = None
-        return
-
-    raise ScimPatchError(f"Unsupported remove path '{path}' for User PATCH")
+    _set_user_field(path, op.value, data, name_data)
 
 
 def _set_user_field(
     path: str,
-    value: ScimPatchValue,
-    ctx: _UserPatchCtx,
-    ignored_paths: frozenset[str],
-    *,
-    strict: bool = True,
+    value: str | bool | dict | list | None,
+    data: dict,
+    name_data: dict,
 ) -> None:
-    """Set a single field on user data by SCIM path.
-
-    Args:
-        strict: When ``False`` (path-less replace), unknown attributes are
-            silently skipped.  When ``True`` (explicit path), they raise.
-    """
-    if path in ignored_paths:
-        return
-
-    # Simple field writes handled by the dispatch table
-    entry = _USER_REPLACE_PATHS.get(path)
-    if entry:
-        key, target = entry
-        target_dict = ctx.data if target == "data" else ctx.name_data
-        target_dict[key] = value
-        return
-
-    # displayName sets both the top-level field and the name.formatted sub-field
-    if path == "displayname":
-        ctx.data["displayName"] = value
-        ctx.name_data["formatted"] = value
-    elif path == "name":
-        if isinstance(value, dict):
-            for k, v in value.items():
-                ctx.name_data[k] = v
-    elif path == "emails":
-        if isinstance(value, list):
-            ctx.data["emails"] = value
-    elif _EMAIL_FILTER_RE.match(path):
-        _update_primary_email(ctx.data, value)
-    elif path.startswith(_ENTERPRISE_URN_LOWER):
-        _set_enterprise_field(path, value, ctx.ent_data)
-    elif not strict:
-        return
+    """Set a single field on user data by SCIM path."""
+    if path == "active":
+        data["active"] = value
+    elif path == "username":
+        data["userName"] = value
+    elif path == "externalid":
+        data["externalId"] = value
+    elif path == "name.givenname":
+        name_data["givenName"] = value
+    elif path == "name.familyname":
+        name_data["familyName"] = value
+    elif path == "name.formatted":
+        name_data["formatted"] = value
+    elif path == "displayname":
+        # Some IdPs send displayName on users; map to formatted name
+        name_data["formatted"] = value
     else:
         raise ScimPatchError(f"Unsupported path '{path}' for User PATCH")
 
 
-def _update_primary_email(data: dict[str, Any], value: ScimPatchValue) -> None:
-    """Update the primary email entry via an email filter path."""
-    emails: list[dict] = data.get("emails") or []
-    for email_entry in emails:
-        if email_entry.get("primary"):
-            email_entry["value"] = value
-            break
-    else:
-        emails.append({"value": value, "type": "work", "primary": True})
-    data["emails"] = emails
-
-
-def _to_dict(value: ScimPatchValue) -> dict | None:
-    """Coerce a SCIM patch value to a plain dict if possible.
-
-    Pydantic may parse raw dicts as ``ScimPatchResourceValue`` (which uses
-    ``extra="allow"``), so we also dump those back to a dict.
-    """
-    if isinstance(value, dict):
-        return value
-    if isinstance(value, ScimPatchResourceValue):
-        return value.model_dump(exclude_unset=True)
-    return None
-
-
-def _set_enterprise_field(
-    path: str,
-    value: ScimPatchValue,
-    ent_data: dict[str, str | None],
-) -> None:
-    """Handle enterprise extension URN paths or value dicts."""
-    # Full URN as key with dict value (path-less PATCH)
-    # e.g. key="urn:...:user", value={"department": "Eng", "manager": {...}}
-    if path == _ENTERPRISE_URN_LOWER:
-        d = _to_dict(value)
-        if d is not None:
-            if "department" in d:
-                ent_data["department"] = d["department"]
-            if "manager" in d:
-                mgr = d["manager"]
-                if isinstance(mgr, dict):
-                    ent_data["manager"] = mgr.get("value")
-        return
-
-    # Dotted URN path, e.g. "urn:...:user:department"
-    suffix = path[len(_ENTERPRISE_URN_LOWER) :].lstrip(":").lower()
-    if suffix == "department":
-        ent_data["department"] = str(value) if value is not None else None
-    elif suffix == "manager":
-        d = _to_dict(value)
-        if d is not None:
-            ent_data["manager"] = d.get("value")
-        elif isinstance(value, str):
-            ent_data["manager"] = value
-    else:
-        # Unknown enterprise attributes are silently ignored rather than
-        # rejected — IdPs may send attributes we don't model yet.
-        logger.warning("Ignoring unknown enterprise extension attribute '%s'", suffix)
-
-
-# ---------------------------------------------------------------------------
-# Group PATCH
-# ---------------------------------------------------------------------------
-
-
 def apply_group_patch(
     operations: list[ScimPatchOperation],
     current: ScimGroupResource,
-    ignored_paths: frozenset[str] = frozenset(),
 ) -> tuple[ScimGroupResource, list[str], list[str]]:
     """Apply SCIM PATCH operations to a group resource.
 
-    Args:
-        operations: The PATCH operations to apply.
-        current: The current group resource state.
-        ignored_paths: SCIM attribute paths to silently skip (from provider).
-
     Returns:
         A tuple of (modified group, added member IDs, removed member IDs).
         The caller uses the member ID lists to update the database.
@@ -319,9 +133,7 @@ def apply_group_patch(
 
     for op in operations:
         if op.op == ScimPatchOperationType.REPLACE:
-            _apply_group_replace(
-                op, data, current_members, added_ids, removed_ids, ignored_paths
-            )
+            _apply_group_replace(op, data, current_members, added_ids, removed_ids)
         elif op.op == ScimPatchOperationType.ADD:
             _apply_group_add(op, current_members, added_ids)
         elif op.op == ScimPatchOperationType.REMOVE:
@@ -342,48 +154,38 @@ def _apply_group_replace(
     current_members: list[dict],
     added_ids: list[str],
     removed_ids: list[str],
-    ignored_paths: frozenset[str],
 ) -> None:
     """Apply a replace operation to group data."""
     path = (op.path or "").lower()
 
     if not path:
-        if isinstance(op.value, ScimPatchResourceValue):
-            dumped = op.value.model_dump(exclude_unset=True)
-            for key, val in dumped.items():
+        if isinstance(op.value, dict):
+            for key, val in op.value.items():
                 if key.lower() == "members":
                     _replace_members(val, current_members, added_ids, removed_ids)
                 else:
-                    _set_group_field(key.lower(), val, data, ignored_paths)
+                    _set_group_field(key.lower(), val, data)
         else:
             raise ScimPatchError("Replace without path requires a dict value")
         return
 
     if path == "members":
-        _replace_members(
-            _members_to_dicts(op.value), current_members, added_ids, removed_ids
-        )
+        _replace_members(op.value, current_members, added_ids, removed_ids)
         return
 
-    _set_group_field(path, op.value, data, ignored_paths)
-
-
-def _members_to_dicts(
-    value: str | bool | list[ScimGroupMember] | ScimPatchResourceValue | None,
-) -> list[dict]:
-    """Convert a member list value to a list of dicts for internal processing."""
-    if not isinstance(value, list):
-        raise ScimPatchError("Replace members requires a list value")
-    return [m.model_dump(exclude_none=True) for m in value]
+    _set_group_field(path, op.value, data)
 
 
 def _replace_members(
-    value: list[dict],
+    value: str | list | dict | bool | None,
     current_members: list[dict],
     added_ids: list[str],
     removed_ids: list[str],
 ) -> None:
     """Replace the entire group member list."""
+    if not isinstance(value, list):
+        raise ScimPatchError("Replace members requires a list value")
+
     old_ids = {m["value"] for m in current_members}
     new_ids = {m.get("value", "") for m in value}
 
@@ -395,21 +197,16 @@ def _replace_members(
 
 def _set_group_field(
     path: str,
-    value: ScimPatchValue,
+    value: str | bool | dict | list | None,
     data: dict,
-    ignored_paths: frozenset[str],
 ) -> None:
     """Set a single field on group data by SCIM path."""
-    if path in ignored_paths:
-        return
-
-    entry = _GROUP_REPLACE_PATHS.get(path)
-    if entry:
-        key, _ = entry
-        data[key] = value
-        return
-
-    raise ScimPatchError(f"Unsupported path '{path}' for Group PATCH")
+    if path == "displayname":
+        data["displayName"] = value
+    elif path == "externalid":
+        data["externalId"] = value
+    else:
+        raise ScimPatchError(f"Unsupported path '{path}' for Group PATCH")
 
 
 def _apply_group_add(
@@ -426,10 +223,8 @@ def _apply_group_add(
     if not isinstance(op.value, list):
         raise ScimPatchError("Add members requires a list value")
 
-    member_dicts = [m.model_dump(exclude_none=True) for m in op.value]
-
     existing_ids = {m["value"] for m in members}
-    for member_data in member_dicts:
+    for member_data in op.value:
         member_id = member_data.get("value", "")
         if member_id and member_id not in existing_ids:
             members.append(member_data)

backend/ee/onyx/server/scim/providers/base.py

@@ -1,212 +0,0 @@
-"""Base SCIM provider abstraction."""
-
-from __future__ import annotations
-
-import json
-import logging
-from abc import ABC
-from abc import abstractmethod
-from uuid import UUID
-
-from pydantic import ValidationError
-
-from ee.onyx.server.scim.models import SCIM_ENTERPRISE_USER_SCHEMA
-from ee.onyx.server.scim.models import SCIM_USER_SCHEMA
-from ee.onyx.server.scim.models import ScimEmail
-from ee.onyx.server.scim.models import ScimEnterpriseExtension
-from ee.onyx.server.scim.models import ScimGroupMember
-from ee.onyx.server.scim.models import ScimGroupResource
-from ee.onyx.server.scim.models import ScimManagerRef
-from ee.onyx.server.scim.models import ScimMappingFields
-from ee.onyx.server.scim.models import ScimMeta
-from ee.onyx.server.scim.models import ScimName
-from ee.onyx.server.scim.models import ScimUserGroupRef
-from ee.onyx.server.scim.models import ScimUserResource
-from onyx.db.models import User
-from onyx.db.models import UserGroup
-
-
-logger = logging.getLogger(__name__)
-
-COMMON_IGNORED_PATCH_PATHS: frozenset[str] = frozenset(
-    {
-        "id",
-        "schemas",
-        "meta",
-    }
-)
-
-
-class ScimProvider(ABC):
-    """Base class for provider-specific SCIM behavior.
-
-    Subclass this to handle IdP-specific quirks. The base class provides
-    RFC 7643-compliant response builders that populate all standard fields.
-    """
-
-    @property
-    @abstractmethod
-    def name(self) -> str:
-        """Short identifier for this provider (e.g. ``"okta"``)."""
-        ...
-
-    @property
-    @abstractmethod
-    def ignored_patch_paths(self) -> frozenset[str]:
-        """SCIM attribute paths to silently skip in PATCH value-object dicts.
-
-        IdPs may include read-only or meta fields alongside actual changes
-        (e.g. Okta sends ``{"id": "...", "active": false}``). Paths listed
-        here are silently dropped instead of raising an error.
-        """
-        ...
-
-    @property
-    def user_schemas(self) -> list[str]:
-        """Schema URIs to include in User resource responses.
-
-        Override in subclasses to advertise additional schemas (e.g. the
-        enterprise extension for Entra ID).
-        """
-        return [SCIM_USER_SCHEMA]
-
-    def build_user_resource(
-        self,
-        user: User,
-        external_id: str | None = None,
-        groups: list[tuple[int, str]] | None = None,
-        scim_username: str | None = None,
-        fields: ScimMappingFields | None = None,
-    ) -> ScimUserResource:
-        """Build a SCIM User response from an Onyx User.
-
-        Args:
-            user: The Onyx user model.
-            external_id: The IdP's external identifier for this user.
-            groups: List of ``(group_id, group_name)`` tuples for the
-                ``groups`` read-only attribute. Pass ``None`` or ``[]``
-                for newly-created users.
-            scim_username: The original-case userName from the IdP. Falls
-                back to ``user.email`` (lowercase) when not available.
-            fields: Stored mapping fields that the IdP expects round-tripped.
-        """
-        f = fields or ScimMappingFields()
-        group_refs = [
-            ScimUserGroupRef(value=str(gid), display=gname)
-            for gid, gname in (groups or [])
-        ]
-
-        username = scim_username or user.email
-
-        # Build enterprise extension when at least one value is present.
-        # Dynamically add the enterprise URN to schemas per RFC 7643 §3.0.
-        enterprise_ext: ScimEnterpriseExtension | None = None
-        schemas = list(self.user_schemas)
-        if f.department is not None or f.manager is not None:
-            manager_ref = (
-                ScimManagerRef(value=f.manager) if f.manager is not None else None
-            )
-            enterprise_ext = ScimEnterpriseExtension(
-                department=f.department,
-                manager=manager_ref,
-            )
-            if SCIM_ENTERPRISE_USER_SCHEMA not in schemas:
-                schemas.append(SCIM_ENTERPRISE_USER_SCHEMA)
-
-        name = self.build_scim_name(user, f)
-        emails = _deserialize_emails(f.scim_emails_json, username)
-
-        resource = ScimUserResource(
-            schemas=schemas,
-            id=str(user.id),
-            externalId=external_id,
-            userName=username,
-            name=name,
-            displayName=user.personal_name,
-            emails=emails,
-            active=user.is_active,
-            groups=group_refs,
-            meta=ScimMeta(resourceType="User"),
-        )
-        resource.enterprise_extension = enterprise_ext
-        return resource
-
-    def build_group_resource(
-        self,
-        group: UserGroup,
-        members: list[tuple[UUID, str | None]],
-        external_id: str | None = None,
-    ) -> ScimGroupResource:
-        """Build a SCIM Group response from an Onyx UserGroup."""
-        scim_members = [
-            ScimGroupMember(value=str(uid), display=email) for uid, email in members
-        ]
-        return ScimGroupResource(
-            id=str(group.id),
-            externalId=external_id,
-            displayName=group.name,
-            members=scim_members,
-            meta=ScimMeta(resourceType="Group"),
-        )
-
-    def build_scim_name(
-        self,
-        user: User,
-        fields: ScimMappingFields,
-    ) -> ScimName:
-        """Build SCIM name components for the response.
-
-        Round-trips stored ``given_name``/``family_name`` when available (so
-        the IdP gets back what it sent). Falls back to splitting
-        ``personal_name`` for users provisioned before we stored components.
-        Always returns a ScimName — Okta's spec tests expect ``name``
-        (with ``givenName``/``familyName``) on every user resource.
-        Providers may override for custom behavior.
-        """
-        if fields.given_name is not None or fields.family_name is not None:
-            return ScimName(
-                givenName=fields.given_name or "",
-                familyName=fields.family_name or "",
-                formatted=user.personal_name or "",
-            )
-        if not user.personal_name:
-            return ScimName(givenName="", familyName="", formatted="")
-        parts = user.personal_name.split(" ", 1)
-        return ScimName(
-            givenName=parts[0],
-            familyName=parts[1] if len(parts) > 1 else "",
-            formatted=user.personal_name,
-        )
-
-
-def _deserialize_emails(stored_json: str | None, username: str) -> list[ScimEmail]:
-    """Deserialize stored email entries or build a default work email."""
-    if stored_json:
-        try:
-            entries = json.loads(stored_json)
-            if isinstance(entries, list) and entries:
-                return [ScimEmail(**e) for e in entries]
-        except (json.JSONDecodeError, TypeError, ValidationError):
-            logger.warning(
-                "Corrupt scim_emails_json, falling back to default: %s", stored_json
-            )
-    return [ScimEmail(value=username, type="work", primary=True)]
-
-
-def serialize_emails(emails: list[ScimEmail]) -> str | None:
-    """Serialize SCIM email entries to JSON for storage."""
-    if not emails:
-        return None
-    return json.dumps([e.model_dump(exclude_none=True) for e in emails])
-
-
-def get_default_provider() -> ScimProvider:
-    """Return the default SCIM provider.
-
-    Currently returns ``OktaProvider`` since Okta is the primary supported
-    IdP. When provider detection is added (via token metadata or tenant
-    config), this can be replaced with dynamic resolution.
-    """
-    from ee.onyx.server.scim.providers.okta import OktaProvider
-
-    return OktaProvider()

backend/ee/onyx/server/scim/providers/entra.py

@@ -1,36 +0,0 @@
-"""Entra ID (Azure AD) SCIM provider."""
-
-from __future__ import annotations
-
-from ee.onyx.server.scim.models import SCIM_ENTERPRISE_USER_SCHEMA
-from ee.onyx.server.scim.models import SCIM_USER_SCHEMA
-from ee.onyx.server.scim.providers.base import COMMON_IGNORED_PATCH_PATHS
-from ee.onyx.server.scim.providers.base import ScimProvider
-
-_ENTRA_IGNORED_PATCH_PATHS = COMMON_IGNORED_PATCH_PATHS
-
-
-class EntraProvider(ScimProvider):
-    """Entra ID (Azure AD) SCIM provider.
-
-    Entra behavioral notes:
-      - Sends capitalized PATCH ops (``"Add"``, ``"Replace"``, ``"Remove"``)
-        — handled by ``ScimPatchOperation.normalize_op`` validator.
-      - Sends the enterprise extension URN as a key in path-less PATCH value
-        dicts — handled by ``_set_enterprise_field`` in ``patch.py`` to
-        store department/manager values.
-      - Expects the enterprise extension schema in ``schemas`` arrays and
-        ``/Schemas`` + ``/ResourceTypes`` discovery endpoints.
-    """
-
-    @property
-    def name(self) -> str:
-        return "entra"
-
-    @property
-    def ignored_patch_paths(self) -> frozenset[str]:
-        return _ENTRA_IGNORED_PATCH_PATHS
-
-    @property
-    def user_schemas(self) -> list[str]:
-        return [SCIM_USER_SCHEMA, SCIM_ENTERPRISE_USER_SCHEMA]

backend/ee/onyx/server/scim/providers/okta.py

@@ -1,26 +0,0 @@
-"""Okta SCIM provider."""
-
-from __future__ import annotations
-
-from ee.onyx.server.scim.providers.base import COMMON_IGNORED_PATCH_PATHS
-from ee.onyx.server.scim.providers.base import ScimProvider
-
-
-class OktaProvider(ScimProvider):
-    """Okta SCIM provider.
-
-    Okta behavioral notes:
-      - Uses ``PATCH {"active": false}`` for deprovisioning (not DELETE)
-      - Sends path-less PATCH with value dicts containing extra fields
-        (``id``, ``schemas``)
-      - Expects ``displayName`` and ``groups`` in user responses
-      - Only uses ``eq`` operator for ``userName`` filter
-    """
-
-    @property
-    def name(self) -> str:
-        return "okta"
-
-    @property
-    def ignored_patch_paths(self) -> frozenset[str]:
-        return COMMON_IGNORED_PATCH_PATHS

backend/ee/onyx/server/scim/schema_definitions.py

@@ -4,7 +4,6 @@ Pre-built at import time — these never change at runtime. Separated from
 api.py to keep the endpoint module focused on request handling.
 """
 
-from ee.onyx.server.scim.models import SCIM_ENTERPRISE_USER_SCHEMA
 from ee.onyx.server.scim.models import SCIM_GROUP_SCHEMA
 from ee.onyx.server.scim.models import SCIM_USER_SCHEMA
 from ee.onyx.server.scim.models import ScimResourceType
@@ -21,9 +20,6 @@ USER_RESOURCE_TYPE = ScimResourceType.model_validate(
         "endpoint": "/scim/v2/Users",
         "description": "SCIM User resource",
         "schema": SCIM_USER_SCHEMA,
-        "schemaExtensions": [
-            {"schema": SCIM_ENTERPRISE_USER_SCHEMA, "required": False}
-        ],
     }
 )
 
@@ -108,31 +104,6 @@ USER_SCHEMA_DEF = ScimSchemaDefinition(
     ],
 )
 
-ENTERPRISE_USER_SCHEMA_DEF = ScimSchemaDefinition(
-    id=SCIM_ENTERPRISE_USER_SCHEMA,
-    name="EnterpriseUser",
-    description="Enterprise User extension (RFC 7643 §4.3)",
-    attributes=[
-        ScimSchemaAttribute(
-            name="department",
-            type="string",
-            description="Department.",
-        ),
-        ScimSchemaAttribute(
-            name="manager",
-            type="complex",
-            description="The user's manager.",
-            subAttributes=[
-                ScimSchemaAttribute(
-                    name="value",
-                    type="string",
-                    description="Manager user ID.",
-                ),
-            ],
-        ),
-    ],
-)
-
 GROUP_SCHEMA_DEF = ScimSchemaDefinition(
     id=SCIM_GROUP_SCHEMA,
     name="Group",

backend/ee/onyx/server/seeding.py

@@ -18,8 +18,8 @@ from ee.onyx.server.enterprise_settings.store import (
     store_settings as store_ee_settings,
 )
 from ee.onyx.server.enterprise_settings.store import upload_logo
+from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
-from onyx.db.llm import fetch_existing_llm_provider
 from onyx.db.llm import update_default_provider
 from onyx.db.llm import upsert_llm_provider
 from onyx.db.models import Tool
@@ -117,38 +117,15 @@ def _seed_custom_tools(db_session: Session, tools: List[CustomToolSeed]) -> None
 def _seed_llms(
     db_session: Session, llm_upsert_requests: list[LLMProviderUpsertRequest]
 ) -> None:
-    if not llm_upsert_requests:
-        return
-
-    logger.notice("Seeding LLMs")
-    for request in llm_upsert_requests:
-        existing = fetch_existing_llm_provider(name=request.name, db_session=db_session)
-        if existing:
-            request.id = existing.id
-    seeded_providers = [
-        upsert_llm_provider(llm_upsert_request, db_session)
-        for llm_upsert_request in llm_upsert_requests
-    ]
-
-    default_provider = next(
-        (p for p in seeded_providers if p.model_configurations), None
-    )
-    if not default_provider:
-        return
-
-    visible_configs = [
-        mc for mc in default_provider.model_configurations if mc.is_visible
-    ]
-    default_config = (
-        visible_configs[0]
-        if visible_configs
-        else default_provider.model_configurations[0]
-    )
-    update_default_provider(
-        provider_id=default_provider.id,
-        model_name=default_config.name,
-        db_session=db_session,
-    )
+    if llm_upsert_requests:
+        logger.notice("Seeding LLMs")
+        seeded_providers = [
+            upsert_llm_provider(llm_upsert_request, db_session)
+            for llm_upsert_request in llm_upsert_requests
+        ]
+        update_default_provider(
+            provider_id=seeded_providers[0].id, db_session=db_session
+        )
 
 
 def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) -> None:
@@ -160,6 +137,12 @@ def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) ->
                     user=None,  # Seeding is done as admin
                     name=persona.name,
                     description=persona.description,
+                    num_chunks=(
+                        persona.num_chunks if persona.num_chunks is not None else 0.0
+                    ),
+                    llm_relevance_filter=persona.llm_relevance_filter,
+                    llm_filter_extraction=persona.llm_filter_extraction,
+                    recency_bias=RecencyBiasSetting.AUTO,
                     document_set_ids=persona.document_set_ids,
                     llm_model_provider_override=persona.llm_model_provider_override,
                     llm_model_version_override=persona.llm_model_version_override,
@@ -171,7 +154,6 @@ def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) ->
                     system_prompt=persona.system_prompt,
                     task_prompt=persona.task_prompt,
                     datetime_aware=persona.datetime_aware,
-                    featured=persona.featured,
                     commit=False,
                 )
             db_session.commit()

backend/ee/onyx/server/settings/api.py

@@ -109,12 +109,6 @@ def apply_license_status_to_settings(settings: Settings) -> Settings:
             if metadata.status == _BLOCKING_STATUS:
                 settings.application_status = metadata.status
                 settings.ee_features_enabled = False
-            elif metadata.used_seats > metadata.seats:
-                # License is valid but seat limit exceeded
-                settings.application_status = ApplicationStatus.SEAT_LIMIT_EXCEEDED
-                settings.seat_count = metadata.seats
-                settings.used_seats = metadata.used_seats
-                settings.ee_features_enabled = True
             else:
                 # Has a valid license (GRACE_PERIOD/PAYMENT_REMINDER still allow EE features)
                 settings.ee_features_enabled = True

backend/ee/onyx/server/tenants/provisioning.py

@@ -33,7 +33,6 @@ from onyx.configs.constants import MilestoneRecordType
 from onyx.db.engine.sql_engine import get_session_with_shared_schema
 from onyx.db.engine.sql_engine import get_session_with_tenant
 from onyx.db.image_generation import create_default_image_gen_config_from_api_key
-from onyx.db.llm import fetch_existing_llm_provider
 from onyx.db.llm import update_default_provider
 from onyx.db.llm import upsert_cloud_embedding_provider
 from onyx.db.llm import upsert_llm_provider
@@ -303,17 +302,12 @@ def configure_default_api_keys(db_session: Session) -> None:
 
     has_set_default_provider = False
 
-    def _upsert(request: LLMProviderUpsertRequest, default_model: str) -> None:
+    def _upsert(request: LLMProviderUpsertRequest) -> None:
         nonlocal has_set_default_provider
         try:
-            existing = fetch_existing_llm_provider(
-                name=request.name, db_session=db_session
-            )
-            if existing:
-                request.id = existing.id
             provider = upsert_llm_provider(request, db_session)
             if not has_set_default_provider:
-                update_default_provider(provider.id, default_model, db_session)
+                update_default_provider(provider.id, db_session)
                 has_set_default_provider = True
         except Exception as e:
             logger.error(f"Failed to configure {request.provider} provider: {e}")
@@ -331,13 +325,14 @@ def configure_default_api_keys(db_session: Session) -> None:
             name="OpenAI",
             provider=OPENAI_PROVIDER_NAME,
             api_key=OPENAI_DEFAULT_API_KEY,
+            default_model_name=default_model_name,
             model_configurations=_build_model_configuration_upsert_requests(
                 OPENAI_PROVIDER_NAME, recommendations
             ),
             api_key_changed=True,
             is_auto_mode=True,
         )
-        _upsert(openai_provider, default_model_name)
+        _upsert(openai_provider)
 
         # Create default image generation config using the OpenAI API key
         try:
@@ -366,13 +361,14 @@ def configure_default_api_keys(db_session: Session) -> None:
             name="Anthropic",
             provider=ANTHROPIC_PROVIDER_NAME,
             api_key=ANTHROPIC_DEFAULT_API_KEY,
+            default_model_name=default_model_name,
             model_configurations=_build_model_configuration_upsert_requests(
                 ANTHROPIC_PROVIDER_NAME, recommendations
             ),
             api_key_changed=True,
             is_auto_mode=True,
         )
-        _upsert(anthropic_provider, default_model_name)
+        _upsert(anthropic_provider)
     else:
         logger.info(
             "ANTHROPIC_DEFAULT_API_KEY not set, skipping Anthropic provider configuration"
@@ -397,13 +393,14 @@ def configure_default_api_keys(db_session: Session) -> None:
             name="Google Vertex AI",
             provider=VERTEXAI_PROVIDER_NAME,
             custom_config=custom_config,
+            default_model_name=default_model_name,
             model_configurations=_build_model_configuration_upsert_requests(
                 VERTEXAI_PROVIDER_NAME, recommendations
             ),
             api_key_changed=True,
             is_auto_mode=True,
         )
-        _upsert(vertexai_provider, default_model_name)
+        _upsert(vertexai_provider)
     else:
         logger.info(
             "VERTEXAI_DEFAULT_CREDENTIALS not set, skipping Vertex AI provider configuration"
@@ -435,11 +432,12 @@ def configure_default_api_keys(db_session: Session) -> None:
             name="OpenRouter",
             provider=OPENROUTER_PROVIDER_NAME,
             api_key=OPENROUTER_DEFAULT_API_KEY,
+            default_model_name=default_model_name,
             model_configurations=model_configurations,
             api_key_changed=True,
             is_auto_mode=True,
         )
-        _upsert(openrouter_provider, default_model_name)
+        _upsert(openrouter_provider)
     else:
         logger.info(
             "OPENROUTER_DEFAULT_API_KEY not set, skipping OpenRouter provider configuration"

backend/ee/onyx/server/user_group/api.py

@@ -37,15 +37,12 @@ def list_user_groups(
     db_session: Session = Depends(get_session),
 ) -> list[UserGroup]:
     if user.role == UserRole.ADMIN:
-        user_groups = fetch_user_groups(
-            db_session, only_up_to_date=False, eager_load_for_snapshot=True
-        )
+        user_groups = fetch_user_groups(db_session, only_up_to_date=False)
     else:
         user_groups = fetch_user_groups_for_user(
             db_session=db_session,
             user_id=user.id,
             only_curator_groups=user.role == UserRole.CURATOR,
-            eager_load_for_snapshot=True,
         )
     return [UserGroup.from_model(user_group) for user_group in user_groups]
 

backend/ee/onyx/server/user_group/models.py

@@ -53,8 +53,7 @@ class UserGroup(BaseModel):
                     id=cc_pair_relationship.cc_pair.id,
                     name=cc_pair_relationship.cc_pair.name,
                     connector=ConnectorSnapshot.from_connector_db_model(
-                        cc_pair_relationship.cc_pair.connector,
-                        credential_ids=[cc_pair_relationship.cc_pair.credential_id],
+                        cc_pair_relationship.cc_pair.connector
                     ),
                     credential=CredentialSnapshot.from_credential_db_model(
                         cc_pair_relationship.cc_pair.credential

backend/onyx/auth/oauth_token_manager.py

@@ -58,27 +58,16 @@ class OAuthTokenManager:
         if not user_token.token_data:
             raise ValueError("No token data available for refresh")
 
-        if (
-            self.oauth_config.client_id is None
-            or self.oauth_config.client_secret is None
-        ):
-            raise ValueError(
-                "OAuth client_id and client_secret are required for token refresh"
-            )
-
         token_data = self._unwrap_token_data(user_token.token_data)
 
-        data: dict[str, str] = {
-            "grant_type": "refresh_token",
-            "refresh_token": token_data["refresh_token"],
-            "client_id": self._unwrap_sensitive_str(self.oauth_config.client_id),
-            "client_secret": self._unwrap_sensitive_str(
-                self.oauth_config.client_secret
-            ),
-        }
         response = requests.post(
             self.oauth_config.token_url,
-            data=data,
+            data={
+                "grant_type": "refresh_token",
+                "refresh_token": token_data["refresh_token"],
+                "client_id": self.oauth_config.client_id,
+                "client_secret": self.oauth_config.client_secret,
+            },
             headers={"Accept": "application/json"},
         )
         response.raise_for_status()
@@ -126,26 +115,15 @@ class OAuthTokenManager:
 
     def exchange_code_for_token(self, code: str, redirect_uri: str) -> dict[str, Any]:
         """Exchange authorization code for access token"""
-        if (
-            self.oauth_config.client_id is None
-            or self.oauth_config.client_secret is None
-        ):
-            raise ValueError(
-                "OAuth client_id and client_secret are required for code exchange"
-            )
-
-        data: dict[str, str] = {
-            "grant_type": "authorization_code",
-            "code": code,
-            "client_id": self._unwrap_sensitive_str(self.oauth_config.client_id),
-            "client_secret": self._unwrap_sensitive_str(
-                self.oauth_config.client_secret
-            ),
-            "redirect_uri": redirect_uri,
-        }
         response = requests.post(
             self.oauth_config.token_url,
-            data=data,
+            data={
+                "grant_type": "authorization_code",
+                "code": code,
+                "client_id": self.oauth_config.client_id,
+                "client_secret": self.oauth_config.client_secret,
+                "redirect_uri": redirect_uri,
+            },
             headers={"Accept": "application/json"},
         )
         response.raise_for_status()
@@ -163,13 +141,8 @@ class OAuthTokenManager:
         oauth_config: OAuthConfig, redirect_uri: str, state: str
     ) -> str:
         """Build OAuth authorization URL"""
-        if oauth_config.client_id is None:
-            raise ValueError("OAuth client_id is required to build authorization URL")
-
         params: dict[str, Any] = {
-            "client_id": OAuthTokenManager._unwrap_sensitive_str(
-                oauth_config.client_id
-            ),
+            "client_id": oauth_config.client_id,
             "redirect_uri": redirect_uri,
             "response_type": "code",
             "state": state,
@@ -188,12 +161,6 @@ class OAuthTokenManager:
 
         return f"{oauth_config.authorization_url}{separator}{urlencode(params)}"
 
-    @staticmethod
-    def _unwrap_sensitive_str(value: SensitiveValue[str] | str) -> str:
-        if isinstance(value, SensitiveValue):
-            return value.get_value(apply_mask=False)
-        return value
-
     @staticmethod
     def _unwrap_token_data(
         token_data: SensitiveValue[dict[str, Any]] | dict[str, Any],

backend/onyx/auth/users.py

@@ -277,32 +277,13 @@ def verify_email_domain(email: str) -> None:
             detail="Email is not valid",
         )
 
-    local_part, domain = email.split("@")
-    domain = domain.lower()
-
-    if AUTH_TYPE == AuthType.CLOUD:
-        # Normalize googlemail.com to gmail.com (they deliver to the same inbox)
-        if domain == "googlemail.com":
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={"reason": "Please use @gmail.com instead of @googlemail.com."},
-            )
-
-        if "+" in local_part and domain != "onyx.app":
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={
-                    "reason": "Email addresses with '+' are not allowed. Please use your base email address."
-                },
-            )
+    domain = email.split("@")[-1].lower()
 
     # Check if email uses a disposable/temporary domain
     if is_disposable_email(email):
         raise HTTPException(
             status_code=status.HTTP_400_BAD_REQUEST,
-            detail={
-                "reason": "Disposable email addresses are not allowed. Please use a permanent email address."
-            },
+            detail="Disposable email addresses are not allowed. Please use a permanent email address.",
         )
 
     # Check domain whitelist if configured
@@ -543,7 +524,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
         result = await db_session.execute(
             select(Persona.id)
             .where(
-                Persona.featured.is_(True),
+                Persona.is_default_persona.is_(True),
                 Persona.is_public.is_(True),
                 Persona.is_visible.is_(True),
                 Persona.deleted.is_(False),
@@ -725,19 +706,11 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
                         if user_by_session:
                             user = user_by_session
 
-                # If the user is inactive, check seat availability before
-                # upgrading role — otherwise they'd become an inactive BASIC
-                # user who still can't log in.
-                if not user.is_active:
-                    with get_session_with_current_tenant() as sync_db:
-                        enforce_seat_limit(sync_db)
-
                 await self.user_db.update(
                     user,
                     {
                         "is_verified": is_verified_by_default,
                         "role": UserRole.BASIC,
-                        **({"is_active": True} if not user.is_active else {}),
                     },
                 )
 
@@ -1698,10 +1671,7 @@ def get_oauth_router(
         if redirect_url is not None:
             authorize_redirect_url = redirect_url
         else:
-            # Use WEB_DOMAIN instead of request.url_for() to prevent host
-            # header poisoning — request.url_for() trusts the Host header.
-            callback_path = request.app.url_path_for(callback_route_name)
-            authorize_redirect_url = f"{WEB_DOMAIN}{callback_path}"
+            authorize_redirect_url = str(request.url_for(callback_route_name))
 
         next_url = request.query_params.get("next", "/")
 

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -241,7 +241,8 @@ _VECTOR_DB_BEAT_TASK_NAMES: set[str] = {
     "check-for-index-attempt-cleanup",
     "check-for-doc-permissions-sync",
     "check-for-external-group-sync",
-    "migrate-chunks-from-vespa-to-opensearch",
+    "check-for-documents-for-opensearch-migration",
+    "migrate-documents-from-vespa-to-opensearch",
 }
 
 if DISABLE_VECTOR_DB:

backend/onyx/background/celery/tasks/opensearch_migration/tasks.py

@@ -48,7 +48,6 @@ from onyx.document_index.opensearch.opensearch_document_index import (
     OpenSearchDocumentIndex,
 )
 from onyx.document_index.vespa.vespa_document_index import VespaDocumentIndex
-from onyx.indexing.models import IndexingSetting
 from onyx.redis.redis_pool import get_redis_client
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.contextvars import get_current_tenant_id
@@ -150,12 +149,8 @@ def migrate_chunks_from_vespa_to_opensearch_task(
             try_insert_opensearch_tenant_migration_record_with_commit(db_session)
             search_settings = get_current_search_settings(db_session)
             tenant_state = TenantState(tenant_id=tenant_id, multitenant=MULTI_TENANT)
-            indexing_setting = IndexingSetting.from_db_model(search_settings)
             opensearch_document_index = OpenSearchDocumentIndex(
-                tenant_state=tenant_state,
-                index_name=search_settings.index_name,
-                embedding_dim=indexing_setting.final_embedding_dim,
-                embedding_precision=indexing_setting.embedding_precision,
+                index_name=search_settings.index_name, tenant_state=tenant_state
             )
             vespa_document_index = VespaDocumentIndex(
                 index_name=search_settings.index_name,

backend/onyx/background/celery/tasks/opensearch_migration/transformer.py

@@ -22,7 +22,6 @@ from onyx.document_index.vespa_constants import HIDDEN
 from onyx.document_index.vespa_constants import IMAGE_FILE_NAME
 from onyx.document_index.vespa_constants import METADATA_LIST
 from onyx.document_index.vespa_constants import METADATA_SUFFIX
-from onyx.document_index.vespa_constants import PERSONAS
 from onyx.document_index.vespa_constants import PRIMARY_OWNERS
 from onyx.document_index.vespa_constants import SECONDARY_OWNERS
 from onyx.document_index.vespa_constants import SEMANTIC_IDENTIFIER
@@ -59,7 +58,6 @@ FIELDS_NEEDED_FOR_TRANSFORMATION: list[str] = [
     METADATA_SUFFIX,
     DOCUMENT_SETS,
     USER_PROJECT,
-    PERSONAS,
     PRIMARY_OWNERS,
     SECONDARY_OWNERS,
     ACCESS_CONTROL_LIST,
@@ -278,7 +276,6 @@ def transform_vespa_chunks_to_opensearch_chunks(
                 )
             )
             user_projects: list[int] | None = vespa_chunk.get(USER_PROJECT)
-            personas: list[int] | None = vespa_chunk.get(PERSONAS)
             primary_owners: list[str] | None = vespa_chunk.get(PRIMARY_OWNERS)
             secondary_owners: list[str] | None = vespa_chunk.get(SECONDARY_OWNERS)
 
@@ -328,7 +325,6 @@ def transform_vespa_chunks_to_opensearch_chunks(
                 metadata_suffix=metadata_suffix,
                 document_sets=document_sets,
                 user_projects=user_projects,
-                personas=personas,
                 primary_owners=primary_owners,
                 secondary_owners=secondary_owners,
                 tenant_id=tenant_state,

backend/onyx/background/celery/tasks/user_file_processing/tasks.py

@@ -5,18 +5,14 @@ from uuid import UUID
 
 import httpx
 import sqlalchemy as sa
-from celery import Celery
 from celery import shared_task
 from celery import Task
-from redis import Redis
 from redis.lock import Lock as RedisLock
 from retry import retry
 from sqlalchemy import select
-from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
-from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_utils import httpx_init_vespa_pool
 from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
 from onyx.configs.app_configs import DISABLE_VECTOR_DB
@@ -25,16 +21,12 @@ from onyx.configs.app_configs import VESPA_CLOUD_CERT_PATH
 from onyx.configs.app_configs import VESPA_CLOUD_KEY_PATH
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT
-from onyx.configs.constants import CELERY_USER_FILE_PROCESSING_TASK_EXPIRES
 from onyx.configs.constants import CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT
-from onyx.configs.constants import CELERY_USER_FILE_PROJECT_SYNC_TASK_EXPIRES
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.configs.constants import USER_FILE_PROCESSING_MAX_QUEUE_DEPTH
-from onyx.configs.constants import USER_FILE_PROJECT_SYNC_MAX_QUEUE_DEPTH
 from onyx.connectors.file.connector import LocalFileConnector
 from onyx.connectors.models import Document
 from onyx.connectors.models import HierarchyNode
@@ -65,73 +57,14 @@ def _user_file_lock_key(user_file_id: str | UUID) -> str:
     return f"{OnyxRedisLocks.USER_FILE_PROCESSING_LOCK_PREFIX}:{user_file_id}"
 
 
-def _user_file_queued_key(user_file_id: str | UUID) -> str:
-    """Key that exists while a process_single_user_file task is sitting in the queue.
-
-    The beat generator sets this with a TTL equal to CELERY_USER_FILE_PROCESSING_TASK_EXPIRES
-    before enqueuing and the worker deletes it as its first action.  This prevents
-    the beat from adding duplicate tasks for files that already have a live task
-    in flight.
-    """
-    return f"{OnyxRedisLocks.USER_FILE_QUEUED_PREFIX}:{user_file_id}"
-
-
-def user_file_project_sync_lock_key(user_file_id: str | UUID) -> str:
+def _user_file_project_sync_lock_key(user_file_id: str | UUID) -> str:
     return f"{OnyxRedisLocks.USER_FILE_PROJECT_SYNC_LOCK_PREFIX}:{user_file_id}"
 
 
-def _user_file_project_sync_queued_key(user_file_id: str | UUID) -> str:
-    return f"{OnyxRedisLocks.USER_FILE_PROJECT_SYNC_QUEUED_PREFIX}:{user_file_id}"
-
-
 def _user_file_delete_lock_key(user_file_id: str | UUID) -> str:
     return f"{OnyxRedisLocks.USER_FILE_DELETE_LOCK_PREFIX}:{user_file_id}"
 
 
-def get_user_file_project_sync_queue_depth(celery_app: Celery) -> int:
-    redis_celery: Redis = celery_app.broker_connection().channel().client  # type: ignore
-    return celery_get_queue_length(
-        OnyxCeleryQueues.USER_FILE_PROJECT_SYNC, redis_celery
-    )
-
-
-def enqueue_user_file_project_sync_task(
-    *,
-    celery_app: Celery,
-    redis_client: Redis,
-    user_file_id: str | UUID,
-    tenant_id: str,
-    priority: OnyxCeleryPriority = OnyxCeleryPriority.HIGH,
-) -> bool:
-    """Enqueue a project-sync task if no matching queued task already exists."""
-    queued_key = _user_file_project_sync_queued_key(user_file_id)
-
-    # NX+EX gives us atomic dedupe and a self-healing TTL.
-    queued_guard_set = redis_client.set(
-        queued_key,
-        1,
-        nx=True,
-        ex=CELERY_USER_FILE_PROJECT_SYNC_TASK_EXPIRES,
-    )
-    if not queued_guard_set:
-        return False
-
-    try:
-        celery_app.send_task(
-            OnyxCeleryTask.PROCESS_SINGLE_USER_FILE_PROJECT_SYNC,
-            kwargs={"user_file_id": str(user_file_id), "tenant_id": tenant_id},
-            queue=OnyxCeleryQueues.USER_FILE_PROJECT_SYNC,
-            priority=priority,
-            expires=CELERY_USER_FILE_PROJECT_SYNC_TASK_EXPIRES,
-        )
-    except Exception:
-        # Roll back the queued guard if task publish fails.
-        redis_client.delete(queued_key)
-        raise
-
-    return True
-
-
 @retry(tries=3, delay=1, backoff=2, jitter=(0.0, 1.0))
 def _visit_chunks(
     *,
@@ -187,24 +120,7 @@ def _get_document_chunk_count(
 def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
     """Scan for user files with PROCESSING status and enqueue per-file tasks.
 
-    Three mechanisms prevent queue runaway:
-
-    1. **Queue depth backpressure** – if the broker queue already has more than
-       USER_FILE_PROCESSING_MAX_QUEUE_DEPTH items we skip this beat cycle
-       entirely.  Workers are clearly behind; adding more tasks would only make
-       the backlog worse.
-
-    2. **Per-file queued guard** – before enqueuing a task we set a short-lived
-       Redis key (TTL = CELERY_USER_FILE_PROCESSING_TASK_EXPIRES).  If that key
-       already exists the file already has a live task in the queue, so we skip
-       it.  The worker deletes the key the moment it picks up the task so the
-       next beat cycle can re-enqueue if the file is still PROCESSING.
-
-    3. **Task expiry** – every enqueued task carries an `expires` value equal to
-       CELERY_USER_FILE_PROCESSING_TASK_EXPIRES.  If a task is still sitting in
-       the queue after that deadline, Celery discards it without touching the DB.
-       This is a belt-and-suspenders defence: even if the guard key is lost (e.g.
-       Redis restart), stale tasks evict themselves rather than piling up forever.
+    Uses direct Redis locks to avoid overlapping runs.
     """
     task_logger.info("check_user_file_processing - Starting")
 
@@ -219,21 +135,7 @@ def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
         return None
 
     enqueued = 0
-    skipped_guard = 0
     try:
-        # --- Protection 1: queue depth backpressure ---
-        r_celery = self.app.broker_connection().channel().client  # type: ignore
-        queue_len = celery_get_queue_length(
-            OnyxCeleryQueues.USER_FILE_PROCESSING, r_celery
-        )
-        if queue_len > USER_FILE_PROCESSING_MAX_QUEUE_DEPTH:
-            task_logger.warning(
-                f"check_user_file_processing - Queue depth {queue_len} exceeds "
-                f"{USER_FILE_PROCESSING_MAX_QUEUE_DEPTH}, skipping enqueue for "
-                f"tenant={tenant_id}"
-            )
-            return None
-
         with get_session_with_current_tenant() as db_session:
             user_file_ids = (
                 db_session.execute(
@@ -246,35 +148,12 @@ def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
             )
 
             for user_file_id in user_file_ids:
-                # --- Protection 2: per-file queued guard ---
-                queued_key = _user_file_queued_key(user_file_id)
-                guard_set = redis_client.set(
-                    queued_key,
-                    1,
-                    ex=CELERY_USER_FILE_PROCESSING_TASK_EXPIRES,
-                    nx=True,
+                self.app.send_task(
+                    OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
+                    kwargs={"user_file_id": str(user_file_id), "tenant_id": tenant_id},
+                    queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
+                    priority=OnyxCeleryPriority.HIGH,
                 )
-                if not guard_set:
-                    skipped_guard += 1
-                    continue
-
-                # --- Protection 3: task expiry ---
-                # If task submission fails, clear the guard immediately so the
-                # next beat cycle can retry enqueuing this file.
-                try:
-                    self.app.send_task(
-                        OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
-                        kwargs={
-                            "user_file_id": str(user_file_id),
-                            "tenant_id": tenant_id,
-                        },
-                        queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
-                        priority=OnyxCeleryPriority.HIGH,
-                        expires=CELERY_USER_FILE_PROCESSING_TASK_EXPIRES,
-                    )
-                except Exception:
-                    redis_client.delete(queued_key)
-                    raise
                 enqueued += 1
 
     finally:
@@ -282,8 +161,7 @@ def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
             lock.release()
 
     task_logger.info(
-        f"check_user_file_processing - Enqueued {enqueued} skipped_guard={skipped_guard} "
-        f"tasks for tenant={tenant_id}"
+        f"check_user_file_processing - Enqueued {enqueued} tasks for tenant={tenant_id}"
     )
     return None
 
@@ -414,31 +292,28 @@ def _process_user_file_with_indexing(
         raise RuntimeError(f"Indexing pipeline failed for user file {user_file_id}")
 
 
-def process_user_file_impl(
-    *, user_file_id: str, tenant_id: str, redis_locking: bool
+@shared_task(
+    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file(
+    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
 ) -> None:
-    """Core implementation for processing a single user file.
-
-    When redis_locking=True, acquires a per-file Redis lock and clears the
-    queued-key guard (Celery path).  When redis_locking=False, skips all Redis
-    operations (BackgroundTask path).
-    """
-    task_logger.info(f"process_user_file_impl - Starting id={user_file_id}")
+    task_logger.info(f"process_single_user_file - Starting id={user_file_id}")
     start = time.monotonic()
 
-    file_lock: RedisLock | None = None
-    if redis_locking:
-        redis_client = get_redis_client(tenant_id=tenant_id)
-        redis_client.delete(_user_file_queued_key(user_file_id))
-        file_lock = redis_client.lock(
-            _user_file_lock_key(user_file_id),
-            timeout=CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT,
+    redis_client = get_redis_client(tenant_id=tenant_id)
+    file_lock: RedisLock = redis_client.lock(
+        _user_file_lock_key(user_file_id),
+        timeout=CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT,
+    )
+
+    if not file_lock.acquire(blocking=False):
+        task_logger.info(
+            f"process_single_user_file - Lock held, skipping user_file_id={user_file_id}"
         )
-        if file_lock is not None and not file_lock.acquire(blocking=False):
-            task_logger.info(
-                f"process_user_file_impl - Lock held, skipping user_file_id={user_file_id}"
-            )
-            return
+        return None
 
     documents: list[Document] = []
     try:
@@ -446,18 +321,15 @@ def process_user_file_impl(
             uf = db_session.get(UserFile, _as_uuid(user_file_id))
             if not uf:
                 task_logger.warning(
-                    f"process_user_file_impl - UserFile not found id={user_file_id}"
+                    f"process_single_user_file - UserFile not found id={user_file_id}"
                 )
-                return
+                return None
 
-            if uf.status not in (
-                UserFileStatus.PROCESSING,
-                UserFileStatus.INDEXING,
-            ):
+            if uf.status != UserFileStatus.PROCESSING:
                 task_logger.info(
-                    f"process_user_file_impl - Skipping id={user_file_id} status={uf.status}"
+                    f"process_single_user_file - Skipping id={user_file_id} status={uf.status}"
                 )
-                return
+                return None
 
             connector = LocalFileConnector(
                 file_locations=[uf.file_id],
@@ -471,6 +343,7 @@ def process_user_file_impl(
                         [doc for doc in batch if not isinstance(doc, HierarchyNode)]
                     )
 
+                # update the document id to userfile id in the documents
                 for document in documents:
                     document.id = str(user_file_id)
                     document.source = DocumentSource.USER_FILE
@@ -492,8 +365,9 @@ def process_user_file_impl(
 
             except Exception as e:
                 task_logger.exception(
-                    f"process_user_file_impl - Error processing file id={user_file_id} - {e.__class__.__name__}"
+                    f"process_single_user_file - Error processing file id={user_file_id} - {e.__class__.__name__}"
                 )
+                # don't update the status if the user file is being deleted
                 current_user_file = db_session.get(UserFile, _as_uuid(user_file_id))
                 if (
                     current_user_file
@@ -502,42 +376,33 @@ def process_user_file_impl(
                     uf.status = UserFileStatus.FAILED
                     db_session.add(uf)
                     db_session.commit()
-                return
+                return None
 
         elapsed = time.monotonic() - start
         task_logger.info(
-            f"process_user_file_impl - Finished id={user_file_id} docs={len(documents)} elapsed={elapsed:.2f}s"
+            f"process_single_user_file - Finished id={user_file_id} docs={len(documents)} elapsed={elapsed:.2f}s"
         )
+        return None
     except Exception as e:
+        # Attempt to mark the file as failed
         with get_session_with_current_tenant() as db_session:
             uf = db_session.get(UserFile, _as_uuid(user_file_id))
             if uf:
+                # don't update the status if the user file is being deleted
                 if uf.status != UserFileStatus.DELETING:
                     uf.status = UserFileStatus.FAILED
                 db_session.add(uf)
                 db_session.commit()
 
         task_logger.exception(
-            f"process_user_file_impl - Error processing file id={user_file_id} - {e.__class__.__name__}"
+            f"process_single_user_file - Error processing file id={user_file_id} - {e.__class__.__name__}"
         )
+        return None
     finally:
-        if file_lock is not None and file_lock.owned():
+        if file_lock.owned():
             file_lock.release()
 
 
-@shared_task(
-    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
-    bind=True,
-    ignore_result=True,
-)
-def process_single_user_file(
-    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
-) -> None:
-    process_user_file_impl(
-        user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True
-    )
-
-
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_USER_FILE_DELETE,
     soft_time_limit=300,
@@ -588,38 +453,36 @@ def check_for_user_file_delete(self: Task, *, tenant_id: str) -> None:
     return None
 
 
-def delete_user_file_impl(
-    *, user_file_id: str, tenant_id: str, redis_locking: bool
+@shared_task(
+    name=OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file_delete(
+    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
 ) -> None:
-    """Core implementation for deleting a single user file.
-
-    When redis_locking=True, acquires a per-file Redis lock (Celery path).
-    When redis_locking=False, skips Redis operations (BackgroundTask path).
-    """
-    task_logger.info(f"delete_user_file_impl - Starting id={user_file_id}")
-
-    file_lock: RedisLock | None = None
-    if redis_locking:
-        redis_client = get_redis_client(tenant_id=tenant_id)
-        file_lock = redis_client.lock(
-            _user_file_delete_lock_key(user_file_id),
-            timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
+    """Process a single user file delete."""
+    task_logger.info(f"process_single_user_file_delete - Starting id={user_file_id}")
+    redis_client = get_redis_client(tenant_id=tenant_id)
+    file_lock: RedisLock = redis_client.lock(
+        _user_file_delete_lock_key(user_file_id),
+        timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
+    )
+    if not file_lock.acquire(blocking=False):
+        task_logger.info(
+            f"process_single_user_file_delete - Lock held, skipping user_file_id={user_file_id}"
         )
-        if file_lock is not None and not file_lock.acquire(blocking=False):
-            task_logger.info(
-                f"delete_user_file_impl - Lock held, skipping user_file_id={user_file_id}"
-            )
-            return
-
+        return None
     try:
         with get_session_with_current_tenant() as db_session:
             user_file = db_session.get(UserFile, _as_uuid(user_file_id))
             if not user_file:
                 task_logger.info(
-                    f"delete_user_file_impl - User file not found id={user_file_id}"
+                    f"process_single_user_file_delete - User file not found id={user_file_id}"
                 )
-                return
+                return None
 
+            # 1) Delete vector DB chunks (skip when disabled)
             if not DISABLE_VECTOR_DB:
                 if MANAGED_VESPA:
                     httpx_init_vespa_pool(
@@ -657,6 +520,7 @@ def delete_user_file_impl(
                         chunk_count=chunk_count,
                     )
 
+            # 2) Delete the user-uploaded file content from filestore (blob + metadata)
             file_store = get_default_file_store()
             try:
                 file_store.delete_file(user_file.file_id)
@@ -664,33 +528,26 @@ def delete_user_file_impl(
                     user_file_id_to_plaintext_file_name(user_file.id)
                 )
             except Exception as e:
+                # This block executed only if the file is not found in the filestore
                 task_logger.exception(
-                    f"delete_user_file_impl - Error deleting file id={user_file.id} - {e.__class__.__name__}"
+                    f"process_single_user_file_delete - Error deleting file id={user_file.id} - {e.__class__.__name__}"
                 )
 
+            # 3) Finally, delete the UserFile row
             db_session.delete(user_file)
             db_session.commit()
-            task_logger.info(f"delete_user_file_impl - Completed id={user_file_id}")
+            task_logger.info(
+                f"process_single_user_file_delete - Completed id={user_file_id}"
+            )
     except Exception as e:
         task_logger.exception(
-            f"delete_user_file_impl - Error processing file id={user_file_id} - {e.__class__.__name__}"
+            f"process_single_user_file_delete - Error processing file id={user_file_id} - {e.__class__.__name__}"
         )
+        return None
     finally:
-        if file_lock is not None and file_lock.owned():
+        if file_lock.owned():
             file_lock.release()
-
-
-@shared_task(
-    name=OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
-    bind=True,
-    ignore_result=True,
-)
-def process_single_user_file_delete(
-    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
-) -> None:
-    delete_user_file_impl(
-        user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True
-    )
+    return None
 
 
 @shared_task(
@@ -700,8 +557,8 @@ def process_single_user_file_delete(
     ignore_result=True,
 )
 def check_for_user_file_project_sync(self: Task, *, tenant_id: str) -> None:
-    """Scan for user files needing project sync and enqueue per-file tasks."""
-    task_logger.info("Starting")
+    """Scan for user files with PROJECT_SYNC status and enqueue per-file tasks."""
+    task_logger.info("check_for_user_file_project_sync - Starting")
 
     redis_client = get_redis_client(tenant_id=tenant_id)
     lock: RedisLock = redis_client.lock(
@@ -713,25 +570,13 @@ def check_for_user_file_project_sync(self: Task, *, tenant_id: str) -> None:
         return None
 
     enqueued = 0
-    skipped_guard = 0
     try:
-        queue_depth = get_user_file_project_sync_queue_depth(self.app)
-        if queue_depth > USER_FILE_PROJECT_SYNC_MAX_QUEUE_DEPTH:
-            task_logger.warning(
-                f"Queue depth {queue_depth} exceeds "
-                f"{USER_FILE_PROJECT_SYNC_MAX_QUEUE_DEPTH}, skipping enqueue for tenant={tenant_id}"
-            )
-            return None
-
         with get_session_with_current_tenant() as db_session:
             user_file_ids = (
                 db_session.execute(
                     select(UserFile.id).where(
                         sa.and_(
-                            sa.or_(
-                                UserFile.needs_project_sync.is_(True),
-                                UserFile.needs_persona_sync.is_(True),
-                            ),
+                            UserFile.needs_project_sync.is_(True),
                             UserFile.status == UserFileStatus.COMPLETED,
                         )
                     )
@@ -741,65 +586,58 @@ def check_for_user_file_project_sync(self: Task, *, tenant_id: str) -> None:
             )
 
             for user_file_id in user_file_ids:
-                if not enqueue_user_file_project_sync_task(
-                    celery_app=self.app,
-                    redis_client=redis_client,
-                    user_file_id=user_file_id,
-                    tenant_id=tenant_id,
+                self.app.send_task(
+                    OnyxCeleryTask.PROCESS_SINGLE_USER_FILE_PROJECT_SYNC,
+                    kwargs={"user_file_id": str(user_file_id), "tenant_id": tenant_id},
+                    queue=OnyxCeleryQueues.USER_FILE_PROJECT_SYNC,
                     priority=OnyxCeleryPriority.HIGH,
-                ):
-                    skipped_guard += 1
-                    continue
+                )
                 enqueued += 1
     finally:
         if lock.owned():
             lock.release()
 
     task_logger.info(
-        f"Enqueued {enqueued} "
-        f"Skipped guard {skipped_guard} tasks for tenant={tenant_id}"
+        f"check_for_user_file_project_sync - Enqueued {enqueued} tasks for tenant={tenant_id}"
     )
     return None
 
 
-def project_sync_user_file_impl(
-    *, user_file_id: str, tenant_id: str, redis_locking: bool
+@shared_task(
+    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE_PROJECT_SYNC,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file_project_sync(
+    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
 ) -> None:
-    """Core implementation for syncing a user file's project/persona metadata.
+    """Process a single user file project sync."""
+    task_logger.info(
+        f"process_single_user_file_project_sync - Starting id={user_file_id}"
+    )
 
-    When redis_locking=True, acquires a per-file Redis lock and clears the
-    queued-key guard (Celery path).  When redis_locking=False, skips Redis
-    operations (BackgroundTask path).
-    """
-    task_logger.info(f"project_sync_user_file_impl - Starting id={user_file_id}")
+    redis_client = get_redis_client(tenant_id=tenant_id)
+    file_lock: RedisLock = redis_client.lock(
+        _user_file_project_sync_lock_key(user_file_id),
+        timeout=CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT,
+    )
 
-    file_lock: RedisLock | None = None
-    if redis_locking:
-        redis_client = get_redis_client(tenant_id=tenant_id)
-        redis_client.delete(_user_file_project_sync_queued_key(user_file_id))
-        file_lock = redis_client.lock(
-            user_file_project_sync_lock_key(user_file_id),
-            timeout=CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT,
+    if not file_lock.acquire(blocking=False):
+        task_logger.info(
+            f"process_single_user_file_project_sync - Lock held, skipping user_file_id={user_file_id}"
         )
-        if file_lock is not None and not file_lock.acquire(blocking=False):
-            task_logger.info(
-                f"project_sync_user_file_impl - Lock held, skipping user_file_id={user_file_id}"
-            )
-            return
+        return None
 
     try:
         with get_session_with_current_tenant() as db_session:
-            user_file = db_session.execute(
-                select(UserFile)
-                .where(UserFile.id == _as_uuid(user_file_id))
-                .options(selectinload(UserFile.assistants))
-            ).scalar_one_or_none()
+            user_file = db_session.get(UserFile, _as_uuid(user_file_id))
             if not user_file:
                 task_logger.info(
-                    f"project_sync_user_file_impl - User file not found id={user_file_id}"
+                    f"process_single_user_file_project_sync - User file not found id={user_file_id}"
                 )
-                return
+                return None
 
+            # Sync project metadata to vector DB (skip when disabled)
             if not DISABLE_VECTOR_DB:
                 if MANAGED_VESPA:
                     httpx_init_vespa_pool(
@@ -820,25 +658,20 @@ def project_sync_user_file_impl(
                 ]
 
                 project_ids = [project.id for project in user_file.projects]
-                persona_ids = [p.id for p in user_file.assistants if not p.deleted]
                 for retry_document_index in retry_document_indices:
                     retry_document_index.update_single(
                         doc_id=str(user_file.id),
                         tenant_id=tenant_id,
                         chunk_count=user_file.chunk_count,
                         fields=None,
-                        user_fields=VespaDocumentUserFields(
-                            user_projects=project_ids,
-                            personas=persona_ids,
-                        ),
+                        user_fields=VespaDocumentUserFields(user_projects=project_ids),
                     )
 
             task_logger.info(
-                f"project_sync_user_file_impl - User file id={user_file_id}"
+                f"process_single_user_file_project_sync - User file id={user_file_id}"
             )
 
             user_file.needs_project_sync = False
-            user_file.needs_persona_sync = False
             user_file.last_project_sync_at = datetime.datetime.now(
                 datetime.timezone.utc
             )
@@ -847,21 +680,11 @@ def project_sync_user_file_impl(
 
     except Exception as e:
         task_logger.exception(
-            f"project_sync_user_file_impl - Error syncing project for file id={user_file_id} - {e.__class__.__name__}"
+            f"process_single_user_file_project_sync - Error syncing project for file id={user_file_id} - {e.__class__.__name__}"
         )
+        return None
     finally:
-        if file_lock is not None and file_lock.owned():
+        if file_lock.owned():
             file_lock.release()
 
-
-@shared_task(
-    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE_PROJECT_SYNC,
-    bind=True,
-    ignore_result=True,
-)
-def process_single_user_file_project_sync(
-    self: Task, *, user_file_id: str, tenant_id: str  # noqa: ARG001
-) -> None:
-    project_sync_user_file_impl(
-        user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True
-    )
+    return None

backend/onyx/background/indexing/run_docfetching.py

@@ -58,8 +58,6 @@ from onyx.file_store.document_batch_storage import DocumentBatchStorage
 from onyx.file_store.document_batch_storage import get_document_batch_storage
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.indexing.indexing_pipeline import index_doc_batch_prepare
-from onyx.indexing.postgres_sanitization import sanitize_document_for_postgres
-from onyx.indexing.postgres_sanitization import sanitize_hierarchy_nodes_for_postgres
 from onyx.redis.redis_hierarchy import cache_hierarchy_nodes_batch
 from onyx.redis.redis_hierarchy import ensure_source_node_exists
 from onyx.redis.redis_hierarchy import get_node_id_from_raw_id
@@ -158,7 +156,36 @@ def strip_null_characters(doc_batch: list[Document]) -> list[Document]:
             logger.warning(
                 f"doc {doc.id} too large, Document size: {sys.getsizeof(doc)}"
             )
-        cleaned_batch.append(sanitize_document_for_postgres(doc))
+        cleaned_doc = doc.model_copy()
+
+        # Postgres cannot handle NUL characters in text fields
+        if "\x00" in cleaned_doc.id:
+            logger.warning(f"NUL characters found in document ID: {cleaned_doc.id}")
+            cleaned_doc.id = cleaned_doc.id.replace("\x00", "")
+
+        if cleaned_doc.title and "\x00" in cleaned_doc.title:
+            logger.warning(
+                f"NUL characters found in document title: {cleaned_doc.title}"
+            )
+            cleaned_doc.title = cleaned_doc.title.replace("\x00", "")
+
+        if "\x00" in cleaned_doc.semantic_identifier:
+            logger.warning(
+                f"NUL characters found in document semantic identifier: {cleaned_doc.semantic_identifier}"
+            )
+            cleaned_doc.semantic_identifier = cleaned_doc.semantic_identifier.replace(
+                "\x00", ""
+            )
+
+        for section in cleaned_doc.sections:
+            if section.link is not None:
+                section.link = section.link.replace("\x00", "")
+
+            # since text can be longer, just replace to avoid double scan
+            if isinstance(section, TextSection) and section.text is not None:
+                section.text = section.text.replace("\x00", "")
+
+        cleaned_batch.append(cleaned_doc)
 
     return cleaned_batch
 
@@ -575,13 +602,10 @@ def connector_document_extraction(
 
                 # Process hierarchy nodes batch - upsert to Postgres and cache in Redis
                 if hierarchy_node_batch:
-                    hierarchy_node_batch_cleaned = (
-                        sanitize_hierarchy_nodes_for_postgres(hierarchy_node_batch)
-                    )
                     with get_session_with_current_tenant() as db_session:
                         upserted_nodes = upsert_hierarchy_nodes_batch(
                             db_session=db_session,
-                            nodes=hierarchy_node_batch_cleaned,
+                            nodes=hierarchy_node_batch,
                             source=db_connector.source,
                             commit=True,
                             is_connector_public=is_connector_public,
@@ -600,7 +624,7 @@ def connector_document_extraction(
                         )
 
                     logger.debug(
-                        f"Persisted and cached {len(hierarchy_node_batch_cleaned)} hierarchy nodes "
+                        f"Persisted and cached {len(hierarchy_node_batch)} hierarchy nodes "
                         f"for attempt={index_attempt_id}"
                     )
 

backend/onyx/background/periodic_poller.py

@@ -1,287 +0,0 @@
-"""Periodic poller for NO_VECTOR_DB deployments.
-
-Replaces Celery Beat and background workers with a lightweight daemon thread
-that runs from the API server process.  Two responsibilities:
-
-1. Recovery polling (every 30 s): re-processes user files stuck in
-   PROCESSING / DELETING / needs_sync states via the drain loops defined
-   in ``task_utils.py``.
-
-2. Periodic task execution (configurable intervals): runs LLM model updates
-   and scheduled evals at their configured cadences, with Postgres advisory
-   lock deduplication across multiple API server instances.
-"""
-
-import threading
-import time
-from collections.abc import Callable
-from dataclasses import dataclass
-from dataclasses import field
-
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-RECOVERY_INTERVAL_SECONDS = 30
-PERIODIC_TASK_LOCK_BASE = 20_000
-PERIODIC_TASK_KV_PREFIX = "periodic_poller:last_claimed:"
-
-
-# ------------------------------------------------------------------
-# Periodic task definitions
-# ------------------------------------------------------------------
-
-
-_NEVER_RAN: float = -1e18
-
-
-@dataclass
-class _PeriodicTaskDef:
-    name: str
-    interval_seconds: float
-    lock_id: int
-    run_fn: Callable[[], None]
-    last_run_at: float = field(default=_NEVER_RAN)
-
-
-def _run_auto_llm_update() -> None:
-    from onyx.configs.app_configs import AUTO_LLM_CONFIG_URL
-
-    if not AUTO_LLM_CONFIG_URL:
-        return
-
-    from onyx.db.engine.sql_engine import get_session_with_current_tenant
-    from onyx.llm.well_known_providers.auto_update_service import (
-        sync_llm_models_from_github,
-    )
-
-    with get_session_with_current_tenant() as db_session:
-        sync_llm_models_from_github(db_session)
-
-
-def _run_scheduled_eval() -> None:
-    from onyx.configs.app_configs import BRAINTRUST_API_KEY
-    from onyx.configs.app_configs import SCHEDULED_EVAL_DATASET_NAMES
-    from onyx.configs.app_configs import SCHEDULED_EVAL_PERMISSIONS_EMAIL
-    from onyx.configs.app_configs import SCHEDULED_EVAL_PROJECT
-
-    if not all(
-        [
-            BRAINTRUST_API_KEY,
-            SCHEDULED_EVAL_PROJECT,
-            SCHEDULED_EVAL_DATASET_NAMES,
-            SCHEDULED_EVAL_PERMISSIONS_EMAIL,
-        ]
-    ):
-        return
-
-    from datetime import datetime
-    from datetime import timezone
-
-    from onyx.evals.eval import run_eval
-    from onyx.evals.models import EvalConfigurationOptions
-
-    run_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-    for dataset_name in SCHEDULED_EVAL_DATASET_NAMES:
-        try:
-            run_eval(
-                configuration=EvalConfigurationOptions(
-                    search_permissions_email=SCHEDULED_EVAL_PERMISSIONS_EMAIL,
-                    dataset_name=dataset_name,
-                    no_send_logs=False,
-                    braintrust_project=SCHEDULED_EVAL_PROJECT,
-                    experiment_name=f"{dataset_name} - {run_timestamp}",
-                ),
-                remote_dataset_name=dataset_name,
-            )
-        except Exception:
-            logger.exception(
-                f"Periodic poller - Failed scheduled eval for dataset {dataset_name}"
-            )
-
-
-def _build_periodic_tasks() -> list[_PeriodicTaskDef]:
-    from onyx.configs.app_configs import AUTO_LLM_CONFIG_URL
-    from onyx.configs.app_configs import AUTO_LLM_UPDATE_INTERVAL_SECONDS
-    from onyx.configs.app_configs import SCHEDULED_EVAL_DATASET_NAMES
-
-    tasks: list[_PeriodicTaskDef] = []
-    if AUTO_LLM_CONFIG_URL:
-        tasks.append(
-            _PeriodicTaskDef(
-                name="auto-llm-update",
-                interval_seconds=AUTO_LLM_UPDATE_INTERVAL_SECONDS,
-                lock_id=PERIODIC_TASK_LOCK_BASE,
-                run_fn=_run_auto_llm_update,
-            )
-        )
-    if SCHEDULED_EVAL_DATASET_NAMES:
-        tasks.append(
-            _PeriodicTaskDef(
-                name="scheduled-eval",
-                interval_seconds=7 * 24 * 3600,
-                lock_id=PERIODIC_TASK_LOCK_BASE + 1,
-                run_fn=_run_scheduled_eval,
-            )
-        )
-    return tasks
-
-
-# ------------------------------------------------------------------
-# Periodic task runner with advisory-lock-guarded claim
-# ------------------------------------------------------------------
-
-
-def _try_claim_task(task_def: _PeriodicTaskDef) -> bool:
-    """Atomically check whether *task_def* should run and record a claim.
-
-    Uses a transaction-scoped advisory lock for atomicity combined with a
-    ``KVStore`` timestamp for cross-instance dedup.  The DB session is held
-    only for this brief claim transaction, not during task execution.
-    """
-    from datetime import datetime
-    from datetime import timezone
-
-    from sqlalchemy import text
-
-    from onyx.db.engine.sql_engine import get_session_with_current_tenant
-    from onyx.db.models import KVStore
-
-    kv_key = PERIODIC_TASK_KV_PREFIX + task_def.name
-
-    with get_session_with_current_tenant() as db_session:
-        acquired = db_session.execute(
-            text("SELECT pg_try_advisory_xact_lock(:id)"),
-            {"id": task_def.lock_id},
-        ).scalar()
-        if not acquired:
-            return False
-
-        row = db_session.query(KVStore).filter_by(key=kv_key).first()
-        if row and row.value is not None:
-            last_claimed = datetime.fromisoformat(str(row.value))
-            elapsed = (datetime.now(timezone.utc) - last_claimed).total_seconds()
-            if elapsed < task_def.interval_seconds:
-                return False
-
-        now_ts = datetime.now(timezone.utc).isoformat()
-        if row:
-            row.value = now_ts
-        else:
-            db_session.add(KVStore(key=kv_key, value=now_ts))
-        db_session.commit()
-
-    return True
-
-
-def _try_run_periodic_task(task_def: _PeriodicTaskDef) -> None:
-    """Run *task_def* if its interval has elapsed and no peer holds the lock."""
-    now = time.monotonic()
-    if now - task_def.last_run_at < task_def.interval_seconds:
-        return
-
-    if not _try_claim_task(task_def):
-        return
-
-    try:
-        task_def.run_fn()
-        task_def.last_run_at = now
-    except Exception:
-        logger.exception(
-            f"Periodic poller - Error running periodic task {task_def.name}"
-        )
-
-
-# ------------------------------------------------------------------
-# Recovery / drain loop runner
-# ------------------------------------------------------------------
-
-
-def _run_drain_loops(tenant_id: str) -> None:
-    from onyx.background.task_utils import drain_delete_loop
-    from onyx.background.task_utils import drain_processing_loop
-    from onyx.background.task_utils import drain_project_sync_loop
-
-    drain_processing_loop(tenant_id)
-    drain_delete_loop(tenant_id)
-    drain_project_sync_loop(tenant_id)
-
-
-# ------------------------------------------------------------------
-# Startup recovery (10g)
-# ------------------------------------------------------------------
-
-
-def recover_stuck_user_files(tenant_id: str) -> None:
-    """Run all drain loops once to re-process files left in intermediate states.
-
-    Called from ``lifespan()`` on startup when ``DISABLE_VECTOR_DB`` is set.
-    """
-    logger.info("recover_stuck_user_files - Checking for stuck user files")
-    try:
-        _run_drain_loops(tenant_id)
-    except Exception:
-        logger.exception("recover_stuck_user_files - Error during recovery")
-
-
-# ------------------------------------------------------------------
-# Daemon thread (10f)
-# ------------------------------------------------------------------
-
-_shutdown_event = threading.Event()
-_poller_thread: threading.Thread | None = None
-
-
-def _poller_loop(tenant_id: str) -> None:
-    from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
-
-    CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
-
-    periodic_tasks = _build_periodic_tasks()
-    logger.info(
-        f"Periodic poller started with {len(periodic_tasks)} periodic task(s): "
-        f"{[t.name for t in periodic_tasks]}"
-    )
-
-    while not _shutdown_event.is_set():
-        try:
-            _run_drain_loops(tenant_id)
-        except Exception:
-            logger.exception("Periodic poller - Error in recovery polling")
-
-        for task_def in periodic_tasks:
-            try:
-                _try_run_periodic_task(task_def)
-            except Exception:
-                logger.exception(
-                    f"Periodic poller - Unhandled error checking task {task_def.name}"
-                )
-
-        _shutdown_event.wait(RECOVERY_INTERVAL_SECONDS)
-
-
-def start_periodic_poller(tenant_id: str) -> None:
-    """Start the periodic poller daemon thread."""
-    global _poller_thread  # noqa: PLW0603
-    _shutdown_event.clear()
-    _poller_thread = threading.Thread(
-        target=_poller_loop,
-        args=(tenant_id,),
-        daemon=True,
-        name="no-vectordb-periodic-poller",
-    )
-    _poller_thread.start()
-    logger.info("Periodic poller thread started")
-
-
-def stop_periodic_poller() -> None:
-    """Signal the periodic poller to stop and wait for it to exit."""
-    global _poller_thread  # noqa: PLW0603
-    if _poller_thread is None:
-        return
-    _shutdown_event.set()
-    _poller_thread.join(timeout=10)
-    if _poller_thread.is_alive():
-        logger.warning("Periodic poller thread did not stop within timeout")
-    _poller_thread = None
-    logger.info("Periodic poller thread stopped")

backend/onyx/background/task_utils.py

@@ -1,33 +1,3 @@
-"""Background task utilities.
-
-Contains query-history report helpers (used by all deployment modes) and
-in-process background task execution helpers for NO_VECTOR_DB mode:
-
-- Atomic claim-and-mark helpers that prevent duplicate processing
-- Drain loops that process all pending user file work
-
-Each claim function runs a short-lived transaction: SELECT ... FOR UPDATE
-SKIP LOCKED, UPDATE the row to remove it from future queries, COMMIT.
-After the commit the row lock is released, but the row is no longer
-eligible for re-claiming.  No long-lived sessions or advisory locks.
-"""
-
-from uuid import UUID
-
-import sqlalchemy as sa
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from onyx.db.enums import UserFileStatus
-from onyx.db.models import UserFile
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-# ------------------------------------------------------------------
-# Query-history report helpers (pre-existing, used by all modes)
-# ------------------------------------------------------------------
-
 QUERY_REPORT_NAME_PREFIX = "query-history"
 
 
@@ -39,142 +9,3 @@ def construct_query_history_report_name(
 
 def extract_task_id_from_query_history_report_name(name: str) -> str:
     return name.removeprefix(f"{QUERY_REPORT_NAME_PREFIX}-").removesuffix(".csv")
-
-
-# ------------------------------------------------------------------
-# Atomic claim-and-mark helpers
-# ------------------------------------------------------------------
-# Each function runs inside a single short-lived session/transaction:
-#   1. SELECT ... FOR UPDATE SKIP LOCKED  (locks one eligible row)
-#   2. UPDATE the row so it is no longer eligible
-#   3. COMMIT  (releases the row lock)
-# After the commit, no other drain loop can claim the same row.
-
-
-def _claim_next_processing_file(db_session: Session) -> UUID | None:
-    """Claim the next PROCESSING file by transitioning it to INDEXING.
-
-    Returns the file id, or None when no eligible files remain.
-    """
-    file_id = db_session.execute(
-        select(UserFile.id)
-        .where(UserFile.status == UserFileStatus.PROCESSING)
-        .order_by(UserFile.created_at)
-        .limit(1)
-        .with_for_update(skip_locked=True)
-    ).scalar_one_or_none()
-    if file_id is None:
-        return None
-
-    db_session.execute(
-        sa.update(UserFile)
-        .where(UserFile.id == file_id)
-        .values(status=UserFileStatus.INDEXING)
-    )
-    db_session.commit()
-    return file_id
-
-
-def _claim_next_deleting_file(db_session: Session) -> UUID | None:
-    """Claim the next DELETING file.
-
-    No status transition needed — the impl deletes the row on success.
-    The short-lived FOR UPDATE lock prevents concurrent claims.
-    """
-    file_id = db_session.execute(
-        select(UserFile.id)
-        .where(UserFile.status == UserFileStatus.DELETING)
-        .order_by(UserFile.created_at)
-        .limit(1)
-        .with_for_update(skip_locked=True)
-    ).scalar_one_or_none()
-    # Commit to release the row lock promptly.
-    db_session.commit()
-    return file_id
-
-
-def _claim_next_sync_file(db_session: Session) -> UUID | None:
-    """Claim the next file needing project/persona sync.
-
-    No status transition needed — the impl clears the sync flags on
-    success.  The short-lived FOR UPDATE lock prevents concurrent claims.
-    """
-    file_id = db_session.execute(
-        select(UserFile.id)
-        .where(
-            sa.and_(
-                sa.or_(
-                    UserFile.needs_project_sync.is_(True),
-                    UserFile.needs_persona_sync.is_(True),
-                ),
-                UserFile.status == UserFileStatus.COMPLETED,
-            )
-        )
-        .order_by(UserFile.created_at)
-        .limit(1)
-        .with_for_update(skip_locked=True)
-    ).scalar_one_or_none()
-    db_session.commit()
-    return file_id
-
-
-# ------------------------------------------------------------------
-# Drain loops — process *all* pending work of each type
-# ------------------------------------------------------------------
-
-
-def drain_processing_loop(tenant_id: str) -> None:
-    """Process all pending PROCESSING user files."""
-    from onyx.background.celery.tasks.user_file_processing.tasks import (
-        process_user_file_impl,
-    )
-    from onyx.db.engine.sql_engine import get_session_with_current_tenant
-
-    while True:
-        with get_session_with_current_tenant() as session:
-            file_id = _claim_next_processing_file(session)
-        if file_id is None:
-            break
-        process_user_file_impl(
-            user_file_id=str(file_id),
-            tenant_id=tenant_id,
-            redis_locking=False,
-        )
-
-
-def drain_delete_loop(tenant_id: str) -> None:
-    """Delete all pending DELETING user files."""
-    from onyx.background.celery.tasks.user_file_processing.tasks import (
-        delete_user_file_impl,
-    )
-    from onyx.db.engine.sql_engine import get_session_with_current_tenant
-
-    while True:
-        with get_session_with_current_tenant() as session:
-            file_id = _claim_next_deleting_file(session)
-        if file_id is None:
-            break
-        delete_user_file_impl(
-            user_file_id=str(file_id),
-            tenant_id=tenant_id,
-            redis_locking=False,
-        )
-
-
-def drain_project_sync_loop(tenant_id: str) -> None:
-    """Sync all pending project/persona metadata for user files."""
-    from onyx.background.celery.tasks.user_file_processing.tasks import (
-        project_sync_user_file_impl,
-    )
-    from onyx.db.engine.sql_engine import get_session_with_current_tenant
-
-    while True:
-        with get_session_with_current_tenant() as session:
-            file_id = _claim_next_sync_file(session)
-        if file_id is None:
-            break
-        project_sync_user_file_impl(
-            user_file_id=str(file_id),
-            tenant_id=tenant_id,
-            redis_locking=False,
-        )

backend/onyx/cache/factory.py

@@ -1,45 +0,0 @@
-from collections.abc import Callable
-
-from onyx.cache.interface import CacheBackend
-from onyx.cache.interface import CacheBackendType
-from onyx.configs.app_configs import CACHE_BACKEND
-
-
-def _build_redis_backend(tenant_id: str) -> CacheBackend:
-    from onyx.cache.redis_backend import RedisCacheBackend
-    from onyx.redis.redis_pool import redis_pool
-
-    return RedisCacheBackend(redis_pool.get_client(tenant_id))
-
-
-_BACKEND_BUILDERS: dict[CacheBackendType, Callable[[str], CacheBackend]] = {
-    CacheBackendType.REDIS: _build_redis_backend,
-    # CacheBackendType.POSTGRES will be added in a follow-up PR.
-}
-
-
-def get_cache_backend(*, tenant_id: str | None = None) -> CacheBackend:
-    """Return a tenant-aware ``CacheBackend``.
-
-    If *tenant_id* is ``None``, the current tenant is read from the
-    thread-local context variable (same behaviour as ``get_redis_client``).
-    """
-    if tenant_id is None:
-        from shared_configs.contextvars import get_current_tenant_id
-
-        tenant_id = get_current_tenant_id()
-
-    builder = _BACKEND_BUILDERS.get(CACHE_BACKEND)
-    if builder is None:
-        raise ValueError(
-            f"Unsupported CACHE_BACKEND={CACHE_BACKEND!r}. "
-            f"Supported values: {[t.value for t in CacheBackendType]}"
-        )
-    return builder(tenant_id)
-
-
-def get_shared_cache_backend() -> CacheBackend:
-    """Return a ``CacheBackend`` in the shared (cross-tenant) namespace."""
-    from shared_configs.configs import DEFAULT_REDIS_PREFIX
-
-    return get_cache_backend(tenant_id=DEFAULT_REDIS_PREFIX)

backend/onyx/cache/interface.py

@@ -1,89 +0,0 @@
-import abc
-from enum import Enum
-
-
-class CacheBackendType(str, Enum):
-    REDIS = "redis"
-    POSTGRES = "postgres"
-
-
-class CacheLock(abc.ABC):
-    """Abstract distributed lock returned by CacheBackend.lock()."""
-
-    @abc.abstractmethod
-    def acquire(
-        self,
-        blocking: bool = True,
-        blocking_timeout: float | None = None,
-    ) -> bool:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def release(self) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def owned(self) -> bool:
-        raise NotImplementedError
-
-
-class CacheBackend(abc.ABC):
-    """Thin abstraction over a key-value cache with TTL, locks, and blocking lists.
-
-    Covers the subset of Redis operations used outside of Celery. When
-    CACHE_BACKEND=postgres, a PostgreSQL-backed implementation is used instead.
-    """
-
-    # -- basic key/value ---------------------------------------------------
-
-    @abc.abstractmethod
-    def get(self, key: str) -> bytes | None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def set(
-        self,
-        key: str,
-        value: str | bytes | int | float,
-        ex: int | None = None,
-    ) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def delete(self, key: str) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def exists(self, key: str) -> bool:
-        raise NotImplementedError
-
-    # -- TTL ---------------------------------------------------------------
-
-    @abc.abstractmethod
-    def expire(self, key: str, seconds: int) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def ttl(self, key: str) -> int:
-        """Return remaining TTL in seconds. -1 if no expiry, -2 if key missing."""
-        raise NotImplementedError
-
-    # -- distributed lock --------------------------------------------------
-
-    @abc.abstractmethod
-    def lock(self, name: str, timeout: float | None = None) -> CacheLock:
-        raise NotImplementedError
-
-    # -- blocking list (used by MCP OAuth BLPOP pattern) -------------------
-
-    @abc.abstractmethod
-    def rpush(self, key: str, value: str | bytes) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def blpop(self, keys: list[str], timeout: int = 0) -> tuple[bytes, bytes] | None:
-        """Block until a value is available on one of *keys*, or *timeout* expires.
-
-        Returns ``(key, value)`` or ``None`` on timeout.
-        """
-        raise NotImplementedError

backend/onyx/cache/redis_backend.py

@@ -1,92 +0,0 @@
-from typing import cast
-
-from redis.client import Redis
-from redis.lock import Lock as RedisLock
-
-from onyx.cache.interface import CacheBackend
-from onyx.cache.interface import CacheLock
-
-
-class RedisCacheLock(CacheLock):
-    """Wraps ``redis.lock.Lock`` behind the ``CacheLock`` interface."""
-
-    def __init__(self, lock: RedisLock) -> None:
-        self._lock = lock
-
-    def acquire(
-        self,
-        blocking: bool = True,
-        blocking_timeout: float | None = None,
-    ) -> bool:
-        return bool(
-            self._lock.acquire(
-                blocking=blocking,
-                blocking_timeout=blocking_timeout,
-            )
-        )
-
-    def release(self) -> None:
-        self._lock.release()
-
-    def owned(self) -> bool:
-        return bool(self._lock.owned())
-
-
-class RedisCacheBackend(CacheBackend):
-    """``CacheBackend`` implementation that delegates to a ``redis.Redis`` client.
-
-    This is a thin pass-through — every method maps 1-to-1 to the underlying
-    Redis command.  ``TenantRedis`` key-prefixing is handled by the client
-    itself (provided by ``get_redis_client``).
-    """
-
-    def __init__(self, redis_client: Redis) -> None:
-        self._r = redis_client
-
-    # -- basic key/value ---------------------------------------------------
-
-    def get(self, key: str) -> bytes | None:
-        val = self._r.get(key)
-        if val is None:
-            return None
-        if isinstance(val, bytes):
-            return val
-        return str(val).encode()
-
-    def set(
-        self,
-        key: str,
-        value: str | bytes | int | float,
-        ex: int | None = None,
-    ) -> None:
-        self._r.set(key, value, ex=ex)
-
-    def delete(self, key: str) -> None:
-        self._r.delete(key)
-
-    def exists(self, key: str) -> bool:
-        return bool(self._r.exists(key))
-
-    # -- TTL ---------------------------------------------------------------
-
-    def expire(self, key: str, seconds: int) -> None:
-        self._r.expire(key, seconds)
-
-    def ttl(self, key: str) -> int:
-        return cast(int, self._r.ttl(key))
-
-    # -- distributed lock --------------------------------------------------
-
-    def lock(self, name: str, timeout: float | None = None) -> CacheLock:
-        return RedisCacheLock(self._r.lock(name, timeout=timeout))
-
-    # -- blocking list (MCP OAuth BLPOP pattern) ---------------------------
-
-    def rpush(self, key: str, value: str | bytes) -> None:
-        self._r.rpush(key, value)
-
-    def blpop(self, keys: list[str], timeout: int = 0) -> tuple[bytes, bytes] | None:
-        result = cast(list[bytes] | None, self._r.blpop(keys, timeout=timeout))
-        if result is None:
-            return None
-        return (result[0], result[1])

backend/onyx/chat/chat_state.py

@@ -3,6 +3,7 @@ import time
 from collections.abc import Callable
 from collections.abc import Generator
 from queue import Empty
+from typing import Any
 
 from onyx.chat.citation_processor import CitationMapping
 from onyx.chat.emitter import Emitter
@@ -162,11 +163,13 @@ class ChatStateContainer:
 
 
 def run_chat_loop_with_state_containers(
-    chat_loop_func: Callable[[Emitter, ChatStateContainer], None],
+    func: Callable[..., None],
     completion_callback: Callable[[ChatStateContainer], None],
     is_connected: Callable[[], bool],
     emitter: Emitter,
     state_container: ChatStateContainer,
+    *args: Any,
+    **kwargs: Any,
 ) -> Generator[Packet, None]:
     """
     Explicit wrapper function that runs a function in a background thread
@@ -177,18 +180,19 @@ def run_chat_loop_with_state_containers(
 
     Args:
         func: The function to wrap (should accept emitter and state_container as first and second args)
-        completion_callback: Callback function to call when the function completes
         emitter: Emitter instance for sending packets
         state_container: ChatStateContainer instance for accumulating state
         is_connected: Callable that returns False when stop signal is set
+        *args: Additional positional arguments for func
+        **kwargs: Additional keyword arguments for func
 
     Usage:
         packets = run_chat_loop_with_state_containers(
             my_func,
-            completion_callback=completion_callback,
             emitter=emitter,
             state_container=state_container,
             is_connected=check_func,
+            arg1, arg2, kwarg1=value1
         )
         for packet in packets:
             # Process packets
@@ -197,7 +201,9 @@ def run_chat_loop_with_state_containers(
 
     def run_with_exception_capture() -> None:
         try:
-            chat_loop_func(emitter, state_container)
+            # Ensure state_container is passed explicitly, removing it from kwargs if present
+            kwargs_with_state = {**kwargs, "state_container": state_container}
+            func(emitter, *args, **kwargs_with_state)
         except Exception as e:
             # If execution fails, emit an exception packet
             emitter.emit(

backend/onyx/chat/chat_utils.py

@@ -1,4 +1,3 @@
-import json
 import re
 from collections.abc import Callable
 from typing import cast
@@ -46,7 +45,6 @@ from onyx.utils.timing import log_function_time
 
 
 logger = setup_logger()
-IMAGE_GENERATION_TOOL_NAME = "generate_image"
 
 
 def create_chat_session_from_request(
@@ -424,44 +422,10 @@ def convert_chat_history_basic(
     return list(reversed(trimmed_reversed))
 
 
-def _build_tool_call_response_history_message(
-    tool_name: str,
-    generated_images: list[dict] | None,
-    tool_call_response: str | None,
-) -> str:
-    if tool_name != IMAGE_GENERATION_TOOL_NAME:
-        return TOOL_CALL_RESPONSE_CROSS_MESSAGE
-
-    if generated_images:
-        llm_image_context: list[dict[str, str]] = []
-        for image in generated_images:
-            file_id = image.get("file_id")
-            revised_prompt = image.get("revised_prompt")
-            if not isinstance(file_id, str):
-                continue
-
-            llm_image_context.append(
-                {
-                    "file_id": file_id,
-                    "revised_prompt": (
-                        revised_prompt if isinstance(revised_prompt, str) else ""
-                    ),
-                }
-            )
-
-        if llm_image_context:
-            return json.dumps(llm_image_context)
-
-    if tool_call_response:
-        return tool_call_response
-
-    return TOOL_CALL_RESPONSE_CROSS_MESSAGE
-
-
 def convert_chat_history(
     chat_history: list[ChatMessage],
     files: list[ChatLoadedFile],
-    context_image_files: list[ChatLoadedFile],
+    project_image_files: list[ChatLoadedFile],
     additional_context: str | None,
     token_counter: Callable[[str], int],
     tool_id_to_name_map: dict[int, str],
@@ -541,11 +505,11 @@ def convert_chat_history(
             )
 
             # Add the user message with image files attached
-            # If this is the last USER message, also include context_image_files
-            # Note: context image file tokens are NOT counted in the token count
+            # If this is the last USER message, also include project_image_files
+            # Note: project image file tokens are NOT counted in the token count
             if idx == last_user_message_idx:
-                if context_image_files:
-                    image_files.extend(context_image_files)
+                if project_image_files:
+                    image_files.extend(project_image_files)
 
                 if additional_context:
                     simple_messages.append(
@@ -618,24 +582,10 @@ def convert_chat_history(
 
                     # Add TOOL_CALL_RESPONSE messages for each tool call in this turn
                     for tool_call in turn_tool_calls:
-                        tool_name = tool_id_to_name_map.get(
-                            tool_call.tool_id, "unknown"
-                        )
-                        tool_response_message = (
-                            _build_tool_call_response_history_message(
-                                tool_name=tool_name,
-                                generated_images=tool_call.generated_images,
-                                tool_call_response=tool_call.tool_call_response,
-                            )
-                        )
                         simple_messages.append(
                             ChatMessageSimple(
-                                message=tool_response_message,
-                                token_count=(
-                                    token_counter(tool_response_message)
-                                    if tool_name == IMAGE_GENERATION_TOOL_NAME
-                                    else 20
-                                ),
+                                message=TOOL_CALL_RESPONSE_CROSS_MESSAGE,
+                                token_count=20,  # Tiny overestimate
                                 message_type=MessageType.TOOL_CALL_RESPONSE,
                                 tool_call_id=tool_call.tool_call_id,
                                 image_files=None,

backend/onyx/chat/llm_loop.py

@@ -1,7 +1,6 @@
 import json
 import time
 from collections.abc import Callable
-from typing import Any
 from typing import Literal
 
 from sqlalchemy.orm import Session
@@ -16,10 +15,10 @@ from onyx.chat.emitter import Emitter
 from onyx.chat.llm_step import extract_tool_calls_from_response_text
 from onyx.chat.llm_step import run_llm_step
 from onyx.chat.models import ChatMessageSimple
-from onyx.chat.models import ContextFileMetadata
-from onyx.chat.models import ExtractedContextFiles
+from onyx.chat.models import ExtractedProjectFiles
 from onyx.chat.models import FileToolMetadata
 from onyx.chat.models import LlmStepResult
+from onyx.chat.models import ProjectFileMetadata
 from onyx.chat.models import ToolCallSimple
 from onyx.chat.prompt_utils import build_reminder_message
 from onyx.chat.prompt_utils import build_system_prompt
@@ -31,7 +30,6 @@ from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import MessageType
 from onyx.context.search.models import SearchDoc
 from onyx.context.search.models import SearchDocsResponse
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
 from onyx.db.memory import add_memory
 from onyx.db.memory import update_memory_at_index
 from onyx.db.memory import UserMemoryContext
@@ -204,17 +202,17 @@ def _try_fallback_tool_extraction(
 MAX_LLM_CYCLES = 6
 
 
-def _build_context_file_citation_mapping(
-    file_metadata: list[ContextFileMetadata],
+def _build_project_file_citation_mapping(
+    project_file_metadata: list[ProjectFileMetadata],
     starting_citation_num: int = 1,
 ) -> CitationMapping:
-    """Build citation mapping for context files.
+    """Build citation mapping for project files.
 
-    Converts context file metadata into SearchDoc objects that can be cited.
+    Converts project file metadata into SearchDoc objects that can be cited.
     Citation numbers start from the provided starting number.
 
     Args:
-        file_metadata: List of context file metadata
+        project_file_metadata: List of project file metadata
         starting_citation_num: Starting citation number (default: 1)
 
     Returns:
@@ -222,7 +220,8 @@ def _build_context_file_citation_mapping(
     """
     citation_mapping: CitationMapping = {}
 
-    for idx, file_meta in enumerate(file_metadata, start=starting_citation_num):
+    for idx, file_meta in enumerate(project_file_metadata, start=starting_citation_num):
+        # Create a SearchDoc for each project file
         search_doc = SearchDoc(
             document_id=file_meta.file_id,
             chunk_ind=0,
@@ -242,28 +241,29 @@ def _build_context_file_citation_mapping(
 
 
 def _build_project_message(
-    context_files: ExtractedContextFiles | None,
+    project_files: ExtractedProjectFiles | None,
     token_counter: Callable[[str], int] | None,
 ) -> list[ChatMessageSimple]:
-    """Build messages for context-injected / tool-backed files.
+    """Build messages for project / tool-backed files.
 
     Returns up to two messages:
-    1. The full-text files message (if file_texts is populated).
+    1. The full-text project files message (if project_file_texts is populated).
     2. A lightweight metadata message for files the LLM should access via the
-       FileReaderTool (e.g. oversized files that don't fit in context).
+       FileReaderTool (e.g. oversized chat-attached files or project files that
+       don't fit in context).
     """
-    if not context_files:
+    if not project_files:
         return []
 
     messages: list[ChatMessageSimple] = []
-    if context_files.file_texts:
+    if project_files.project_file_texts:
         messages.append(
-            _create_context_files_message(context_files, token_counter=None)
+            _create_project_files_message(project_files, token_counter=None)
         )
-    if context_files.file_metadata_for_tool and token_counter:
+    if project_files.file_metadata_for_tool and token_counter:
         messages.append(
             _create_file_tool_metadata_message(
-                context_files.file_metadata_for_tool, token_counter
+                project_files.file_metadata_for_tool, token_counter
             )
         )
     return messages
@@ -274,7 +274,7 @@ def construct_message_history(
     custom_agent_prompt: ChatMessageSimple | None,
     simple_chat_history: list[ChatMessageSimple],
     reminder_message: ChatMessageSimple | None,
-    context_files: ExtractedContextFiles | None,
+    project_files: ExtractedProjectFiles | None,
     available_tokens: int,
     last_n_user_messages: int | None = None,
     token_counter: Callable[[str], int] | None = None,
@@ -288,7 +288,7 @@ def construct_message_history(
 
     # Build the project / file-metadata messages up front so we can use their
     # actual token counts for the budget.
-    project_messages = _build_project_message(context_files, token_counter)
+    project_messages = _build_project_message(project_files, token_counter)
     project_messages_tokens = sum(m.token_count for m in project_messages)
 
     history_token_budget = available_tokens
@@ -444,17 +444,17 @@ def construct_message_history(
                     )
 
     # Attach project images to the last user message
-    if context_files and context_files.image_files:
+    if project_files and project_files.project_image_files:
         existing_images = last_user_message.image_files or []
         last_user_message = ChatMessageSimple(
             message=last_user_message.message,
             token_count=last_user_message.token_count,
             message_type=last_user_message.message_type,
-            image_files=existing_images + context_files.image_files,
+            image_files=existing_images + project_files.project_image_files,
         )
 
     # Build the final message list according to README ordering:
-    # [system], [history_before_last_user], [custom_agent], [context_files],
+    # [system], [history_before_last_user], [custom_agent], [project_files],
     # [forgotten_files], [last_user_message], [messages_after_last_user], [reminder]
     result = [system_prompt] if system_prompt else []
 
@@ -465,14 +465,14 @@ def construct_message_history(
     if custom_agent_prompt:
         result.append(custom_agent_prompt)
 
-    # 3. Add context files / file-metadata messages (inserted before last user message)
+    # 3. Add project files / file-metadata messages (inserted before last user message)
     result.extend(project_messages)
 
     # 4. Add forgotten-files metadata (right before the user's question)
     if forgotten_files_message:
         result.append(forgotten_files_message)
 
-    # 5. Add last user message (with context images attached)
+    # 5. Add last user message (with project images attached)
     result.append(last_user_message)
 
     # 6. Add messages after last user message (tool calls, responses, etc.)
@@ -531,13 +531,11 @@ def _create_file_tool_metadata_message(
     """
     lines = [
         "You have access to the following files. Use the read_file tool to "
-        "read sections of any file. You MUST pass the file_id UUID (not the "
-        "filename) to read_file:"
+        "read sections of any file:"
     ]
     for meta in file_metadata:
         lines.append(
-            f'- file_id="{meta.file_id}" filename="{meta.filename}" '
-            f"(~{meta.approx_char_count:,} chars)"
+            f'- {meta.file_id}: "{meta.filename}" (~{meta.approx_char_count:,} chars)'
         )
 
     message_content = "\n".join(lines)
@@ -548,11 +546,11 @@ def _create_file_tool_metadata_message(
     )
 
 
-def _create_context_files_message(
-    context_files: ExtractedContextFiles,
+def _create_project_files_message(
+    project_files: ExtractedProjectFiles,
     token_counter: Callable[[str], int] | None,  # noqa: ARG001
 ) -> ChatMessageSimple:
-    """Convert context files to a ChatMessageSimple message.
+    """Convert project files to a ChatMessageSimple message.
 
     Format follows the README specification for document representation.
     """
@@ -560,25 +558,21 @@ def _create_context_files_message(
 
     # Format as documents JSON as described in README
     documents_list = []
-    for idx, file_text in enumerate(context_files.file_texts, start=1):
-        title = (
-            context_files.file_metadata[idx - 1].filename
-            if idx - 1 < len(context_files.file_metadata)
-            else None
+    for idx, file_text in enumerate(project_files.project_file_texts, start=1):
+        documents_list.append(
+            {
+                "document": idx,
+                "contents": file_text,
+            }
         )
-        entry: dict[str, Any] = {"document": idx}
-        if title:
-            entry["title"] = title
-        entry["contents"] = file_text
-        documents_list.append(entry)
 
     documents_json = json.dumps({"documents": documents_list}, indent=2)
     message_content = f"Here are some documents provided for context, they may not all be relevant:\n{documents_json}"
 
-    # Use pre-calculated token count from context_files
+    # Use pre-calculated token count from project_files
     return ChatMessageSimple(
         message=message_content,
-        token_count=context_files.total_token_count,
+        token_count=project_files.total_token_count,
         message_type=MessageType.USER,
     )
 
@@ -589,7 +583,7 @@ def run_llm_loop(
     simple_chat_history: list[ChatMessageSimple],
     tools: list[Tool],
     custom_agent_prompt: str | None,
-    context_files: ExtractedContextFiles,
+    project_files: ExtractedProjectFiles,
     persona: Persona | None,
     user_memory_context: UserMemoryContext | None,
     llm: LLM,
@@ -632,9 +626,9 @@ def run_llm_loop(
 
         # Add project file citation mappings if project files are present
         project_citation_mapping: CitationMapping = {}
-        if context_files.file_metadata:
-            project_citation_mapping = _build_context_file_citation_mapping(
-                context_files.file_metadata
+        if project_files.project_file_metadata:
+            project_citation_mapping = _build_project_file_citation_mapping(
+                project_files.project_file_metadata
             )
             citation_processor.update_citation_mapping(project_citation_mapping)
 
@@ -652,7 +646,7 @@ def run_llm_loop(
         # TODO allow citing of images in Projects. Since attached to the last user message, it has no text associated with it.
         # One future workaround is to include the images as separate user messages with citation information and process those.
         always_cite_documents: bool = bool(
-            context_files.use_as_search_filter or context_files.file_texts
+            project_files.project_as_filter or project_files.project_file_texts
         )
         should_cite_documents: bool = False
         ran_image_gen: bool = False
@@ -662,12 +656,7 @@ def run_llm_loop(
         fallback_extraction_attempted: bool = False
         citation_mapping: dict[int, str] = {}  # Maps citation_num -> document_id/URL
 
-        # Fetch this in a short-lived session so the long-running stream loop does
-        # not pin a connection just to keep read state alive.
-        with get_session_with_current_tenant() as prompt_db_session:
-            default_base_system_prompt: str = get_default_base_system_prompt(
-                prompt_db_session
-            )
+        default_base_system_prompt: str = get_default_base_system_prompt(db_session)
         system_prompt = None
         custom_agent_prompt_msg = None
 
@@ -793,7 +782,7 @@ def run_llm_loop(
                 custom_agent_prompt=custom_agent_prompt_msg,
                 simple_chat_history=simple_chat_history,
                 reminder_message=reminder_msg,
-                context_files=context_files,
+                project_files=project_files,
                 available_tokens=available_tokens,
                 token_counter=token_counter,
                 all_injected_file_metadata=all_injected_file_metadata,

backend/onyx/chat/models.py

@@ -31,6 +31,13 @@ class CustomToolResponse(BaseModel):
     tool_name: str
 
 
+class ProjectSearchConfig(BaseModel):
+    """Configuration for search tool availability in project context."""
+
+    search_usage: SearchToolUsage
+    disable_forced_tool: bool
+
+
 class CreateChatSessionID(BaseModel):
     chat_session_id: UUID
 
@@ -125,8 +132,8 @@ class ChatMessageSimple(BaseModel):
     file_id: str | None = None
 
 
-class ContextFileMetadata(BaseModel):
-    """Metadata for a context-injected file to enable citation support."""
+class ProjectFileMetadata(BaseModel):
+    """Metadata for a project file to enable citation support."""
 
     file_id: str
     filename: str
@@ -160,28 +167,20 @@ class ChatHistoryResult(BaseModel):
     all_injected_file_metadata: dict[str, FileToolMetadata]
 
 
-class ExtractedContextFiles(BaseModel):
-    """Result of attempting to load user files (from a project or persona) into context."""
-
-    file_texts: list[str]
-    image_files: list[ChatLoadedFile]
-    use_as_search_filter: bool
+class ExtractedProjectFiles(BaseModel):
+    project_file_texts: list[str]
+    project_image_files: list[ChatLoadedFile]
+    project_as_filter: bool
     total_token_count: int
+    # Metadata for project files to enable citations
+    project_file_metadata: list[ProjectFileMetadata]
+    # None if not a project
+    project_uncapped_token_count: int | None
     # Lightweight metadata for files exposed via FileReaderTool
-    # (populated when files don't fit in context and vector DB is disabled).
-    file_metadata: list[ContextFileMetadata]
-    uncapped_token_count: int | None
+    # (populated when files don't fit in context and vector DB is disabled)
     file_metadata_for_tool: list[FileToolMetadata] = []
 
 
-class SearchParams(BaseModel):
-    """Resolved search filter IDs and search-tool usage for a chat turn."""
-
-    search_project_id: int | None
-    search_persona_id: int | None
-    search_usage: SearchToolUsage
-
-
 class LlmStepResult(BaseModel):
     reasoning: str | None
     answer: str | None

backend/onyx/chat/process_message.py

@@ -3,7 +3,6 @@ IMPORTANT: familiarize yourself with the design concepts prior to contributing t
 An overview can be found in the README.md file in this directory.
 """
 
-import io
 import re
 import traceback
 from collections.abc import Callable
@@ -34,11 +33,11 @@ from onyx.chat.models import ChatBasicResponse
 from onyx.chat.models import ChatFullResponse
 from onyx.chat.models import ChatLoadedFile
 from onyx.chat.models import ChatMessageSimple
-from onyx.chat.models import ContextFileMetadata
 from onyx.chat.models import CreateChatSessionID
-from onyx.chat.models import ExtractedContextFiles
+from onyx.chat.models import ExtractedProjectFiles
 from onyx.chat.models import FileToolMetadata
-from onyx.chat.models import SearchParams
+from onyx.chat.models import ProjectFileMetadata
+from onyx.chat.models import ProjectSearchConfig
 from onyx.chat.models import StreamingError
 from onyx.chat.models import ToolCallResponse
 from onyx.chat.prompt_utils import calculate_reserved_tokens
@@ -63,12 +62,11 @@ from onyx.db.models import ChatSession
 from onyx.db.models import Persona
 from onyx.db.models import User
 from onyx.db.models import UserFile
+from onyx.db.projects import get_project_token_count
 from onyx.db.projects import get_user_files_from_project
 from onyx.db.tools import get_tools
 from onyx.deep_research.dr_loop import run_deep_research_llm_loop
-from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_store.models import ChatFileType
-from onyx.file_store.models import InMemoryChatFile
 from onyx.file_store.utils import load_in_memory_chat_files
 from onyx.file_store.utils import verify_user_files
 from onyx.llm.factory import get_llm_for_persona
@@ -141,12 +139,12 @@ def _collect_available_file_ids(
                 pass
 
     if project_id:
-        user_files = get_user_files_from_project(
+        project_files = get_user_files_from_project(
             project_id=project_id,
             user_id=user_id,
             db_session=db_session,
         )
-        for uf in user_files:
+        for uf in project_files:
             user_file_ids.add(uf.id)
 
     return _AvailableFiles(
@@ -194,67 +192,9 @@ def _convert_loaded_files_to_chat_files(
     return chat_files
 
 
-def resolve_context_user_files(
-    persona: Persona,
+def _extract_project_file_texts_and_images(
     project_id: int | None,
     user_id: UUID | None,
-    db_session: Session,
-) -> list[UserFile]:
-    """Apply the precedence rule to decide which user files to load.
-
-    A custom persona fully supersedes the project.  When a chat uses a
-    custom persona, the project is purely organisational — its files are
-    never loaded and never made searchable.
-
-    Custom persona → persona's own user_files (may be empty).
-    Default persona inside a project → project files.
-    Otherwise → empty list.
-    """
-    if persona.id != DEFAULT_PERSONA_ID:
-        return list(persona.user_files) if persona.user_files else []
-    if project_id:
-        return get_user_files_from_project(
-            project_id=project_id,
-            user_id=user_id,
-            db_session=db_session,
-        )
-    return []
-
-
-def _empty_extracted_context_files() -> ExtractedContextFiles:
-    return ExtractedContextFiles(
-        file_texts=[],
-        image_files=[],
-        use_as_search_filter=False,
-        total_token_count=0,
-        file_metadata=[],
-        uncapped_token_count=None,
-    )
-
-
-def _extract_text_from_in_memory_file(f: InMemoryChatFile) -> str | None:
-    """Extract text content from an InMemoryChatFile.
-
-    PLAIN_TEXT: the content is pre-extracted UTF-8 plaintext stored during
-    ingestion — decode directly.
-    DOC / CSV / other text types: the content is the original file bytes —
-    use extract_file_text which handles encoding detection and format parsing.
-    """
-    try:
-        if f.file_type == ChatFileType.PLAIN_TEXT:
-            return f.content.decode("utf-8", errors="ignore").replace("\x00", "")
-        return extract_file_text(
-            file=io.BytesIO(f.content),
-            file_name=f.filename or "",
-            break_on_unprocessable=False,
-        )
-    except Exception:
-        logger.warning(f"Failed to extract text from file {f.file_id}", exc_info=True)
-        return None
-
-
-def extract_context_files(
-    user_files: list[UserFile],
     llm_max_context_window: int,
     reserved_token_count: int,
     db_session: Session,
@@ -263,12 +203,8 @@ def extract_context_files(
     # 60% of the LLM's max context window. The other benefit is that for projects with
     # more files, this makes it so that we don't throw away the history too quickly every time.
     max_llm_context_percentage: float = 0.6,
-) -> ExtractedContextFiles:
-    """Load user files into context if they fit; otherwise flag for search.
-
-    The caller is responsible for deciding *which* user files to pass in
-    (project files, persona files, etc.).  This function only cares about
-    the all-or-nothing fit check and the actual content loading.
+) -> ExtractedProjectFiles:
+    """Extract text content from project files if they fit within the context window.
 
     Args:
         project_id: The project ID to load files from
@@ -277,95 +213,160 @@ def extract_context_files(
         reserved_token_count: Number of tokens to reserve for other content
         db_session: Database session
         max_llm_context_percentage: Maximum percentage of the LLM context window to use.
+
     Returns:
-        ExtractedContextFiles containing:
-        - List of text content strings from context files (text files only)
-        - List of image files from context (ChatLoadedFile objects)
+        ExtractedProjectFiles containing:
+        - List of text content strings from project files (text files only)
+        - List of image files from project (ChatLoadedFile objects)
+        - Project id if the the project should be provided as a filter in search or None if not.
         - Total token count of all extracted files
-        - File metadata for context files
-        - Uncapped token count of all extracted files
-        - File metadata for files that don't fit in context and vector DB is disabled
     """
-    # TODO(yuhong): I believe this is not handling all file types correctly.
+    # TODO I believe this is not handling all file types correctly.
+    project_as_filter = False
+    if not project_id:
+        return ExtractedProjectFiles(
+            project_file_texts=[],
+            project_image_files=[],
+            project_as_filter=False,
+            total_token_count=0,
+            project_file_metadata=[],
+            project_uncapped_token_count=None,
+        )
 
-    if not user_files:
-        return _empty_extracted_context_files()
-
-    aggregate_tokens = sum(uf.token_count or 0 for uf in user_files)
     max_actual_tokens = (
         llm_max_context_window - reserved_token_count
     ) * max_llm_context_percentage
 
-    if aggregate_tokens >= max_actual_tokens:
-        tool_metadata = []
-        use_as_search_filter = not DISABLE_VECTOR_DB
-        if DISABLE_VECTOR_DB:
-            tool_metadata = _build_file_tool_metadata_for_user_files(user_files)
-        return ExtractedContextFiles(
-            file_texts=[],
-            image_files=[],
-            use_as_search_filter=use_as_search_filter,
-            total_token_count=0,
-            file_metadata=[],
-            uncapped_token_count=aggregate_tokens,
-            file_metadata_for_tool=tool_metadata,
-        )
-
-    # Files fit — load them into context
-    user_file_map = {str(uf.id): uf for uf in user_files}
-    in_memory_files = load_in_memory_chat_files(
-        user_file_ids=[uf.id for uf in user_files],
+    # Calculate total token count for all user files in the project
+    project_tokens = get_project_token_count(
+        project_id=project_id,
+        user_id=user_id,
         db_session=db_session,
     )
 
-    file_texts: list[str] = []
-    image_files: list[ChatLoadedFile] = []
-    file_metadata: list[ContextFileMetadata] = []
+    project_file_texts: list[str] = []
+    project_image_files: list[ChatLoadedFile] = []
+    project_file_metadata: list[ProjectFileMetadata] = []
     total_token_count = 0
+    if project_tokens < max_actual_tokens:
+        # Load project files into memory using cached plaintext when available
+        project_user_files = get_user_files_from_project(
+            project_id=project_id,
+            user_id=user_id,
+            db_session=db_session,
+        )
+        if project_user_files:
+            # Create a mapping from file_id to UserFile for token count lookup
+            user_file_map = {str(file.id): file for file in project_user_files}
 
-    for f in in_memory_files:
-        uf = user_file_map.get(str(f.file_id))
-        if f.file_type.is_text_file():
-            text_content = _extract_text_from_in_memory_file(f)
-            if not text_content:
-                continue
-            file_texts.append(text_content)
-            file_metadata.append(
-                ContextFileMetadata(
-                    file_id=str(f.file_id),
-                    filename=f.filename or f"file_{f.file_id}",
-                    file_content=text_content,
-                )
-            )
-            if uf and uf.token_count:
-                total_token_count += uf.token_count
-        elif f.file_type == ChatFileType.IMAGE:
-            token_count = uf.token_count if uf and uf.token_count else 0
-            total_token_count += token_count
-            image_files.append(
-                ChatLoadedFile(
-                    file_id=f.file_id,
-                    content=f.content,
-                    file_type=f.file_type,
-                    filename=f.filename,
-                    content_text=None,
-                    token_count=token_count,
-                )
+            project_file_ids = [file.id for file in project_user_files]
+            in_memory_project_files = load_in_memory_chat_files(
+                user_file_ids=project_file_ids,
+                db_session=db_session,
             )
 
-    return ExtractedContextFiles(
-        file_texts=file_texts,
-        image_files=image_files,
-        use_as_search_filter=False,
+            # Extract text content from loaded files
+            for file in in_memory_project_files:
+                if file.file_type.is_text_file():
+                    try:
+                        text_content = file.content.decode("utf-8", errors="ignore")
+                        # Strip null bytes
+                        text_content = text_content.replace("\x00", "")
+                        if text_content:
+                            project_file_texts.append(text_content)
+                            # Add metadata for citation support
+                            project_file_metadata.append(
+                                ProjectFileMetadata(
+                                    file_id=str(file.file_id),
+                                    filename=file.filename or f"file_{file.file_id}",
+                                    file_content=text_content,
+                                )
+                            )
+                            # Add token count for text file
+                            user_file = user_file_map.get(str(file.file_id))
+                            if user_file and user_file.token_count:
+                                total_token_count += user_file.token_count
+                    except Exception:
+                        # Skip files that can't be decoded
+                        pass
+                elif file.file_type == ChatFileType.IMAGE:
+                    # Convert InMemoryChatFile to ChatLoadedFile
+                    user_file = user_file_map.get(str(file.file_id))
+                    token_count = (
+                        user_file.token_count
+                        if user_file and user_file.token_count
+                        else 0
+                    )
+                    total_token_count += token_count
+                    chat_loaded_file = ChatLoadedFile(
+                        file_id=file.file_id,
+                        content=file.content,
+                        file_type=file.file_type,
+                        filename=file.filename,
+                        content_text=None,  # Images don't have text content
+                        token_count=token_count,
+                    )
+                    project_image_files.append(chat_loaded_file)
+    else:
+        if DISABLE_VECTOR_DB:
+            # Without a vector DB we can't use project-as-filter search.
+            # Instead, build lightweight metadata so the LLM can call the
+            # FileReaderTool to inspect individual files on demand.
+            file_metadata_for_tool = _build_file_tool_metadata_for_project(
+                project_id=project_id,
+                user_id=user_id,
+                db_session=db_session,
+            )
+            return ExtractedProjectFiles(
+                project_file_texts=[],
+                project_image_files=[],
+                project_as_filter=False,
+                total_token_count=0,
+                project_file_metadata=[],
+                project_uncapped_token_count=project_tokens,
+                file_metadata_for_tool=file_metadata_for_tool,
+            )
+        project_as_filter = True
+
+    return ExtractedProjectFiles(
+        project_file_texts=project_file_texts,
+        project_image_files=project_image_files,
+        project_as_filter=project_as_filter,
         total_token_count=total_token_count,
-        file_metadata=file_metadata,
-        uncapped_token_count=aggregate_tokens,
+        project_file_metadata=project_file_metadata,
+        project_uncapped_token_count=project_tokens,
     )
 
 
 APPROX_CHARS_PER_TOKEN = 4
 
 
+def _build_file_tool_metadata_for_project(
+    project_id: int,
+    user_id: UUID | None,
+    db_session: Session,
+) -> list[FileToolMetadata]:
+    """Build lightweight FileToolMetadata for every file in a project.
+
+    Used when files are too large to fit in context and the vector DB is
+    disabled, so the LLM needs to know which files it can read via the
+    FileReaderTool.
+    """
+    project_user_files = get_user_files_from_project(
+        project_id=project_id,
+        user_id=user_id,
+        db_session=db_session,
+    )
+    return [
+        FileToolMetadata(
+            file_id=str(uf.id),
+            filename=uf.name,
+            approx_char_count=(uf.token_count or 0) * APPROX_CHARS_PER_TOKEN,
+        )
+        for uf in project_user_files
+    ]
+
+
 def _build_file_tool_metadata_for_user_files(
     user_files: list[UserFile],
 ) -> list[FileToolMetadata]:
@@ -380,46 +381,55 @@ def _build_file_tool_metadata_for_user_files(
     ]
 
 
-def determine_search_params(
-    persona_id: int,
+def _get_project_search_availability(
     project_id: int | None,
-    extracted_context_files: ExtractedContextFiles,
-) -> SearchParams:
-    """Decide which search filter IDs and search-tool usage apply for a chat turn.
+    persona_id: int | None,
+    loaded_project_files: bool,
+    project_has_files: bool,
+    forced_tool_id: int | None,
+    search_tool_id: int | None,
+) -> ProjectSearchConfig:
+    """Determine search tool availability based on project context.
 
-    A custom persona fully supersedes the project — project files are never
-    searchable and the search tool config is entirely controlled by the
-    persona.  The project_id filter is only set for the default persona.
+    Search is disabled when ALL of the following are true:
+    - User is in a project
+    - Using the default persona (not a custom agent)
+    - Project files are already loaded in context
 
-    For the default persona inside a project:
-      - Files overflow  → ENABLED  (vector DB scopes to these files)
-      - Files fit       → DISABLED (content already in prompt)
-      - No files at all → DISABLED (nothing to search)
+    When search is disabled and the user tried to force the search tool,
+    that forcing is also disabled.
+
+    Returns AUTO (follow persona config) in all other cases.
     """
-    is_custom_persona = persona_id != DEFAULT_PERSONA_ID
+    # Not in a project, this should have no impact on search tool availability
+    if not project_id:
+        return ProjectSearchConfig(
+            search_usage=SearchToolUsage.AUTO, disable_forced_tool=False
+        )
 
-    search_project_id: int | None = None
-    search_persona_id: int | None = None
-    if extracted_context_files.use_as_search_filter:
-        if is_custom_persona:
-            search_persona_id = persona_id
-        else:
-            search_project_id = project_id
+    # Custom persona in project - let persona config decide
+    # Even if there are no files in the project, it's still guided by the persona config.
+    if persona_id != DEFAULT_PERSONA_ID:
+        return ProjectSearchConfig(
+            search_usage=SearchToolUsage.AUTO, disable_forced_tool=False
+        )
 
-    search_usage = SearchToolUsage.AUTO
-    if not is_custom_persona and project_id:
-        has_context_files = bool(extracted_context_files.uncapped_token_count)
-        files_loaded_in_context = bool(extracted_context_files.file_texts)
+    # If in a project with the default persona and the files have been already loaded into the context or
+    # there are no files in the project, disable search as there is nothing to search for.
+    if loaded_project_files or not project_has_files:
+        user_forced_search = (
+            forced_tool_id is not None
+            and search_tool_id is not None
+            and forced_tool_id == search_tool_id
+        )
+        return ProjectSearchConfig(
+            search_usage=SearchToolUsage.DISABLED,
+            disable_forced_tool=user_forced_search,
+        )
 
-        if extracted_context_files.use_as_search_filter:
-            search_usage = SearchToolUsage.ENABLED
-        elif files_loaded_in_context or not has_context_files:
-            search_usage = SearchToolUsage.DISABLED
-
-    return SearchParams(
-        search_project_id=search_project_id,
-        search_persona_id=search_persona_id,
-        search_usage=search_usage,
+    # Default persona in a project with files, but also the files have not been loaded into the context already.
+    return ProjectSearchConfig(
+        search_usage=SearchToolUsage.ENABLED, disable_forced_tool=False
     )
 
 
@@ -651,37 +661,26 @@ def handle_stream_message_objects(
             user_memory_context=prompt_memory_context,
         )
 
-        # Determine which user files to use.  A custom persona fully
-        # supersedes the project — project files are never loaded or
-        # searchable when a custom persona is in play.  Only the default
-        # persona inside a project uses the project's files.
-        context_user_files = resolve_context_user_files(
-            persona=persona,
+        # Process projects, if all of the files fit in the context, it doesn't need to use RAG
+        extracted_project_files = _extract_project_file_texts_and_images(
             project_id=chat_session.project_id,
             user_id=user_id,
-            db_session=db_session,
-        )
-
-        extracted_context_files = extract_context_files(
-            user_files=context_user_files,
             llm_max_context_window=llm.config.max_input_tokens,
             reserved_token_count=reserved_token_count,
             db_session=db_session,
         )
 
-        search_params = determine_search_params(
-            persona_id=persona.id,
-            project_id=chat_session.project_id,
-            extracted_context_files=extracted_context_files,
-        )
-
-        # Also grant access to persona-attached user files for FileReaderTool
-        if persona.user_files:
-            existing = set(available_files.user_file_ids)
-            for uf in persona.user_files:
-                if uf.id not in existing:
-                    available_files.user_file_ids.append(uf.id)
+        # When the vector DB is disabled, persona-attached user_files have no
+        # search pipeline path. Inject them as file_metadata_for_tool so the
+        # LLM can read them via the FileReaderTool.
+        if DISABLE_VECTOR_DB and persona.user_files:
+            persona_file_metadata = _build_file_tool_metadata_for_user_files(
+                persona.user_files
+            )
+            # Merge persona file metadata into the extracted project files
+            extracted_project_files.file_metadata_for_tool.extend(persona_file_metadata)
 
+        # Build a mapping of tool_id to tool_name for history reconstruction
         all_tools = get_tools(db_session)
         tool_id_to_name_map = {tool.id: tool.name for tool in all_tools}
 
@@ -690,17 +689,30 @@ def handle_stream_message_objects(
             None,
         )
 
+        # Determine if search should be disabled for this project context
         forced_tool_id = new_msg_req.forced_tool_id
-        if (
-            search_params.search_usage == SearchToolUsage.DISABLED
-            and forced_tool_id is not None
-            and search_tool_id is not None
-            and forced_tool_id == search_tool_id
-        ):
+        project_search_config = _get_project_search_availability(
+            project_id=chat_session.project_id,
+            persona_id=persona.id,
+            loaded_project_files=bool(extracted_project_files.project_file_texts),
+            project_has_files=bool(
+                extracted_project_files.project_uncapped_token_count
+            ),
+            forced_tool_id=new_msg_req.forced_tool_id,
+            search_tool_id=search_tool_id,
+        )
+        if project_search_config.disable_forced_tool:
             forced_tool_id = None
 
         emitter = get_default_emitter()
 
+        # Also grant access to persona-attached user files
+        if persona.user_files:
+            existing = set(available_files.user_file_ids)
+            for uf in persona.user_files:
+                if uf.id not in existing:
+                    available_files.user_file_ids.append(uf.id)
+
         # Construct tools based on the persona configurations
         tool_dict = construct_tools(
             persona=persona,
@@ -710,8 +722,11 @@ def handle_stream_message_objects(
             llm=llm,
             search_tool_config=SearchToolConfig(
                 user_selected_filters=new_msg_req.internal_search_filters,
-                project_id=search_params.search_project_id,
-                persona_id=search_params.search_persona_id,
+                project_id=(
+                    chat_session.project_id
+                    if extracted_project_files.project_as_filter
+                    else None
+                ),
                 bypass_acl=bypass_acl,
                 slack_context=slack_context,
                 enable_slack_search=_should_enable_slack_search(
@@ -729,7 +744,7 @@ def handle_stream_message_objects(
                 chat_file_ids=available_files.chat_file_ids,
             ),
             allowed_tool_ids=new_msg_req.allowed_tool_ids,
-            search_usage_forcing_setting=search_params.search_usage,
+            search_usage_forcing_setting=project_search_config.search_usage,
         )
         tools: list[Tool] = []
         for tool_list in tool_dict.values():
@@ -768,7 +783,7 @@ def handle_stream_message_objects(
         chat_history_result = convert_chat_history(
             chat_history=chat_history,
             files=files,
-            context_image_files=extracted_context_files.image_files,
+            project_image_files=extracted_project_files.project_image_files,
             additional_context=additional_context,
             token_counter=token_counter,
             tool_id_to_name_map=tool_id_to_name_map,
@@ -841,11 +856,6 @@ def handle_stream_message_objects(
                 reserved_tokens=reserved_token_count,
             )
 
-        # Release any read transaction before entering the long-running LLM stream.
-        # Without this, the request-scoped session can keep a connection checked out
-        # for the full stream duration.
-        db_session.commit()
-
         # The stream generator can resume on a different worker thread after early yields.
         # Set this right before launching the LLM loop so run_in_background copies the right context.
         if new_msg_req.mock_llm_response is not None:
@@ -864,54 +874,46 @@ def handle_stream_message_objects(
             # (user has already responded to a clarification question)
             skip_clarification = is_last_assistant_message_clarification(chat_history)
 
-            # NOTE: we _could_ pass in a zero argument function since emitter and state_container
-            # are just passed in immediately anyways, but the abstraction is cleaner this way.
             yield from run_chat_loop_with_state_containers(
-                lambda emitter, state_container: run_deep_research_llm_loop(
-                    emitter=emitter,
-                    state_container=state_container,
-                    simple_chat_history=simple_chat_history,
-                    tools=tools,
-                    custom_agent_prompt=custom_agent_prompt,
-                    llm=llm,
-                    token_counter=token_counter,
-                    db_session=db_session,
-                    skip_clarification=skip_clarification,
-                    user_identity=user_identity,
-                    chat_session_id=str(chat_session.id),
-                    all_injected_file_metadata=all_injected_file_metadata,
-                ),
+                run_deep_research_llm_loop,
                 llm_loop_completion_callback,
                 is_connected=check_is_connected,
                 emitter=emitter,
                 state_container=state_container,
+                simple_chat_history=simple_chat_history,
+                tools=tools,
+                custom_agent_prompt=custom_agent_prompt,
+                llm=llm,
+                token_counter=token_counter,
+                db_session=db_session,
+                skip_clarification=skip_clarification,
+                user_identity=user_identity,
+                chat_session_id=str(chat_session.id),
+                all_injected_file_metadata=all_injected_file_metadata,
             )
         else:
             yield from run_chat_loop_with_state_containers(
-                lambda emitter, state_container: run_llm_loop(
-                    emitter=emitter,
-                    state_container=state_container,
-                    simple_chat_history=simple_chat_history,
-                    tools=tools,
-                    custom_agent_prompt=custom_agent_prompt,
-                    context_files=extracted_context_files,
-                    persona=persona,
-                    user_memory_context=user_memory_context,
-                    llm=llm,
-                    token_counter=token_counter,
-                    db_session=db_session,
-                    forced_tool_id=forced_tool_id,
-                    user_identity=user_identity,
-                    chat_session_id=str(chat_session.id),
-                    chat_files=chat_files_for_tools,
-                    include_citations=new_msg_req.include_citations,
-                    all_injected_file_metadata=all_injected_file_metadata,
-                    inject_memories_in_prompt=user.use_memories,
-                ),
+                run_llm_loop,
                 llm_loop_completion_callback,
                 is_connected=check_is_connected,  # Not passed through to run_llm_loop
                 emitter=emitter,
                 state_container=state_container,
+                simple_chat_history=simple_chat_history,
+                tools=tools,
+                custom_agent_prompt=custom_agent_prompt,
+                project_files=extracted_project_files,
+                persona=persona,
+                user_memory_context=user_memory_context,
+                llm=llm,
+                token_counter=token_counter,
+                db_session=db_session,
+                forced_tool_id=forced_tool_id,
+                user_identity=user_identity,
+                chat_session_id=str(chat_session.id),
+                chat_files=chat_files_for_tools,
+                include_citations=new_msg_req.include_citations,
+                all_injected_file_metadata=all_injected_file_metadata,
+                inject_memories_in_prompt=user.use_memories,
             )
 
     except ValueError as e:

backend/onyx/chat/prompt_utils.py

@@ -190,7 +190,7 @@ def _build_user_information_section(
     if not sections:
         return ""
 
-    return USER_INFORMATION_HEADER + "\n".join(sections)
+    return USER_INFORMATION_HEADER + "".join(sections)
 
 
 def build_system_prompt(
@@ -228,21 +228,23 @@ def build_system_prompt(
         system_prompt += REQUIRE_CITATION_GUIDANCE
 
     if include_all_guidance:
-        tool_sections = [
-            TOOL_DESCRIPTION_SEARCH_GUIDANCE,
-            INTERNAL_SEARCH_GUIDANCE,
-            WEB_SEARCH_GUIDANCE.format(
+        system_prompt += (
+            TOOL_SECTION_HEADER
+            + TOOL_DESCRIPTION_SEARCH_GUIDANCE
+            + INTERNAL_SEARCH_GUIDANCE
+            + WEB_SEARCH_GUIDANCE.format(
                 site_colon_disabled=WEB_SEARCH_SITE_DISABLED_GUIDANCE
-            ),
-            OPEN_URLS_GUIDANCE,
-            PYTHON_TOOL_GUIDANCE,
-            GENERATE_IMAGE_GUIDANCE,
-            MEMORY_GUIDANCE,
-        ]
-        system_prompt += TOOL_SECTION_HEADER + "\n".join(tool_sections)
+            )
+            + OPEN_URLS_GUIDANCE
+            + PYTHON_TOOL_GUIDANCE
+            + GENERATE_IMAGE_GUIDANCE
+            + MEMORY_GUIDANCE
+        )
         return system_prompt
 
     if tools:
+        system_prompt += TOOL_SECTION_HEADER
+
         has_web_search = any(isinstance(tool, WebSearchTool) for tool in tools)
         has_internal_search = any(isinstance(tool, SearchTool) for tool in tools)
         has_open_urls = any(isinstance(tool, OpenURLTool) for tool in tools)
@@ -252,14 +254,12 @@ def build_system_prompt(
         )
         has_memory = any(isinstance(tool, MemoryTool) for tool in tools)
 
-        tool_guidance_sections: list[str] = []
-
         if has_web_search or has_internal_search or include_all_guidance:
-            tool_guidance_sections.append(TOOL_DESCRIPTION_SEARCH_GUIDANCE)
+            system_prompt += TOOL_DESCRIPTION_SEARCH_GUIDANCE
 
         # These are not included at the Tool level because the ordering may matter.
         if has_internal_search or include_all_guidance:
-            tool_guidance_sections.append(INTERNAL_SEARCH_GUIDANCE)
+            system_prompt += INTERNAL_SEARCH_GUIDANCE
 
         if has_web_search or include_all_guidance:
             site_disabled_guidance = ""
@@ -269,23 +269,20 @@ def build_system_prompt(
                 )
                 if web_search_tool and not web_search_tool.supports_site_filter:
                     site_disabled_guidance = WEB_SEARCH_SITE_DISABLED_GUIDANCE
-            tool_guidance_sections.append(
-                WEB_SEARCH_GUIDANCE.format(site_colon_disabled=site_disabled_guidance)
+            system_prompt += WEB_SEARCH_GUIDANCE.format(
+                site_colon_disabled=site_disabled_guidance
             )
 
         if has_open_urls or include_all_guidance:
-            tool_guidance_sections.append(OPEN_URLS_GUIDANCE)
+            system_prompt += OPEN_URLS_GUIDANCE
 
         if has_python or include_all_guidance:
-            tool_guidance_sections.append(PYTHON_TOOL_GUIDANCE)
+            system_prompt += PYTHON_TOOL_GUIDANCE
 
         if has_generate_image or include_all_guidance:
-            tool_guidance_sections.append(GENERATE_IMAGE_GUIDANCE)
+            system_prompt += GENERATE_IMAGE_GUIDANCE
 
         if has_memory or include_all_guidance:
-            tool_guidance_sections.append(MEMORY_GUIDANCE)
-
-        if tool_guidance_sections:
-            system_prompt += TOOL_SECTION_HEADER + "\n".join(tool_guidance_sections)
+            system_prompt += MEMORY_GUIDANCE
 
     return system_prompt

backend/onyx/configs/app_configs.py

@@ -6,7 +6,6 @@ from datetime import timezone
 from typing import cast
 
 from onyx.auth.schemas import AuthBackend
-from onyx.cache.interface import CacheBackendType
 from onyx.configs.constants import AuthType
 from onyx.configs.constants import QueryHistoryType
 from onyx.file_processing.enums import HtmlBasedConnectorTransformLinksStrategy
@@ -55,12 +54,6 @@ DISABLE_USER_KNOWLEDGE = os.environ.get("DISABLE_USER_KNOWLEDGE", "").lower() ==
 # are disabled but core chat, tools, user file uploads, and Projects still work.
 DISABLE_VECTOR_DB = os.environ.get("DISABLE_VECTOR_DB", "").lower() == "true"
 
-# Which backend to use for caching, locks, and ephemeral state.
-# "redis" (default) or "postgres" (only valid when DISABLE_VECTOR_DB=true).
-CACHE_BACKEND = CacheBackendType(
-    os.environ.get("CACHE_BACKEND", CacheBackendType.REDIS)
-)
-
 # Maximum token count for a single uploaded file. Files exceeding this are rejected.
 # Defaults to 100k tokens (or 10M when vector DB is disabled).
 _DEFAULT_FILE_TOKEN_LIMIT = 10_000_000 if DISABLE_VECTOR_DB else 100_000
@@ -217,10 +210,10 @@ AUTH_COOKIE_EXPIRE_TIME_SECONDS = int(
 REQUIRE_EMAIL_VERIFICATION = (
     os.environ.get("REQUIRE_EMAIL_VERIFICATION", "").lower() == "true"
 )
-SMTP_SERVER = os.environ.get("SMTP_SERVER") or ""
+SMTP_SERVER = os.environ.get("SMTP_SERVER") or "smtp.gmail.com"
 SMTP_PORT = int(os.environ.get("SMTP_PORT") or "587")
-SMTP_USER = os.environ.get("SMTP_USER") or ""
-SMTP_PASS = os.environ.get("SMTP_PASS") or ""
+SMTP_USER = os.environ.get("SMTP_USER", "your-email@gmail.com")
+SMTP_PASS = os.environ.get("SMTP_PASS", "your-gmail-password")
 EMAIL_FROM = os.environ.get("EMAIL_FROM") or SMTP_USER
 
 SENDGRID_API_KEY = os.environ.get("SENDGRID_API_KEY") or ""
@@ -258,9 +251,7 @@ DEFAULT_OPENSEARCH_QUERY_TIMEOUT_S = int(
     os.environ.get("DEFAULT_OPENSEARCH_QUERY_TIMEOUT_S") or 50
 )
 OPENSEARCH_ADMIN_USERNAME = os.environ.get("OPENSEARCH_ADMIN_USERNAME", "admin")
-OPENSEARCH_ADMIN_PASSWORD = os.environ.get(
-    "OPENSEARCH_ADMIN_PASSWORD", "StrongPassword123!"
-)
+OPENSEARCH_ADMIN_PASSWORD = os.environ.get("OPENSEARCH_ADMIN_PASSWORD", "")
 USING_AWS_MANAGED_OPENSEARCH = (
     os.environ.get("USING_AWS_MANAGED_OPENSEARCH", "").lower() == "true"
 )
@@ -291,9 +282,6 @@ OPENSEARCH_TEXT_ANALYZER = os.environ.get("OPENSEARCH_TEXT_ANALYZER") or "englis
 ENABLE_OPENSEARCH_INDEXING_FOR_ONYX = (
     os.environ.get("ENABLE_OPENSEARCH_INDEXING_FOR_ONYX", "").lower() == "true"
 )
-# NOTE: This effectively does nothing anymore, admins can now toggle whether
-# retrieval is through OpenSearch. This value is only used as a final fallback
-# in case that doesn't work for whatever reason.
 # Given that the "base" config above is true, this enables whether we want to
 # retrieve from OpenSearch or Vespa. We want to be able to quickly toggle this
 # in the event we see issues with OpenSearch retrieval in our dev environments.
@@ -301,12 +289,6 @@ ENABLE_OPENSEARCH_RETRIEVAL_FOR_ONYX = (
     ENABLE_OPENSEARCH_INDEXING_FOR_ONYX
     and os.environ.get("ENABLE_OPENSEARCH_RETRIEVAL_FOR_ONYX", "").lower() == "true"
 )
-# Whether we should check for and create an index if necessary every time we
-# instantiate an OpenSearchDocumentIndex on multitenant cloud. Defaults to True.
-VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT = (
-    os.environ.get("VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT", "true").lower()
-    == "true"
-)
 
 VESPA_HOST = os.environ.get("VESPA_HOST") or "localhost"
 # NOTE: this is used if and only if the vespa config server is accessible via a
@@ -655,14 +637,6 @@ SHAREPOINT_CONNECTOR_SIZE_THRESHOLD = int(
     os.environ.get("SHAREPOINT_CONNECTOR_SIZE_THRESHOLD", 20 * 1024 * 1024)
 )
 
-# When True, group sync enumerates every Azure AD group in the tenant (expensive).
-# When False (default), only groups found in site role assignments are synced.
-# Can be overridden per-connector via the "exhaustive_ad_enumeration" key in
-# connector_specific_config.
-SHAREPOINT_EXHAUSTIVE_AD_ENUMERATION = (
-    os.environ.get("SHAREPOINT_EXHAUSTIVE_AD_ENUMERATION", "").lower() == "true"
-)
-
 BLOB_STORAGE_SIZE_THRESHOLD = int(
     os.environ.get("BLOB_STORAGE_SIZE_THRESHOLD", 20 * 1024 * 1024)
 )

backend/onyx/configs/constants.py

@@ -157,25 +157,6 @@ CELERY_EXTERNAL_GROUP_SYNC_LOCK_TIMEOUT = 300  # 5 min
 
 CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT = 30 * 60  # 30 minutes (in seconds)
 
-# How long a queued user-file task is valid before workers discard it.
-# Should be longer than the beat interval (20 s) but short enough to prevent
-# indefinite queue growth.  Workers drop tasks older than this without touching
-# the DB, so a shorter value = faster drain of stale duplicates.
-CELERY_USER_FILE_PROCESSING_TASK_EXPIRES = 60  # 1 minute (in seconds)
-
-# Maximum number of tasks allowed in the user-file-processing queue before the
-# beat generator stops adding more.  Prevents unbounded queue growth when workers
-# fall behind.
-USER_FILE_PROCESSING_MAX_QUEUE_DEPTH = 500
-# How long a queued user-file-project-sync task remains valid.
-# Should be short enough to discard stale queue entries under load while still
-# allowing workers enough time to pick up new tasks.
-CELERY_USER_FILE_PROJECT_SYNC_TASK_EXPIRES = 60  # 1 minute (in seconds)
-
-# Max queue depth before user-file-project-sync producers stop enqueuing.
-# This applies backpressure when workers are falling behind.
-USER_FILE_PROJECT_SYNC_MAX_QUEUE_DEPTH = 500
-
 CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT = 5 * 60  # 5 minutes (in seconds)
 
 CELERY_SANDBOX_FILE_SYNC_LOCK_TIMEOUT = 5 * 60  # 5 minutes (in seconds)
@@ -462,12 +443,8 @@ class OnyxRedisLocks:
     # User file processing
     USER_FILE_PROCESSING_BEAT_LOCK = "da_lock:check_user_file_processing_beat"
     USER_FILE_PROCESSING_LOCK_PREFIX = "da_lock:user_file_processing"
-    # Short-lived key set when a task is enqueued; cleared when the worker picks it up.
-    # Prevents the beat from re-enqueuing the same file while a task is already queued.
-    USER_FILE_QUEUED_PREFIX = "da_lock:user_file_queued"
     USER_FILE_PROJECT_SYNC_BEAT_LOCK = "da_lock:check_user_file_project_sync_beat"
     USER_FILE_PROJECT_SYNC_LOCK_PREFIX = "da_lock:user_file_project_sync"
-    USER_FILE_PROJECT_SYNC_QUEUED_PREFIX = "da_lock:user_file_project_sync_queued"
     USER_FILE_DELETE_BEAT_LOCK = "da_lock:check_user_file_delete_beat"
     USER_FILE_DELETE_LOCK_PREFIX = "da_lock:user_file_delete"
 

backend/onyx/connectors/gong/connector.py

@@ -32,8 +32,6 @@ class GongConnector(LoadConnector, PollConnector):
     BASE_URL = "https://api.gong.io"
     MAX_CALL_DETAILS_ATTEMPTS = 6
     CALL_DETAILS_DELAY = 30  # in seconds
-    # Gong API limit is 3 calls/sec — stay safely under it
-    MIN_REQUEST_INTERVAL = 0.5  # seconds between requests
 
     def __init__(
         self,
@@ -47,13 +45,9 @@ class GongConnector(LoadConnector, PollConnector):
         self.continue_on_fail = continue_on_fail
         self.auth_token_basic: str | None = None
         self.hide_user_info = hide_user_info
-        self._last_request_time: float = 0.0
 
-        # urllib3 Retry already respects the Retry-After header by default
-        # (respect_retry_after_header=True), so on 429 it will sleep for the
-        # duration Gong specifies before retrying.
         retry_strategy = Retry(
-            total=10,
+            total=5,
             backoff_factor=2,
             status_forcelist=[429, 500, 502, 503, 504],
         )
@@ -67,24 +61,8 @@ class GongConnector(LoadConnector, PollConnector):
         url = f"{GongConnector.BASE_URL}{endpoint}"
         return url
 
-    def _throttled_request(
-        self, method: str, url: str, **kwargs: Any
-    ) -> requests.Response:
-        """Rate-limited request wrapper. Enforces MIN_REQUEST_INTERVAL between
-        calls to stay under Gong's 3 calls/sec limit and avoid triggering 429s."""
-        now = time.monotonic()
-        elapsed = now - self._last_request_time
-        if elapsed < self.MIN_REQUEST_INTERVAL:
-            time.sleep(self.MIN_REQUEST_INTERVAL - elapsed)
-
-        response = self._session.request(method, url, **kwargs)
-        self._last_request_time = time.monotonic()
-        return response
-
     def _get_workspace_id_map(self) -> dict[str, str]:
-        response = self._throttled_request(
-            "GET", GongConnector.make_url("/v2/workspaces")
-        )
+        response = self._session.get(GongConnector.make_url("/v2/workspaces"))
         response.raise_for_status()
 
         workspaces_details = response.json().get("workspaces")
@@ -128,8 +106,8 @@ class GongConnector(LoadConnector, PollConnector):
                     del body["filter"]["workspaceId"]
 
             while True:
-                response = self._throttled_request(
-                    "POST", GongConnector.make_url("/v2/calls/transcript"), json=body
+                response = self._session.post(
+                    GongConnector.make_url("/v2/calls/transcript"), json=body
                 )
                 # If no calls in the range, just break out
                 if response.status_code == 404:
@@ -164,8 +142,8 @@ class GongConnector(LoadConnector, PollConnector):
             "contentSelector": {"exposedFields": {"parties": True}},
         }
 
-        response = self._throttled_request(
-            "POST", GongConnector.make_url("/v2/calls/extensive"), json=body
+        response = self._session.post(
+            GongConnector.make_url("/v2/calls/extensive"), json=body
         )
         response.raise_for_status()
 
@@ -216,8 +194,7 @@ class GongConnector(LoadConnector, PollConnector):
             # There's a likely race condition in the API where a transcript will have a
             # call id but the call to v2/calls/extensive will not return all of the id's
             # retry with exponential backoff has been observed to mitigate this
-            # in ~2 minutes. After max attempts, proceed with whatever we have —
-            # the per-call loop below will skip missing IDs gracefully.
+            # in ~2 minutes
             current_attempt = 0
             while True:
                 current_attempt += 1
@@ -236,14 +213,11 @@ class GongConnector(LoadConnector, PollConnector):
                     f"missing_call_ids={missing_call_ids}"
                 )
                 if current_attempt >= self.MAX_CALL_DETAILS_ATTEMPTS:
-                    logger.error(
-                        f"Giving up on missing call id's after "
-                        f"{self.MAX_CALL_DETAILS_ATTEMPTS} attempts: "
-                        f"missing_call_ids={missing_call_ids} — "
-                        f"proceeding with {len(call_details_map)} of "
-                        f"{len(transcript_call_ids)} calls"
+                    raise RuntimeError(
+                        f"Attempt count exceeded for _get_call_details_by_ids: "
+                        f"missing_call_ids={missing_call_ids} "
+                        f"max_attempts={self.MAX_CALL_DETAILS_ATTEMPTS}"
                     )
-                    break
 
                 wait_seconds = self.CALL_DETAILS_DELAY * pow(2, current_attempt - 1)
                 logger.warning(

backend/onyx/connectors/google_utils/google_utils.py

@@ -16,22 +16,6 @@ from onyx.utils.retry_wrapper import retry_builder
 
 logger = setup_logger()
 
-_RATE_LIMIT_REASONS = {"userRateLimitExceeded", "rateLimitExceeded"}
-
-
-def _is_rate_limit_error(error: HttpError) -> bool:
-    """Google sometimes returns rate-limit errors as 403 with reason
-    'userRateLimitExceeded' instead of 429. This helper detects both."""
-    if error.resp.status == 429:
-        return True
-    if error.resp.status != 403:
-        return False
-    error_details = getattr(error, "error_details", None) or []
-    for detail in error_details:
-        if isinstance(detail, dict) and detail.get("reason") in _RATE_LIMIT_REASONS:
-            return True
-    return "userRateLimitExceeded" in str(error) or "rateLimitExceeded" in str(error)
-
 
 # Google Drive APIs are quite flakey and may 500 for an
 # extended period of time. This is now addressed by checkpointing.
@@ -73,7 +57,7 @@ def _execute_with_retry(request: Any) -> Any:
         except HttpError as error:
             attempt += 1
 
-            if _is_rate_limit_error(error):
+            if error.resp.status == 429:
                 # Attempt to get 'Retry-After' from headers
                 retry_after = error.resp.get("Retry-After")
                 if retry_after:
@@ -156,16 +140,16 @@ def _execute_single_retrieval(
                 )
             logger.error(f"Error executing request: {e}")
             raise e
-        elif _is_rate_limit_error(e):
-            results = _execute_with_retry(
-                lambda: retrieval_function(**request_kwargs).execute()
-            )
         elif e.resp.status == 404 or e.resp.status == 403:
             if continue_on_404_or_403:
                 logger.debug(f"Error executing request: {e}")
                 results = {}
             else:
                 raise e
+        elif e.resp.status == 429:
+            results = _execute_with_retry(
+                lambda: retrieval_function(**request_kwargs).execute()
+            )
         else:
             logger.exception("Error executing request:")
             raise e

backend/onyx/connectors/microsoft_graph_env.py

@@ -1,96 +0,0 @@
-"""Inverse mapping from user-facing Microsoft host URLs to the SDK's AzureEnvironment.
-
-The office365 library's GraphClient requires an ``AzureEnvironment`` string
-(e.g. ``"Global"``, ``"GCC High"``) to route requests to the correct national
-cloud.  Our connectors instead expose free-text ``authority_host`` and
-``graph_api_host`` fields so the frontend doesn't need to know about SDK
-internals.
-
-This module bridges the gap: given the two host URLs the user configured, it
-resolves the matching ``AzureEnvironment`` value (and the implied SharePoint
-domain suffix) so callers can pass ``environment=…`` to ``GraphClient``.
-"""
-
-from office365.graph_client import AzureEnvironment  # type: ignore[import-untyped]
-from pydantic import BaseModel
-
-from onyx.connectors.exceptions import ConnectorValidationError
-
-
-class MicrosoftGraphEnvironment(BaseModel):
-    """One row of the inverse mapping."""
-
-    environment: str
-    graph_host: str
-    authority_host: str
-    sharepoint_domain_suffix: str
-
-
-_ENVIRONMENTS: list[MicrosoftGraphEnvironment] = [
-    MicrosoftGraphEnvironment(
-        environment=AzureEnvironment.Global,
-        graph_host="https://graph.microsoft.com",
-        authority_host="https://login.microsoftonline.com",
-        sharepoint_domain_suffix="sharepoint.com",
-    ),
-    MicrosoftGraphEnvironment(
-        environment=AzureEnvironment.USGovernmentHigh,
-        graph_host="https://graph.microsoft.us",
-        authority_host="https://login.microsoftonline.us",
-        sharepoint_domain_suffix="sharepoint.us",
-    ),
-    MicrosoftGraphEnvironment(
-        environment=AzureEnvironment.USGovernmentDoD,
-        graph_host="https://dod-graph.microsoft.us",
-        authority_host="https://login.microsoftonline.us",
-        sharepoint_domain_suffix="sharepoint.us",
-    ),
-    MicrosoftGraphEnvironment(
-        environment=AzureEnvironment.China,
-        graph_host="https://microsoftgraph.chinacloudapi.cn",
-        authority_host="https://login.chinacloudapi.cn",
-        sharepoint_domain_suffix="sharepoint.cn",
-    ),
-    MicrosoftGraphEnvironment(
-        environment=AzureEnvironment.Germany,
-        graph_host="https://graph.microsoft.de",
-        authority_host="https://login.microsoftonline.de",
-        sharepoint_domain_suffix="sharepoint.de",
-    ),
-]
-
-_GRAPH_HOST_INDEX: dict[str, MicrosoftGraphEnvironment] = {
-    env.graph_host: env for env in _ENVIRONMENTS
-}
-
-
-def resolve_microsoft_environment(
-    graph_api_host: str,
-    authority_host: str,
-) -> MicrosoftGraphEnvironment:
-    """Return the ``MicrosoftGraphEnvironment`` that matches the supplied hosts.
-
-    Raises ``ConnectorValidationError`` when the combination is unknown or
-    internally inconsistent (e.g. a GCC-High graph host paired with a
-    commercial authority host).
-    """
-    graph_api_host = graph_api_host.rstrip("/")
-    authority_host = authority_host.rstrip("/")
-
-    env = _GRAPH_HOST_INDEX.get(graph_api_host)
-    if env is None:
-        known = ", ".join(sorted(_GRAPH_HOST_INDEX))
-        raise ConnectorValidationError(
-            f"Unsupported Microsoft Graph API host '{graph_api_host}'. "
-            f"Recognised hosts: {known}"
-        )
-
-    if env.authority_host != authority_host:
-        raise ConnectorValidationError(
-            f"Authority host '{authority_host}' is inconsistent with "
-            f"graph API host '{graph_api_host}'. "
-            f"Expected authority host '{env.authority_host}' "
-            f"for the {env.environment} environment."
-        )
-
-    return env

backend/onyx/connectors/models.py

@@ -6,7 +6,6 @@ from typing import cast
 
 from pydantic import BaseModel
 from pydantic import Field
-from pydantic import field_validator
 from pydantic import model_validator
 
 from onyx.access.models import ExternalAccess
@@ -168,14 +167,6 @@ class DocumentBase(BaseModel):
     # list of strings.
     metadata: dict[str, str | list[str]]
 
-    @field_validator("metadata", mode="before")
-    @classmethod
-    def _coerce_metadata_values(cls, v: dict[str, Any]) -> dict[str, str | list[str]]:
-        return {
-            key: [str(item) for item in val] if isinstance(val, list) else str(val)
-            for key, val in v.items()
-        }
-
     # UTC time
     doc_updated_at: datetime | None = None
     chunk_count: int | None = None

backend/onyx/connectors/sharepoint/connector.py

@@ -23,6 +23,7 @@ from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.serialization import pkcs12
 from office365.graph_client import GraphClient  # type: ignore[import-untyped]
+from office365.intune.organizations.organization import Organization  # type: ignore[import-untyped]
 from office365.onedrive.driveitems.driveItem import DriveItem  # type: ignore[import-untyped]
 from office365.onedrive.sites.site import Site  # type: ignore[import-untyped]
 from office365.onedrive.sites.sites_with_root import SitesWithRoot  # type: ignore[import-untyped]
@@ -46,7 +47,6 @@ from onyx.connectors.interfaces import GenerateSlimDocumentOutput
 from onyx.connectors.interfaces import IndexingHeartbeatInterface
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnectorWithPermSync
-from onyx.connectors.microsoft_graph_env import resolve_microsoft_environment
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorCheckpoint
 from onyx.connectors.models import ConnectorFailure
@@ -83,11 +83,7 @@ SHARED_DOCUMENTS_MAP_REVERSE = {v: k for k, v in SHARED_DOCUMENTS_MAP.items()}
 
 ASPX_EXTENSION = ".aspx"
 
-DEFAULT_AUTHORITY_HOST = "https://login.microsoftonline.com"
-DEFAULT_GRAPH_API_HOST = "https://graph.microsoft.com"
-DEFAULT_SHAREPOINT_DOMAIN_SUFFIX = "sharepoint.com"
-
-GRAPH_API_BASE = f"{DEFAULT_GRAPH_API_HOST}/v1.0"
+GRAPH_API_BASE = "https://graph.microsoft.com/v1.0"
 GRAPH_API_MAX_RETRIES = 5
 GRAPH_API_RETRYABLE_STATUSES = frozenset({429, 500, 502, 503, 504})
 
@@ -146,9 +142,7 @@ class DriveItemData(BaseModel):
             self.id,
             ResourcePath("items", ResourcePath(self.drive_id, ResourcePath("drives"))),
         )
-        item = DriveItem(graph_client, path)
-        item.set_property("id", self.id)
-        return item
+        return DriveItem(graph_client, path)
 
 
 # The office365 library's ClientContext caches the access token from its
@@ -182,25 +176,6 @@ class CertificateData(BaseModel):
     thumbprint: str
 
 
-def _site_page_in_time_window(
-    page: dict[str, Any],
-    start: datetime | None,
-    end: datetime | None,
-) -> bool:
-    """Return True if the page's lastModifiedDateTime falls within [start, end]."""
-    if start is None and end is None:
-        return True
-    raw = page.get("lastModifiedDateTime")
-    if not raw:
-        return True
-    if not isinstance(raw, str):
-        raise ValueError(f"lastModifiedDateTime is not a string: {raw}")
-    last_modified = datetime.fromisoformat(raw.replace("Z", "+00:00"))
-    return (start is None or last_modified >= start) and (
-        end is None or last_modified <= end
-    )
-
-
 def sleep_and_retry(
     query_obj: ClientQuery, method_name: str, max_retries: int = 3
 ) -> Any:
@@ -246,12 +221,6 @@ class SharepointConnectorCheckpoint(ConnectorCheckpoint):
     current_drive_name: str | None = None
     # Drive's web_url from the API - used as raw_node_id for DRIVE hierarchy nodes
     current_drive_web_url: str | None = None
-    # Resolved drive ID — avoids re-resolving on checkpoint resume
-    current_drive_id: str | None = None
-    # Next delta API page URL for per-page checkpointing within a drive.
-    # When set, Phase 3b fetches one page at a time so progress is persisted
-    # between pages.  None means BFS path or no active delta traversal.
-    current_drive_delta_next_link: str | None = None
 
     process_site_pages: bool = False
 
@@ -297,12 +266,10 @@ def load_certificate_from_pfx(pfx_data: bytes, password: str) -> CertificateData
 
 
 def acquire_token_for_rest(
-    msal_app: msal.ConfidentialClientApplication,
-    sp_tenant_domain: str,
-    sharepoint_domain_suffix: str,
+    msal_app: msal.ConfidentialClientApplication, sp_tenant_domain: str
 ) -> TokenResponse:
     token = msal_app.acquire_token_for_client(
-        scopes=[f"https://{sp_tenant_domain}.{sharepoint_domain_suffix}/.default"]
+        scopes=[f"https://{sp_tenant_domain}.sharepoint.com/.default"]
     )
     return TokenResponse.from_json(token)
 
@@ -417,13 +384,12 @@ def _download_via_graph_api(
     drive_id: str,
     item_id: str,
     bytes_allowed: int,
-    graph_api_base: str,
 ) -> bytes:
     """Download a drive item via the Graph API /content endpoint with a byte cap.
 
     Raises SizeCapExceeded if the cap is exceeded.
     """
-    url = f"{graph_api_base}/drives/{drive_id}/items/{item_id}/content"
+    url = f"{GRAPH_API_BASE}/drives/{drive_id}/items/{item_id}/content"
     headers = {"Authorization": f"Bearer {access_token}"}
     with requests.get(
         url, headers=headers, stream=True, timeout=REQUEST_TIMEOUT_SECONDS
@@ -444,7 +410,6 @@ def _convert_driveitem_to_document_with_permissions(
     drive_name: str,
     ctx: ClientContext | None,
     graph_client: GraphClient,
-    graph_api_base: str,
     include_permissions: bool = False,
     parent_hierarchy_raw_node_id: str | None = None,
     access_token: str | None = None,
@@ -501,7 +466,6 @@ def _convert_driveitem_to_document_with_permissions(
                 driveitem.drive_id,
                 driveitem.id,
                 SHAREPOINT_CONNECTOR_SIZE_THRESHOLD,
-                graph_api_base=graph_api_base,
             )
         except SizeCapExceeded:
             logger.warning(
@@ -821,9 +785,6 @@ class SharepointConnector(
         sites: list[str] = [],
         include_site_pages: bool = True,
         include_site_documents: bool = True,
-        authority_host: str = DEFAULT_AUTHORITY_HOST,
-        graph_api_host: str = DEFAULT_GRAPH_API_HOST,
-        sharepoint_domain_suffix: str = DEFAULT_SHAREPOINT_DOMAIN_SUFFIX,
     ) -> None:
         self.batch_size = batch_size
         self.sites = list(sites)
@@ -840,20 +801,6 @@ class SharepointConnector(
         self._cached_rest_ctx_url: str | None = None
         self._cached_rest_ctx_created_at: float = 0.0
 
-        resolved_env = resolve_microsoft_environment(graph_api_host, authority_host)
-        self._azure_environment = resolved_env.environment
-        self.authority_host = resolved_env.authority_host
-        self.graph_api_host = resolved_env.graph_host
-        self.graph_api_base = f"{self.graph_api_host}/v1.0"
-        self.sharepoint_domain_suffix = resolved_env.sharepoint_domain_suffix
-        if sharepoint_domain_suffix != resolved_env.sharepoint_domain_suffix:
-            logger.warning(
-                f"Configured sharepoint_domain_suffix '{sharepoint_domain_suffix}' "
-                f"differs from the expected suffix '{resolved_env.sharepoint_domain_suffix}' "
-                f"for the {resolved_env.environment} environment. "
-                f"Using '{resolved_env.sharepoint_domain_suffix}'."
-            )
-
     def validate_connector_settings(self) -> None:
         # Validate that at least one content type is enabled
         if not self.include_site_documents and not self.include_site_pages:
@@ -871,56 +818,6 @@ class SharepointConnector(
                     "Site URLs must be full Sharepoint URLs (e.g. https://your-tenant.sharepoint.com/sites/your-site or https://your-tenant.sharepoint.com/teams/your-team)"
                 )
 
-    def _extract_tenant_domain_from_sites(self) -> str | None:
-        """Extract the tenant domain from configured site URLs.
-
-        Site URLs look like https://{tenant}.sharepoint.com/sites/... so the
-        tenant domain is the first label of the hostname.
-        """
-        for site_url in self.sites:
-            try:
-                hostname = urlsplit(site_url.strip()).hostname
-            except ValueError:
-                continue
-            if not hostname:
-                continue
-            tenant = hostname.split(".")[0]
-            if tenant:
-                return tenant
-        logger.warning(f"No tenant domain found from {len(self.sites)} sites")
-        return None
-
-    def _resolve_tenant_domain_from_root_site(self) -> str:
-        """Resolve tenant domain via GET /v1.0/sites/root which only requires
-        Sites.Read.All (a permission the connector already needs)."""
-        root_site = self.graph_client.sites.root.get().execute_query()
-        hostname = root_site.site_collection.hostname
-        if not hostname:
-            raise ConnectorValidationError(
-                "Could not determine tenant domain from root site"
-            )
-        tenant_domain = hostname.split(".")[0]
-        logger.info(
-            "Resolved tenant domain '%s' from root site hostname '%s'",
-            tenant_domain,
-            hostname,
-        )
-        return tenant_domain
-
-    def _resolve_tenant_domain(self) -> str:
-        """Determine the tenant domain, preferring site URLs over a Graph API
-        call to avoid needing extra permissions."""
-        from_sites = self._extract_tenant_domain_from_sites()
-        if from_sites:
-            logger.info(
-                "Resolved tenant domain '%s' from site URLs",
-                from_sites,
-            )
-            return from_sites
-
-        logger.info("No site URLs available; resolving tenant domain from root site")
-        return self._resolve_tenant_domain_from_root_site()
-
     @property
     def graph_client(self) -> GraphClient:
         if self._graph_client is None:
@@ -959,9 +856,8 @@ class SharepointConnector(
 
         msal_app = self.msal_app
         sp_tenant_domain = self.sp_tenant_domain
-        sp_domain_suffix = self.sharepoint_domain_suffix
         self._cached_rest_ctx = ClientContext(site_url).with_access_token(
-            lambda: acquire_token_for_rest(msal_app, sp_tenant_domain, sp_domain_suffix)
+            lambda: acquire_token_for_rest(msal_app, sp_tenant_domain)
         )
         self._cached_rest_ctx_url = site_url
         self._cached_rest_ctx_created_at = time.monotonic()
@@ -1221,36 +1117,76 @@ class SharepointConnector(
         site_descriptor: SiteDescriptor,
         start: datetime | None = None,
         end: datetime | None = None,
-    ) -> Generator[dict[str, Any], None, None]:
-        """Yield SharePoint site pages (.aspx files) one at a time.
+    ) -> list[dict[str, Any]]:
+        """Fetch SharePoint site pages (.aspx files) using the SharePoint Pages API."""
 
-        Pages are fetched via the Graph Pages API and yielded lazily as each
-        API page arrives, so memory stays bounded regardless of total page count.
-        Time-window filtering is applied per-item before yielding.
-        """
+        # Get the site to extract the site ID
         site = self.graph_client.sites.get_by_url(site_descriptor.url)
-        site.execute_query()
+        site.execute_query()  # Execute the query to actually fetch the data
         site_id = site.id
 
-        page_url: str | None = (
-            f"{self.graph_api_base}/sites/{site_id}" f"/pages/microsoft.graph.sitePage"
+        # Get the token acquisition function from the GraphClient
+        token_data = self._acquire_token()
+        access_token = token_data.get("access_token")
+        if not access_token:
+            raise RuntimeError("Failed to acquire access token")
+
+        # Construct the SharePoint Pages API endpoint
+        # Using API directly, since the Graph Client doesn't support the Pages API
+        pages_endpoint = f"https://graph.microsoft.com/v1.0/sites/{site_id}/pages/microsoft.graph.sitePage"
+
+        headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Content-Type": "application/json",
+        }
+
+        # Add expand parameter to get canvas layout content
+        params = {"$expand": "canvasLayout"}
+
+        response = requests.get(
+            pages_endpoint,
+            headers=headers,
+            params=params,
+            timeout=REQUEST_TIMEOUT_SECONDS,
         )
-        params: dict[str, str] | None = {"$expand": "canvasLayout"}
-        total_yielded = 0
+        response.raise_for_status()
+        pages_data = response.json()
+        all_pages = pages_data.get("value", [])
 
-        while page_url:
-            data = self._graph_api_get_json(page_url, params)
-            params = None  # nextLink already embeds query params
+        # Handle pagination if there are more pages
+        # TODO: This accumulates all pages in memory and can be heavy on large tenants.
+        #       We should process each page incrementally to avoid unbounded growth.
+        while "@odata.nextLink" in pages_data:
+            next_url = pages_data["@odata.nextLink"]
+            response = requests.get(
+                next_url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS
+            )
+            response.raise_for_status()
+            pages_data = response.json()
+            all_pages.extend(pages_data.get("value", []))
 
-            for page in data.get("value", []):
-                if not _site_page_in_time_window(page, start, end):
-                    continue
-                total_yielded += 1
-                yield page
+        logger.debug(f"Found {len(all_pages)} site pages in {site_descriptor.url}")
 
-            page_url = data.get("@odata.nextLink")
+        # Filter pages based on time window if specified
+        if start is not None or end is not None:
+            filtered_pages: list[dict[str, Any]] = []
+            for page in all_pages:
+                page_modified = page.get("lastModifiedDateTime")
+                if page_modified:
+                    if isinstance(page_modified, str):
+                        page_modified = datetime.fromisoformat(
+                            page_modified.replace("Z", "+00:00")
+                        )
 
-        logger.debug(f"Yielded {total_yielded} site pages for {site_descriptor.url}")
+                    if start is not None and page_modified < start:
+                        continue
+                    if end is not None and page_modified > end:
+                        continue
+
+                filtered_pages.append(page)
+            all_pages = filtered_pages
+
+        return all_pages
 
     def _acquire_token(self) -> dict[str, Any]:
         """
@@ -1260,7 +1196,7 @@ class SharepointConnector(
             raise RuntimeError("MSAL app is not initialized")
 
         token = self.msal_app.acquire_token_for_client(
-            scopes=[f"{self.graph_api_host}/.default"]
+            scopes=["https://graph.microsoft.com/.default"]
         )
         return token
 
@@ -1333,10 +1269,9 @@ class SharepointConnector(
         Performs BFS folder traversal manually, fetching one page of children
         at a time so that memory usage stays bounded regardless of drive size.
         """
-        base = f"{self.graph_api_base}/drives/{drive_id}"
+        base = f"{GRAPH_API_BASE}/drives/{drive_id}"
         if folder_path:
-            encoded_path = quote(folder_path, safe="/")
-            start_url = f"{base}/root:/{encoded_path}:/children"
+            start_url = f"{base}/root:/{folder_path}:/children"
         else:
             start_url = f"{base}/root/children"
 
@@ -1394,7 +1329,7 @@ class SharepointConnector(
         """
         use_timestamp_token = start is not None and start > _EPOCH
 
-        initial_url = f"{self.graph_api_base}/drives/{drive_id}/root/delta"
+        initial_url = f"{GRAPH_API_BASE}/drives/{drive_id}/root/delta"
         if use_timestamp_token:
             assert start is not None  # mypy
             token = quote(start.isoformat(timespec="seconds"))
@@ -1440,7 +1375,7 @@ class SharepointConnector(
                         drive_id,
                     )
                     yield from self._iter_delta_pages(
-                        initial_url=f"{self.graph_api_base}/drives/{drive_id}/root/delta",
+                        initial_url=f"{GRAPH_API_BASE}/drives/{drive_id}/root/delta",
                         drive_id=drive_id,
                         start=start,
                         end=end,
@@ -1471,87 +1406,6 @@ class SharepointConnector(
             if not page_url:
                 break
 
-    def _build_delta_start_url(
-        self,
-        drive_id: str,
-        start: datetime | None = None,
-        page_size: int = 200,
-    ) -> str:
-        """Build the initial delta API URL with query parameters embedded.
-
-        Embeds ``$top`` (and optionally a timestamp ``token``) directly in the
-        URL so that the returned string is fully self-contained and can be
-        stored in a checkpoint without needing a separate params dict.
-        """
-        base_url = f"{self.graph_api_base}/drives/{drive_id}/root/delta"
-        params = [f"$top={page_size}"]
-        if start is not None and start > _EPOCH:
-            token = quote(start.isoformat(timespec="seconds"))
-            params.append(f"token={token}")
-        return f"{base_url}?{'&'.join(params)}"
-
-    def _fetch_one_delta_page(
-        self,
-        page_url: str,
-        drive_id: str,
-        start: datetime | None = None,
-        end: datetime | None = None,
-        page_size: int = 200,
-    ) -> tuple[list[DriveItemData], str | None]:
-        """Fetch a single page of delta API results.
-
-        Returns ``(items, next_page_url)``.  *next_page_url* is ``None`` when
-        the delta enumeration is complete (deltaLink with no nextLink).
-
-        On 410 Gone (expired token) returns ``([], full_resync_url)`` so
-        the caller can store the resync URL in the checkpoint and retry on
-        the next cycle.
-        """
-        try:
-            data = self._graph_api_get_json(page_url)
-        except requests.HTTPError as e:
-            if e.response is not None and e.response.status_code == 410:
-                logger.warning(
-                    "Delta token expired (410 Gone) for drive '%s'. "
-                    "Will restart with full delta enumeration.",
-                    drive_id,
-                )
-                full_url = (
-                    f"{self.graph_api_base}/drives/{drive_id}/root/delta"
-                    f"?$top={page_size}"
-                )
-                return [], full_url
-            raise
-
-        items: list[DriveItemData] = []
-        for item in data.get("value", []):
-            if "folder" in item or "deleted" in item:
-                continue
-            if start is not None or end is not None:
-                raw_ts = item.get("lastModifiedDateTime")
-                if raw_ts:
-                    mod_dt = datetime.fromisoformat(raw_ts.replace("Z", "+00:00"))
-                    if start is not None and mod_dt < start:
-                        continue
-                    if end is not None and mod_dt > end:
-                        continue
-            items.append(DriveItemData.from_graph_json(item))
-
-        next_url = data.get("@odata.nextLink")
-        if next_url:
-            return items, next_url
-        return items, None
-
-    @staticmethod
-    def _clear_drive_checkpoint_state(
-        checkpoint: "SharepointConnectorCheckpoint",
-    ) -> None:
-        """Reset all drive-level fields in the checkpoint."""
-        checkpoint.current_drive_name = None
-        checkpoint.current_drive_id = None
-        checkpoint.current_drive_web_url = None
-        checkpoint.current_drive_delta_next_link = None
-
     def _fetch_slim_documents_from_sharepoint(self) -> GenerateSlimDocumentOutput:
         site_descriptors = self.site_descriptors or self.fetch_sites()
 
@@ -1638,12 +1492,7 @@ class SharepointConnector(
         sp_private_key = credentials.get("sp_private_key")
         sp_certificate_password = credentials.get("sp_certificate_password")
 
-        if not sp_client_id:
-            raise ConnectorValidationError("Client ID is required")
-        if not sp_directory_id:
-            raise ConnectorValidationError("Directory (tenant) ID is required")
-
-        authority_url = f"{self.authority_host}/{sp_directory_id}"
+        authority_url = f"https://login.microsoftonline.com/{sp_directory_id}"
 
         if auth_method == SharepointAuthMethod.CERTIFICATE.value:
             logger.info("Using certificate authentication")
@@ -1659,7 +1508,6 @@ class SharepointConnector(
             if certificate_data is None:
                 raise RuntimeError("Failed to load certificate")
 
-            logger.info(f"Creating MSAL app with authority url {authority_url}")
             self.msal_app = msal.ConfidentialClientApplication(
                 authority=authority_url,
                 client_id=sp_client_id,
@@ -1685,17 +1533,29 @@ class SharepointConnector(
                 raise ConnectorValidationError("MSAL app is not initialized")
 
             token = self.msal_app.acquire_token_for_client(
-                scopes=[f"{self.graph_api_host}/.default"]
+                scopes=["https://graph.microsoft.com/.default"]
             )
             if token is None:
                 raise ConnectorValidationError("Failed to acquire token for graph")
             return token
 
-        self._graph_client = GraphClient(
-            _acquire_token_for_graph, environment=self._azure_environment
-        )
+        self._graph_client = GraphClient(_acquire_token_for_graph)
         if auth_method == SharepointAuthMethod.CERTIFICATE.value:
-            self.sp_tenant_domain = self._resolve_tenant_domain()
+            org = self.graph_client.organization.get().execute_query()
+            if not org or len(org) == 0:
+                raise ConnectorValidationError("No organization found")
+
+            tenant_info: Organization = org[
+                0
+            ]  # Access first item directly from collection
+            if not tenant_info.verified_domains:
+                raise ConnectorValidationError("No verified domains found for tenant")
+
+            sp_tenant_domain = tenant_info.verified_domains[0].name
+            if not sp_tenant_domain:
+                raise ConnectorValidationError("No verified domains found for tenant")
+            # remove the .onmicrosoft.com part
+            self.sp_tenant_domain = sp_tenant_domain.split(".")[0]
         return None
 
     def _get_drive_names_for_site(self, site_url: str) -> list[str]:
@@ -1987,13 +1847,14 @@ class SharepointConnector(
             # Return checkpoint to allow persistence after drive initialization
             return checkpoint
 
-        # Phase 3a: Initialize the next drive for processing
+        # Phase 3: Process documents from current drive
         if (
             checkpoint.current_site_descriptor
             and checkpoint.cached_drive_names
             and len(checkpoint.cached_drive_names) > 0
             and checkpoint.current_drive_name is None
         ):
+
             checkpoint.current_drive_name = checkpoint.cached_drive_names.popleft()
 
             start_dt = datetime.fromtimestamp(start, tz=timezone.utc)
@@ -2001,8 +1862,7 @@ class SharepointConnector(
             site_descriptor = checkpoint.current_site_descriptor
 
             logger.info(
-                f"Processing drive '{checkpoint.current_drive_name}' "
-                f"in site: {site_descriptor.url}"
+                f"Processing drive '{checkpoint.current_drive_name}' in site: {site_descriptor.url}"
             )
             logger.debug(f"Time range: {start_dt} to {end_dt}")
 
@@ -2011,35 +1871,35 @@ class SharepointConnector(
                 logger.warning("Current drive name is None, skipping")
                 return checkpoint
 
+            driveitems: Iterable[DriveItemData] = iter(())
+            drive_web_url: str | None = None
             try:
                 logger.info(
                     f"Fetching drive items for drive name: {current_drive_name}"
                 )
                 result = self._resolve_drive(site_descriptor, current_drive_name)
-                if result is None:
-                    logger.warning(f"Drive '{current_drive_name}' not found, skipping")
-                    self._clear_drive_checkpoint_state(checkpoint)
-                    return checkpoint
-
-                drive_id, drive_web_url = result
-                checkpoint.current_drive_id = drive_id
-                checkpoint.current_drive_web_url = drive_web_url
+                if result is not None:
+                    drive_id, drive_web_url = result
+                    driveitems = self._get_drive_items_for_drive_id(
+                        site_descriptor, drive_id, start_dt, end_dt
+                    )
+                    checkpoint.current_drive_web_url = drive_web_url
             except Exception as e:
                 logger.error(
-                    f"Failed to retrieve items from drive '{current_drive_name}' "
-                    f"in site: {site_descriptor.url}: {e}"
+                    f"Failed to retrieve items from drive '{current_drive_name}' in site: {site_descriptor.url}: {e}"
                 )
                 yield _create_entity_failure(
                     f"{site_descriptor.url}|{current_drive_name}",
-                    f"Failed to access drive '{current_drive_name}' "
-                    f"in site '{site_descriptor.url}': {str(e)}",
+                    f"Failed to access drive '{current_drive_name}' in site '{site_descriptor.url}': {str(e)}",
                     (start_dt, end_dt),
                     e,
                 )
-                self._clear_drive_checkpoint_state(checkpoint)
+                checkpoint.current_drive_name = None
+                checkpoint.current_drive_web_url = None
                 return checkpoint
 
-            display_drive_name = SHARED_DOCUMENTS_MAP.get(
+            # Normalize drive name (e.g., "Documents" -> "Shared Documents")
+            current_drive_name = SHARED_DOCUMENTS_MAP.get(
                 current_drive_name, current_drive_name
             )
 
@@ -2047,74 +1907,10 @@ class SharepointConnector(
                 yield from self._yield_drive_hierarchy_node(
                     site_descriptor.url,
                     drive_web_url,
-                    display_drive_name,
+                    current_drive_name,
                     checkpoint,
                 )
 
-            # For non-folder-scoped drives, use delta API with per-page
-            # checkpointing.  Build the initial URL and fall through to 3b.
-            if not site_descriptor.folder_path:
-                checkpoint.current_drive_delta_next_link = self._build_delta_start_url(
-                    drive_id, start_dt
-                )
-            # else: BFS path — delta_next_link stays None;
-            # Phase 3b will use _iter_drive_items_paged.
-
-        # Phase 3b: Process items from the current drive
-        if (
-            checkpoint.current_site_descriptor
-            and checkpoint.current_drive_name is not None
-            and checkpoint.current_drive_id is not None
-        ):
-            site_descriptor = checkpoint.current_site_descriptor
-            start_dt = datetime.fromtimestamp(start, tz=timezone.utc)
-            end_dt = datetime.fromtimestamp(end, tz=timezone.utc)
-            current_drive_name = SHARED_DOCUMENTS_MAP.get(
-                checkpoint.current_drive_name, checkpoint.current_drive_name
-            )
-            drive_web_url = checkpoint.current_drive_web_url
-
-            # --- determine item source ---
-            driveitems: Iterable[DriveItemData]
-            has_more_delta_pages = False
-
-            if checkpoint.current_drive_delta_next_link:
-                # Delta path: fetch one page at a time for checkpointing
-                try:
-                    page_items, next_url = self._fetch_one_delta_page(
-                        page_url=checkpoint.current_drive_delta_next_link,
-                        drive_id=checkpoint.current_drive_id,
-                        start=start_dt,
-                        end=end_dt,
-                    )
-                except Exception as e:
-                    logger.error(
-                        f"Failed to fetch delta page for drive "
-                        f"'{current_drive_name}': {e}"
-                    )
-                    yield _create_entity_failure(
-                        f"{site_descriptor.url}|{current_drive_name}",
-                        f"Failed to fetch delta page for drive "
-                        f"'{current_drive_name}': {str(e)}",
-                        (start_dt, end_dt),
-                        e,
-                    )
-                    self._clear_drive_checkpoint_state(checkpoint)
-                    return checkpoint
-
-                driveitems = page_items
-                has_more_delta_pages = next_url is not None
-                if next_url:
-                    checkpoint.current_drive_delta_next_link = next_url
-            else:
-                # BFS path (folder-scoped): process all items at once
-                driveitems = self._iter_drive_items_paged(
-                    drive_id=checkpoint.current_drive_id,
-                    folder_path=site_descriptor.folder_path,
-                    start=start_dt,
-                    end=end_dt,
-                )
-
             item_count = 0
             for driveitem in driveitems:
                 item_count += 1
@@ -2156,6 +1952,8 @@ class SharepointConnector(
                     if include_permissions:
                         ctx = self._create_rest_client_context(site_descriptor.url)
 
+                    # Re-acquire token in case it expired during a long traversal
+                    # MSAL has a cache that returns the same token while still valid.
                     access_token = self._get_graph_access_token()
                     doc_or_failure = _convert_driveitem_to_document_with_permissions(
                         driveitem,
@@ -2164,7 +1962,6 @@ class SharepointConnector(
                         self.graph_client,
                         include_permissions=include_permissions,
                         parent_hierarchy_raw_node_id=parent_hierarchy_url,
-                        graph_api_base=self.graph_api_base,
                         access_token=access_token,
                     )
 
@@ -2191,11 +1988,8 @@ class SharepointConnector(
                     )
 
             logger.info(f"Processed {item_count} items in drive '{current_drive_name}'")
-
-            if has_more_delta_pages:
-                return checkpoint
-
-            self._clear_drive_checkpoint_state(checkpoint)
+            checkpoint.current_drive_name = None
+            checkpoint.current_drive_web_url = None
 
         # Phase 4: Progression logic - determine next step
         # If we have more drives in current site, continue with current site

backend/onyx/connectors/slab/connector.py

@@ -11,7 +11,6 @@ from dateutil import parser
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
 from onyx.connectors.interfaces import LoadConnector
@@ -259,21 +258,3 @@ class SlabConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):
                 slim_doc_batch = []
         if slim_doc_batch:
             yield slim_doc_batch
-
-    def validate_connector_settings(self) -> None:
-        """
-        Very basic validation, we could do more here
-        """
-        if not self.base_url.startswith("https://") and not self.base_url.startswith(
-            "http://"
-        ):
-            raise ConnectorValidationError(
-                "Base URL must start with https:// or http://"
-            )
-
-        try:
-            get_all_post_ids(self.slab_bot_token)
-        except ConnectorMissingCredentialError:
-            raise
-        except Exception as e:
-            raise ConnectorValidationError(f"Failed to fetch posts from Slab: {e}")

backend/onyx/connectors/teams/connector.py

@@ -23,7 +23,6 @@ from onyx.connectors.interfaces import CheckpointOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnectorWithPermSync
-from onyx.connectors.microsoft_graph_env import resolve_microsoft_environment
 from onyx.connectors.models import ConnectorCheckpoint
 from onyx.connectors.models import ConnectorFailure
 from onyx.connectors.models import ConnectorMissingCredentialError
@@ -51,15 +50,12 @@ class TeamsCheckpoint(ConnectorCheckpoint):
     todo_team_ids: list[str] | None = None
 
 
-DEFAULT_AUTHORITY_HOST = "https://login.microsoftonline.com"
-DEFAULT_GRAPH_API_HOST = "https://graph.microsoft.com"
-
-
 class TeamsConnector(
     CheckpointedConnectorWithPermSync[TeamsCheckpoint],
     SlimConnectorWithPermSync,
 ):
     MAX_WORKERS = 10
+    AUTHORITY_URL_PREFIX = "https://login.microsoftonline.com/"
 
     def __init__(
         self,
@@ -67,19 +63,12 @@ class TeamsConnector(
         # are not necessarily guaranteed to be unique
         teams: list[str] = [],
         max_workers: int = MAX_WORKERS,
-        authority_host: str = DEFAULT_AUTHORITY_HOST,
-        graph_api_host: str = DEFAULT_GRAPH_API_HOST,
     ) -> None:
         self.graph_client: GraphClient | None = None
         self.msal_app: msal.ConfidentialClientApplication | None = None
         self.max_workers = max_workers
         self.requested_team_list: list[str] = teams
 
-        resolved_env = resolve_microsoft_environment(graph_api_host, authority_host)
-        self._azure_environment = resolved_env.environment
-        self.authority_host = resolved_env.authority_host
-        self.graph_api_host = resolved_env.graph_host
-
     # impls for BaseConnector
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
@@ -87,7 +76,7 @@ class TeamsConnector(
         teams_client_secret = credentials["teams_client_secret"]
         teams_directory_id = credentials["teams_directory_id"]
 
-        authority_url = f"{self.authority_host}/{teams_directory_id}"
+        authority_url = f"{TeamsConnector.AUTHORITY_URL_PREFIX}{teams_directory_id}"
         self.msal_app = msal.ConfidentialClientApplication(
             authority=authority_url,
             client_id=teams_client_id,
@@ -102,7 +91,7 @@ class TeamsConnector(
                 raise RuntimeError("MSAL app is not initialized")
 
             token = self.msal_app.acquire_token_for_client(
-                scopes=[f"{self.graph_api_host}/.default"]
+                scopes=["https://graph.microsoft.com/.default"]
             )
 
             if not isinstance(token, dict):
@@ -110,9 +99,7 @@ class TeamsConnector(
 
             return token
 
-        self.graph_client = GraphClient(
-            _acquire_token_func, environment=self._azure_environment
-        )
+        self.graph_client = GraphClient(_acquire_token_func)
         return None
 
     def validate_connector_settings(self) -> None:

backend/onyx/context/search/federated/slack_search.py

@@ -32,7 +32,6 @@ from onyx.context.search.federated.slack_search_utils import should_include_mess
 from onyx.context.search.models import ChunkIndexRequest
 from onyx.context.search.models import InferenceChunk
 from onyx.db.document import DocumentSource
-from onyx.db.models import SearchSettings
 from onyx.db.search_settings import get_current_search_settings
 from onyx.document_index.document_index_utils import (
     get_multipass_config,
@@ -906,15 +905,13 @@ def convert_slack_score(slack_score: float) -> float:
 def slack_retrieval(
     query: ChunkIndexRequest,
     access_token: str,
-    db_session: Session | None = None,
+    db_session: Session,
     connector: FederatedConnectorDetail | None = None,  # noqa: ARG001
     entities: dict[str, Any] | None = None,
     limit: int | None = None,
     slack_event_context: SlackContext | None = None,
     bot_token: str | None = None,  # Add bot token parameter
     team_id: str | None = None,
-    # Pre-fetched data — when provided, avoids DB query (no session needed)
-    search_settings: SearchSettings | None = None,
 ) -> list[InferenceChunk]:
     """
     Main entry point for Slack federated search with entity filtering.
@@ -928,7 +925,7 @@ def slack_retrieval(
     Args:
         query: Search query object
         access_token: User OAuth access token
-        db_session: Database session (optional if search_settings provided)
+        db_session: Database session
         connector: Federated connector detail (unused, kept for backwards compat)
         entities: Connector-level config (entity filtering configuration)
         limit: Maximum number of results
@@ -1156,10 +1153,7 @@ def slack_retrieval(
 
     # chunk index docs into doc aware chunks
     # a single index doc can get split into multiple chunks
-    if search_settings is None:
-        if db_session is None:
-            raise ValueError("Either db_session or search_settings must be provided")
-        search_settings = get_current_search_settings(db_session)
+    search_settings = get_current_search_settings(db_session)
     embedder = DefaultIndexingEmbedder.from_db_search_settings(
         search_settings=search_settings
     )

backend/onyx/context/search/models.py

@@ -72,7 +72,6 @@ class BaseFilters(BaseModel):
 class UserFileFilters(BaseModel):
     user_file_ids: list[UUID] | None = None
     project_id: int | None = None
-    persona_id: int | None = None
 
 
 class AssistantKnowledgeFilters(BaseModel):

backend/onyx/context/search/pipeline.py

@@ -18,10 +18,8 @@ from onyx.context.search.utils import inference_section_from_chunks
 from onyx.db.models import Persona
 from onyx.db.models import User
 from onyx.document_index.interfaces import DocumentIndex
-from onyx.federated_connectors.federated_retrieval import FederatedRetrievalInfo
 from onyx.llm.interfaces import LLM
 from onyx.natural_language_processing.english_stopwords import strip_stopwords
-from onyx.natural_language_processing.search_nlp_models import EmbeddingModel
 from onyx.secondary_llm_flows.source_filter import extract_source_filter
 from onyx.secondary_llm_flows.time_filter import extract_time_filter
 from onyx.utils.logger import setup_logger
@@ -40,11 +38,10 @@ def _build_index_filters(
     user_provided_filters: BaseFilters | None,
     user: User,  # Used for ACLs, anonymous users only see public docs
     project_id: int | None,
-    persona_id: int | None,
     user_file_ids: list[UUID] | None,
     persona_document_sets: list[str] | None,
     persona_time_cutoff: datetime | None,
-    db_session: Session | None = None,
+    db_session: Session,
     auto_detect_filters: bool = False,
     query: str | None = None,
     llm: LLM | None = None,
@@ -52,19 +49,18 @@ def _build_index_filters(
     # Assistant knowledge filters
     attached_document_ids: list[str] | None = None,
     hierarchy_node_ids: list[int] | None = None,
-    # Pre-fetched ACL filters (skips DB query when provided)
-    acl_filters: list[str] | None = None,
 ) -> IndexFilters:
     if auto_detect_filters and (llm is None or query is None):
         raise RuntimeError("LLM and query are required for auto detect filters")
 
     base_filters = user_provided_filters or BaseFilters()
 
-    document_set_filter = (
-        base_filters.document_set
-        if base_filters.document_set is not None
-        else persona_document_sets
-    )
+    if (
+        user_provided_filters
+        and user_provided_filters.document_set is None
+        and persona_document_sets is not None
+    ):
+        base_filters.document_set = persona_document_sets
 
     time_filter = base_filters.time_cutoff or persona_time_cutoff
     source_filter = base_filters.source_type
@@ -107,21 +103,15 @@ def _build_index_filters(
             source_filter = list(source_filter) + [DocumentSource.USER_FILE]
             logger.debug("Added USER_FILE to source_filter for user knowledge search")
 
-    if bypass_acl:
-        user_acl_filters = None
-    elif acl_filters is not None:
-        user_acl_filters = acl_filters
-    else:
-        if db_session is None:
-            raise ValueError("Either db_session or acl_filters must be provided")
-        user_acl_filters = build_access_filters_for_user(user, db_session)
+    user_acl_filters = (
+        None if bypass_acl else build_access_filters_for_user(user, db_session)
+    )
 
     final_filters = IndexFilters(
         user_file_ids=user_file_ids,
         project_id=project_id,
-        persona_id=persona_id,
         source_type=source_filter,
-        document_set=document_set_filter,
+        document_set=persona_document_sets,
         time_cutoff=time_filter,
         tags=base_filters.tags,
         access_control_list=user_acl_filters,
@@ -262,17 +252,11 @@ def search_pipeline(
     user: User,
     # Used for default filters and settings
     persona: Persona | None,
-    db_session: Session | None = None,
+    db_session: Session,
     auto_detect_filters: bool = False,
     llm: LLM | None = None,
     # If a project ID is provided, it will be exclusively scoped to that project
     project_id: int | None = None,
-    # If a persona_id is provided, search scopes to files attached to this persona
-    persona_id: int | None = None,
-    # Pre-fetched data — when provided, avoids DB queries (no session needed)
-    acl_filters: list[str] | None = None,
-    embedding_model: EmbeddingModel | None = None,
-    prefetched_federated_retrieval_infos: list[FederatedRetrievalInfo] | None = None,
 ) -> list[InferenceChunk]:
     user_uploaded_persona_files: list[UUID] | None = (
         [user_file.id for user_file in persona.user_files] if persona else None
@@ -303,7 +287,6 @@ def search_pipeline(
         user_provided_filters=chunk_search_request.user_selected_filters,
         user=user,
         project_id=project_id,
-        persona_id=persona_id,
         user_file_ids=user_uploaded_persona_files,
         persona_document_sets=persona_document_sets,
         persona_time_cutoff=persona_time_cutoff,
@@ -314,7 +297,6 @@ def search_pipeline(
         bypass_acl=chunk_search_request.bypass_acl,
         attached_document_ids=attached_document_ids,
         hierarchy_node_ids=hierarchy_node_ids,
-        acl_filters=acl_filters,
     )
 
     query_keywords = strip_stopwords(chunk_search_request.query)
@@ -333,8 +315,6 @@ def search_pipeline(
         user_id=user.id if user else None,
         document_index=document_index,
         db_session=db_session,
-        embedding_model=embedding_model,
-        prefetched_federated_retrieval_infos=prefetched_federated_retrieval_infos,
     )
 
     # For some specific connectors like Salesforce, a user that has access to an object doesn't mean

backend/onyx/context/search/retrieval/search_runner.py

@@ -14,11 +14,9 @@ from onyx.context.search.utils import get_query_embedding
 from onyx.context.search.utils import inference_section_from_chunks
 from onyx.document_index.interfaces import DocumentIndex
 from onyx.document_index.interfaces import VespaChunkRequest
-from onyx.federated_connectors.federated_retrieval import FederatedRetrievalInfo
 from onyx.federated_connectors.federated_retrieval import (
     get_federated_retrieval_functions,
 )
-from onyx.natural_language_processing.search_nlp_models import EmbeddingModel
 from onyx.utils.logger import setup_logger
 from onyx.utils.threadpool_concurrency import run_functions_tuples_in_parallel
 
@@ -52,14 +50,9 @@ def combine_retrieval_results(
 def _embed_and_search(
     query_request: ChunkIndexRequest,
     document_index: DocumentIndex,
-    db_session: Session | None = None,
-    embedding_model: EmbeddingModel | None = None,
+    db_session: Session,
 ) -> list[InferenceChunk]:
-    query_embedding = get_query_embedding(
-        query_request.query,
-        db_session=db_session,
-        embedding_model=embedding_model,
-    )
+    query_embedding = get_query_embedding(query_request.query, db_session)
 
     hybrid_alpha = query_request.hybrid_alpha or HYBRID_ALPHA
 
@@ -85,9 +78,7 @@ def search_chunks(
     query_request: ChunkIndexRequest,
     user_id: UUID | None,
     document_index: DocumentIndex,
-    db_session: Session | None = None,
-    embedding_model: EmbeddingModel | None = None,
-    prefetched_federated_retrieval_infos: list[FederatedRetrievalInfo] | None = None,
+    db_session: Session,
 ) -> list[InferenceChunk]:
     run_queries: list[tuple[Callable, tuple]] = []
 
@@ -97,22 +88,14 @@ def search_chunks(
         else None
     )
 
-    # Federated retrieval — use pre-fetched if available, otherwise query DB
-    if prefetched_federated_retrieval_infos is not None:
-        federated_retrieval_infos = prefetched_federated_retrieval_infos
-    else:
-        if db_session is None:
-            raise ValueError(
-                "Either db_session or prefetched_federated_retrieval_infos "
-                "must be provided"
-            )
-        federated_retrieval_infos = get_federated_retrieval_functions(
-            db_session=db_session,
-            user_id=user_id,
-            source_types=list(source_filters) if source_filters else None,
-            document_set_names=query_request.filters.document_set,
-            user_file_ids=query_request.filters.user_file_ids,
-        )
+    # Federated retrieval
+    federated_retrieval_infos = get_federated_retrieval_functions(
+        db_session=db_session,
+        user_id=user_id,
+        source_types=list(source_filters) if source_filters else None,
+        document_set_names=query_request.filters.document_set,
+        user_file_ids=query_request.filters.user_file_ids,
+    )
 
     federated_sources = set(
         federated_retrieval_info.source.to_non_federated_source()
@@ -131,10 +114,7 @@ def search_chunks(
 
     if normal_search_enabled:
         run_queries.append(
-            (
-                _embed_and_search,
-                (query_request, document_index, db_session, embedding_model),
-            )
+            (_embed_and_search, (query_request, document_index, db_session))
         )
 
     parallel_search_results = run_functions_tuples_in_parallel(run_queries)

backend/onyx/context/search/utils.py

@@ -64,34 +64,23 @@ def inference_section_from_single_chunk(
     )
 
 
-def get_query_embeddings(
-    queries: list[str],
-    db_session: Session | None = None,
-    embedding_model: EmbeddingModel | None = None,
-) -> list[Embedding]:
-    if embedding_model is None:
-        if db_session is None:
-            raise ValueError("Either db_session or embedding_model must be provided")
-        search_settings = get_current_search_settings(db_session)
-        embedding_model = EmbeddingModel.from_db_model(
-            search_settings=search_settings,
-            server_host=MODEL_SERVER_HOST,
-            server_port=MODEL_SERVER_PORT,
-        )
+def get_query_embeddings(queries: list[str], db_session: Session) -> list[Embedding]:
+    search_settings = get_current_search_settings(db_session)
 
-    query_embedding = embedding_model.encode(queries, text_type=EmbedTextType.QUERY)
+    model = EmbeddingModel.from_db_model(
+        search_settings=search_settings,
+        # The below are globally set, this flow always uses the indexing one
+        server_host=MODEL_SERVER_HOST,
+        server_port=MODEL_SERVER_PORT,
+    )
+
+    query_embedding = model.encode(queries, text_type=EmbedTextType.QUERY)
     return query_embedding
 
 
 @log_function_time(print_only=True, debug_only=True)
-def get_query_embedding(
-    query: str,
-    db_session: Session | None = None,
-    embedding_model: EmbeddingModel | None = None,
-) -> Embedding:
-    return get_query_embeddings(
-        [query], db_session=db_session, embedding_model=embedding_model
-    )[0]
+def get_query_embedding(query: str, db_session: Session) -> Embedding:
+    return get_query_embeddings([query], db_session)[0]
 
 
 def convert_inference_sections_to_search_docs(

backend/onyx/db/api_key.py

@@ -4,7 +4,6 @@ from fastapi_users.password import PasswordHelper
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
-from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
 from onyx.auth.api_key import ApiKeyDescriptor
@@ -55,7 +54,6 @@ async def fetch_user_for_api_key(
         select(User)
         .join(ApiKey, ApiKey.user_id == User.id)
         .where(ApiKey.hashed_api_key == hashed_api_key)
-        .options(selectinload(User.memories))
     )
 
 

backend/onyx/db/auth.py

@@ -13,7 +13,6 @@ from sqlalchemy import func
 from sqlalchemy import Select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
-from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
 from onyx.auth.schemas import UserRole
@@ -98,11 +97,6 @@ async def get_user_count(only_admin_users: bool = False) -> int:
 
 # Need to override this because FastAPI Users doesn't give flexibility for backend field creation logic in OAuth flow
 class SQLAlchemyUserAdminDB(SQLAlchemyUserDatabase[UP, ID]):
-    async def _get_user(self, statement: Select) -> UP | None:
-        statement = statement.options(selectinload(User.memories))
-        results = await self.session.execute(statement)
-        return results.unique().scalar_one_or_none()
-
     async def create(
         self,
         create_dict: Dict[str, Any],

backend/onyx/db/chat.py

@@ -98,7 +98,6 @@ def get_chat_sessions_by_user(
     db_session: Session,
     include_onyxbot_flows: bool = False,
     limit: int = 50,
-    before: datetime | None = None,
     project_id: int | None = None,
     only_non_project_chats: bool = False,
     include_failed_chats: bool = False,
@@ -113,9 +112,6 @@ def get_chat_sessions_by_user(
     if deleted is not None:
         stmt = stmt.where(ChatSession.deleted == deleted)
 
-    if before is not None:
-        stmt = stmt.where(ChatSession.time_updated < before)
-
     if limit:
         stmt = stmt.limit(limit)
 

backend/onyx/db/code_interpreter.py

@@ -1,21 +0,0 @@
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from onyx.db.models import CodeInterpreterServer
-
-
-def fetch_code_interpreter_server(
-    db_session: Session,
-) -> CodeInterpreterServer:
-    server = db_session.scalars(select(CodeInterpreterServer)).one()
-    return server
-
-
-def update_code_interpreter_server_enabled(
-    db_session: Session,
-    enabled: bool,
-) -> CodeInterpreterServer:
-    server = db_session.scalars(select(CodeInterpreterServer)).one()
-    server.server_enabled = enabled
-    db_session.commit()
-    return server

backend/onyx/db/connector_credential_pair.py

@@ -116,15 +116,12 @@ def get_connector_credential_pairs_for_user(
     order_by_desc: bool = False,
     source: DocumentSource | None = None,
     processing_mode: ProcessingMode | None = ProcessingMode.REGULAR,
-    defer_connector_config: bool = False,
 ) -> list[ConnectorCredentialPair]:
     """Get connector credential pairs for a user.
 
     Args:
         processing_mode: Filter by processing mode. Defaults to REGULAR to hide
             FILE_SYSTEM connectors from standard admin UI. Pass None to get all.
-        defer_connector_config: If True, skips loading Connector.connector_specific_config
-            to avoid fetching large JSONB blobs when they aren't needed.
     """
     if eager_load_user:
         assert (
@@ -133,10 +130,7 @@ def get_connector_credential_pairs_for_user(
     stmt = select(ConnectorCredentialPair).distinct()
 
     if eager_load_connector:
-        connector_load = selectinload(ConnectorCredentialPair.connector)
-        if defer_connector_config:
-            connector_load = connector_load.defer(Connector.connector_specific_config)
-        stmt = stmt.options(connector_load)
+        stmt = stmt.options(selectinload(ConnectorCredentialPair.connector))
 
     if eager_load_credential:
         load_opts = selectinload(ConnectorCredentialPair.credential)
@@ -176,7 +170,6 @@ def get_connector_credential_pairs_for_user_parallel(
     order_by_desc: bool = False,
     source: DocumentSource | None = None,
     processing_mode: ProcessingMode | None = ProcessingMode.REGULAR,
-    defer_connector_config: bool = False,
 ) -> list[ConnectorCredentialPair]:
     with get_session_with_current_tenant() as db_session:
         return get_connector_credential_pairs_for_user(
@@ -190,7 +183,6 @@ def get_connector_credential_pairs_for_user_parallel(
             order_by_desc=order_by_desc,
             source=source,
             processing_mode=processing_mode,
-            defer_connector_config=defer_connector_config,
         )
 
 

backend/onyx/db/document_set.py

@@ -554,19 +554,10 @@ def fetch_all_document_sets_for_user(
     stmt = (
         select(DocumentSetDBModel)
         .distinct()
-        .options(
-            selectinload(DocumentSetDBModel.connector_credential_pairs).selectinload(
-                ConnectorCredentialPair.connector
-            ),
-            selectinload(DocumentSetDBModel.users),
-            selectinload(DocumentSetDBModel.groups),
-            selectinload(DocumentSetDBModel.federated_connectors).selectinload(
-                FederatedConnector__DocumentSet.federated_connector
-            ),
-        )
+        .options(selectinload(DocumentSetDBModel.federated_connectors))
     )
     stmt = _add_user_filters(stmt, user, get_editable=get_editable)
-    return db_session.scalars(stmt).unique().all()
+    return db_session.scalars(stmt).all()
 
 
 def fetch_documents_for_document_set_paginated(

backend/onyx/db/engine/async_sql_engine.py

@@ -21,8 +21,8 @@ from onyx.configs.app_configs import POSTGRES_POOL_RECYCLE
 from onyx.configs.app_configs import POSTGRES_PORT
 from onyx.configs.app_configs import POSTGRES_USE_NULL_POOL
 from onyx.configs.app_configs import POSTGRES_USER
-from onyx.db.engine.iam_auth import create_ssl_context_if_iam
 from onyx.db.engine.iam_auth import get_iam_auth_token
+from onyx.db.engine.iam_auth import ssl_context
 from onyx.db.engine.sql_engine import ASYNC_DB_API
 from onyx.db.engine.sql_engine import build_connection_string
 from onyx.db.engine.sql_engine import is_valid_schema_name
@@ -66,7 +66,7 @@ def get_sqlalchemy_async_engine() -> AsyncEngine:
         if app_name:
             connect_args["server_settings"] = {"application_name": app_name}
 
-        connect_args["ssl"] = create_ssl_context_if_iam()
+        connect_args["ssl"] = ssl_context
 
         engine_kwargs = {
             "connect_args": connect_args,
@@ -97,7 +97,7 @@ def get_sqlalchemy_async_engine() -> AsyncEngine:
                 user = POSTGRES_USER
                 token = get_iam_auth_token(host, port, user, AWS_REGION_NAME)
                 cparams["password"] = token
-                cparams["ssl"] = create_ssl_context_if_iam()
+                cparams["ssl"] = ssl_context
 
     return _ASYNC_ENGINE
 

backend/onyx/db/engine/iam_auth.py

@@ -1,4 +1,3 @@
-import functools
 import os
 import ssl
 from typing import Any
@@ -49,9 +48,11 @@ def provide_iam_token(
         configure_psycopg2_iam_auth(cparams, host, port, user, region)
 
 
-@functools.cache
 def create_ssl_context_if_iam() -> ssl.SSLContext | None:
     """Create an SSL context if IAM authentication is enabled, else return None."""
     if USE_IAM_AUTH:
         return ssl.create_default_context(cafile=SSL_CERT_FILE)
     return None
+
+
+ssl_context = create_ssl_context_if_iam()

backend/onyx/db/engine/tenant_utils.py

@@ -1,102 +1,11 @@
 from sqlalchemy import text
 
 from onyx.db.engine.sql_engine import get_session_with_shared_schema
-from onyx.db.engine.sql_engine import SqlEngine
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.configs import TENANT_ID_PREFIX
 
 
-def get_schemas_needing_migration(
-    tenant_schemas: list[str], head_rev: str
-) -> list[str]:
-    """Return only schemas whose current alembic version is not at head.
-
-    Uses a server-side PL/pgSQL loop to collect each schema's alembic version
-    into a temp table one at a time. This avoids building a massive UNION ALL
-    query (which locks the DB and times out at 17k+ schemas) and instead
-    acquires locks sequentially, one schema per iteration.
-    """
-    if not tenant_schemas:
-        return []
-
-    engine = SqlEngine.get_engine()
-
-    with engine.connect() as conn:
-        # Populate a temp input table with exactly the schemas we care about.
-        # The DO block reads from this table so it only iterates the requested
-        # schemas instead of every tenant_% schema in the database.
-        conn.execute(text("DROP TABLE IF EXISTS _alembic_version_snapshot"))
-        conn.execute(text("DROP TABLE IF EXISTS _tenant_schemas_input"))
-        conn.execute(text("CREATE TEMP TABLE _tenant_schemas_input (schema_name text)"))
-        conn.execute(
-            text(
-                "INSERT INTO _tenant_schemas_input (schema_name) "
-                "SELECT unnest(CAST(:schemas AS text[]))"
-            ),
-            {"schemas": tenant_schemas},
-        )
-        conn.execute(
-            text(
-                "CREATE TEMP TABLE _alembic_version_snapshot "
-                "(schema_name text, version_num text)"
-            )
-        )
-
-        conn.execute(
-            text(
-                """
-                DO $$
-                DECLARE
-                    s        text;
-                    schemas  text[];
-                BEGIN
-                    SELECT array_agg(schema_name) INTO schemas
-                    FROM _tenant_schemas_input;
-
-                    IF schemas IS NULL THEN
-                        RAISE NOTICE 'No tenant schemas found.';
-                        RETURN;
-                    END IF;
-
-                    FOREACH s IN ARRAY schemas LOOP
-                        BEGIN
-                            EXECUTE format(
-                                'INSERT INTO _alembic_version_snapshot
-                                 SELECT %L, version_num FROM %I.alembic_version',
-                                s, s
-                            );
-                        EXCEPTION
-                            -- undefined_table: schema exists but has no alembic_version
-                            --   table yet (new tenant, not yet migrated).
-                            -- invalid_schema_name: tenant is registered but its
-                            --   PostgreSQL schema does not exist yet (e.g. provisioning
-                            --   incomplete). Both cases mean no version is available and
-                            --   the schema will be included in the migration list.
-                            WHEN undefined_table THEN NULL;
-                            WHEN invalid_schema_name THEN NULL;
-                        END;
-                    END LOOP;
-                END;
-                $$
-                """
-            )
-        )
-
-        rows = conn.execute(
-            text("SELECT schema_name, version_num FROM _alembic_version_snapshot")
-        )
-        version_by_schema = {row[0]: row[1] for row in rows}
-
-        conn.execute(text("DROP TABLE IF EXISTS _alembic_version_snapshot"))
-        conn.execute(text("DROP TABLE IF EXISTS _tenant_schemas_input"))
-
-    # Schemas missing from the snapshot have no alembic_version table yet and
-    # also need migration. version_by_schema.get(s) returns None for those,
-    # and None != head_rev, so they are included automatically.
-    return [s for s in tenant_schemas if version_by_schema.get(s) != head_rev]
-
-
 def get_all_tenant_ids() -> list[str]:
     """Returning [None] means the only tenant is the 'public' or self hosted tenant."""
 

backend/onyx/db/enums.py

@@ -186,7 +186,6 @@ class EmbeddingPrecision(str, PyEnum):
 
 class UserFileStatus(str, PyEnum):
     PROCESSING = "PROCESSING"
-    INDEXING = "INDEXING"
     COMPLETED = "COMPLETED"
     FAILED = "FAILED"
     CANCELED = "CANCELED"

backend/onyx/db/image_generation.py

@@ -202,6 +202,7 @@ def create_default_image_gen_config_from_api_key(
             api_key=api_key,
             api_base=None,
             api_version=None,
+            default_model_name=model_name,
             deployment_name=None,
             is_public=True,
         )

backend/onyx/db/llm.py

@@ -109,38 +109,45 @@ def can_user_access_llm_provider(
         is_admin: If True, bypass user group restrictions but still respect persona restrictions
 
     Access logic:
-    - is_public controls USER access (group bypass): when True, all users can access
-      regardless of group membership. When False, user must be in a whitelisted group
-      (or be admin).
-    - Persona restrictions are ALWAYS enforced when set, regardless of is_public.
-      This allows admins to make a provider available to all users while still
-      restricting which personas (assistants) can use it.
-
-    Decision matrix:
-    1. is_public=True, no personas set → everyone has access
-    2. is_public=True, personas set → all users, but only whitelisted personas
-    3. is_public=False, groups+personas set → must satisfy BOTH (admins bypass groups)
-    4. is_public=False, only groups set → must be in group (admins bypass)
-    5. is_public=False, only personas set → must use whitelisted persona
-    6. is_public=False, neither set → admin-only (locked)
+    1. If is_public=True → everyone has access (public override)
+    2. If is_public=False:
+       - Both groups AND personas set → must satisfy BOTH (AND logic, admins bypass group check)
+       - Only groups set → must be in one of the groups (OR across groups, admins bypass)
+       - Only personas set → must use one of the personas (OR across personas, applies to admins)
+       - Neither set → NOBODY has access unless admin (locked, admin-only)
     """
-    provider_group_ids = {g.id for g in (provider.groups or [])}
-    provider_persona_ids = {p.id for p in (provider.personas or [])}
-    has_groups = bool(provider_group_ids)
-    has_personas = bool(provider_persona_ids)
-
-    # Persona restrictions are always enforced when set, regardless of is_public
-    if has_personas and not (persona and persona.id in provider_persona_ids):
-        return False
-
+    # Public override - everyone has access
     if provider.is_public:
         return True
 
+    # Extract IDs once to avoid multiple iterations
+    provider_group_ids = (
+        {group.id for group in provider.groups} if provider.groups else set()
+    )
+    provider_persona_ids = (
+        {p.id for p in provider.personas} if provider.personas else set()
+    )
+
+    has_groups = bool(provider_group_ids)
+    has_personas = bool(provider_persona_ids)
+
+    # Both groups AND personas set → AND logic (must satisfy both)
+    if has_groups and has_personas:
+        # Admins bypass group check but still must satisfy persona restrictions
+        user_in_group = is_admin or bool(user_group_ids & provider_group_ids)
+        persona_allowed = persona.id in provider_persona_ids if persona else False
+        return user_in_group and persona_allowed
+
+    # Only groups set → user must be in one of the groups (admins bypass)
     if has_groups:
         return is_admin or bool(user_group_ids & provider_group_ids)
 
-    # No groups: either persona-whitelisted (already passed) or admin-only if locked
-    return has_personas or is_admin
+    # Only personas set → persona must be in allowed list (applies to admins too)
+    if has_personas:
+        return persona.id in provider_persona_ids if persona else False
+
+    # Neither groups nor personas set, and not public → admins can access
+    return is_admin
 
 
 def validate_persona_ids_exist(
@@ -206,29 +213,11 @@ def upsert_llm_provider(
     llm_provider_upsert_request: LLMProviderUpsertRequest,
     db_session: Session,
 ) -> LLMProviderView:
-    existing_llm_provider: LLMProviderModel | None = None
-    if llm_provider_upsert_request.id:
-        existing_llm_provider = fetch_existing_llm_provider_by_id(
-            id=llm_provider_upsert_request.id, db_session=db_session
-        )
-        if not existing_llm_provider:
-            raise ValueError(
-                f"LLM provider with id {llm_provider_upsert_request.id} not found"
-            )
+    existing_llm_provider = fetch_existing_llm_provider(
+        name=llm_provider_upsert_request.name, db_session=db_session
+    )
 
-        if existing_llm_provider.name != llm_provider_upsert_request.name:
-            raise ValueError(
-                f"LLM provider with id {llm_provider_upsert_request.id} name change not allowed"
-            )
-    else:
-        existing_llm_provider = fetch_existing_llm_provider(
-            name=llm_provider_upsert_request.name, db_session=db_session
-        )
-        if existing_llm_provider:
-            raise ValueError(
-                f"LLM provider with name '{llm_provider_upsert_request.name}'"
-                " already exists"
-            )
+    if not existing_llm_provider:
         existing_llm_provider = LLMProviderModel(name=llm_provider_upsert_request.name)
         db_session.add(existing_llm_provider)
 
@@ -249,7 +238,11 @@ def upsert_llm_provider(
     existing_llm_provider.api_base = api_base
     existing_llm_provider.api_version = llm_provider_upsert_request.api_version
     existing_llm_provider.custom_config = custom_config
-
+    # TODO: Remove default model name on api change
+    # Needed due to /provider/{id}/default endpoint not disclosing the default model name
+    existing_llm_provider.default_model_name = (
+        llm_provider_upsert_request.default_model_name
+    )
     existing_llm_provider.is_public = llm_provider_upsert_request.is_public
     existing_llm_provider.is_auto_mode = llm_provider_upsert_request.is_auto_mode
     existing_llm_provider.deployment_name = llm_provider_upsert_request.deployment_name
@@ -313,6 +306,15 @@ def upsert_llm_provider(
                 display_name=model_config.display_name,
             )
 
+    default_model = fetch_default_model(db_session, LLMModelFlowType.CHAT)
+    if default_model and default_model.llm_provider_id == existing_llm_provider.id:
+        _update_default_model(
+            db_session=db_session,
+            provider_id=existing_llm_provider.id,
+            model=existing_llm_provider.default_model_name,
+            flow_type=LLMModelFlowType.CHAT,
+        )
+
     # Make sure the relationship table stays up to date
     update_group_llm_provider_relationships__no_commit(
         llm_provider_id=existing_llm_provider.id,
@@ -486,22 +488,6 @@ def fetch_existing_llm_provider(
     return provider_model
 
 
-def fetch_existing_llm_provider_by_id(
-    id: int, db_session: Session
-) -> LLMProviderModel | None:
-    provider_model = db_session.scalar(
-        select(LLMProviderModel)
-        .where(LLMProviderModel.id == id)
-        .options(
-            selectinload(LLMProviderModel.model_configurations),
-            selectinload(LLMProviderModel.groups),
-            selectinload(LLMProviderModel.personas),
-        )
-    )
-
-    return provider_model
-
-
 def fetch_embedding_provider(
     db_session: Session, provider_type: EmbeddingProvider
 ) -> CloudEmbeddingProviderModel | None:
@@ -618,13 +604,22 @@ def remove_llm_provider__no_commit(db_session: Session, provider_id: int) -> Non
     db_session.flush()
 
 
-def update_default_provider(
-    provider_id: int, model_name: str, db_session: Session
-) -> None:
+def update_default_provider(provider_id: int, db_session: Session) -> None:
+    # Attempt to get the default_model_name from the provider first
+    # TODO: Remove default_model_name check
+    provider = db_session.scalar(
+        select(LLMProviderModel).where(
+            LLMProviderModel.id == provider_id,
+        )
+    )
+
+    if provider is None:
+        raise ValueError(f"LLM Provider with id={provider_id} does not exist")
+
     _update_default_model(
         db_session,
         provider_id,
-        model_name,
+        provider.default_model_name,
         LLMModelFlowType.CHAT,
     )
 
@@ -810,6 +805,12 @@ def sync_auto_mode_models(
             )
             changes += 1
 
+    # In Auto mode, default model is always set from GitHub config
+    default_model = llm_recommendations.get_default_model(provider.provider)
+    if default_model and provider.default_model_name != default_model.name:
+        provider.default_model_name = default_model.name
+        changes += 1
+
     db_session.commit()
     return changes
 

backend/onyx/db/models.py

@@ -103,6 +103,7 @@ from onyx.utils.encryption import encrypt_string_to_bytes
 from onyx.utils.sensitive import SensitiveValue
 from onyx.utils.headers import HeaderItemDict
 from shared_configs.enums import EmbeddingProvider
+from onyx.context.search.enums import RecencyBiasSetting
 
 # TODO: After anonymous user migration has been deployed, make user_id columns NOT NULL
 # and update Mapped[User | None] relationships to Mapped[User] where needed.
@@ -286,7 +287,7 @@ class User(SQLAlchemyBaseUserTableUUID, Base):
 
     # relationships
     credentials: Mapped[list["Credential"]] = relationship(
-        "Credential", back_populates="user"
+        "Credential", back_populates="user", lazy="joined"
     )
     chat_sessions: Mapped[list["ChatSession"]] = relationship(
         "ChatSession", back_populates="user"
@@ -320,6 +321,7 @@ class User(SQLAlchemyBaseUserTableUUID, Base):
         "Memory",
         back_populates="user",
         cascade="all, delete-orphan",
+        lazy="selectin",
         order_by="desc(Memory.id)",
     )
     oauth_user_tokens: Mapped[list["OAuthUserToken"]] = relationship(
@@ -2821,9 +2823,14 @@ class LLMProvider(Base):
     custom_config: Mapped[dict[str, str] | None] = mapped_column(
         postgresql.JSONB(), nullable=True
     )
+    default_model_name: Mapped[str] = mapped_column(String)
 
     deployment_name: Mapped[str | None] = mapped_column(String, nullable=True)
 
+    # should only be set for a single provider
+    is_default_provider: Mapped[bool | None] = mapped_column(Boolean, unique=True)
+    is_default_vision_provider: Mapped[bool | None] = mapped_column(Boolean)
+    default_vision_model: Mapped[str | None] = mapped_column(String, nullable=True)
     # EE only
     is_public: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
     # Auto mode: models, visibility, and defaults are managed by GitHub config
@@ -2873,7 +2880,6 @@ class ModelConfiguration(Base):
     # - The end-user is configuring a model and chooses not to set a max-input-tokens limit.
     max_input_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
 
-    # Deprecated: use LLMModelFlow with VISION flow type instead
     supports_image_input: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
 
     # Human-readable display name for the model.
@@ -3255,6 +3261,19 @@ class Persona(Base):
     )
     name: Mapped[str] = mapped_column(String)
     description: Mapped[str] = mapped_column(String)
+    # Number of chunks to pass to the LLM for generation.
+    num_chunks: Mapped[float | None] = mapped_column(Float, nullable=True)
+    chunks_above: Mapped[int] = mapped_column(Integer)
+    chunks_below: Mapped[int] = mapped_column(Integer)
+    # Pass every chunk through LLM for evaluation, fairly expensive
+    # Can be turned off globally by admin, in which case, this setting is ignored
+    llm_relevance_filter: Mapped[bool] = mapped_column(Boolean)
+    # Enables using LLM to extract time and source type filters
+    # Can also be admin disabled globally
+    llm_filter_extraction: Mapped[bool] = mapped_column(Boolean)
+    recency_bias: Mapped[RecencyBiasSetting] = mapped_column(
+        Enum(RecencyBiasSetting, native_enum=False)
+    )
 
     # Allows the persona to specify a specific default LLM model
     # NOTE: only is applied on the actual response generation - is not used for things like
@@ -3281,8 +3300,11 @@ class Persona(Base):
     # Treated specially (cannot be user edited etc.)
     builtin_persona: Mapped[bool] = mapped_column(Boolean, default=False)
 
-    # Featured personas are highlighted in the UI
-    featured: Mapped[bool] = mapped_column(Boolean, default=False)
+    # Default personas are personas created by admins and are automatically added
+    # to all users' assistants list.
+    is_default_persona: Mapped[bool] = mapped_column(
+        Boolean, default=False, nullable=False
+    )
     # controls whether the persona is available to be selected by users
     is_visible: Mapped[bool] = mapped_column(Boolean, default=True)
     # controls the ordering of personas in the UI
@@ -4249,9 +4271,6 @@ class UserFile(Base):
     needs_project_sync: Mapped[bool] = mapped_column(
         Boolean, nullable=False, default=False
     )
-    needs_persona_sync: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=False
-    )
     last_project_sync_at: Mapped[datetime.datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )
@@ -4921,12 +4940,6 @@ class ScimUserMapping(Base):
     user_id: Mapped[UUID] = mapped_column(
         ForeignKey("user.id", ondelete="CASCADE"), unique=True, nullable=False
     )
-    scim_username: Mapped[str | None] = mapped_column(String, nullable=True)
-    department: Mapped[str | None] = mapped_column(String, nullable=True)
-    manager: Mapped[str | None] = mapped_column(String, nullable=True)
-    given_name: Mapped[str | None] = mapped_column(String, nullable=True)
-    family_name: Mapped[str | None] = mapped_column(String, nullable=True)
-    scim_emails_json: Mapped[str | None] = mapped_column(Text, nullable=True)
 
     created_at: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True), server_default=func.now(), nullable=False
@@ -4965,12 +4978,3 @@ class ScimGroupMapping(Base):
     user_group: Mapped[UserGroup] = relationship(
         "UserGroup", foreign_keys=[user_group_id]
     )
-
-
-class CodeInterpreterServer(Base):
-    """Details about the code interpreter server"""
-
-    __tablename__ = "code_interpreter_server"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    server_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)

backend/onyx/db/pat.py

@@ -8,7 +8,6 @@ from uuid import UUID
 from sqlalchemy import select
 from sqlalchemy import update
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
 from onyx.auth.pat import build_displayable_pat
@@ -32,59 +31,53 @@ async def fetch_user_for_pat(
 
     NOTE: This is async since it's used during auth (which is necessarily async due to FastAPI Users).
     NOTE: Expired includes both naturally expired and user-revoked tokens (revocation sets expires_at=NOW()).
-
-    Uses select(User) as primary entity so that joined-eager relationships (e.g. oauth_accounts)
-    are loaded correctly — matching the pattern in fetch_user_for_api_key.
     """
+    # Single joined query with all filters pushed to database
     now = datetime.now(timezone.utc)
-
-    user = await async_db_session.scalar(
-        select(User)
-        .join(PersonalAccessToken, PersonalAccessToken.user_id == User.id)
+    result = await async_db_session.execute(
+        select(PersonalAccessToken, User)
+        .join(User, PersonalAccessToken.user_id == User.id)
         .where(PersonalAccessToken.hashed_token == hashed_token)
         .where(User.is_active)  # type: ignore
         .where(
             (PersonalAccessToken.expires_at.is_(None))
             | (PersonalAccessToken.expires_at > now)
         )
-        .options(selectinload(User.memories))
+        .limit(1)
     )
-    if not user:
+    row = result.first()
+
+    if not row:
         return None
 
-    _schedule_pat_last_used_update(hashed_token, now)
-    return user
+    pat, user = row
 
+    # Throttle last_used_at updates to reduce DB load (5-minute granularity sufficient for auditing)
+    # For request-level auditing, use application logs or a dedicated audit table
+    should_update = (
+        pat.last_used_at is None or (now - pat.last_used_at).total_seconds() > 300
+    )
 
-def _schedule_pat_last_used_update(hashed_token: str, now: datetime) -> None:
-    """Fire-and-forget update of last_used_at, throttled to 5-minute granularity."""
-
-    async def _update() -> None:
-        try:
-            tenant_id = get_current_tenant_id()
-            async with get_async_session_context_manager(tenant_id) as session:
-                pat = await session.scalar(
-                    select(PersonalAccessToken).where(
-                        PersonalAccessToken.hashed_token == hashed_token
+    if should_update:
+        # Update in separate session to avoid transaction coupling (fire-and-forget)
+        async def _update_last_used() -> None:
+            try:
+                tenant_id = get_current_tenant_id()
+                async with get_async_session_context_manager(
+                    tenant_id
+                ) as separate_session:
+                    await separate_session.execute(
+                        update(PersonalAccessToken)
+                        .where(PersonalAccessToken.hashed_token == hashed_token)
+                        .values(last_used_at=now)
                     )
-                )
-                if not pat:
-                    return
-                if (
-                    pat.last_used_at is not None
-                    and (now - pat.last_used_at).total_seconds() <= 300
-                ):
-                    return
-                await session.execute(
-                    update(PersonalAccessToken)
-                    .where(PersonalAccessToken.hashed_token == hashed_token)
-                    .values(last_used_at=now)
-                )
-                await session.commit()
-        except Exception as e:
-            logger.warning(f"Failed to update last_used_at for PAT: {e}")
+                    await separate_session.commit()
+            except Exception as e:
+                logger.warning(f"Failed to update last_used_at for PAT: {e}")
 
-    asyncio.create_task(_update())
+        asyncio.create_task(_update_last_used())
+
+    return user
 
 
 def create_pat(

backend/onyx/db/persona.py

@@ -18,14 +18,16 @@ from sqlalchemy.orm import Session
 from onyx.access.hierarchy_access import get_user_external_group_ids
 from onyx.auth.schemas import UserRole
 from onyx.configs.app_configs import CURATORS_CANNOT_VIEW_OR_EDIT_NON_OWNED_ASSISTANTS
+from onyx.configs.chat_configs import CONTEXT_CHUNKS_ABOVE
+from onyx.configs.chat_configs import CONTEXT_CHUNKS_BELOW
 from onyx.configs.constants import DEFAULT_PERSONA_ID
 from onyx.configs.constants import NotificationType
+from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.constants import SLACK_BOT_PERSONA_PREFIX
 from onyx.db.document_access import get_accessible_documents_by_ids
 from onyx.db.models import ConnectorCredentialPair
 from onyx.db.models import Document
 from onyx.db.models import DocumentSet
-from onyx.db.models import FederatedConnector__DocumentSet
 from onyx.db.models import HierarchyNode
 from onyx.db.models import Persona
 from onyx.db.models import Persona__User
@@ -251,15 +253,16 @@ def create_update_persona(
     # Permission to actually use these is checked later
 
     try:
-        # Featured persona validation
-        if create_persona_request.featured:
+        # Default persona validation
+        if create_persona_request.is_default_persona:
+            if not create_persona_request.is_public:
+                raise ValueError("Cannot make a default persona non public")
 
-            # Curators can edit featured personas, but not make them
-            # TODO this will be reworked soon with RBAC permissions feature
+            # Curators can edit default personas, but not make them
             if user.role == UserRole.CURATOR or user.role == UserRole.GLOBAL_CURATOR:
                 pass
             elif user.role != UserRole.ADMIN:
-                raise ValueError("Only admins can make a featured persona")
+                raise ValueError("Only admins can make a default persona")
 
         # Convert incoming string UUIDs to UUID objects for DB operations
         converted_user_file_ids = None
@@ -280,6 +283,7 @@ def create_update_persona(
             document_set_ids=create_persona_request.document_set_ids,
             tool_ids=create_persona_request.tool_ids,
             is_public=create_persona_request.is_public,
+            recency_bias=create_persona_request.recency_bias,
             llm_model_provider_override=create_persona_request.llm_model_provider_override,
             llm_model_version_override=create_persona_request.llm_model_version_override,
             starter_messages=create_persona_request.starter_messages,
@@ -293,7 +297,10 @@ def create_update_persona(
             remove_image=create_persona_request.remove_image,
             search_start_date=create_persona_request.search_start_date,
             label_ids=create_persona_request.label_ids,
-            featured=create_persona_request.featured,
+            num_chunks=create_persona_request.num_chunks,
+            llm_relevance_filter=create_persona_request.llm_relevance_filter,
+            llm_filter_extraction=create_persona_request.llm_filter_extraction,
+            is_default_persona=create_persona_request.is_default_persona,
             user_file_ids=converted_user_file_ids,
             commit=False,
             hierarchy_node_ids=create_persona_request.hierarchy_node_ids,
@@ -327,7 +334,6 @@ def update_persona_shared(
     db_session: Session,
     group_ids: list[int] | None = None,
     is_public: bool | None = None,
-    label_ids: list[int] | None = None,
 ) -> None:
     """Simplified version of `create_update_persona` which only touches the
     accessibility rather than any of the logic (e.g. prompt, connected data sources,
@@ -337,7 +343,9 @@ def update_persona_shared(
     )
 
     if user and user.role != UserRole.ADMIN and persona.user_id != user.id:
-        raise PermissionError("You don't have permission to modify this persona")
+        raise HTTPException(
+            status_code=403, detail="You don't have permission to modify this persona"
+        )
 
     versioned_update_persona_access = fetch_versioned_implementation(
         "onyx.db.persona", "update_persona_access"
@@ -351,15 +359,6 @@ def update_persona_shared(
         group_ids=group_ids,
     )
 
-    if label_ids is not None:
-        labels = (
-            db_session.query(PersonaLabel).filter(PersonaLabel.id.in_(label_ids)).all()
-        )
-        if len(labels) != len(label_ids):
-            raise ValueError("Some label IDs were not found in the database")
-        persona.labels.clear()
-        persona.labels = labels
-
     db_session.commit()
 
 
@@ -421,16 +420,9 @@ def get_minimal_persona_snapshots_for_user(
     stmt = stmt.options(
         selectinload(Persona.tools),
         selectinload(Persona.labels),
-        selectinload(Persona.document_sets).options(
-            selectinload(DocumentSet.connector_credential_pairs).selectinload(
-                ConnectorCredentialPair.connector
-            ),
-            selectinload(DocumentSet.users),
-            selectinload(DocumentSet.groups),
-            selectinload(DocumentSet.federated_connectors).selectinload(
-                FederatedConnector__DocumentSet.federated_connector
-            ),
-        ),
+        selectinload(Persona.document_sets)
+        .selectinload(DocumentSet.connector_credential_pairs)
+        .selectinload(ConnectorCredentialPair.connector),
         selectinload(Persona.hierarchy_nodes),
         selectinload(Persona.attached_documents).selectinload(
             Document.parent_hierarchy_node
@@ -461,16 +453,7 @@ def get_persona_snapshots_for_user(
             Document.parent_hierarchy_node
         ),
         selectinload(Persona.labels),
-        selectinload(Persona.document_sets).options(
-            selectinload(DocumentSet.connector_credential_pairs).selectinload(
-                ConnectorCredentialPair.connector
-            ),
-            selectinload(DocumentSet.users),
-            selectinload(DocumentSet.groups),
-            selectinload(DocumentSet.federated_connectors).selectinload(
-                FederatedConnector__DocumentSet.federated_connector
-            ),
-        ),
+        selectinload(Persona.document_sets),
         selectinload(Persona.user),
         selectinload(Persona.user_files),
         selectinload(Persona.users),
@@ -567,16 +550,9 @@ def get_minimal_persona_snapshots_paginated(
             Document.parent_hierarchy_node
         ),
         selectinload(Persona.labels),
-        selectinload(Persona.document_sets).options(
-            selectinload(DocumentSet.connector_credential_pairs).selectinload(
-                ConnectorCredentialPair.connector
-            ),
-            selectinload(DocumentSet.users),
-            selectinload(DocumentSet.groups),
-            selectinload(DocumentSet.federated_connectors).selectinload(
-                FederatedConnector__DocumentSet.federated_connector
-            ),
-        ),
+        selectinload(Persona.document_sets)
+        .selectinload(DocumentSet.connector_credential_pairs)
+        .selectinload(ConnectorCredentialPair.connector),
         selectinload(Persona.user),
     )
 
@@ -635,16 +611,7 @@ def get_persona_snapshots_paginated(
             Document.parent_hierarchy_node
         ),
         selectinload(Persona.labels),
-        selectinload(Persona.document_sets).options(
-            selectinload(DocumentSet.connector_credential_pairs).selectinload(
-                ConnectorCredentialPair.connector
-            ),
-            selectinload(DocumentSet.users),
-            selectinload(DocumentSet.groups),
-            selectinload(DocumentSet.federated_connectors).selectinload(
-                FederatedConnector__DocumentSet.federated_connector
-            ),
-        ),
+        selectinload(Persona.document_sets),
         selectinload(Persona.user),
         selectinload(Persona.user_files),
         selectinload(Persona.users),
@@ -765,9 +732,6 @@ def mark_persona_as_deleted(
 ) -> None:
     persona = get_persona_by_id(persona_id=persona_id, user=user, db_session=db_session)
     persona.deleted = True
-    affected_file_ids = [uf.id for uf in persona.user_files]
-    if affected_file_ids:
-        _mark_files_need_persona_sync(db_session, affected_file_ids)
     db_session.commit()
 
 
@@ -779,13 +743,11 @@ def mark_persona_as_not_deleted(
     persona = get_persona_by_id(
         persona_id=persona_id, user=user, db_session=db_session, include_deleted=True
     )
-    if not persona.deleted:
+    if persona.deleted:
+        persona.deleted = False
+        db_session.commit()
+    else:
         raise ValueError(f"Persona with ID {persona_id} is not deleted.")
-    persona.deleted = False
-    affected_file_ids = [uf.id for uf in persona.user_files]
-    if affected_file_ids:
-        _mark_files_need_persona_sync(db_session, affected_file_ids)
-    db_session.commit()
 
 
 def mark_delete_persona_by_name(
@@ -851,24 +813,14 @@ def update_personas_display_priority(
         db_session.commit()
 
 
-def _mark_files_need_persona_sync(
-    db_session: Session,
-    user_file_ids: list[UUID],
-) -> None:
-    """Flag the given UserFile rows so the background sync task picks them up
-    and updates their persona metadata in the vector DB."""
-    if not user_file_ids:
-        return
-    db_session.query(UserFile).filter(UserFile.id.in_(user_file_ids)).update(
-        {UserFile.needs_persona_sync: True},
-        synchronize_session=False,
-    )
-
-
 def upsert_persona(
     user: User | None,
     name: str,
     description: str,
+    num_chunks: float,
+    llm_relevance_filter: bool,
+    llm_filter_extraction: bool,
+    recency_bias: RecencyBiasSetting,
     llm_model_provider_override: str | None,
     llm_model_version_override: str | None,
     starter_messages: list[StarterMessage] | None,
@@ -889,11 +841,13 @@ def upsert_persona(
     remove_image: bool | None = None,
     search_start_date: datetime | None = None,
     builtin_persona: bool = False,
-    featured: bool | None = None,
+    is_default_persona: bool | None = None,
     label_ids: list[int] | None = None,
     user_file_ids: list[UUID] | None = None,
     hierarchy_node_ids: list[int] | None = None,
     document_ids: list[str] | None = None,
+    chunks_above: int = CONTEXT_CHUNKS_ABOVE,
+    chunks_below: int = CONTEXT_CHUNKS_BELOW,
     replace_base_system_prompt: bool = False,
 ) -> Persona:
     """
@@ -959,8 +913,6 @@ def upsert_persona(
         labels = (
             db_session.query(PersonaLabel).filter(PersonaLabel.id.in_(label_ids)).all()
         )
-        if len(labels) != len(label_ids):
-            raise ValueError("Some label IDs were not found in the database")
 
     # Fetch and attach hierarchy_nodes by IDs
     hierarchy_nodes = None
@@ -1004,6 +956,12 @@ def upsert_persona(
         # `default` and `built-in` properties can only be set when creating a persona.
         existing_persona.name = name
         existing_persona.description = description
+        existing_persona.num_chunks = num_chunks
+        existing_persona.chunks_above = chunks_above
+        existing_persona.chunks_below = chunks_below
+        existing_persona.llm_relevance_filter = llm_relevance_filter
+        existing_persona.llm_filter_extraction = llm_filter_extraction
+        existing_persona.recency_bias = recency_bias
         existing_persona.llm_model_provider_override = llm_model_provider_override
         existing_persona.llm_model_version_override = llm_model_version_override
         existing_persona.starter_messages = starter_messages
@@ -1017,8 +975,10 @@ def upsert_persona(
         if label_ids is not None:
             existing_persona.labels.clear()
             existing_persona.labels = labels or []
-        existing_persona.featured = (
-            featured if featured is not None else existing_persona.featured
+        existing_persona.is_default_persona = (
+            is_default_persona
+            if is_default_persona is not None
+            else existing_persona.is_default_persona
         )
         # Update embedded prompt fields if provided
         if system_prompt is not None:
@@ -1041,13 +1001,8 @@ def upsert_persona(
             existing_persona.tools = tools or []
 
         if user_file_ids is not None:
-            old_file_ids = {uf.id for uf in existing_persona.user_files}
-            new_file_ids = {uf.id for uf in (user_files or [])}
-            affected_file_ids = old_file_ids | new_file_ids
             existing_persona.user_files.clear()
             existing_persona.user_files = user_files or []
-            if affected_file_ids:
-                _mark_files_need_persona_sync(db_session, list(affected_file_ids))
 
         if hierarchy_node_ids is not None:
             existing_persona.hierarchy_nodes.clear()
@@ -1071,6 +1026,12 @@ def upsert_persona(
             is_public=is_public,
             name=name,
             description=description,
+            num_chunks=num_chunks,
+            chunks_above=chunks_above,
+            chunks_below=chunks_below,
+            llm_relevance_filter=llm_relevance_filter,
+            llm_filter_extraction=llm_filter_extraction,
+            recency_bias=recency_bias,
             builtin_persona=builtin_persona,
             system_prompt=system_prompt or "",
             task_prompt=task_prompt or "",
@@ -1086,15 +1047,15 @@ def upsert_persona(
             display_priority=display_priority,
             is_visible=is_visible,
             search_start_date=search_start_date,
-            featured=(featured if featured is not None else False),
+            is_default_persona=(
+                is_default_persona if is_default_persona is not None else False
+            ),
             user_files=user_files or [],
             labels=labels or [],
             hierarchy_nodes=hierarchy_nodes or [],
             attached_documents=attached_documents or [],
         )
         db_session.add(new_persona)
-        if user_files:
-            _mark_files_need_persona_sync(db_session, [uf.id for uf in user_files])
         persona = new_persona
     if commit:
         db_session.commit()
@@ -1131,9 +1092,9 @@ def delete_old_default_personas(
     db_session.commit()
 
 
-def update_persona_featured(
+def update_persona_is_default(
     persona_id: int,
-    featured: bool,
+    is_default: bool,
     db_session: Session,
     user: User,
 ) -> None:
@@ -1141,7 +1102,10 @@ def update_persona_featured(
         db_session=db_session, persona_id=persona_id, user=user, get_editable=True
     )
 
-    persona.featured = featured
+    if not persona.is_public:
+        persona.is_public = True
+
+    persona.is_default_persona = is_default
     db_session.commit()
 
 

backend/onyx/db/projects.py

@@ -9,9 +9,8 @@ from pydantic import BaseModel
 from pydantic import ConfigDict
 from sqlalchemy import func
 from sqlalchemy.orm import Session
-from starlette.background import BackgroundTasks
 
-from onyx.configs.app_configs import DISABLE_VECTOR_DB
+from onyx.background.celery.versioned_apps.client import app as client_app
 from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
@@ -106,8 +105,8 @@ def upload_files_to_user_files_with_indexing(
     user: User,
     temp_id_map: dict[str, str] | None,
     db_session: Session,
-    background_tasks: BackgroundTasks | None = None,
 ) -> CategorizedFilesResult:
+    # Validate project ownership if a project_id is provided
     if project_id is not None and user is not None:
         if not check_project_ownership(project_id, user.id, db_session):
             raise HTTPException(status_code=404, detail="Project not found")
@@ -128,27 +127,16 @@ def upload_files_to_user_files_with_indexing(
         logger.warning(
             f"File {rejected_file.filename} rejected for {rejected_file.reason}"
         )
-
-    if DISABLE_VECTOR_DB and background_tasks is not None:
-        from onyx.background.task_utils import drain_processing_loop
-
-        background_tasks.add_task(drain_processing_loop, tenant_id)
-        for user_file in user_files:
-            logger.info(f"Queued in-process processing for user_file_id={user_file.id}")
-    else:
-        from onyx.background.celery.versioned_apps.client import app as client_app
-
-        for user_file in user_files:
-            task = client_app.send_task(
-                OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
-                kwargs={"user_file_id": user_file.id, "tenant_id": tenant_id},
-                queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
-                priority=OnyxCeleryPriority.HIGH,
-            )
-            logger.info(
-                f"Triggered indexing for user_file_id={user_file.id} "
-                f"with task_id={task.id}"
-            )
+    for user_file in user_files:
+        task = client_app.send_task(
+            OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
+            kwargs={"user_file_id": user_file.id, "tenant_id": tenant_id},
+            queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
+            priority=OnyxCeleryPriority.HIGH,
+        )
+        logger.info(
+            f"Triggered indexing for user_file_id={user_file.id} with task_id={task.id}"
+        )
 
     return CategorizedFilesResult(
         user_files=user_files,

backend/onyx/db/seeding/chat_history_seeding.py

@@ -2,7 +2,6 @@ import random
 from datetime import datetime
 from datetime import timedelta
 from logging import getLogger
-from uuid import UUID
 
 from onyx.configs.constants import MessageType
 from onyx.db.chat import create_chat_session
@@ -14,26 +13,18 @@ from onyx.db.models import ChatSession
 logger = getLogger(__name__)
 
 
-def seed_chat_history(
-    num_sessions: int,
-    num_messages: int,
-    days: int,
-    user_id: UUID | None = None,
-    persona_id: int | None = None,
-) -> None:
+def seed_chat_history(num_sessions: int, num_messages: int, days: int) -> None:
     """Utility function to seed chat history for testing.
 
     num_sessions: the number of sessions to seed
     num_messages: the number of messages to seed per sessions
     days: the number of days looking backwards from the current time over which to randomize
     the times.
-    user_id: optional user to associate with sessions
-    persona_id: optional persona/assistant to associate with sessions
     """
     with get_session_with_current_tenant() as db_session:
         logger.info(f"Seeding {num_sessions} sessions.")
         for y in range(0, num_sessions):
-            create_chat_session(db_session, f"pytest_session_{y}", user_id, persona_id)
+            create_chat_session(db_session, f"pytest_session_{y}", None, None)
 
         # randomize all session times
         logger.info(f"Seeding {num_messages} messages per session.")

backend/onyx/db/slack_channel_config.py

@@ -5,6 +5,8 @@ from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm import Session
 
+from onyx.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT
+from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.constants import DEFAULT_PERSONA_SLACK_CHANNEL_NAME
 from onyx.db.constants import SLACK_BOT_PERSONA_PREFIX
 from onyx.db.models import ChannelConfig
@@ -43,6 +45,8 @@ def create_slack_channel_persona(
     channel_name: str | None,
     document_set_ids: list[int],
     existing_persona_id: int | None = None,
+    num_chunks: float = MAX_CHUNKS_FED_TO_CHAT,
+    enable_auto_filters: bool = False,
 ) -> Persona:
     """NOTE: does not commit changes"""
 
@@ -69,13 +73,17 @@ def create_slack_channel_persona(
         system_prompt="",
         task_prompt="",
         datetime_aware=True,
+        num_chunks=num_chunks,
+        llm_relevance_filter=True,
+        llm_filter_extraction=enable_auto_filters,
+        recency_bias=RecencyBiasSetting.AUTO,
         tool_ids=[search_tool.id],
         document_set_ids=document_set_ids,
         llm_model_provider_override=None,
         llm_model_version_override=None,
         starter_messages=None,
         is_public=True,
-        featured=False,
+        is_default_persona=False,
         db_session=db_session,
         commit=False,
     )

backend/onyx/db/user_file.py

@@ -3,10 +3,8 @@ from uuid import UUID
 
 from sqlalchemy import func
 from sqlalchemy import select
-from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
-from onyx.db.models import Project__UserFile
 from onyx.db.models import UserFile
 
 
@@ -58,34 +56,10 @@ def fetch_user_project_ids_for_user_files(
     db_session: Session,
 ) -> dict[str, list[int]]:
     """Fetch user project ids for specified user files"""
-    user_file_uuid_ids = [UUID(user_file_id) for user_file_id in user_file_ids]
-    stmt = select(Project__UserFile.user_file_id, Project__UserFile.project_id).where(
-        Project__UserFile.user_file_id.in_(user_file_uuid_ids)
-    )
-    rows = db_session.execute(stmt).all()
-
-    user_file_id_to_project_ids: dict[str, list[int]] = {
-        user_file_id: [] for user_file_id in user_file_ids
-    }
-    for user_file_id, project_id in rows:
-        user_file_id_to_project_ids[str(user_file_id)].append(project_id)
-
-    return user_file_id_to_project_ids
-
-
-def fetch_persona_ids_for_user_files(
-    user_file_ids: list[str],
-    db_session: Session,
-) -> dict[str, list[int]]:
-    """Fetch persona (assistant) ids for specified user files."""
-    stmt = (
-        select(UserFile)
-        .where(UserFile.id.in_(user_file_ids))
-        .options(selectinload(UserFile.assistants))
-    )
+    stmt = select(UserFile).where(UserFile.id.in_(user_file_ids))
     results = db_session.execute(stmt).scalars().all()
     return {
-        str(user_file.id): [persona.id for persona in user_file.assistants]
+        str(user_file.id): [project.id for project in user_file.projects]
         for user_file in results
     }
 

backend/onyx/deep_research/dr_loop.py

@@ -139,7 +139,7 @@ def generate_final_report(
             custom_agent_prompt=None,
             simple_chat_history=history,
             reminder_message=reminder_message,
-            context_files=None,
+            project_files=None,
             available_tokens=llm.config.max_input_tokens,
             all_injected_file_metadata=all_injected_file_metadata,
         )
@@ -257,7 +257,7 @@ def run_deep_research_llm_loop(
                     custom_agent_prompt=None,
                     simple_chat_history=simple_chat_history,
                     reminder_message=None,
-                    context_files=None,
+                    project_files=None,
                     available_tokens=available_tokens,
                     last_n_user_messages=MAX_USER_MESSAGES_FOR_CONTEXT,
                     all_injected_file_metadata=all_injected_file_metadata,
@@ -321,7 +321,7 @@ def run_deep_research_llm_loop(
                 custom_agent_prompt=None,
                 simple_chat_history=simple_chat_history + [reminder_message],
                 reminder_message=None,
-                context_files=None,
+                project_files=None,
                 available_tokens=available_tokens,
                 last_n_user_messages=MAX_USER_MESSAGES_FOR_CONTEXT + 1,
                 all_injected_file_metadata=all_injected_file_metadata,
@@ -485,7 +485,7 @@ def run_deep_research_llm_loop(
                     custom_agent_prompt=None,
                     simple_chat_history=simple_chat_history,
                     reminder_message=first_cycle_reminder_message,
-                    context_files=None,
+                    project_files=None,
                     available_tokens=available_tokens,
                     last_n_user_messages=MAX_USER_MESSAGES_FOR_CONTEXT,
                     all_injected_file_metadata=all_injected_file_metadata,