nit

.git-blame-ignore-revs

@@ -6,4 +6,3 @@
 
 3134e5f840c12c8f32613ce520101a047c89dcc2  # refactor(whitespace): rm temporary react fragments (#7161)
 ed3f72bc75f3e3a9ae9e4d8cd38278f9c97e78b4  # refactor(whitespace): rm react fragment #7190
-7b927e79c25f4ddfd18a067f489e122acd2c89de  # chore(format): format files where `ruff` and `black` agree (#9339)

.github/actions/slack-notify/action.yml

@@ -10,9 +10,6 @@ inputs:
   failed-jobs:
     description: "Deprecated alias for details"
     required: false
-  mention:
-    description: "GitHub username to resolve to a Slack @-mention. Replaces {mention} in details."
-    required: false
   title:
     description: "Title for the notification"
     required: false
@@ -29,7 +26,6 @@ runs:
         SLACK_WEBHOOK_URL: ${{ inputs.webhook-url }}
         DETAILS: ${{ inputs.details }}
         FAILED_JOBS: ${{ inputs.failed-jobs }}
-        MENTION_USER: ${{ inputs.mention }}
         TITLE: ${{ inputs.title }}
         REF_NAME: ${{ inputs.ref-name }}
         REPO: ${{ github.repository }}
@@ -56,27 +52,6 @@ runs:
           DETAILS="$FAILED_JOBS"
         fi
 
-        # Resolve {mention} placeholder if a GitHub username was provided.
-        # Looks up the username in user-mappings.json (co-located with this action)
-        # and replaces {mention} with <@SLACK_ID> for a Slack @-mention.
-        # Falls back to the plain GitHub username if not found in the mapping.
-        if [ -n "$MENTION_USER" ]; then
-          MAPPINGS_FILE="${GITHUB_ACTION_PATH}/user-mappings.json"
-          slack_id="$(jq -r --arg gh "$MENTION_USER" 'to_entries[] | select(.value | ascii_downcase == ($gh | ascii_downcase)) | .key' "$MAPPINGS_FILE" 2>/dev/null | head -1)"
-
-          if [ -n "$slack_id" ]; then
-            mention_text="<@${slack_id}>"
-          else
-            mention_text="${MENTION_USER}"
-          fi
-
-          DETAILS="${DETAILS//\{mention\}/$mention_text}"
-          TITLE="${TITLE//\{mention\}/}"
-        else
-          DETAILS="${DETAILS//\{mention\}/}"
-          TITLE="${TITLE//\{mention\}/}"
-        fi
-
         normalize_multiline() {
           printf '%s' "$1" | awk 'BEGIN { ORS=""; first=1 } { if (!first) printf "\\n"; printf "%s", $0; first=0 }'
         }

.github/actions/slack-notify/user-mappings.json

@@ -1,18 +0,0 @@
-{
-  "U05SAGZPEA1": "yuhongsun96",
-  "U05SAH6UGUD": "Weves",
-  "U07PWEQB7A5": "evan-onyx",
-  "U07V1SM68KF": "joachim-danswer",
-  "U08JZ9N3QNN": "raunakab",
-  "U08L24NCLJE": "Subash-Mohan",
-  "U090B9M07B2": "wenxi-onyx",
-  "U094RASDP0Q": "duo-onyx",
-  "U096L8ZQ85B": "justin-tahara",
-  "U09AHV8UBQX": "jessicasingh7",
-  "U09KAL5T3C2": "nmgarza5",
-  "U09KPGVQ70R": "acaprau",
-  "U09QR8KTSJH": "rohoswagger",
-  "U09RB4NTXA4": "jmelahman",
-  "U0A6K9VCY6A": "Danelegend",
-  "U0AGC4KH71A": "Bo-Onyx"
-}

.github/workflows/deployment.yml

@@ -44,7 +44,7 @@ jobs:
           fetch-tags: true
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # ratchet:astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
         with:
           version: "0.9.9"
           enable-cache: false
@@ -165,7 +165,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # ratchet:astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
         with:
           version: "0.9.9"
           # NOTE: This isn't caching much and zizmor suggests this could be poisoned, so disable.
@@ -307,7 +307,7 @@ jobs:
             xdg-utils
 
       - name: setup node
-        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # ratchet:actions/setup-node@v6.3.0
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v6.2.0
         with:
           node-version: 24
           package-manager-cache: false
@@ -455,7 +455,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -529,7 +529,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -607,7 +607,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -668,7 +668,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -750,7 +750,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -836,7 +836,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -894,7 +894,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -967,7 +967,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -1044,7 +1044,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -1105,7 +1105,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY_IMAGE }}
           flavor: |
@@ -1178,7 +1178,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY_IMAGE }}
           flavor: |
@@ -1256,7 +1256,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY_IMAGE }}
           flavor: |
@@ -1317,7 +1317,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -1397,7 +1397,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |
@@ -1480,7 +1480,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
           flavor: |

.github/workflows/post-merge-beta-cherry-pick.yml

@@ -114,7 +114,7 @@ jobs:
           ref: main
 
       - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # ratchet:astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
         with:
           enable-cache: false
           version: "0.9.9"
@@ -207,7 +207,7 @@ jobs:
           CHERRY_PICK_PR_URL: ${{ needs.cherry-pick-to-latest-release.outputs.cherry_pick_pr_url }}
         run: |
           source_pr_url="https://github.com/${GITHUB_REPOSITORY}/pull/${SOURCE_PR_NUMBER}"
-          details="*Cherry-pick PR opened successfully.*\\n• author: {mention}\\n• source PR: ${source_pr_url}"
+          details="*Cherry-pick PR opened successfully.*\\n• source PR: ${source_pr_url}"
           if [ -n "${CHERRY_PICK_PR_URL}" ]; then
             details="${details}\\n• cherry-pick PR: ${CHERRY_PICK_PR_URL}"
           fi
@@ -221,7 +221,6 @@ jobs:
         uses: ./.github/actions/slack-notify
         with:
           webhook-url: ${{ secrets.CHERRY_PICK_PRS_WEBHOOK }}
-          mention: ${{ needs.resolve-cherry-pick-request.outputs.merged_by }}
           details: ${{ steps.success-summary.outputs.details }}
           title: "✅ Automated Cherry-Pick PR Opened"
           ref-name: ${{ github.event.pull_request.base.ref }}
@@ -276,21 +275,20 @@ jobs:
           else
             failed_job_label="cherry-pick-to-latest-release"
           fi
-          details="• author: {mention}\\n• ${failed_job_label}\\n• source PR: ${source_pr_url}\\n• reason: ${reason_text}"
+          failed_jobs="• ${failed_job_label}\\n• source PR: ${source_pr_url}\\n• reason: ${reason_text}"
           if [ -n "${MERGE_COMMIT_SHA}" ]; then
-            details="${details}\\n• merge SHA: ${MERGE_COMMIT_SHA}"
+            failed_jobs="${failed_jobs}\\n• merge SHA: ${MERGE_COMMIT_SHA}"
           fi
           if [ -n "${details_excerpt}" ]; then
-            details="${details}\\n• excerpt: ${details_excerpt}"
+            failed_jobs="${failed_jobs}\\n• excerpt: ${details_excerpt}"
           fi
 
-          echo "details=${details}" >> "$GITHUB_OUTPUT"
+          echo "jobs=${failed_jobs}" >> "$GITHUB_OUTPUT"
 
       - name: Notify #cherry-pick-prs about cherry-pick failure
         uses: ./.github/actions/slack-notify
         with:
           webhook-url: ${{ secrets.CHERRY_PICK_PRS_WEBHOOK }}
-          mention: ${{ needs.resolve-cherry-pick-request.outputs.merged_by }}
-          details: ${{ steps.failure-summary.outputs.details }}
+          details: ${{ steps.failure-summary.outputs.jobs }}
           title: "🚨 Automated Cherry-Pick Failed"
           ref-name: ${{ github.event.pull_request.base.ref }}

.github/workflows/pr-desktop-build.yml

@@ -50,7 +50,7 @@ jobs:
           persist-credentials: false
 
       - name: Setup node
-        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238
         with:
           node-version: 24
           cache: "npm" # zizmor: ignore[cache-poisoning]
@@ -105,7 +105,7 @@ jobs:
 
       - name: Upload build artifacts
         if: always()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: desktop-build-${{ matrix.platform }}-${{ github.run_id }}
           path: |

.github/workflows/pr-external-dependency-unit-tests.yml

@@ -7,15 +7,6 @@ on:
   merge_group:
   pull_request:
     branches: [main]
-    paths:
-      - "backend/**"
-      - "pyproject.toml"
-      - "uv.lock"
-      - ".github/workflows/pr-external-dependency-unit-tests.yml"
-      - ".github/actions/setup-python-and-install-dependencies/**"
-      - ".github/actions/setup-playwright/**"
-      - "deployment/docker_compose/docker-compose.yml"
-      - "deployment/docker_compose/docker-compose.dev.yml"
   push:
     tags:
       - "v*.*.*"
@@ -183,7 +174,7 @@ jobs:
 
       - name: Upload Docker logs
         if: failure()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: docker-logs-${{ matrix.test-dir }}
           path: docker-logs/

.github/workflows/pr-golang-tests.yml

@@ -25,7 +25,7 @@ jobs:
     outputs:
       modules: ${{ steps.set-modules.outputs.modules }}
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
         with:
           persist-credentials: false
       - id: set-modules
@@ -39,7 +39,7 @@ jobs:
       matrix:
         modules: ${{ fromJSON(needs.detect-modules.outputs.modules) }}
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
         with:
           persist-credentials: false
       - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # zizmor: ignore[cache-poisoning]

.github/workflows/pr-integration-tests.yml

@@ -466,7 +466,7 @@ jobs:
 
       - name: Upload logs
         if: always()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: docker-all-logs-${{ matrix.edition }}-${{ matrix.test-dir.name }}
           path: ${{ github.workspace }}/docker-compose.log
@@ -587,7 +587,7 @@ jobs:
 
       - name: Upload logs (onyx-lite)
         if: always()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: docker-all-logs-onyx-lite
           path: ${{ github.workspace }}/docker-compose-onyx-lite.log
@@ -725,7 +725,7 @@ jobs:
 
       - name: Upload logs (multi-tenant)
         if: always()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: docker-all-logs-multitenant
           path: ${{ github.workspace }}/docker-compose-multitenant.log

.github/workflows/pr-jest-tests.yml

@@ -28,7 +28,7 @@ jobs:
           persist-credentials: false
 
       - name: Setup node
-        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # ratchet:actions/setup-node@v4
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v4
         with:
           node-version: 22
           cache: "npm" # zizmor: ignore[cache-poisoning] test-only workflow; no deploy artifacts
@@ -44,7 +44,7 @@ jobs:
 
       - name: Upload coverage reports
         if: always()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: jest-coverage-${{ github.run_id }}
           path: ./web/coverage

.github/workflows/pr-playwright-tests.yml

@@ -272,7 +272,7 @@ jobs:
 
       - name: Setup node
         # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
-        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # ratchet:actions/setup-node@v4
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v4
         with:
           node-version: 22
           cache: "npm" # zizmor: ignore[cache-poisoning]
@@ -445,7 +445,7 @@ jobs:
         run: |
           npx playwright test --project ${PROJECT}
 
-      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         if: always()
         with:
           # Includes test results and trace.zip files
@@ -454,7 +454,7 @@ jobs:
           retention-days: 30
 
       - name: Upload screenshots
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         if: always()
         with:
           name: playwright-screenshots-${{ matrix.project }}-${{ github.run_id }}
@@ -471,7 +471,7 @@ jobs:
 
       - name: Install the latest version of uv
         if: always()
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # ratchet:astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
         with:
           enable-cache: false
           version: "0.9.9"
@@ -534,7 +534,7 @@ jobs:
             "s3://${PLAYWRIGHT_S3_BUCKET}/reports/pr-${PR_NUMBER}/${RUN_ID}/${PROJECT}/"
 
       - name: Upload visual diff summary
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         if: always()
         with:
           name: screenshot-diff-summary-${{ matrix.project }}
@@ -543,7 +543,7 @@ jobs:
           retention-days: 5
 
       - name: Upload visual diff report artifact
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         if: always()
         with:
           name: screenshot-diff-report-${{ matrix.project }}-${{ github.run_id }}
@@ -590,7 +590,7 @@ jobs:
 
       - name: Upload logs
         if: success() || failure()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: docker-logs-${{ matrix.project }}-${{ github.run_id }}
           path: ${{ github.workspace }}/docker-compose.log
@@ -614,7 +614,7 @@ jobs:
 
       - name: Setup node
         # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
-        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # ratchet:actions/setup-node@v4
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v4
         with:
           node-version: 22
           cache: "npm" # zizmor: ignore[cache-poisoning]
@@ -674,7 +674,7 @@ jobs:
         working-directory: ./web
         run: npx playwright test --project lite
 
-      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         if: always()
         with:
           name: playwright-test-results-lite-${{ github.run_id }}
@@ -692,7 +692,7 @@ jobs:
 
       - name: Upload logs
         if: success() || failure()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: docker-logs-lite-${{ github.run_id }}
           path: ${{ github.workspace }}/docker-compose.log

.github/workflows/pr-python-connector-tests.yml

@@ -7,13 +7,6 @@ on:
   merge_group:
   pull_request:
     branches: [main]
-    paths:
-      - "backend/**"
-      - "pyproject.toml"
-      - "uv.lock"
-      - ".github/workflows/pr-python-connector-tests.yml"
-      - ".github/actions/setup-python-and-install-dependencies/**"
-      - ".github/actions/setup-playwright/**"
   push:
     tags:
       - "v*.*.*"

.github/workflows/pr-python-model-tests.yml

@@ -73,7 +73,7 @@ jobs:
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
 
       - name: Build and load
-        uses: docker/bake-action@82490499d2e5613fcead7e128237ef0b0ea210f7 # ratchet:docker/bake-action@v7.0.0
+        uses: docker/bake-action@5be5f02ff8819ecd3092ea6b2e6261c31774f2b4 # ratchet:docker/bake-action@v6
         env:
           TAG: model-server-${{ github.run_id }}
         with:
@@ -122,7 +122,7 @@ jobs:
 
       - name: Upload logs
         if: always()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: docker-all-logs
           path: ${{ github.workspace }}/docker-compose.log

.github/workflows/pr-quality-checks.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # ratchet:hashicorp/setup-terraform@v4.0.0
       - name: Setup node
-        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # ratchet:actions/setup-node@v6
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v6
         with: # zizmor: ignore[cache-poisoning]
           node-version: 22
           cache: "npm"

.github/workflows/preview.yml

@@ -22,7 +22,7 @@ jobs:
           persist-credentials: false
 
       - name: Setup node
-        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # ratchet:actions/setup-node@v4
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v4
         with:
           node-version: 22
           cache: "npm"

.github/workflows/release-cli.yml

@@ -26,7 +26,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
         with:
           persist-credentials: false
-      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # ratchet:astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
         with:
           enable-cache: false
           version: "0.9.9"

.github/workflows/release-devtools.yml

@@ -26,7 +26,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
         with:
           persist-credentials: false
-      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # ratchet:astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
         with:
           enable-cache: false
           version: "0.9.9"

.github/workflows/reusable-nightly-llm-provider-chat.yml

@@ -319,7 +319,7 @@ jobs:
 
       - name: Upload logs
         if: always()
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:
           name: docker-all-logs-nightly-${{ matrix.provider }}-llm-provider
           path: |

.github/workflows/sandbox-deployment.yml

@@ -125,7 +125,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY_IMAGE }}
           flavor: |
@@ -195,7 +195,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY_IMAGE }}
           flavor: |
@@ -268,7 +268,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY_IMAGE }}
           flavor: |

.github/workflows/storybook-deploy.yml

@@ -32,7 +32,7 @@ jobs:
           persist-credentials: false
 
       - name: Setup node
-        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # ratchet:actions/setup-node@v4
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v4
         with:
           node-version: 22
           cache: "npm"

.github/workflows/zizmor.yml

@@ -24,7 +24,7 @@ jobs:
           persist-credentials: false
 
       - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # ratchet:astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
         with:
           enable-cache: false
           version: "0.9.9"

AGENTS.md

@@ -167,7 +167,284 @@ web/
 
 ## Frontend Standards
 
-Frontend standards for the `web/` and `desktop/` projects live in `web/AGENTS.md`.
+### 1. Import Standards
+
+**Always use absolute imports with the `@` prefix.**
+
+**Reason:** Moving files around becomes easier since you don't also have to update those import statements. This makes modifications to the codebase much nicer.
+
+```typescript
+// ✅ Good
+import { Button } from "@/components/ui/button";
+import { useAuth } from "@/hooks/useAuth";
+import { Text } from "@/refresh-components/texts/Text";
+
+// ❌ Bad
+import { Button } from "../../../components/ui/button";
+import { useAuth } from "./hooks/useAuth";
+```
+
+### 2. React Component Functions
+
+**Prefer regular functions over arrow functions for React components.**
+
+**Reason:** Functions just become easier to read.
+
+```typescript
+// ✅ Good
+function UserProfile({ userId }: UserProfileProps) {
+  return <div>User Profile</div>
+}
+
+// ❌ Bad
+const UserProfile = ({ userId }: UserProfileProps) => {
+  return <div>User Profile</div>
+}
+```
+
+### 3. Props Interface Extraction
+
+**Extract prop types into their own interface definitions.**
+
+**Reason:** Functions just become easier to read.
+
+```typescript
+// ✅ Good
+interface UserCardProps {
+  user: User
+  showActions?: boolean
+  onEdit?: (userId: string) => void
+}
+
+function UserCard({ user, showActions = false, onEdit }: UserCardProps) {
+  return <div>User Card</div>
+}
+
+// ❌ Bad
+function UserCard({
+  user,
+  showActions = false,
+  onEdit
+}: {
+  user: User
+  showActions?: boolean
+  onEdit?: (userId: string) => void
+}) {
+  return <div>User Card</div>
+}
+```
+
+### 4. Spacing Guidelines
+
+**Prefer padding over margins for spacing.**
+
+**Reason:** We want to consolidate usage to paddings instead of margins.
+
+```typescript
+// ✅ Good
+<div className="p-4 space-y-2">
+  <div className="p-2">Content</div>
+</div>
+
+// ❌ Bad
+<div className="m-4 space-y-2">
+  <div className="m-2">Content</div>
+</div>
+```
+
+### 5. Tailwind Dark Mode
+
+**Strictly forbid using the `dark:` modifier in Tailwind classes, except for logo icon handling.**
+
+**Reason:** The `colors.css` file already, VERY CAREFULLY, defines what the exact opposite colour of each light-mode colour is. Overriding this behaviour is VERY bad and will lead to horrible UI breakages.
+
+**Exception:** The `createLogoIcon` helper in `web/src/components/icons/icons.tsx` uses `dark:` modifiers (`dark:invert`, `dark:hidden`, `dark:block`) to handle third-party logo icons that cannot automatically adapt through `colors.css`. This is the ONLY acceptable use of dark mode modifiers.
+
+```typescript
+// ✅ Good - Standard components use `tailwind-themes/tailwind.config.js` / `src/app/css/colors.css`
+<div className="bg-background-neutral-03 text-text-02">
+  Content
+</div>
+
+// ✅ Good - Logo icons with dark mode handling via createLogoIcon
+export const GithubIcon = createLogoIcon(githubLightIcon, {
+  monochromatic: true,  // Will apply dark:invert internally
+});
+
+export const GitbookIcon = createLogoIcon(gitbookLightIcon, {
+  darkSrc: gitbookDarkIcon,  // Will use dark:hidden/dark:block internally
+});
+
+// ❌ Bad - Manual dark mode overrides
+<div className="bg-white dark:bg-black text-black dark:text-white">
+  Content
+</div>
+```
+
+### 6. Class Name Utilities
+
+**Use the `cn` utility instead of raw string formatting for classNames.**
+
+**Reason:** `cn`s are easier to read. They also allow for more complex types (i.e., string-arrays) to get formatted properly (it flattens each element in that string array down). As a result, it can allow things such as conditionals (i.e., `myCondition && "some-tailwind-class"`, which evaluates to `false` when `myCondition` is `false`) to get filtered out.
+
+```typescript
+import { cn } from '@/lib/utils'
+
+// ✅ Good
+<div className={cn(
+  'base-class',
+  isActive && 'active-class',
+  className
+)}>
+  Content
+</div>
+
+// ❌ Bad
+<div className={`base-class ${isActive ? 'active-class' : ''} ${className}`}>
+  Content
+</div>
+```
+
+### 7. Custom Hooks Organization
+
+**Follow a "hook-per-file" layout. Each hook should live in its own file within `web/src/hooks`.**
+
+**Reason:** This is just a layout preference. Keeps code clean.
+
+```typescript
+// web/src/hooks/useUserData.ts
+export function useUserData(userId: string) {
+  // hook implementation
+}
+
+// web/src/hooks/useLocalStorage.ts
+export function useLocalStorage<T>(key: string, initialValue: T) {
+  // hook implementation
+}
+```
+
+### 8. Icon Usage
+
+**ONLY use icons from the `web/src/icons` directory. Do NOT use icons from `react-icons`, `lucide`, or other external libraries.**
+
+**Reason:** We have a very carefully curated selection of icons that match our Onyx guidelines. We do NOT want to muddy those up with different aesthetic stylings.
+
+```typescript
+// ✅ Good
+import SvgX from "@/icons/x";
+import SvgMoreHorizontal from "@/icons/more-horizontal";
+
+// ❌ Bad
+import { User } from "lucide-react";
+import { FiSearch } from "react-icons/fi";
+```
+
+**Missing Icons**: If an icon is needed but doesn't exist in the `web/src/icons` directory, import it from Figma using the Figma MCP tool and add it to the icons directory.
+If you need help with this step, reach out to `raunak@onyx.app`.
+
+### 9. Text Rendering
+
+**Prefer using the `refresh-components/texts/Text` component for all text rendering. Avoid "naked" text nodes.**
+
+**Reason:** The `Text` component is fully compliant with the stylings provided in Figma. It provides easy utilities to specify the text-colour and font-size in the form of flags. Super duper easy.
+
+```typescript
+// ✅ Good
+import { Text } from '@/refresh-components/texts/Text'
+
+function UserCard({ name }: { name: string }) {
+  return (
+    <Text
+      {/* The `text03` flag makes the text it renders to be coloured the 3rd-scale grey */}
+      text03
+      {/* The `mainAction` flag makes the text it renders to be "main-action" font + line-height + weightage, as described in the Figma */}
+      mainAction
+    >
+      {name}
+    </Text>
+  )
+}
+
+// ❌ Bad
+function UserCard({ name }: { name: string }) {
+  return (
+    <div>
+      <h2>{name}</h2>
+      <p>User details</p>
+    </div>
+  )
+}
+```
+
+### 10. Component Usage
+
+**Heavily avoid raw HTML input components. Always use components from the `web/src/refresh-components` or `web/lib/opal/src` directory.**
+
+**Reason:** We've put in a lot of effort to unify the components that are rendered in the Onyx app. Using raw components breaks the entire UI of the application, and leaves it in a muddier state than before.
+
+```typescript
+// ✅ Good
+import Button from '@/refresh-components/buttons/Button'
+import InputTypeIn from '@/refresh-components/inputs/InputTypeIn'
+import SvgPlusCircle from '@/icons/plus-circle'
+
+function ContactForm() {
+  return (
+    <form>
+      <InputTypeIn placeholder="Search..." />
+      <Button type="submit" leftIcon={SvgPlusCircle}>Submit</Button>
+    </form>
+  )
+}
+
+// ❌ Bad
+function ContactForm() {
+  return (
+    <form>
+      <input placeholder="Name" />
+      <textarea placeholder="Message" />
+      <button type="submit">Submit</button>
+    </form>
+  )
+}
+```
+
+### 11. Colors
+
+**Always use custom overrides for colors and borders rather than built in Tailwind CSS colors. These overrides live in `web/tailwind-themes/tailwind.config.js`.**
+
+**Reason:** Our custom color system uses CSS variables that automatically handle dark mode and maintain design consistency across the app. Standard Tailwind colors bypass this system.
+
+**Available color categories:**
+
+- **Text:** `text-01` through `text-05`, `text-inverted-XX`
+- **Backgrounds:** `background-neutral-XX`, `background-tint-XX` (and inverted variants)
+- **Borders:** `border-01` through `border-05`, `border-inverted-XX`
+- **Actions:** `action-link-XX`, `action-danger-XX`
+- **Status:** `status-info-XX`, `status-success-XX`, `status-warning-XX`, `status-error-XX`
+- **Theme:** `theme-primary-XX`, `theme-red-XX`, `theme-blue-XX`, etc.
+
+```typescript
+// ✅ Good - Use custom Onyx color classes
+<div className="bg-background-neutral-01 border border-border-02" />
+<div className="bg-background-tint-02 border border-border-01" />
+<div className="bg-status-success-01" />
+<div className="bg-action-link-01" />
+<div className="bg-theme-primary-05" />
+
+// ❌ Bad - Do NOT use standard Tailwind colors
+<div className="bg-gray-100 border border-gray-300 text-gray-600" />
+<div className="bg-white border border-slate-200" />
+<div className="bg-green-100 text-green-700" />
+<div className="bg-blue-100 text-blue-600" />
+<div className="bg-indigo-500" />
+```
+
+### 12. Data Fetching
+
+**Prefer using `useSWR` for data fetching. Data should generally be fetched on the client side. Components that need data should display a loader / placeholder while waiting for that data. Prefer loading data within the component that needs it rather than at the top level and passing it down.**
+
+**Reason:** Client side fetching allows us to load the skeleton of the page without waiting for data to load, leading to a snappier UX. Loading data where needed reduces dependencies between a component and its parent component(s).
 
 ## Database & Migrations
 

backend/Dockerfile

@@ -47,8 +47,6 @@ RUN apt-get update && \
         gcc \
         nano \
         vim \
-        # Install procps so kubernetes exec sessions can use ps aux for debugging
-        procps \
         libjemalloc2 \
         && \
     rm -rf /var/lib/apt/lists/* && \

backend/alembic/versions/689433b0d8de_add_hook_and_hook_execution_log_tables.py

@@ -1,103 +0,0 @@
-"""add_hook_and_hook_execution_log_tables
-
-Revision ID: 689433b0d8de
-Revises: 93a2e195e25c
-Create Date: 2026-03-13 11:25:06.547474
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects.postgresql import UUID as PGUUID
-
-
-# revision identifiers, used by Alembic.
-revision = "689433b0d8de"
-down_revision = "93a2e195e25c"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "hook",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("name", sa.String(), nullable=False),
-        sa.Column(
-            "hook_point",
-            sa.Enum("document_ingestion", "query_processing", native_enum=False),
-            nullable=False,
-        ),
-        sa.Column("endpoint_url", sa.Text(), nullable=True),
-        sa.Column("api_key", sa.LargeBinary(), nullable=True),
-        sa.Column("is_reachable", sa.Boolean(), nullable=True),
-        sa.Column(
-            "fail_strategy",
-            sa.Enum("hard", "soft", native_enum=False),
-            nullable=False,
-        ),
-        sa.Column("timeout_seconds", sa.Float(), nullable=False),
-        sa.Column(
-            "is_active", sa.Boolean(), nullable=False, server_default=sa.text("false")
-        ),
-        sa.Column(
-            "deleted", sa.Boolean(), nullable=False, server_default=sa.text("false")
-        ),
-        sa.Column("creator_id", PGUUID(as_uuid=True), nullable=True),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=False,
-        ),
-        sa.Column(
-            "updated_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=False,
-        ),
-        sa.ForeignKeyConstraint(["creator_id"], ["user.id"], ondelete="SET NULL"),
-        sa.PrimaryKeyConstraint("id"),
-    )
-    op.create_index(
-        "ix_hook_one_non_deleted_per_point",
-        "hook",
-        ["hook_point"],
-        unique=True,
-        postgresql_where=sa.text("deleted = false"),
-    )
-
-    op.create_table(
-        "hook_execution_log",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("hook_id", sa.Integer(), nullable=False),
-        sa.Column(
-            "is_success",
-            sa.Boolean(),
-            nullable=False,
-        ),
-        sa.Column("error_message", sa.Text(), nullable=True),
-        sa.Column("status_code", sa.Integer(), nullable=True),
-        sa.Column("duration_ms", sa.Integer(), nullable=True),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=False,
-        ),
-        sa.ForeignKeyConstraint(["hook_id"], ["hook.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("id"),
-    )
-    op.create_index("ix_hook_execution_log_hook_id", "hook_execution_log", ["hook_id"])
-    op.create_index(
-        "ix_hook_execution_log_created_at", "hook_execution_log", ["created_at"]
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_hook_execution_log_created_at", table_name="hook_execution_log")
-    op.drop_index("ix_hook_execution_log_hook_id", table_name="hook_execution_log")
-    op.drop_table("hook_execution_log")
-
-    op.drop_index("ix_hook_one_non_deleted_per_point", table_name="hook")
-    op.drop_table("hook")

backend/alembic/versions/b728689f45b1_rename_persona_is_visible_to_is_listed_.py

@@ -1,26 +0,0 @@
-"""rename persona is_visible to is_listed and featured to is_featured
-
-Revision ID: b728689f45b1
-Revises: 689433b0d8de
-Create Date: 2026-03-23 12:36:26.607305
-
-"""
-
-from alembic import op
-
-
-# revision identifiers, used by Alembic.
-revision = "b728689f45b1"
-down_revision = "689433b0d8de"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.alter_column("persona", "is_visible", new_column_name="is_listed")
-    op.alter_column("persona", "featured", new_column_name="is_featured")
-
-
-def downgrade() -> None:
-    op.alter_column("persona", "is_listed", new_column_name="is_visible")
-    op.alter_column("persona", "is_featured", new_column_name="featured")

backend/alembic/versions/e7f8a9b0c1d2_create_anonymous_user.py

@@ -36,56 +36,6 @@ TABLES_WITH_USER_ID = [
 ]
 
 
-def _dedupe_null_notifications(connection: sa.Connection) -> None:
-    # Multiple NULL-owned notifications can exist because the unique index treats
-    # NULL user_id values as distinct. Before migrating them to the anonymous
-    # user, collapse duplicates and remove rows that would conflict with an
-    # already-existing anonymous notification.
-    result = connection.execute(
-        sa.text(
-            """
-            WITH ranked_null_notifications AS (
-                SELECT
-                    id,
-                    ROW_NUMBER() OVER (
-                        PARTITION BY notif_type, COALESCE(additional_data, '{}'::jsonb)
-                        ORDER BY first_shown DESC, last_shown DESC, id DESC
-                    ) AS row_num
-                FROM notification
-                WHERE user_id IS NULL
-            )
-            DELETE FROM notification
-            WHERE id IN (
-                SELECT id
-                FROM ranked_null_notifications
-                WHERE row_num > 1
-            )
-            """
-        )
-    )
-    if result.rowcount > 0:
-        print(f"Deleted {result.rowcount} duplicate NULL-owned notifications")
-
-    result = connection.execute(
-        sa.text(
-            """
-            DELETE FROM notification AS null_owned
-            USING notification AS anonymous_owned
-            WHERE null_owned.user_id IS NULL
-              AND anonymous_owned.user_id = :user_id
-              AND null_owned.notif_type = anonymous_owned.notif_type
-              AND COALESCE(null_owned.additional_data, '{}'::jsonb) =
-                  COALESCE(anonymous_owned.additional_data, '{}'::jsonb)
-            """
-        ),
-        {"user_id": ANONYMOUS_USER_UUID},
-    )
-    if result.rowcount > 0:
-        print(
-            f"Deleted {result.rowcount} NULL-owned notifications that conflict with existing anonymous-owned notifications"
-        )
-
-
 def upgrade() -> None:
     """
     Create the anonymous user for anonymous access feature.
@@ -115,12 +65,7 @@ def upgrade() -> None:
 
     # Migrate any remaining user_id=NULL records to anonymous user
     for table in TABLES_WITH_USER_ID:
-        # Dedup notifications outside the savepoint so deletions persist
-        # even if the subsequent UPDATE rolls back
-        if table == "notification":
-            _dedupe_null_notifications(connection)
-
-        with connection.begin_nested():
+        try:
             # Exclude public credential (id=0) which must remain user_id=NULL
             # Exclude builtin tools (in_code_tool_id IS NOT NULL) which must remain user_id=NULL
             # Exclude builtin personas (builtin_persona=True) which must remain user_id=NULL
@@ -135,7 +80,6 @@ def upgrade() -> None:
                 condition = "user_id IS NULL AND is_public = false"
             else:
                 condition = "user_id IS NULL"
-
             result = connection.execute(
                 sa.text(
                     f"""
@@ -148,19 +92,19 @@ def upgrade() -> None:
             )
             if result.rowcount > 0:
                 print(f"Updated {result.rowcount} rows in {table} to anonymous user")
+        except Exception as e:
+            print(f"Skipping {table}: {e}")
 
 
 def downgrade() -> None:
     """
     Set anonymous user's records back to NULL and delete the anonymous user.
-
-    Note: Duplicate NULL-owned notifications removed during upgrade are not restored.
     """
     connection = op.get_bind()
 
     # Set records back to NULL
     for table in TABLES_WITH_USER_ID:
-        with connection.begin_nested():
+        try:
             connection.execute(
                 sa.text(
                     f"""
@@ -171,6 +115,8 @@ def downgrade() -> None:
                 ),
                 {"user_id": ANONYMOUS_USER_UUID},
             )
+        except Exception:
+            pass
 
     # Delete the anonymous user
     connection.execute(

backend/ee/onyx/background/celery/tasks/tenant_provisioning/tasks.py

@@ -25,13 +25,13 @@ from onyx.redis.redis_pool import get_redis_client
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import TENANT_ID_PREFIX
 
-# Maximum tenants to provision in a single task run.
-# Each tenant takes ~80s (alembic migrations), so 5 tenants ≈ 7 minutes.
-_MAX_TENANTS_PER_RUN = 5
+# Default number of pre-provisioned tenants to maintain
+DEFAULT_TARGET_AVAILABLE_TENANTS = 5
 
-# Time limits sized for worst-case batch: _MAX_TENANTS_PER_RUN × ~90s + buffer.
-_TENANT_PROVISIONING_SOFT_TIME_LIMIT = 60 * 10  # 10 minutes
-_TENANT_PROVISIONING_TIME_LIMIT = 60 * 15  # 15 minutes
+# Soft time limit for tenant pre-provisioning tasks (in seconds)
+_TENANT_PROVISIONING_SOFT_TIME_LIMIT = 60 * 5  # 5 minutes
+# Hard time limit for tenant pre-provisioning tasks (in seconds)
+_TENANT_PROVISIONING_TIME_LIMIT = 60 * 10  # 10 minutes
 
 
 @shared_task(
@@ -58,7 +58,7 @@ def check_available_tenants(self: Task) -> None:  # noqa: ARG001
     r = get_redis_client(tenant_id=ONYX_CLOUD_TENANT_ID)
     lock_check: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_AVAILABLE_TENANTS_LOCK,
-        timeout=_TENANT_PROVISIONING_TIME_LIMIT,
+        timeout=_TENANT_PROVISIONING_SOFT_TIME_LIMIT,
     )
 
     # These tasks should never overlap
@@ -74,7 +74,9 @@ def check_available_tenants(self: Task) -> None:  # noqa: ARG001
             num_available_tenants = db_session.query(AvailableTenant).count()
 
         # Get the target number of available tenants
-        num_minimum_available_tenants = TARGET_AVAILABLE_TENANTS
+        num_minimum_available_tenants = getattr(
+            TARGET_AVAILABLE_TENANTS, "value", DEFAULT_TARGET_AVAILABLE_TENANTS
+        )
 
         # Calculate how many new tenants we need to provision
         if num_available_tenants < num_minimum_available_tenants:
@@ -88,46 +90,22 @@ def check_available_tenants(self: Task) -> None:  # noqa: ARG001
             f"To provision: {tenants_to_provision}"
         )
 
-        batch_size = min(tenants_to_provision, _MAX_TENANTS_PER_RUN)
-        if batch_size < tenants_to_provision:
-            task_logger.info(
-                f"Capping batch to {batch_size} "
-                f"(need {tenants_to_provision}, will catch up next cycle)"
-            )
-
-        provisioned = 0
-        for i in range(batch_size):
-            task_logger.info(f"Provisioning tenant {i + 1}/{batch_size}")
-            try:
-                if pre_provision_tenant():
-                    provisioned += 1
-            except Exception:
-                task_logger.exception(
-                    f"Failed to provision tenant {i + 1}/{batch_size}, "
-                    "continuing with remaining tenants"
-                )
-
-        task_logger.info(f"Provisioning complete: {provisioned}/{batch_size} succeeded")
+        # just provision one tenant each time we run this ... increase if needed.
+        if tenants_to_provision > 0:
+            pre_provision_tenant()
 
     except Exception:
         task_logger.exception("Error in check_available_tenants task")
 
     finally:
-        try:
-            lock_check.release()
-        except Exception:
-            task_logger.warning(
-                "Could not release check lock (likely expired), continuing"
-            )
+        lock_check.release()
 
 
-def pre_provision_tenant() -> bool:
+def pre_provision_tenant() -> None:
     """
     Pre-provision a new tenant and store it in the NewAvailableTenant table.
     This function fully sets up the tenant with all necessary configurations,
     so it's ready to be assigned to a user immediately.
-
-    Returns True if a tenant was successfully provisioned, False otherwise.
     """
     # The MULTI_TENANT check is now done at the caller level (check_available_tenants)
     # rather than inside this function
@@ -135,15 +113,15 @@ def pre_provision_tenant() -> bool:
     r = get_redis_client(tenant_id=ONYX_CLOUD_TENANT_ID)
     lock_provision: RedisLock = r.lock(
         OnyxRedisLocks.CLOUD_PRE_PROVISION_TENANT_LOCK,
-        timeout=_TENANT_PROVISIONING_TIME_LIMIT,
+        timeout=_TENANT_PROVISIONING_SOFT_TIME_LIMIT,
     )
 
     # Allow multiple pre-provisioning tasks to run, but ensure they don't overlap
     if not lock_provision.acquire(blocking=False):
-        task_logger.warning(
-            "Skipping pre_provision_tenant — could not acquire provision lock"
+        task_logger.debug(
+            "Skipping pre_provision_tenant task because it is already running"
         )
-        return False
+        return
 
     tenant_id: str | None = None
     try:
@@ -183,7 +161,6 @@ def pre_provision_tenant() -> bool:
                 db_session.add(new_tenant)
                 db_session.commit()
                 task_logger.info(f"Successfully pre-provisioned tenant: {tenant_id}")
-                return True
             except Exception:
                 db_session.rollback()
                 task_logger.error(
@@ -207,11 +184,5 @@ def pre_provision_tenant() -> bool:
                 asyncio.run(rollback_tenant_provisioning(tenant_id))
             except Exception:
                 task_logger.exception(f"Error during rollback for tenant: {tenant_id}")
-        return False
     finally:
-        try:
-            lock_provision.release()
-        except Exception:
-            task_logger.warning(
-                "Could not release provision lock (likely expired), continuing"
-            )
+        lock_provision.release()

backend/ee/onyx/configs/app_configs.py

@@ -118,7 +118,9 @@ JWT_PUBLIC_KEY_URL: str | None = os.getenv("JWT_PUBLIC_KEY_URL", None)
 SUPER_USERS = json.loads(os.environ.get("SUPER_USERS", "[]"))
 SUPER_CLOUD_API_KEY = os.environ.get("SUPER_CLOUD_API_KEY", "api_key")
 
-POSTHOG_API_KEY = os.environ.get("POSTHOG_API_KEY")
+# The posthog client does not accept empty API keys or hosts however it fails silently
+# when the capture is called. These defaults prevent Posthog issues from breaking the Onyx app
+POSTHOG_API_KEY = os.environ.get("POSTHOG_API_KEY") or "FooBar"
 POSTHOG_HOST = os.environ.get("POSTHOG_HOST") or "https://us.i.posthog.com"
 POSTHOG_DEBUG_LOGS_ENABLED = (
     os.environ.get("POSTHOG_DEBUG_LOGS_ENABLED", "").lower() == "true"

backend/ee/onyx/db/user_group.py

@@ -800,33 +800,6 @@ def update_user_group(
     return db_user_group
 
 
-def rename_user_group(
-    db_session: Session,
-    user_group_id: int,
-    new_name: str,
-) -> UserGroup:
-    stmt = select(UserGroup).where(UserGroup.id == user_group_id)
-    db_user_group = db_session.scalar(stmt)
-    if db_user_group is None:
-        raise ValueError(f"UserGroup with id '{user_group_id}' not found")
-
-    _check_user_group_is_modifiable(db_user_group)
-
-    db_user_group.name = new_name
-    db_user_group.time_last_modified_by_user = func.now()
-
-    # CC pair documents in Vespa contain the group name, so we need to
-    # trigger a sync to update them with the new name.
-    _mark_user_group__cc_pair_relationships_outdated__no_commit(
-        db_session=db_session, user_group_id=user_group_id
-    )
-    if not DISABLE_VECTOR_DB:
-        db_user_group.is_up_to_date = False
-
-    db_session.commit()
-    return db_user_group
-
-
 def prepare_user_group_for_deletion(db_session: Session, user_group_id: int) -> None:
     stmt = select(UserGroup).where(UserGroup.id == user_group_id)
     db_user_group = db_session.scalar(stmt)

backend/ee/onyx/feature_flags/posthog_provider.py

@@ -34,9 +34,6 @@ class PostHogFeatureFlagProvider(FeatureFlagProvider):
         Returns:
             True if the feature is enabled for the user, False otherwise.
         """
-        if not posthog:
-            return False
-
         try:
             posthog.set(
                 distinct_id=user_id,

backend/ee/onyx/server/enterprise_settings/api.py

@@ -157,11 +157,7 @@ def fetch_logo_helper(db_session: Session) -> Response:  # noqa: ARG001
             detail="No logo file found",
         )
     else:
-        return Response(
-            content=onyx_file.data,
-            media_type=onyx_file.mime_type,
-            headers={"Cache-Control": "no-cache"},
-        )
+        return Response(content=onyx_file.data, media_type=onyx_file.mime_type)
 
 
 def fetch_logotype_helper(db_session: Session) -> Response:  # noqa: ARG001

backend/ee/onyx/server/seeding.py

@@ -178,7 +178,7 @@ def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) ->
                     system_prompt=persona.system_prompt,
                     task_prompt=persona.task_prompt,
                     datetime_aware=persona.datetime_aware,
-                    is_featured=persona.is_featured,
+                    featured=persona.featured,
                     commit=False,
                 )
             db_session.commit()

backend/ee/onyx/server/tenants/provisioning.py

@@ -29,6 +29,7 @@ from onyx.configs.app_configs import OPENAI_DEFAULT_API_KEY
 from onyx.configs.app_configs import OPENROUTER_DEFAULT_API_KEY
 from onyx.configs.app_configs import VERTEXAI_DEFAULT_CREDENTIALS
 from onyx.configs.app_configs import VERTEXAI_DEFAULT_LOCATION
+from onyx.configs.constants import MilestoneRecordType
 from onyx.db.engine.sql_engine import get_session_with_shared_schema
 from onyx.db.engine.sql_engine import get_session_with_tenant
 from onyx.db.image_generation import create_default_image_gen_config_from_api_key
@@ -58,6 +59,7 @@ from onyx.server.manage.llm.models import LLMProviderUpsertRequest
 from onyx.server.manage.llm.models import ModelConfigurationUpsertRequest
 from onyx.setup import setup_onyx
 from onyx.utils.logger import setup_logger
+from onyx.utils.telemetry import mt_cloud_telemetry
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.configs import TENANT_ID_PREFIX
@@ -69,9 +71,7 @@ logger = setup_logger()
 
 
 async def get_or_provision_tenant(
-    email: str,
-    referral_source: str | None = None,
-    request: Request | None = None,
+    email: str, referral_source: str | None = None, request: Request | None = None
 ) -> str:
     """
     Get existing tenant ID for an email or create a new tenant if none exists.
@@ -693,6 +693,12 @@ async def assign_tenant_to_user(
 
     try:
         add_users_to_tenant([email], tenant_id)
+
+        mt_cloud_telemetry(
+            tenant_id=tenant_id,
+            distinct_id=email,
+            event=MilestoneRecordType.TENANT_CREATED,
+        )
     except Exception:
         logger.exception(f"Failed to assign tenant {tenant_id} to user {email}")
         raise Exception("Failed to assign tenant to user")

backend/ee/onyx/server/user_group/api.py

@@ -4,7 +4,6 @@ from fastapi import HTTPException
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
-from ee.onyx.db.persona import update_persona_access
 from ee.onyx.db.user_group import add_users_to_user_group
 from ee.onyx.db.user_group import delete_user_group as db_delete_user_group
 from ee.onyx.db.user_group import fetch_user_group
@@ -12,16 +11,13 @@ from ee.onyx.db.user_group import fetch_user_groups
 from ee.onyx.db.user_group import fetch_user_groups_for_user
 from ee.onyx.db.user_group import insert_user_group
 from ee.onyx.db.user_group import prepare_user_group_for_deletion
-from ee.onyx.db.user_group import rename_user_group
 from ee.onyx.db.user_group import update_user_curator_relationship
 from ee.onyx.db.user_group import update_user_group
 from ee.onyx.server.user_group.models import AddUsersToUserGroupRequest
 from ee.onyx.server.user_group.models import MinimalUserGroupSnapshot
 from ee.onyx.server.user_group.models import SetCuratorRequest
-from ee.onyx.server.user_group.models import UpdateGroupAgentsRequest
 from ee.onyx.server.user_group.models import UserGroup
 from ee.onyx.server.user_group.models import UserGroupCreate
-from ee.onyx.server.user_group.models import UserGroupRename
 from ee.onyx.server.user_group.models import UserGroupUpdate
 from onyx.auth.users import current_admin_user
 from onyx.auth.users import current_curator_or_admin_user
@@ -31,9 +27,6 @@ from onyx.configs.constants import PUBLIC_API_TAGS
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.models import User
 from onyx.db.models import UserRole
-from onyx.db.persona import get_persona_by_id
-from onyx.error_handling.error_codes import OnyxErrorCode
-from onyx.error_handling.exceptions import OnyxError
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -94,32 +87,6 @@ def create_user_group(
     return UserGroup.from_model(db_user_group)
 
 
-@router.patch("/admin/user-group/rename")
-def rename_user_group_endpoint(
-    rename_request: UserGroupRename,
-    _: User = Depends(current_admin_user),
-    db_session: Session = Depends(get_session),
-) -> UserGroup:
-    try:
-        return UserGroup.from_model(
-            rename_user_group(
-                db_session=db_session,
-                user_group_id=rename_request.id,
-                new_name=rename_request.name,
-            )
-        )
-    except IntegrityError:
-        raise OnyxError(
-            OnyxErrorCode.DUPLICATE_RESOURCE,
-            f"User group with name '{rename_request.name}' already exists.",
-        )
-    except ValueError as e:
-        msg = str(e)
-        if "not found" in msg.lower():
-            raise OnyxError(OnyxErrorCode.NOT_FOUND, msg)
-        raise OnyxError(OnyxErrorCode.CONFLICT, msg)
-
-
 @router.patch("/admin/user-group/{user_group_id}")
 def patch_user_group(
     user_group_id: int,
@@ -194,38 +161,3 @@ def delete_user_group(
         user_group = fetch_user_group(db_session, user_group_id)
         if user_group:
             db_delete_user_group(db_session, user_group)
-
-
-@router.patch("/admin/user-group/{user_group_id}/agents")
-def update_group_agents(
-    user_group_id: int,
-    request: UpdateGroupAgentsRequest,
-    user: User = Depends(current_admin_user),
-    db_session: Session = Depends(get_session),
-) -> None:
-    for agent_id in request.added_agent_ids:
-        persona = get_persona_by_id(
-            persona_id=agent_id, user=user, db_session=db_session
-        )
-        current_group_ids = [g.id for g in persona.groups]
-        if user_group_id not in current_group_ids:
-            update_persona_access(
-                persona_id=agent_id,
-                creator_user_id=user.id,
-                db_session=db_session,
-                group_ids=current_group_ids + [user_group_id],
-            )
-
-    for agent_id in request.removed_agent_ids:
-        persona = get_persona_by_id(
-            persona_id=agent_id, user=user, db_session=db_session
-        )
-        current_group_ids = [g.id for g in persona.groups]
-        update_persona_access(
-            persona_id=agent_id,
-            creator_user_id=user.id,
-            db_session=db_session,
-            group_ids=[gid for gid in current_group_ids if gid != user_group_id],
-        )
-
-    db_session.commit()

backend/ee/onyx/server/user_group/models.py

@@ -104,16 +104,6 @@ class AddUsersToUserGroupRequest(BaseModel):
     user_ids: list[UUID]
 
 
-class UserGroupRename(BaseModel):
-    id: int
-    name: str
-
-
 class SetCuratorRequest(BaseModel):
     user_id: UUID
     is_curator: bool
-
-
-class UpdateGroupAgentsRequest(BaseModel):
-    added_agent_ids: list[int]
-    removed_agent_ids: list[int]

backend/ee/onyx/utils/posthog_client.py

@@ -9,7 +9,6 @@ from ee.onyx.configs.app_configs import POSTHOG_API_KEY
 from ee.onyx.configs.app_configs import POSTHOG_DEBUG_LOGS_ENABLED
 from ee.onyx.configs.app_configs import POSTHOG_HOST
 from onyx.utils.logger import setup_logger
-from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 
@@ -19,19 +18,12 @@ def posthog_on_error(error: Any, items: Any) -> None:
     logger.error(f"PostHog error: {error}, items: {items}")
 
 
-posthog: Posthog | None = None
-if POSTHOG_API_KEY:
-    posthog = Posthog(
-        project_api_key=POSTHOG_API_KEY,
-        host=POSTHOG_HOST,
-        debug=POSTHOG_DEBUG_LOGS_ENABLED,
-        on_error=posthog_on_error,
-    )
-elif MULTI_TENANT:
-    logger.warning(
-        "POSTHOG_API_KEY is not set but MULTI_TENANT is enabled — "
-        "PostHog telemetry and feature flags will be disabled"
-    )
+posthog = Posthog(
+    project_api_key=POSTHOG_API_KEY,
+    host=POSTHOG_HOST,
+    debug=POSTHOG_DEBUG_LOGS_ENABLED,
+    on_error=posthog_on_error,
+)
 
 # For cross referencing between cloud and www Onyx sites
 # NOTE: These clients are separate because they are separate posthog projects.
@@ -68,7 +60,7 @@ def capture_and_sync_with_alternate_posthog(
         logger.error(f"Error capturing marketing posthog event: {e}")
 
     try:
-        if posthog and (cloud_user_id := props.get("onyx_cloud_user_id")):
+        if cloud_user_id := props.get("onyx_cloud_user_id"):
             cloud_props = props.copy()
             cloud_props.pop("onyx_cloud_user_id", None)
 
@@ -80,45 +72,15 @@ def capture_and_sync_with_alternate_posthog(
         logger.error(f"Error identifying cloud posthog user: {e}")
 
 
-def alias_user(distinct_id: str, anonymous_id: str) -> None:
-    """Link an anonymous distinct_id to an identified user, merging person profiles.
-
-    No-ops when the IDs match (e.g. returning users whose PostHog cookie
-    already contains their identified user ID).
-    """
-    if not posthog or anonymous_id == distinct_id:
-        return
-
-    try:
-        posthog.alias(previous_id=anonymous_id, distinct_id=distinct_id)
-        posthog.flush()
-    except Exception as e:
-        logger.error(f"Error aliasing PostHog user: {e}")
-
-
-def get_anon_id_from_request(request: Any) -> str | None:
-    """Extract the anonymous distinct_id from the app PostHog cookie on a request."""
-    if not POSTHOG_API_KEY:
-        return None
-
-    cookie_name = f"ph_{POSTHOG_API_KEY}_posthog"
-    if (cookie_value := request.cookies.get(cookie_name)) and (
-        parsed := parse_posthog_cookie(cookie_value)
-    ):
-        return parsed.get("distinct_id")
-
-    return None
-
-
 def get_marketing_posthog_cookie_name() -> str | None:
     if not MARKETING_POSTHOG_API_KEY:
         return None
     return f"onyx_custom_ph_{MARKETING_POSTHOG_API_KEY}_posthog"
 
 
-def parse_posthog_cookie(cookie_value: str) -> dict[str, Any] | None:
+def parse_marketing_cookie(cookie_value: str) -> dict[str, Any] | None:
     """
-    Parse a URL-encoded JSON PostHog cookie
+    Parse the URL-encoded JSON marketing cookie.
 
     Expected format (URL-encoded):
     {"distinct_id":"...", "featureFlags":{"landing_page_variant":"..."}, ...}
@@ -132,7 +94,7 @@ def parse_posthog_cookie(cookie_value: str) -> dict[str, Any] | None:
         cookie_data = json.loads(decoded_cookie)
 
         distinct_id = cookie_data.get("distinct_id")
-        if not distinct_id or not isinstance(distinct_id, str):
+        if not distinct_id:
             return None
 
         return cookie_data

backend/ee/onyx/utils/telemetry.py

@@ -1,5 +1,3 @@
-from typing import Any
-
 from ee.onyx.utils.posthog_client import posthog
 from onyx.utils.logger import setup_logger
 
@@ -7,27 +5,12 @@ logger = setup_logger()
 
 
 def event_telemetry(
-    distinct_id: str, event: str, properties: dict[str, Any] | None = None
+    distinct_id: str, event: str, properties: dict | None = None
 ) -> None:
     """Capture and send an event to PostHog, flushing immediately."""
-    if not posthog:
-        return
-
     logger.info(f"Capturing PostHog event: {distinct_id} {event} {properties}")
     try:
         posthog.capture(distinct_id, event, properties)
         posthog.flush()
     except Exception as e:
         logger.error(f"Error capturing PostHog event: {e}")
-
-
-def identify_user(distinct_id: str, properties: dict[str, Any] | None = None) -> None:
-    """Create/update a PostHog person profile, flushing immediately."""
-    if not posthog:
-        return
-
-    try:
-        posthog.identify(distinct_id, properties)
-        posthog.flush()
-    except Exception as e:
-        logger.error(f"Error identifying PostHog user: {e}")

backend/onyx/auth/users.py

@@ -19,7 +19,6 @@ from typing import Optional
 from typing import Protocol
 from typing import Tuple
 from typing import TypeVar
-from urllib.parse import urlparse
 
 import jwt
 from email_validator import EmailNotValidError
@@ -135,9 +134,6 @@ from onyx.redis.redis_pool import retrieve_ws_token_data
 from onyx.server.settings.store import load_settings
 from onyx.server.utils import BasicAuthenticationError
 from onyx.utils.logger import setup_logger
-from onyx.utils.telemetry import mt_cloud_alias
-from onyx.utils.telemetry import mt_cloud_get_anon_id
-from onyx.utils.telemetry import mt_cloud_identify
 from onyx.utils.telemetry import mt_cloud_telemetry
 from onyx.utils.telemetry import optional_telemetry
 from onyx.utils.telemetry import RecordType
@@ -253,12 +249,18 @@ def verify_email_is_invited(email: str) -> None:
     whitelist = get_invited_users()
 
     if not email:
-        raise OnyxError(OnyxErrorCode.INVALID_INPUT, "Email must be specified")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={"reason": "Email must be specified"},
+        )
 
     try:
         email_info = validate_email(email, check_deliverability=False)
     except EmailUndeliverableError:
-        raise OnyxError(OnyxErrorCode.INVALID_INPUT, "Email is not valid")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={"reason": "Email is not valid"},
+        )
 
     for email_whitelist in whitelist:
         try:
@@ -275,9 +277,12 @@ def verify_email_is_invited(email: str) -> None:
         if email_info.normalized.lower() == email_info_whitelist.normalized.lower():
             return
 
-    raise OnyxError(
-        OnyxErrorCode.UNAUTHORIZED,
-        "This workspace is invite-only. Please ask your admin to invite you.",
+    raise HTTPException(
+        status_code=status.HTTP_403_FORBIDDEN,
+        detail={
+            "code": REGISTER_INVITE_ONLY_CODE,
+            "reason": "This workspace is invite-only. Please ask your admin to invite you.",
+        },
     )
 
 
@@ -287,47 +292,48 @@ def verify_email_in_whitelist(email: str, tenant_id: str) -> None:
             verify_email_is_invited(email)
 
 
-def verify_email_domain(email: str, *, is_registration: bool = False) -> None:
+def verify_email_domain(email: str) -> None:
     if email.count("@") != 1:
-        raise OnyxError(OnyxErrorCode.INVALID_INPUT, "Email is not valid")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Email is not valid",
+        )
 
     local_part, domain = email.split("@")
     domain = domain.lower()
-    local_part = local_part.lower()
 
     if AUTH_TYPE == AuthType.CLOUD:
         # Normalize googlemail.com to gmail.com (they deliver to the same inbox)
         if domain == "googlemail.com":
-            raise OnyxError(
-                OnyxErrorCode.INVALID_INPUT,
-                "Please use @gmail.com instead of @googlemail.com.",
-            )
-
-        # Only block dotted Gmail on new signups — existing users must still be
-        # able to sign in with the address they originally registered with.
-        if is_registration and domain == "gmail.com" and "." in local_part:
-            raise OnyxError(
-                OnyxErrorCode.INVALID_INPUT,
-                "Gmail addresses with '.' are not allowed. Please use your base email address.",
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail={"reason": "Please use @gmail.com instead of @googlemail.com."},
             )
 
         if "+" in local_part and domain != "onyx.app":
-            raise OnyxError(
-                OnyxErrorCode.INVALID_INPUT,
-                "Email addresses with '+' are not allowed. Please use your base email address.",
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail={
+                    "reason": "Email addresses with '+' are not allowed. Please use your base email address."
+                },
             )
 
     # Check if email uses a disposable/temporary domain
     if is_disposable_email(email):
-        raise OnyxError(
-            OnyxErrorCode.INVALID_INPUT,
-            "Disposable email addresses are not allowed. Please use a permanent email address.",
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={
+                "reason": "Disposable email addresses are not allowed. Please use a permanent email address."
+            },
         )
 
     # Check domain whitelist if configured
     if VALID_EMAIL_DOMAINS:
         if domain not in VALID_EMAIL_DOMAINS:
-            raise OnyxError(OnyxErrorCode.INVALID_INPUT, "Email domain is not valid")
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="Email domain is not valid",
+            )
 
 
 def enforce_seat_limit(db_session: Session, seats_needed: int = 1) -> None:
@@ -343,7 +349,7 @@ def enforce_seat_limit(db_session: Session, seats_needed: int = 1) -> None:
     )(db_session, seats_needed=seats_needed)
 
     if result is not None and not result.available:
-        raise OnyxError(OnyxErrorCode.SEAT_LIMIT_EXCEEDED, result.error_message)
+        raise HTTPException(status_code=402, detail=result.error_message)
 
 
 class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
@@ -396,7 +402,10 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
                     captcha_token or "", expected_action="signup"
                 )
             except CaptchaVerificationError as e:
-                raise OnyxError(OnyxErrorCode.INVALID_INPUT, str(e))
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail={"reason": str(e)},
+                )
 
         # We verify the password here to make sure it's valid before we proceed
         await self.validate_password(
@@ -406,10 +415,13 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
         # Check for disposable emails BEFORE provisioning tenant
         # This prevents creating tenants for throwaway email addresses
         try:
-            verify_email_domain(user_create.email, is_registration=True)
-        except OnyxError as e:
+            verify_email_domain(user_create.email)
+        except HTTPException as e:
             # Log blocked disposable email attempts
-            if "Disposable email" in e.detail:
+            if (
+                e.status_code == status.HTTP_400_BAD_REQUEST
+                and "Disposable email" in str(e.detail)
+            ):
                 domain = (
                     user_create.email.split("@")[-1]
                     if "@" in user_create.email
@@ -553,9 +565,9 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
         result = await db_session.execute(
             select(Persona.id)
             .where(
-                Persona.is_featured.is_(True),
+                Persona.featured.is_(True),
                 Persona.is_public.is_(True),
-                Persona.is_listed.is_(True),
+                Persona.is_visible.is_(True),
                 Persona.deleted.is_(False),
             )
             .order_by(
@@ -683,8 +695,6 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
                         raise exceptions.UserNotExists()
 
                 except exceptions.UserNotExists:
-                    verify_email_domain(account_email, is_registration=True)
-
                     # Check seat availability before creating (single-tenant only)
                     with get_session_with_current_tenant() as sync_db:
                         enforce_seat_limit(sync_db)
@@ -782,18 +792,6 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
         except Exception:
             logger.exception("Error deleting anonymous user cookie")
 
-        tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
-
-        # Link the anonymous PostHog session to the identified user so that
-        # pre-login session recordings and events merge into one person profile.
-        if anon_id := mt_cloud_get_anon_id(request):
-            mt_cloud_alias(distinct_id=str(user.id), anonymous_id=anon_id)
-
-        mt_cloud_identify(
-            distinct_id=str(user.id),
-            properties={"email": user.email, "tenant_id": tenant_id},
-        )
-
     async def on_after_register(
         self, user: User, request: Optional[Request] = None
     ) -> None:
@@ -812,30 +810,12 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             user_count = await get_user_count()
             logger.debug(f"Current tenant user count: {user_count}")
 
-            # Link the anonymous PostHog session to the identified user so
-            # that pre-signup session recordings merge into one person profile.
-            if anon_id := mt_cloud_get_anon_id(request):
-                mt_cloud_alias(distinct_id=str(user.id), anonymous_id=anon_id)
-
-            # Ensure a PostHog person profile exists for this user.
-            mt_cloud_identify(
-                distinct_id=str(user.id),
-                properties={"email": user.email, "tenant_id": tenant_id},
-            )
-
             mt_cloud_telemetry(
                 tenant_id=tenant_id,
-                distinct_id=str(user.id),
+                distinct_id=user.email,
                 event=MilestoneRecordType.USER_SIGNED_UP,
             )
 
-            if user_count == 1:
-                mt_cloud_telemetry(
-                    tenant_id=tenant_id,
-                    distinct_id=str(user.id),
-                    event=MilestoneRecordType.TENANT_CREATED,
-                )
-
         finally:
             CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
@@ -845,9 +825,9 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             attribute="get_marketing_posthog_cookie_name",
             noop_return_value=None,
         )
-        parse_posthog_cookie = fetch_ee_implementation_or_noop(
+        parse_marketing_cookie = fetch_ee_implementation_or_noop(
             module="onyx.utils.posthog_client",
-            attribute="parse_posthog_cookie",
+            attribute="parse_marketing_cookie",
             noop_return_value=None,
         )
         capture_and_sync_with_alternate_posthog = fetch_ee_implementation_or_noop(
@@ -861,7 +841,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             and user_count is not None
             and (marketing_cookie_name := get_marketing_posthog_cookie_name())
             and (marketing_cookie_value := request.cookies.get(marketing_cookie_name))
-            and (parsed_cookie := parse_posthog_cookie(marketing_cookie_value))
+            and (parsed_cookie := parse_marketing_cookie(marketing_cookie_value))
         ):
             marketing_anonymous_id = parsed_cookie["distinct_id"]
 
@@ -1672,33 +1652,6 @@ async def _get_user_from_token_data(token_data: dict) -> User | None:
         return user
 
 
-_LOOPBACK_HOSTNAMES = frozenset({"localhost", "127.0.0.1", "::1"})
-
-
-def _is_same_origin(actual: str, expected: str) -> bool:
-    """Compare two origins for the WebSocket CSWSH check.
-
-    Scheme and hostname must match exactly.  Port must also match, except
-    when the hostname is a loopback address (localhost / 127.0.0.1 / ::1),
-    where port is ignored.  On loopback, all ports belong to the same
-    operator, so port differences carry no security significance — the
-    CSWSH threat is remote origins, not local ones.
-    """
-    a = urlparse(actual.rstrip("/"))
-    e = urlparse(expected.rstrip("/"))
-
-    if a.scheme != e.scheme or a.hostname != e.hostname:
-        return False
-
-    if a.hostname in _LOOPBACK_HOSTNAMES:
-        return True
-
-    actual_port = a.port or (443 if a.scheme == "https" else 80)
-    expected_port = e.port or (443 if e.scheme == "https" else 80)
-
-    return actual_port == expected_port
-
-
 async def current_user_from_websocket(
     websocket: WebSocket,
     token: str = Query(..., description="WebSocket authentication token"),
@@ -1718,15 +1671,19 @@ async def current_user_from_websocket(
 
     This applies the same auth checks as current_user() for HTTP endpoints.
     """
-    # Check Origin header to prevent Cross-Site WebSocket Hijacking (CSWSH).
-    # Browsers always send Origin on WebSocket connections.
+    # Check Origin header to prevent Cross-Site WebSocket Hijacking (CSWSH)
+    # Browsers always send Origin on WebSocket connections
     origin = websocket.headers.get("origin")
+    expected_origin = WEB_DOMAIN.rstrip("/")
     if not origin:
         logger.warning("WS auth: missing Origin header")
         raise BasicAuthenticationError(detail="Access denied. Missing origin.")
 
-    if not _is_same_origin(origin, WEB_DOMAIN):
-        logger.warning(f"WS auth: origin mismatch. Expected {WEB_DOMAIN}, got {origin}")
+    actual_origin = origin.rstrip("/")
+    if actual_origin != expected_origin:
+        logger.warning(
+            f"WS auth: origin mismatch. Expected {expected_origin}, got {actual_origin}"
+        )
         raise BasicAuthenticationError(detail="Access denied. Invalid origin.")
 
     # Validate WS token in Redis (single-use, deleted after retrieval)

backend/onyx/background/celery/apps/primary.py

@@ -317,7 +317,6 @@ celery_app.autodiscover_tasks(
             "onyx.background.celery.tasks.docprocessing",
             "onyx.background.celery.tasks.evals",
             "onyx.background.celery.tasks.hierarchyfetching",
-            "onyx.background.celery.tasks.hooks",
             "onyx.background.celery.tasks.periodic",
             "onyx.background.celery.tasks.pruning",
             "onyx.background.celery.tasks.shared",

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -14,7 +14,6 @@ from onyx.configs.constants import ONYX_CLOUD_CELERY_TASK_PREFIX
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.hooks.utils import HOOKS_AVAILABLE
 from shared_configs.configs import MULTI_TENANT
 
 # choosing 15 minutes because it roughly gives us enough time to process many tasks
@@ -362,19 +361,6 @@ if not MULTI_TENANT:
 
     tasks_to_schedule.extend(beat_task_templates)
 
-if HOOKS_AVAILABLE:
-    tasks_to_schedule.append(
-        {
-            "name": "hook-execution-log-cleanup",
-            "task": OnyxCeleryTask.HOOK_EXECUTION_LOG_CLEANUP_TASK,
-            "schedule": timedelta(days=1),
-            "options": {
-                "priority": OnyxCeleryPriority.LOW,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        }
-    )
-
 
 def generate_cloud_tasks(
     beat_tasks: list[dict], beat_templates: list[dict], beat_multiplier: float

backend/onyx/background/celery/tasks/hierarchyfetching/tasks.py

@@ -29,8 +29,6 @@ from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.connectors.factory import ConnectorMissingException
-from onyx.connectors.factory import identify_connector_class
 from onyx.connectors.factory import instantiate_connector
 from onyx.connectors.interfaces import HierarchyConnector
 from onyx.connectors.models import HierarchyNode as PydanticHierarchyNode
@@ -57,26 +55,6 @@ logger = setup_logger()
 HIERARCHY_FETCH_INTERVAL_SECONDS = 24 * 60 * 60
 
 
-def _connector_supports_hierarchy_fetching(
-    cc_pair: ConnectorCredentialPair,
-) -> bool:
-    """Return True only for connectors whose class implements HierarchyConnector."""
-    try:
-        connector_class = identify_connector_class(
-            cc_pair.connector.source,
-        )
-    except ConnectorMissingException as e:
-        task_logger.warning(
-            "Skipping hierarchy fetching enqueue for source=%s input_type=%s: %s",
-            cc_pair.connector.source,
-            cc_pair.connector.input_type,
-            str(e),
-        )
-        return False
-
-    return issubclass(connector_class, HierarchyConnector)
-
-
 def _is_hierarchy_fetching_due(cc_pair: ConnectorCredentialPair) -> bool:
     """Returns boolean indicating if hierarchy fetching is due for this connector.
 
@@ -208,10 +186,7 @@ def check_for_hierarchy_fetching(self: Task, *, tenant_id: str) -> int | None:
                     cc_pair_id=cc_pair_id,
                 )
 
-                if not cc_pair or not _connector_supports_hierarchy_fetching(cc_pair):
-                    continue
-
-                if not _is_hierarchy_fetching_due(cc_pair):
+                if not cc_pair or not _is_hierarchy_fetching_due(cc_pair):
                     continue
 
                 task_id = _try_creating_hierarchy_fetching_task(

backend/onyx/background/celery/tasks/hooks/tasks.py

@@ -1,35 +0,0 @@
-from celery import shared_task
-
-from onyx.configs.app_configs import JOB_TIMEOUT
-from onyx.configs.constants import OnyxCeleryTask
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
-from onyx.db.hook import cleanup_old_execution_logs__no_commit
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-_HOOK_EXECUTION_LOG_RETENTION_DAYS: int = 30
-
-
-@shared_task(
-    name=OnyxCeleryTask.HOOK_EXECUTION_LOG_CLEANUP_TASK,
-    ignore_result=True,
-    soft_time_limit=JOB_TIMEOUT,
-    trail=False,
-)
-def hook_execution_log_cleanup_task(*, tenant_id: str) -> None:  # noqa: ARG001
-    try:
-        with get_session_with_current_tenant() as db_session:
-            deleted: int = cleanup_old_execution_logs__no_commit(
-                db_session=db_session,
-                max_age_days=_HOOK_EXECUTION_LOG_RETENTION_DAYS,
-            )
-            db_session.commit()
-            if deleted:
-                logger.info(
-                    f"Deleted {deleted} hook execution log(s) older than "
-                    f"{_HOOK_EXECUTION_LOG_RETENTION_DAYS} days."
-                )
-    except Exception:
-        logger.exception("Failed to clean up hook execution logs")
-        raise

backend/onyx/background/celery/tasks/user_file_processing/tasks.py

@@ -24,7 +24,6 @@ from onyx.configs.app_configs import MANAGED_VESPA
 from onyx.configs.app_configs import VESPA_CLOUD_CERT_PATH
 from onyx.configs.app_configs import VESPA_CLOUD_KEY_PATH
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
-from onyx.configs.constants import CELERY_USER_FILE_DELETE_TASK_EXPIRES
 from onyx.configs.constants import CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT
 from onyx.configs.constants import CELERY_USER_FILE_PROCESSING_TASK_EXPIRES
 from onyx.configs.constants import CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT
@@ -34,7 +33,6 @@ from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.configs.constants import USER_FILE_DELETE_MAX_QUEUE_DEPTH
 from onyx.configs.constants import USER_FILE_PROCESSING_MAX_QUEUE_DEPTH
 from onyx.configs.constants import USER_FILE_PROJECT_SYNC_MAX_QUEUE_DEPTH
 from onyx.connectors.file.connector import LocalFileConnector
@@ -93,17 +91,6 @@ def _user_file_delete_lock_key(user_file_id: str | UUID) -> str:
     return f"{OnyxRedisLocks.USER_FILE_DELETE_LOCK_PREFIX}:{user_file_id}"
 
 
-def _user_file_delete_queued_key(user_file_id: str | UUID) -> str:
-    """Key that exists while a delete_single_user_file task is sitting in the queue.
-
-    The beat generator sets this with a TTL equal to CELERY_USER_FILE_DELETE_TASK_EXPIRES
-    before enqueuing and the worker deletes it as its first action.  This prevents
-    the beat from adding duplicate tasks for files that already have a live task
-    in flight.
-    """
-    return f"{OnyxRedisLocks.USER_FILE_DELETE_QUEUED_PREFIX}:{user_file_id}"
-
-
 def get_user_file_project_sync_queue_depth(celery_app: Celery) -> int:
     redis_celery: Redis = celery_app.broker_connection().channel().client  # type: ignore
     return celery_get_queue_length(
@@ -559,23 +546,7 @@ def process_single_user_file(
     ignore_result=True,
 )
 def check_for_user_file_delete(self: Task, *, tenant_id: str) -> None:
-    """Scan for user files with DELETING status and enqueue per-file tasks.
-
-    Three mechanisms prevent queue runaway (mirrors check_user_file_processing):
-
-    1. **Queue depth backpressure** – if the broker queue already has more than
-       USER_FILE_DELETE_MAX_QUEUE_DEPTH items we skip this beat cycle entirely.
-
-    2. **Per-file queued guard** – before enqueuing a task we set a short-lived
-       Redis key (TTL = CELERY_USER_FILE_DELETE_TASK_EXPIRES).  If that key
-       already exists the file already has a live task in the queue, so we skip
-       it.  The worker deletes the key the moment it picks up the task so the
-       next beat cycle can re-enqueue if the file is still DELETING.
-
-    3. **Task expiry** – every enqueued task carries an `expires` value equal to
-       CELERY_USER_FILE_DELETE_TASK_EXPIRES.  If a task is still sitting in
-       the queue after that deadline, Celery discards it without touching the DB.
-    """
+    """Scan for user files with DELETING status and enqueue per-file tasks."""
     task_logger.info("check_for_user_file_delete - Starting")
     redis_client = get_redis_client(tenant_id=tenant_id)
     lock: RedisLock = redis_client.lock(
@@ -584,23 +555,8 @@ def check_for_user_file_delete(self: Task, *, tenant_id: str) -> None:
     )
     if not lock.acquire(blocking=False):
         return None
-
     enqueued = 0
-    skipped_guard = 0
     try:
-        # --- Protection 1: queue depth backpressure ---
-        # NOTE: must use the broker's Redis client (not redis_client) because
-        # Celery queues live on a separate Redis DB with CELERY_SEPARATOR keys.
-        r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
-        queue_len = celery_get_queue_length(OnyxCeleryQueues.USER_FILE_DELETE, r_celery)
-        if queue_len > USER_FILE_DELETE_MAX_QUEUE_DEPTH:
-            task_logger.warning(
-                f"check_for_user_file_delete - Queue depth {queue_len} exceeds "
-                f"{USER_FILE_DELETE_MAX_QUEUE_DEPTH}, skipping enqueue for "
-                f"tenant={tenant_id}"
-            )
-            return None
-
         with get_session_with_current_tenant() as db_session:
             user_file_ids = (
                 db_session.execute(
@@ -612,40 +568,23 @@ def check_for_user_file_delete(self: Task, *, tenant_id: str) -> None:
                 .all()
             )
             for user_file_id in user_file_ids:
-                # --- Protection 2: per-file queued guard ---
-                queued_key = _user_file_delete_queued_key(user_file_id)
-                guard_set = redis_client.set(
-                    queued_key,
-                    1,
-                    ex=CELERY_USER_FILE_DELETE_TASK_EXPIRES,
-                    nx=True,
+                self.app.send_task(
+                    OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
+                    kwargs={"user_file_id": str(user_file_id), "tenant_id": tenant_id},
+                    queue=OnyxCeleryQueues.USER_FILE_DELETE,
+                    priority=OnyxCeleryPriority.HIGH,
                 )
-                if not guard_set:
-                    skipped_guard += 1
-                    continue
-
-                # --- Protection 3: task expiry ---
-                try:
-                    self.app.send_task(
-                        OnyxCeleryTask.DELETE_SINGLE_USER_FILE,
-                        kwargs={
-                            "user_file_id": str(user_file_id),
-                            "tenant_id": tenant_id,
-                        },
-                        queue=OnyxCeleryQueues.USER_FILE_DELETE,
-                        priority=OnyxCeleryPriority.HIGH,
-                        expires=CELERY_USER_FILE_DELETE_TASK_EXPIRES,
-                    )
-                except Exception:
-                    redis_client.delete(queued_key)
-                    raise
                 enqueued += 1
+    except Exception as e:
+        task_logger.exception(
+            f"check_for_user_file_delete - Error enqueuing deletes - {e.__class__.__name__}"
+        )
+        return None
     finally:
         if lock.owned():
             lock.release()
-
     task_logger.info(
-        f"check_for_user_file_delete - Enqueued {enqueued} tasks, skipped_guard={skipped_guard} for tenant={tenant_id}"
+        f"check_for_user_file_delete - Enqueued {enqueued} tasks for tenant={tenant_id}"
     )
     return None
 
@@ -663,9 +602,6 @@ def delete_user_file_impl(
     file_lock: RedisLock | None = None
     if redis_locking:
         redis_client = get_redis_client(tenant_id=tenant_id)
-        # Clear the queued guard so the beat can re-enqueue if deletion fails
-        # and the file remains in DELETING status.
-        redis_client.delete(_user_file_delete_queued_key(user_file_id))
         file_lock = redis_client.lock(
             _user_file_delete_lock_key(user_file_id),
             timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,

backend/onyx/cache/postgres_backend.py

@@ -297,9 +297,7 @@ class PostgresCacheBackend(CacheBackend):
 
     def _lock_id_for(self, name: str) -> int:
         """Map *name* to a 64-bit signed int for ``pg_advisory_lock``."""
-        h = hashlib.md5(
-            f"{self._tenant_id}:{name}".encode(), usedforsecurity=False
-        ).digest()
+        h = hashlib.md5(f"{self._tenant_id}:{name}".encode()).digest()
         return struct.unpack("q", h[:8])[0]
 
 

backend/onyx/chat/chat_utils.py

@@ -30,8 +30,6 @@ from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_store.file_store import get_default_file_store
 from onyx.file_store.models import ChatFileType
 from onyx.file_store.models import FileDescriptor
-from onyx.file_store.utils import plaintext_file_name_for_id
-from onyx.file_store.utils import store_plaintext
 from onyx.kg.models import KGException
 from onyx.kg.setup.kg_default_entity_definitions import (
     populate_missing_default_entity_types__commit,
@@ -291,33 +289,6 @@ def process_kg_commands(
         raise KGException("KG setup done")
 
 
-def _get_or_extract_plaintext(
-    file_id: str,
-    extract_fn: Callable[[], str],
-) -> str:
-    """Load cached plaintext for a file, or extract and store it.
-
-    Tries to read pre-stored plaintext from the file store.  On a miss,
-    calls extract_fn to produce the text, then stores the result so
-    future calls skip the expensive extraction.
-    """
-    file_store = get_default_file_store()
-    plaintext_key = plaintext_file_name_for_id(file_id)
-
-    # Try cached plaintext first.
-    try:
-        plaintext_io = file_store.read_file(plaintext_key, mode="b")
-        return plaintext_io.read().decode("utf-8")
-    except Exception:
-        logger.exception(f"Error when reading file, id={file_id}")
-
-    # Cache miss — extract and store.
-    content_text = extract_fn()
-    if content_text:
-        store_plaintext(file_id, content_text)
-    return content_text
-
-
 @log_function_time(print_only=True)
 def load_chat_file(
     file_descriptor: FileDescriptor, db_session: Session
@@ -332,23 +303,12 @@ def load_chat_file(
     file_type = ChatFileType(file_descriptor["type"])
 
     if file_type.is_text_file():
-        file_id = file_descriptor["id"]
-
-        def _extract() -> str:
-            return extract_file_text(
+        try:
+            content_text = extract_file_text(
                 file=file_io,
                 file_name=file_descriptor.get("name") or "",
                 break_on_unprocessable=False,
             )
-
-        # Use the user_file_id as cache key when available (matches what
-        # the celery indexing worker stores), otherwise fall back to the
-        # file store id (covers code-interpreter-generated files, etc.).
-        user_file_id_str = file_descriptor.get("user_file_id")
-        cache_key = user_file_id_str or file_id
-
-        try:
-            content_text = _get_or_extract_plaintext(cache_key, _extract)
         except Exception as e:
             logger.warning(
                 f"Failed to retrieve content for file {file_descriptor['id']}: {str(e)}"

backend/onyx/chat/llm_loop.py

@@ -36,11 +36,9 @@ from onyx.db.memory import add_memory
 from onyx.db.memory import update_memory_at_index
 from onyx.db.memory import UserMemoryContext
 from onyx.db.models import Persona
-from onyx.llm.constants import LlmProviderNames
 from onyx.llm.interfaces import LLM
 from onyx.llm.interfaces import LLMUserIdentity
 from onyx.llm.interfaces import ToolChoiceOptions
-from onyx.llm.utils import is_true_openai_model
 from onyx.prompts.chat_prompts import IMAGE_GEN_REMINDER
 from onyx.prompts.chat_prompts import OPEN_URL_REMINDER
 from onyx.server.query_and_chat.placement import Placement
@@ -74,70 +72,6 @@ from shared_configs.contextvars import get_current_tenant_id
 logger = setup_logger()
 
 
-class EmptyLLMResponseError(RuntimeError):
-    """Raised when the streamed LLM response completes without a usable answer."""
-
-    def __init__(
-        self,
-        *,
-        provider: str,
-        model: str,
-        tool_choice: ToolChoiceOptions,
-        client_error_msg: str,
-        error_code: str = "EMPTY_LLM_RESPONSE",
-        is_retryable: bool = True,
-    ) -> None:
-        super().__init__(client_error_msg)
-        self.provider = provider
-        self.model = model
-        self.tool_choice = tool_choice
-        self.client_error_msg = client_error_msg
-        self.error_code = error_code
-        self.is_retryable = is_retryable
-
-
-def _build_empty_llm_response_error(
-    llm: LLM,
-    llm_step_result: LlmStepResult,
-    tool_choice: ToolChoiceOptions,
-) -> EmptyLLMResponseError:
-    provider = llm.config.model_provider
-    model = llm.config.model_name
-
-    # OpenAI quota exhaustion has reached us as a streamed "stop" with zero content.
-    # When the stream is completely empty and there is no reasoning/tool output, surface
-    # the likely account-level cause instead of a generic tool-calling error.
-    if (
-        not llm_step_result.reasoning
-        and provider == LlmProviderNames.OPENAI
-        and is_true_openai_model(provider, model)
-    ):
-        return EmptyLLMResponseError(
-            provider=provider,
-            model=model,
-            tool_choice=tool_choice,
-            client_error_msg=(
-                "The selected OpenAI model returned an empty streamed response "
-                "before producing any tokens. This commonly happens when the API "
-                "key or project has no remaining quota or billing is not enabled. "
-                "Verify quota and billing for this key and try again."
-            ),
-            error_code="BUDGET_EXCEEDED",
-            is_retryable=False,
-        )
-
-    return EmptyLLMResponseError(
-        provider=provider,
-        model=model,
-        tool_choice=tool_choice,
-        client_error_msg=(
-            "The selected model returned no final answer before the stream "
-            "completed. No text or tool calls were received from the upstream "
-            "provider."
-        ),
-    )
-
-
 def _looks_like_xml_tool_call_payload(text: str | None) -> bool:
     """Detect XML-style marshaled tool calls emitted as plain text."""
     if not text:
@@ -679,12 +613,7 @@ def run_llm_loop(
             )
             citation_processor.update_citation_mapping(project_citation_mapping)
 
-        llm_step_result = LlmStepResult(
-            reasoning=None,
-            answer=None,
-            tool_calls=None,
-            raw_answer=None,
-        )
+        llm_step_result: LlmStepResult | None = None
 
         # Pass the total budget to construct_message_history, which will handle token allocation
         available_tokens = llm.config.max_input_tokens
@@ -1155,18 +1084,12 @@ def run_llm_loop(
                 # As long as 1 tool with citeable documents is called at any point, we ask the LLM to try to cite
                 should_cite_documents = True
 
-        if not llm_step_result.answer and not llm_step_result.tool_calls:
-            raise _build_empty_llm_response_error(
-                llm=llm,
-                llm_step_result=llm_step_result,
-                tool_choice=tool_choice,
-            )
-
-        if not llm_step_result.answer:
+        if not llm_step_result or not llm_step_result.answer:
             raise RuntimeError(
-                "The LLM did not return a final answer after tool execution. "
-                "Typically this indicates invalid tool-call output, a model/provider mismatch, "
-                "or serving API misconfiguration."
+                "The LLM did not return an answer. "
+                "Typically this is an issue with LLMs that do not support tool calling natively, "
+                "or the model serving API is not configured correctly. "
+                "This may also happen with models that are lower quality outputting invalid tool calls."
             )
 
         emitter.emit(

backend/onyx/chat/llm_step.py

@@ -1013,10 +1013,6 @@ def run_llm_step_pkt_generator(
     accumulated_reasoning = ""
     accumulated_answer = ""
     accumulated_raw_answer = ""
-    stream_chunk_count = 0
-    actionable_chunk_count = 0
-    empty_chunk_count = 0
-    finish_reasons: set[str] = set()
     xml_tool_call_content_filter = _XmlToolCallContentFilter()
 
     processor_state: Any = None
@@ -1149,7 +1145,6 @@ def run_llm_step_pkt_generator(
             user_identity=user_identity,
             timeout_override=timeout_override,
         ):
-            stream_chunk_count += 1
             if packet.usage:
                 usage = packet.usage
                 span_generation.span_data.usage = {
@@ -1159,21 +1154,16 @@ def run_llm_step_pkt_generator(
                     "cache_creation_input_tokens": usage.cache_creation_input_tokens,
                 }
                 # Note: LLM cost tracking is now handled in multi_llm.py
-            finish_reason = packet.choice.finish_reason
-            if finish_reason:
-                finish_reasons.add(str(finish_reason))
             delta = packet.choice.delta
 
             # Weird behavior from some model providers, just log and ignore for now
             if (
-                not delta.content
+                delta.content is None
                 and delta.reasoning_content is None
-                and not delta.tool_calls
+                and delta.tool_calls is None
             ):
-                empty_chunk_count += 1
                 logger.warning(
-                    "LLM packet is empty (no content, reasoning, or tool calls). "
-                    f"finish_reason={finish_reason}. Skipping: {packet}"
+                    f"LLM packet is empty (no contents, reasoning or tool calls). Skipping: {packet}"
                 )
                 continue
 
@@ -1182,8 +1172,6 @@ def run_llm_step_pkt_generator(
                     time.monotonic() - stream_start_time
                 )
                 first_action_recorded = True
-            if _delta_has_action(delta):
-                actionable_chunk_count += 1
 
             if custom_token_processor:
                 # The custom token processor can modify the deltas for specific custom logic
@@ -1319,15 +1307,6 @@ def run_llm_step_pkt_generator(
         else:
             logger.debug("Tool calls: []")
 
-    if actionable_chunk_count == 0:
-        logger.warning(
-            "LLM stream completed with no actionable deltas. "
-            f"chunks={stream_chunk_count}, empty_chunks={empty_chunk_count}, "
-            f"finish_reasons={sorted(finish_reasons)}, "
-            f"provider={llm.config.model_provider}, model={llm.config.model_name}, "
-            f"tool_choice={tool_choice}, tools_sent={len(tool_definitions)}"
-        )
-
     return (
         LlmStepResult(
             reasoning=accumulated_reasoning if accumulated_reasoning else None,

backend/onyx/chat/models.py

@@ -177,8 +177,8 @@ class ExtractedContextFiles(BaseModel):
 class SearchParams(BaseModel):
     """Resolved search filter IDs and search-tool usage for a chat turn."""
 
-    project_id_filter: int | None
-    persona_id_filter: int | None
+    search_project_id: int | None
+    search_persona_id: int | None
     search_usage: SearchToolUsage
 
 

backend/onyx/chat/process_message.py

@@ -29,7 +29,6 @@ from onyx.chat.compression import compress_chat_history
 from onyx.chat.compression import find_summary_for_branch
 from onyx.chat.compression import get_compression_params
 from onyx.chat.emitter import get_default_emitter
-from onyx.chat.llm_loop import EmptyLLMResponseError
 from onyx.chat.llm_loop import run_llm_loop
 from onyx.chat.models import AnswerStream
 from onyx.chat.models import ChatBasicResponse
@@ -59,7 +58,6 @@ from onyx.db.chat import create_new_chat_message
 from onyx.db.chat import get_chat_session_by_id
 from onyx.db.chat import get_or_create_root_message
 from onyx.db.chat import reserve_message_id
-from onyx.db.enums import HookPoint
 from onyx.db.memory import get_memories
 from onyx.db.models import ChatMessage
 from onyx.db.models import ChatSession
@@ -69,19 +67,11 @@ from onyx.db.models import UserFile
 from onyx.db.projects import get_user_files_from_project
 from onyx.db.tools import get_tools
 from onyx.deep_research.dr_loop import run_deep_research_llm_loop
-from onyx.error_handling.error_codes import OnyxErrorCode
-from onyx.error_handling.exceptions import log_onyx_error
-from onyx.error_handling.exceptions import OnyxError
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_store.models import ChatFileType
 from onyx.file_store.models import InMemoryChatFile
 from onyx.file_store.utils import load_in_memory_chat_files
 from onyx.file_store.utils import verify_user_files
-from onyx.hooks.executor import execute_hook
-from onyx.hooks.executor import HookSkipped
-from onyx.hooks.executor import HookSoftFailed
-from onyx.hooks.points.query_processing import QueryProcessingPayload
-from onyx.hooks.points.query_processing import QueryProcessingResponse
 from onyx.llm.factory import get_llm_for_persona
 from onyx.llm.factory import get_llm_token_counter
 from onyx.llm.interfaces import LLM
@@ -408,13 +398,13 @@ def determine_search_params(
     """
     is_custom_persona = persona_id != DEFAULT_PERSONA_ID
 
-    project_id_filter: int | None = None
-    persona_id_filter: int | None = None
+    search_project_id: int | None = None
+    search_persona_id: int | None = None
     if extracted_context_files.use_as_search_filter:
         if is_custom_persona:
-            persona_id_filter = persona_id
+            search_persona_id = persona_id
         else:
-            project_id_filter = project_id
+            search_project_id = project_id
 
     search_usage = SearchToolUsage.AUTO
     if not is_custom_persona and project_id:
@@ -427,34 +417,12 @@ def determine_search_params(
             search_usage = SearchToolUsage.DISABLED
 
     return SearchParams(
-        project_id_filter=project_id_filter,
-        persona_id_filter=persona_id_filter,
+        search_project_id=search_project_id,
+        search_persona_id=search_persona_id,
         search_usage=search_usage,
     )
 
 
-def _resolve_query_processing_hook_result(
-    hook_result: QueryProcessingResponse | HookSkipped | HookSoftFailed,
-    message_text: str,
-) -> str:
-    """Apply the Query Processing hook result to the message text.
-
-    Returns the (possibly rewritten) message text, or raises OnyxError with
-    QUERY_REJECTED if the hook signals rejection (query is null or empty).
-    HookSkipped and HookSoftFailed are pass-throughs — the original text is
-    returned unchanged.
-    """
-    if isinstance(hook_result, (HookSkipped, HookSoftFailed)):
-        return message_text
-    if not (hook_result.query and hook_result.query.strip()):
-        raise OnyxError(
-            OnyxErrorCode.QUERY_REJECTED,
-            hook_result.rejection_message
-            or "The hook extension for query processing did not return a valid query. No rejection reason was provided.",
-        )
-    return hook_result.query.strip()
-
-
 def handle_stream_message_objects(
     new_msg_req: SendMessageRequest,
     user: User,
@@ -505,24 +473,16 @@ def handle_stream_message_objects(
                 db_session=db_session,
             )
             yield CreateChatSessionID(chat_session_id=chat_session.id)
-            chat_session = get_chat_session_by_id(
-                chat_session_id=chat_session.id,
-                user_id=user_id,
-                db_session=db_session,
-                eager_load_persona=True,
-            )
         else:
             chat_session = get_chat_session_by_id(
                 chat_session_id=new_msg_req.chat_session_id,
                 user_id=user_id,
                 db_session=db_session,
-                eager_load_persona=True,
             )
 
         persona = chat_session.persona
 
         message_text = new_msg_req.message
-
         user_identity = LLMUserIdentity(
             user_id=llm_user_identifier, session_id=str(chat_session.id)
         )
@@ -530,13 +490,13 @@ def handle_stream_message_objects(
         # Milestone tracking, most devs using the API don't need to understand this
         mt_cloud_telemetry(
             tenant_id=tenant_id,
-            distinct_id=str(user.id) if not user.is_anonymous else tenant_id,
+            distinct_id=user.email if not user.is_anonymous else tenant_id,
             event=MilestoneRecordType.MULTIPLE_ASSISTANTS,
         )
 
         mt_cloud_telemetry(
             tenant_id=tenant_id,
-            distinct_id=str(user.id) if not user.is_anonymous else tenant_id,
+            distinct_id=user.email if not user.is_anonymous else tenant_id,
             event=MilestoneRecordType.USER_MESSAGE_SENT,
             properties={
                 "origin": new_msg_req.origin.value,
@@ -614,28 +574,6 @@ def handle_stream_message_objects(
         if parent_message.message_type == MessageType.USER:
             user_message = parent_message
         else:
-            # New message — run the Query Processing hook before saving to DB.
-            # Skipped on regeneration: the message already exists and was accepted previously.
-            # Skip the hook for empty/whitespace-only messages — no meaningful query
-            # to process, and SendMessageRequest.message has no min_length guard.
-            if message_text.strip():
-                hook_result = execute_hook(
-                    db_session=db_session,
-                    hook_point=HookPoint.QUERY_PROCESSING,
-                    payload=QueryProcessingPayload(
-                        query=message_text,
-                        # Pass None for anonymous users or authenticated users without an email
-                        # (e.g. some SSO flows). QueryProcessingPayload.user_email is str | None,
-                        # so None is accepted and serialised as null in both cases.
-                        user_email=None if user.is_anonymous else user.email,
-                        chat_session_id=str(chat_session.id),
-                    ).model_dump(),
-                    response_type=QueryProcessingResponse,
-                )
-                message_text = _resolve_query_processing_hook_result(
-                    hook_result, message_text
-                )
-
             user_message = create_new_chat_message(
                 chat_session_id=chat_session.id,
                 parent_message=parent_message,
@@ -772,8 +710,8 @@ def handle_stream_message_objects(
             llm=llm,
             search_tool_config=SearchToolConfig(
                 user_selected_filters=new_msg_req.internal_search_filters,
-                project_id_filter=search_params.project_id_filter,
-                persona_id_filter=search_params.persona_id_filter,
+                project_id=search_params.search_project_id,
+                persona_id=search_params.search_persona_id,
                 bypass_acl=bypass_acl,
                 slack_context=slack_context,
                 enable_slack_search=_should_enable_slack_search(
@@ -975,17 +913,6 @@ def handle_stream_message_objects(
                 state_container=state_container,
             )
 
-    except OnyxError as e:
-        if e.error_code is not OnyxErrorCode.QUERY_REJECTED:
-            log_onyx_error(e)
-        yield StreamingError(
-            error=e.detail,
-            error_code=e.error_code.code,
-            is_retryable=e.status_code >= 500,
-        )
-        db_session.rollback()
-        return
-
     except ValueError as e:
         logger.exception("Failed to process chat message.")
 
@@ -998,28 +925,9 @@ def handle_stream_message_objects(
         db_session.rollback()
         return
 
-    except EmptyLLMResponseError as e:
-        stack_trace = traceback.format_exc()
-
-        logger.warning(
-            "LLM returned an empty response "
-            f"(provider={e.provider}, model={e.model}, tool_choice={e.tool_choice})"
-        )
-
-        yield StreamingError(
-            error=e.client_error_msg,
-            stack_trace=stack_trace,
-            error_code=e.error_code,
-            is_retryable=e.is_retryable,
-            details={
-                "model": e.model,
-                "provider": e.provider,
-                "tool_choice": e.tool_choice.value,
-            },
-        )
-        db_session.rollback()
     except Exception as e:
         logger.exception(f"Failed to process chat message due to {e}")
+        error_msg = str(e)
         stack_trace = traceback.format_exc()
 
         if llm:
@@ -1138,46 +1046,10 @@ def llm_loop_completion_handle(
         )
 
 
-_CITATION_LINK_START_PATTERN = re.compile(r"\s*\[\[\d+\]\]\(")
-
-
-def _find_markdown_link_end(text: str, destination_start: int) -> int | None:
-    depth = 0
-    i = destination_start
-
-    while i < len(text):
-        curr = text[i]
-        if curr == "\\":
-            i += 2
-            continue
-
-        if curr == "(":
-            depth += 1
-        elif curr == ")":
-            if depth == 0:
-                return i
-            depth -= 1
-
-        i += 1
-
-    return None
-
-
 def remove_answer_citations(answer: str) -> str:
-    stripped_parts: list[str] = []
-    cursor = 0
+    pattern = r"\s*\[\[\d+\]\]\(http[s]?://[^\s]+\)"
 
-    while match := _CITATION_LINK_START_PATTERN.search(answer, cursor):
-        stripped_parts.append(answer[cursor : match.start()])
-        link_end = _find_markdown_link_end(answer, match.end())
-        if link_end is None:
-            stripped_parts.append(answer[match.start() :])
-            return "".join(stripped_parts)
-
-        cursor = link_end + 1
-
-    stripped_parts.append(answer[cursor:])
-    return "".join(stripped_parts)
+    return re.sub(pattern, "", answer)
 
 
 @log_function_time()
@@ -1215,11 +1087,8 @@ def gather_stream(
         raise ValueError("Message ID is required")
 
     if answer is None:
-        if error_msg is not None:
-            answer = ""
-        else:
-            # This should never be the case as these non-streamed flows do not have a stop-generation signal
-            raise RuntimeError("Answer was not generated")
+        # This should never be the case as these non-streamed flows do not have a stop-generation signal
+        raise RuntimeError("Answer was not generated")
 
     return ChatBasicResponse(
         answer=answer,

backend/onyx/configs/app_configs.py

@@ -278,17 +278,14 @@ USING_AWS_MANAGED_OPENSEARCH = (
 OPENSEARCH_PROFILING_DISABLED = (
     os.environ.get("OPENSEARCH_PROFILING_DISABLED", "").lower() == "true"
 )
-# Whether to disable match highlights for OpenSearch. Defaults to True for now
-# as we investigate query performance.
-OPENSEARCH_MATCH_HIGHLIGHTS_DISABLED = (
-    os.environ.get("OPENSEARCH_MATCH_HIGHLIGHTS_DISABLED", "true").lower() == "true"
-)
+
 # When enabled, OpenSearch returns detailed score breakdowns for each hit.
 # Useful for debugging and tuning search relevance. Has ~10-30% performance overhead according to documentation.
 # Seems for Hybrid Search in practice, the impact is actually more like 1000x slower.
 OPENSEARCH_EXPLAIN_ENABLED = (
     os.environ.get("OPENSEARCH_EXPLAIN_ENABLED", "").lower() == "true"
 )
+
 # Analyzer used for full-text fields (title, content). Use OpenSearch built-in analyzer
 # names (e.g. "english", "standard", "german"). Affects stemming and tokenization;
 # existing indices need reindexing after a change.
@@ -321,16 +318,8 @@ VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT = (
 OPENSEARCH_MIGRATION_GET_VESPA_CHUNKS_PAGE_SIZE = int(
     os.environ.get("OPENSEARCH_MIGRATION_GET_VESPA_CHUNKS_PAGE_SIZE") or 500
 )
-# If set, will override the default number of shards and replicas for the index.
-OPENSEARCH_INDEX_NUM_SHARDS: int | None = (
-    int(os.environ["OPENSEARCH_INDEX_NUM_SHARDS"])
-    if os.environ.get("OPENSEARCH_INDEX_NUM_SHARDS", None) is not None
-    else None
-)
-OPENSEARCH_INDEX_NUM_REPLICAS: int | None = (
-    int(os.environ["OPENSEARCH_INDEX_NUM_REPLICAS"])
-    if os.environ.get("OPENSEARCH_INDEX_NUM_REPLICAS", None) is not None
-    else None
+OPENSEARCH_OVERRIDE_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES = int(
+    os.environ.get("OPENSEARCH_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES") or 0
 )
 
 VESPA_HOST = os.environ.get("VESPA_HOST") or "localhost"
@@ -1057,8 +1046,6 @@ POD_NAMESPACE = os.environ.get("POD_NAMESPACE")
 
 DEV_MODE = os.environ.get("DEV_MODE", "").lower() == "true"
 
-HOOK_ENABLED = os.environ.get("HOOK_ENABLED", "").lower() == "true"
-
 INTEGRATION_TESTS_MODE = os.environ.get("INTEGRATION_TESTS_MODE", "").lower() == "true"
 
 #####

backend/onyx/configs/constants.py

@@ -177,14 +177,6 @@ USER_FILE_PROJECT_SYNC_MAX_QUEUE_DEPTH = 500
 
 CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT = 5 * 60  # 5 minutes (in seconds)
 
-# How long a queued user-file-delete task is valid before workers discard it.
-# Mirrors the processing task expiry to prevent indefinite queue growth when
-# files are stuck in DELETING status and the beat keeps re-enqueuing them.
-CELERY_USER_FILE_DELETE_TASK_EXPIRES = 60  # 1 minute (in seconds)
-
-# Max queue depth before the delete beat stops enqueuing more delete tasks.
-USER_FILE_DELETE_MAX_QUEUE_DEPTH = 500
-
 CELERY_SANDBOX_FILE_SYNC_LOCK_TIMEOUT = 5 * 60  # 5 minutes (in seconds)
 
 DANSWER_REDIS_FUNCTION_LOCK_PREFIX = "da_function_lock:"
@@ -477,9 +469,6 @@ class OnyxRedisLocks:
     USER_FILE_PROJECT_SYNC_QUEUED_PREFIX = "da_lock:user_file_project_sync_queued"
     USER_FILE_DELETE_BEAT_LOCK = "da_lock:check_user_file_delete_beat"
     USER_FILE_DELETE_LOCK_PREFIX = "da_lock:user_file_delete"
-    # Short-lived key set when a delete task is enqueued; cleared when the worker picks it up.
-    # Prevents the beat from re-enqueuing the same file while a delete task is already queued.
-    USER_FILE_DELETE_QUEUED_PREFIX = "da_lock:user_file_delete_queued"
 
     # Release notes
     RELEASE_NOTES_FETCH_LOCK = "da_lock:release_notes_fetch"
@@ -608,9 +597,6 @@ class OnyxCeleryTask:
     EXPORT_QUERY_HISTORY_TASK = "export_query_history_task"
     EXPORT_QUERY_HISTORY_CLEANUP_TASK = "export_query_history_cleanup_task"
 
-    # Hook execution log retention
-    HOOK_EXECUTION_LOG_CLEANUP_TASK = "hook_execution_log_cleanup_task"
-
     # Sandbox cleanup
     CLEANUP_IDLE_SANDBOXES = "cleanup_idle_sandboxes"
     CLEANUP_OLD_SNAPSHOTS = "cleanup_old_snapshots"

backend/onyx/connectors/google_utils/google_utils.py

@@ -157,7 +157,9 @@ def _execute_single_retrieval(
             logger.error(f"Error executing request: {e}")
             raise e
         elif _is_rate_limit_error(e):
-            results = _execute_with_retry(retrieval_function(**request_kwargs))
+            results = _execute_with_retry(
+                lambda: retrieval_function(**request_kwargs).execute()
+            )
         elif e.resp.status == 404 or e.resp.status == 403:
             if continue_on_404_or_403:
                 logger.debug(f"Error executing request: {e}")

backend/onyx/connectors/web/connector.py

@@ -88,9 +88,8 @@ WEB_CONNECTOR_MAX_SCROLL_ATTEMPTS = 20
 IFRAME_TEXT_LENGTH_THRESHOLD = 700
 # Message indicating JavaScript is disabled, which often appears when scraping fails
 JAVASCRIPT_DISABLED_MESSAGE = "You have JavaScript disabled in your browser"
-# Grace period after page navigation to allow bot-detection challenges
-# and SPA content rendering to complete
-PAGE_RENDER_TIMEOUT_MS = 5000
+# Grace period after page navigation to allow bot-detection challenges to complete
+BOT_DETECTION_GRACE_PERIOD_MS = 5000
 
 # Define common headers that mimic a real browser
 DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
@@ -548,15 +547,7 @@ class WebConnector(LoadConnector):
             )
             # Give the page a moment to start rendering after navigation commits.
             # Allows CloudFlare and other bot-detection challenges to complete.
-            page.wait_for_timeout(PAGE_RENDER_TIMEOUT_MS)
-
-            # Wait for network activity to settle so SPAs that fetch content
-            # asynchronously after the initial JS bundle have time to render.
-            try:
-                # A bit of extra time to account for long-polling, websockets, etc.
-                page.wait_for_load_state("networkidle", timeout=PAGE_RENDER_TIMEOUT_MS)
-            except TimeoutError:
-                pass
+            page.wait_for_timeout(BOT_DETECTION_GRACE_PERIOD_MS)
 
             last_modified = (
                 page_response.header_value("Last-Modified") if page_response else None
@@ -585,7 +576,7 @@ class WebConnector(LoadConnector):
                     # (e.g., CloudFlare protection keeps making requests)
                     try:
                         page.wait_for_load_state(
-                            "networkidle", timeout=PAGE_RENDER_TIMEOUT_MS
+                            "networkidle", timeout=BOT_DETECTION_GRACE_PERIOD_MS
                         )
                     except TimeoutError:
                         # If networkidle times out, just give it a moment for content to render

backend/onyx/context/search/models.py

@@ -2,6 +2,7 @@ from collections.abc import Sequence
 from datetime import datetime
 from enum import Enum
 from typing import Any
+from uuid import UUID
 
 from pydantic import BaseModel
 from pydantic import Field
@@ -69,13 +70,9 @@ class BaseFilters(BaseModel):
 
 
 class UserFileFilters(BaseModel):
-    # Scopes search to user files tagged with a given project/persona in Vespa.
-    # These are NOT simply the IDs of the current project or persona — they are
-    # only set when the persona's/project's user files overflowed the LLM
-    # context window and must be searched via vector DB instead of being loaded
-    # directly into the prompt.
-    project_id_filter: int | None = None
-    persona_id_filter: int | None = None
+    user_file_ids: list[UUID] | None = None
+    project_id: int | None = None
+    persona_id: int | None = None
 
 
 class AssistantKnowledgeFilters(BaseModel):

backend/onyx/context/search/pipeline.py

@@ -1,5 +1,6 @@
 from collections import defaultdict
 from datetime import datetime
+from uuid import UUID
 
 from sqlalchemy.orm import Session
 
@@ -38,8 +39,9 @@ logger = setup_logger()
 def _build_index_filters(
     user_provided_filters: BaseFilters | None,
     user: User,  # Used for ACLs, anonymous users only see public docs
-    project_id_filter: int | None,
-    persona_id_filter: int | None,
+    project_id: int | None,
+    persona_id: int | None,
+    user_file_ids: list[UUID] | None,
     persona_document_sets: list[str] | None,
     persona_time_cutoff: datetime | None,
     db_session: Session | None = None,
@@ -95,6 +97,16 @@ def _build_index_filters(
     if not source_filter and detected_source_filter:
         source_filter = detected_source_filter
 
+    # CRITICAL FIX: If user_file_ids are present, we must ensure "user_file"
+    # source type is included in the filter, otherwise user files will be excluded!
+    if user_file_ids and source_filter:
+        from onyx.configs.constants import DocumentSource
+
+        # Add user_file to the source filter if not already present
+        if DocumentSource.USER_FILE not in source_filter:
+            source_filter = list(source_filter) + [DocumentSource.USER_FILE]
+            logger.debug("Added USER_FILE to source_filter for user knowledge search")
+
     if bypass_acl:
         user_acl_filters = None
     elif acl_filters is not None:
@@ -105,8 +117,9 @@ def _build_index_filters(
         user_acl_filters = build_access_filters_for_user(user, db_session)
 
     final_filters = IndexFilters(
-        project_id_filter=project_id_filter,
-        persona_id_filter=persona_id_filter,
+        user_file_ids=user_file_ids,
+        project_id=project_id,
+        persona_id=persona_id,
         source_type=source_filter,
         document_set=document_set_filter,
         time_cutoff=time_filter,
@@ -252,16 +265,19 @@ def search_pipeline(
     db_session: Session | None = None,
     auto_detect_filters: bool = False,
     llm: LLM | None = None,
-    # Vespa metadata filters for overflowing user files.  NOT the raw IDs
-    # of the current project/persona — only set when user files couldn't fit
-    # in the LLM context and need to be searched via vector DB.
-    project_id_filter: int | None = None,
-    persona_id_filter: int | None = None,
+    # If a project ID is provided, it will be exclusively scoped to that project
+    project_id: int | None = None,
+    # If a persona_id is provided, search scopes to files attached to this persona
+    persona_id: int | None = None,
     # Pre-fetched data — when provided, avoids DB queries (no session needed)
     acl_filters: list[str] | None = None,
     embedding_model: EmbeddingModel | None = None,
     prefetched_federated_retrieval_infos: list[FederatedRetrievalInfo] | None = None,
 ) -> list[InferenceChunk]:
+    user_uploaded_persona_files: list[UUID] | None = (
+        [user_file.id for user_file in persona.user_files] if persona else None
+    )
+
     persona_document_sets: list[str] | None = (
         [persona_document_set.name for persona_document_set in persona.document_sets]
         if persona
@@ -286,8 +302,9 @@ def search_pipeline(
     filters = _build_index_filters(
         user_provided_filters=chunk_search_request.user_selected_filters,
         user=user,
-        project_id_filter=project_id_filter,
-        persona_id_filter=persona_id_filter,
+        project_id=project_id,
+        persona_id=persona_id,
+        user_file_ids=user_uploaded_persona_files,
         persona_document_sets=persona_document_sets,
         persona_time_cutoff=persona_time_cutoff,
         db_session=db_session,

backend/onyx/context/search/retrieval/search_runner.py

@@ -110,6 +110,7 @@ def search_chunks(
             user_id=user_id,
             source_types=list(source_filters) if source_filters else None,
             document_set_names=query_request.filters.document_set,
+            user_file_ids=query_request.filters.user_file_ids,
         )
 
     federated_sources = set(

backend/onyx/db/chat.py

@@ -16,7 +16,6 @@ from sqlalchemy import Row
 from sqlalchemy import select
 from sqlalchemy import update
 from sqlalchemy.exc import MultipleResultsFound
-from sqlalchemy.orm import joinedload
 from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
@@ -29,7 +28,6 @@ from onyx.db.models import ChatMessage
 from onyx.db.models import ChatMessage__SearchDoc
 from onyx.db.models import ChatSession
 from onyx.db.models import ChatSessionSharedStatus
-from onyx.db.models import Persona
 from onyx.db.models import SearchDoc as DBSearchDoc
 from onyx.db.models import ToolCall
 from onyx.db.models import User
@@ -55,19 +53,9 @@ def get_chat_session_by_id(
     db_session: Session,
     include_deleted: bool = False,
     is_shared: bool = False,
-    eager_load_persona: bool = False,
 ) -> ChatSession:
     stmt = select(ChatSession).where(ChatSession.id == chat_session_id)
 
-    if eager_load_persona:
-        stmt = stmt.options(
-            joinedload(ChatSession.persona).options(
-                selectinload(Persona.tools),
-                selectinload(Persona.user_files),
-            ),
-            joinedload(ChatSession.project),
-        )
-
     if is_shared:
         stmt = stmt.where(ChatSession.shared_status == ChatSessionSharedStatus.PUBLIC)
     else:

backend/onyx/db/connector_credential_pair.py

@@ -511,7 +511,7 @@ def add_credential_to_connector(
     user: User,
     connector_id: int,
     credential_id: int,
-    cc_pair_name: str,
+    cc_pair_name: str | None,
     access_type: AccessType,
     groups: list[int] | None,
     auto_sync_options: dict | None = None,

backend/onyx/db/enums.py

@@ -304,13 +304,3 @@ class LLMModelFlowType(str, PyEnum):
     CHAT = "chat"
     VISION = "vision"
     CONTEXTUAL_RAG = "contextual_rag"
-
-
-class HookPoint(str, PyEnum):
-    DOCUMENT_INGESTION = "document_ingestion"
-    QUERY_PROCESSING = "query_processing"
-
-
-class HookFailStrategy(str, PyEnum):
-    HARD = "hard"  # exception propagates, pipeline aborts
-    SOFT = "soft"  # log error, return original input, pipeline continues

backend/onyx/db/hook.py

@@ -1,233 +0,0 @@
-import datetime
-from uuid import UUID
-
-from sqlalchemy import delete
-from sqlalchemy import select
-from sqlalchemy.engine import CursorResult
-from sqlalchemy.exc import IntegrityError
-from sqlalchemy.orm import selectinload
-from sqlalchemy.orm import Session
-
-from onyx.db.constants import UNSET
-from onyx.db.constants import UnsetType
-from onyx.db.enums import HookFailStrategy
-from onyx.db.enums import HookPoint
-from onyx.db.models import Hook
-from onyx.db.models import HookExecutionLog
-from onyx.error_handling.error_codes import OnyxErrorCode
-from onyx.error_handling.exceptions import OnyxError
-
-
-# ── Hook CRUD ────────────────────────────────────────────────────────────
-
-
-def get_hook_by_id(
-    *,
-    db_session: Session,
-    hook_id: int,
-    include_deleted: bool = False,
-    include_creator: bool = False,
-) -> Hook | None:
-    stmt = select(Hook).where(Hook.id == hook_id)
-    if not include_deleted:
-        stmt = stmt.where(Hook.deleted.is_(False))
-    if include_creator:
-        stmt = stmt.options(selectinload(Hook.creator))
-    return db_session.scalar(stmt)
-
-
-def get_non_deleted_hook_by_hook_point(
-    *,
-    db_session: Session,
-    hook_point: HookPoint,
-    include_creator: bool = False,
-) -> Hook | None:
-    stmt = (
-        select(Hook).where(Hook.hook_point == hook_point).where(Hook.deleted.is_(False))
-    )
-    if include_creator:
-        stmt = stmt.options(selectinload(Hook.creator))
-    return db_session.scalar(stmt)
-
-
-def get_hooks(
-    *,
-    db_session: Session,
-    include_deleted: bool = False,
-    include_creator: bool = False,
-) -> list[Hook]:
-    stmt = select(Hook)
-    if not include_deleted:
-        stmt = stmt.where(Hook.deleted.is_(False))
-    if include_creator:
-        stmt = stmt.options(selectinload(Hook.creator))
-    stmt = stmt.order_by(Hook.hook_point, Hook.created_at.desc())
-    return list(db_session.scalars(stmt).all())
-
-
-def create_hook__no_commit(
-    *,
-    db_session: Session,
-    name: str,
-    hook_point: HookPoint,
-    endpoint_url: str | None = None,
-    api_key: str | None = None,
-    fail_strategy: HookFailStrategy,
-    timeout_seconds: float,
-    is_active: bool = False,
-    creator_id: UUID | None = None,
-) -> Hook:
-    """Create a new hook for the given hook point.
-
-    At most one non-deleted hook per hook point is allowed. Raises
-    OnyxError(CONFLICT) if a hook already exists, including under concurrent
-    duplicate creates where the partial unique index fires an IntegrityError.
-    """
-    existing = get_non_deleted_hook_by_hook_point(
-        db_session=db_session, hook_point=hook_point
-    )
-    if existing:
-        raise OnyxError(
-            OnyxErrorCode.CONFLICT,
-            f"A hook for '{hook_point.value}' already exists (id={existing.id}).",
-        )
-
-    hook = Hook(
-        name=name,
-        hook_point=hook_point,
-        endpoint_url=endpoint_url,
-        api_key=api_key,
-        fail_strategy=fail_strategy,
-        timeout_seconds=timeout_seconds,
-        is_active=is_active,
-        creator_id=creator_id,
-    )
-    # Use a savepoint so that a failed insert only rolls back this operation,
-    # not the entire outer transaction.
-    savepoint = db_session.begin_nested()
-    try:
-        db_session.add(hook)
-        savepoint.commit()
-    except IntegrityError as exc:
-        savepoint.rollback()
-        if "ix_hook_one_non_deleted_per_point" in str(exc.orig):
-            raise OnyxError(
-                OnyxErrorCode.CONFLICT,
-                f"A hook for '{hook_point.value}' already exists.",
-            )
-        raise  # re-raise unrelated integrity errors (FK violations, etc.)
-    return hook
-
-
-def update_hook__no_commit(
-    *,
-    db_session: Session,
-    hook_id: int,
-    name: str | None = None,
-    endpoint_url: str | None | UnsetType = UNSET,
-    api_key: str | None | UnsetType = UNSET,
-    fail_strategy: HookFailStrategy | None = None,
-    timeout_seconds: float | None = None,
-    is_active: bool | None = None,
-    is_reachable: bool | None = None,
-    include_creator: bool = False,
-) -> Hook:
-    """Update hook fields.
-
-    Sentinel conventions:
-    - endpoint_url, api_key: pass UNSET to leave unchanged; pass None to clear.
-    - name, fail_strategy, timeout_seconds, is_active, is_reachable: pass None to leave unchanged.
-    """
-    hook = get_hook_by_id(
-        db_session=db_session, hook_id=hook_id, include_creator=include_creator
-    )
-    if hook is None:
-        raise OnyxError(OnyxErrorCode.NOT_FOUND, f"Hook with id {hook_id} not found.")
-
-    if name is not None:
-        hook.name = name
-    if not isinstance(endpoint_url, UnsetType):
-        hook.endpoint_url = endpoint_url
-    if not isinstance(api_key, UnsetType):
-        hook.api_key = api_key  # type: ignore[assignment]  # EncryptedString coerces str → SensitiveValue at the ORM level
-    if fail_strategy is not None:
-        hook.fail_strategy = fail_strategy
-    if timeout_seconds is not None:
-        hook.timeout_seconds = timeout_seconds
-    if is_active is not None:
-        hook.is_active = is_active
-    if is_reachable is not None:
-        hook.is_reachable = is_reachable
-
-    db_session.flush()
-    return hook
-
-
-def delete_hook__no_commit(
-    *,
-    db_session: Session,
-    hook_id: int,
-) -> None:
-    hook = get_hook_by_id(db_session=db_session, hook_id=hook_id)
-    if hook is None:
-        raise OnyxError(OnyxErrorCode.NOT_FOUND, f"Hook with id {hook_id} not found.")
-
-    hook.deleted = True
-    hook.is_active = False
-    db_session.flush()
-
-
-# ── HookExecutionLog CRUD ────────────────────────────────────────────────
-
-
-def create_hook_execution_log__no_commit(
-    *,
-    db_session: Session,
-    hook_id: int,
-    is_success: bool,
-    error_message: str | None = None,
-    status_code: int | None = None,
-    duration_ms: int | None = None,
-) -> HookExecutionLog:
-    log = HookExecutionLog(
-        hook_id=hook_id,
-        is_success=is_success,
-        error_message=error_message,
-        status_code=status_code,
-        duration_ms=duration_ms,
-    )
-    db_session.add(log)
-    db_session.flush()
-    return log
-
-
-def get_hook_execution_logs(
-    *,
-    db_session: Session,
-    hook_id: int,
-    limit: int,
-) -> list[HookExecutionLog]:
-    stmt = (
-        select(HookExecutionLog)
-        .where(HookExecutionLog.hook_id == hook_id)
-        .order_by(HookExecutionLog.created_at.desc())
-        .limit(limit)
-    )
-    return list(db_session.scalars(stmt).all())
-
-
-def cleanup_old_execution_logs__no_commit(
-    *,
-    db_session: Session,
-    max_age_days: int,
-) -> int:
-    """Delete execution logs older than max_age_days. Returns the number of rows deleted."""
-    cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
-        days=max_age_days
-    )
-    result: CursorResult = db_session.execute(  # type: ignore[assignment]
-        delete(HookExecutionLog)
-        .where(HookExecutionLog.created_at < cutoff)
-        .execution_options(synchronize_session=False)
-    )
-    return result.rowcount

backend/onyx/db/index_attempt.py

@@ -583,67 +583,6 @@ def get_latest_index_attempt_for_cc_pair_id(
     return db_session.execute(stmt).scalar_one_or_none()
 
 
-def get_latest_successful_index_attempt_for_cc_pair_id(
-    db_session: Session,
-    connector_credential_pair_id: int,
-    secondary_index: bool = False,
-) -> IndexAttempt | None:
-    """Returns the most recent successful index attempt for the given cc pair,
-    filtered to the current (or future) search settings.
-    Uses MAX(id) semantics to match get_latest_index_attempts_by_status."""
-    status = IndexModelStatus.FUTURE if secondary_index else IndexModelStatus.PRESENT
-    stmt = (
-        select(IndexAttempt)
-        .where(
-            IndexAttempt.connector_credential_pair_id == connector_credential_pair_id,
-            IndexAttempt.status.in_(
-                [IndexingStatus.SUCCESS, IndexingStatus.COMPLETED_WITH_ERRORS]
-            ),
-        )
-        .join(SearchSettings)
-        .where(SearchSettings.status == status)
-        .order_by(desc(IndexAttempt.id))
-        .limit(1)
-    )
-    return db_session.execute(stmt).scalar_one_or_none()
-
-
-def get_latest_successful_index_attempts_parallel(
-    secondary_index: bool = False,
-) -> Sequence[IndexAttempt]:
-    """Batch version: returns the latest successful index attempt per cc pair.
-    Covers both SUCCESS and COMPLETED_WITH_ERRORS (matching is_successful())."""
-    model_status = (
-        IndexModelStatus.FUTURE if secondary_index else IndexModelStatus.PRESENT
-    )
-    with get_session_with_current_tenant() as db_session:
-        latest_ids = (
-            select(
-                IndexAttempt.connector_credential_pair_id,
-                func.max(IndexAttempt.id).label("max_id"),
-            )
-            .join(SearchSettings, IndexAttempt.search_settings_id == SearchSettings.id)
-            .where(
-                SearchSettings.status == model_status,
-                IndexAttempt.status.in_(
-                    [IndexingStatus.SUCCESS, IndexingStatus.COMPLETED_WITH_ERRORS]
-                ),
-            )
-            .group_by(IndexAttempt.connector_credential_pair_id)
-            .subquery()
-        )
-
-        stmt = select(IndexAttempt).join(
-            latest_ids,
-            (
-                IndexAttempt.connector_credential_pair_id
-                == latest_ids.c.connector_credential_pair_id
-            )
-            & (IndexAttempt.id == latest_ids.c.max_id),
-        )
-        return db_session.execute(stmt).scalars().all()
-
-
 def count_index_attempts_for_cc_pair(
     db_session: Session,
     cc_pair_id: int,

backend/onyx/db/models.py

@@ -64,8 +64,6 @@ from onyx.db.enums import (
     BuildSessionStatus,
     EmbeddingPrecision,
     HierarchyNodeType,
-    HookFailStrategy,
-    HookPoint,
     IndexingMode,
     OpenSearchDocumentMigrationStatus,
     OpenSearchTenantMigrationStatus,
@@ -3467,9 +3465,9 @@ class Persona(Base):
     builtin_persona: Mapped[bool] = mapped_column(Boolean, default=False)
 
     # Featured personas are highlighted in the UI
-    is_featured: Mapped[bool] = mapped_column(Boolean, default=False)
-    # controls whether the persona is listed in user-facing agent lists
-    is_listed: Mapped[bool] = mapped_column(Boolean, default=True)
+    featured: Mapped[bool] = mapped_column(Boolean, default=False)
+    # controls whether the persona is available to be selected by users
+    is_visible: Mapped[bool] = mapped_column(Boolean, default=True)
     # controls the ordering of personas in the UI
     # higher priority personas are displayed first, ties are resolved by the ID,
     # where lower value IDs (e.g. created earlier) are displayed first
@@ -5180,90 +5178,3 @@ class CacheStore(Base):
     expires_at: Mapped[datetime.datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )
-
-
-class Hook(Base):
-    """Pairs a HookPoint with a customer-provided API endpoint.
-
-    At most one non-deleted Hook per HookPoint is allowed, enforced by a
-    partial unique index on (hook_point) where deleted=false.
-    """
-
-    __tablename__ = "hook"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    name: Mapped[str] = mapped_column(String, nullable=False)
-    hook_point: Mapped[HookPoint] = mapped_column(
-        Enum(HookPoint, native_enum=False), nullable=False
-    )
-    endpoint_url: Mapped[str | None] = mapped_column(Text, nullable=True)
-    api_key: Mapped[SensitiveValue[str] | None] = mapped_column(
-        EncryptedString(), nullable=True
-    )
-    is_reachable: Mapped[bool | None] = mapped_column(
-        Boolean, nullable=True, default=None
-    )  # null = never validated, true = last check passed, false = last check failed
-    fail_strategy: Mapped[HookFailStrategy] = mapped_column(
-        Enum(HookFailStrategy, native_enum=False),
-        nullable=False,
-        default=HookFailStrategy.HARD,
-    )
-    timeout_seconds: Mapped[float] = mapped_column(Float, nullable=False, default=30.0)
-    is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    creator_id: Mapped[UUID | None] = mapped_column(
-        PGUUID(as_uuid=True),
-        ForeignKey("user.id", ondelete="SET NULL"),
-        nullable=True,
-    )
-    created_at: Mapped[datetime.datetime] = mapped_column(
-        DateTime(timezone=True), server_default=func.now(), nullable=False
-    )
-    updated_at: Mapped[datetime.datetime] = mapped_column(
-        DateTime(timezone=True),
-        server_default=func.now(),
-        onupdate=func.now(),
-        nullable=False,
-    )
-
-    creator: Mapped["User | None"] = relationship("User", foreign_keys=[creator_id])
-    execution_logs: Mapped[list["HookExecutionLog"]] = relationship(
-        "HookExecutionLog", back_populates="hook", cascade="all, delete-orphan"
-    )
-
-    __table_args__ = (
-        Index(
-            "ix_hook_one_non_deleted_per_point",
-            "hook_point",
-            unique=True,
-            postgresql_where=(deleted == False),  # noqa: E712
-        ),
-    )
-
-
-class HookExecutionLog(Base):
-    """Records hook executions for health monitoring and debugging.
-
-    Currently only failures are logged; the is_success column exists so
-    success logging can be added later without a schema change.
-    Retention: rows older than 30 days are deleted by a nightly Celery task.
-    """
-
-    __tablename__ = "hook_execution_log"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    hook_id: Mapped[int] = mapped_column(
-        Integer,
-        ForeignKey("hook.id", ondelete="CASCADE"),
-        nullable=False,
-        index=True,
-    )
-    is_success: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
-    status_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
-    duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
-    created_at: Mapped[datetime.datetime] = mapped_column(
-        DateTime(timezone=True), server_default=func.now(), nullable=False, index=True
-    )
-
-    hook: Mapped["Hook"] = relationship("Hook", back_populates="execution_logs")

backend/onyx/db/persona.py

@@ -126,7 +126,7 @@ def _add_user_filters(
     else:
         # Group the public persona conditions
         public_condition = (Persona.is_public == True) & (  # noqa: E712
-            Persona.is_listed == True  # noqa: E712
+            Persona.is_visible == True  # noqa: E712
         )
 
         where_clause |= public_condition
@@ -260,7 +260,7 @@ def create_update_persona(
 
     try:
         # Featured persona validation
-        if create_persona_request.is_featured:
+        if create_persona_request.featured:
             # Curators can edit featured personas, but not make them
             # TODO this will be reworked soon with RBAC permissions feature
             if user.role == UserRole.CURATOR or user.role == UserRole.GLOBAL_CURATOR:
@@ -300,7 +300,7 @@ def create_update_persona(
             remove_image=create_persona_request.remove_image,
             search_start_date=create_persona_request.search_start_date,
             label_ids=create_persona_request.label_ids,
-            is_featured=create_persona_request.is_featured,
+            featured=create_persona_request.featured,
             user_file_ids=converted_user_file_ids,
             commit=False,
             hierarchy_node_ids=create_persona_request.hierarchy_node_ids,
@@ -910,11 +910,11 @@ def upsert_persona(
     uploaded_image_id: str | None = None,
     icon_name: str | None = None,
     display_priority: int | None = None,
-    is_listed: bool = True,
+    is_visible: bool = True,
     remove_image: bool | None = None,
     search_start_date: datetime | None = None,
     builtin_persona: bool = False,
-    is_featured: bool | None = None,
+    featured: bool | None = None,
     label_ids: list[int] | None = None,
     user_file_ids: list[UUID] | None = None,
     hierarchy_node_ids: list[int] | None = None,
@@ -1037,13 +1037,13 @@ def upsert_persona(
         if remove_image or uploaded_image_id:
             existing_persona.uploaded_image_id = uploaded_image_id
         existing_persona.icon_name = icon_name
-        existing_persona.is_listed = is_listed
+        existing_persona.is_visible = is_visible
         existing_persona.search_start_date = search_start_date
         if label_ids is not None:
             existing_persona.labels.clear()
             existing_persona.labels = labels or []
-        existing_persona.is_featured = (
-            is_featured if is_featured is not None else existing_persona.is_featured
+        existing_persona.featured = (
+            featured if featured is not None else existing_persona.featured
         )
         # Update embedded prompt fields if provided
         if system_prompt is not None:
@@ -1109,9 +1109,9 @@ def upsert_persona(
             uploaded_image_id=uploaded_image_id,
             icon_name=icon_name,
             display_priority=display_priority,
-            is_listed=is_listed,
+            is_visible=is_visible,
             search_start_date=search_start_date,
-            is_featured=(is_featured if is_featured is not None else False),
+            featured=(featured if featured is not None else False),
             user_files=user_files or [],
             labels=labels or [],
             hierarchy_nodes=hierarchy_nodes or [],
@@ -1158,7 +1158,7 @@ def delete_old_default_personas(
 
 def update_persona_featured(
     persona_id: int,
-    is_featured: bool,
+    featured: bool,
     db_session: Session,
     user: User,
 ) -> None:
@@ -1166,13 +1166,13 @@ def update_persona_featured(
         db_session=db_session, persona_id=persona_id, user=user, get_editable=True
     )
 
-    persona.is_featured = is_featured
+    persona.featured = featured
     db_session.commit()
 
 
 def update_persona_visibility(
     persona_id: int,
-    is_listed: bool,
+    is_visible: bool,
     db_session: Session,
     user: User,
 ) -> None:
@@ -1180,7 +1180,7 @@ def update_persona_visibility(
         db_session=db_session, persona_id=persona_id, user=user, get_editable=True
     )
 
-    persona.is_listed = is_listed
+    persona.is_visible = is_visible
     db_session.commit()
 
 

backend/onyx/db/projects.py

@@ -12,7 +12,6 @@ from sqlalchemy.orm import Session
 from starlette.background import BackgroundTasks
 
 from onyx.configs.app_configs import DISABLE_VECTOR_DB
-from onyx.configs.constants import CELERY_USER_FILE_PROCESSING_TASK_EXPIRES
 from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
@@ -145,7 +144,6 @@ def upload_files_to_user_files_with_indexing(
                 kwargs={"user_file_id": user_file.id, "tenant_id": tenant_id},
                 queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
                 priority=OnyxCeleryPriority.HIGH,
-                expires=CELERY_USER_FILE_PROCESSING_TASK_EXPIRES,
             )
             logger.info(
                 f"Triggered indexing for user_file_id={user_file.id} with task_id={task.id}"

backend/onyx/db/slack_channel_config.py

@@ -75,7 +75,7 @@ def create_slack_channel_persona(
         llm_model_version_override=None,
         starter_messages=None,
         is_public=True,
-        is_featured=False,
+        featured=False,
         db_session=db_session,
         commit=False,
     )

backend/onyx/db/swap_index.py

@@ -2,7 +2,6 @@ import time
 
 from sqlalchemy.orm import Session
 
-from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.configs.app_configs import VESPA_NUM_ATTEMPTS_ON_STARTUP
 from onyx.configs.constants import KV_REINDEX_KEY
 from onyx.db.connector_credential_pair import get_connector_credential_pairs
@@ -150,9 +149,6 @@ def check_and_perform_index_swap(db_session: Session) -> SearchSettings | None:
     Returns None if search settings did not change, or the old search settings if they
     did change.
     """
-    if DISABLE_VECTOR_DB:
-        return None
-
     # Default CC-pair created for Ingestion API unused here
     all_cc_pairs = get_connector_credential_pairs(db_session)
     cc_pair_count = max(len(all_cc_pairs) - 1, 0)

backend/onyx/document_index/FILTER_SEMANTICS.md

@@ -10,8 +10,8 @@ How `IndexFilters` fields combine into the final query filter. Applies to both V
 | **Tenant** | `tenant_id` | AND (multi-tenant only) |
 | **ACL** | `access_control_list` | OR within, AND with rest |
 | **Narrowing** | `source_type`, `tags`, `time_cutoff` | Each OR within, AND with rest |
-| **Knowledge scope** | `document_set`, `attached_document_ids`, `hierarchy_node_ids`, `persona_id_filter` | OR within group, AND with rest |
-| **Additive scope** | `project_id_filter` | OR'd into knowledge scope **only when** a knowledge scope filter already exists |
+| **Knowledge scope** | `document_set`, `user_file_ids`, `attached_document_ids`, `hierarchy_node_ids` | OR within group, AND with rest |
+| **Additive scope** | `project_id`, `persona_id` | OR'd into knowledge scope **only when** a knowledge scope filter already exists |
 
 ## How filters combine
 
@@ -31,22 +31,12 @@ AND time >= cutoff                      -- if set
 
 The knowledge scope filter controls **what knowledge an assistant can access**.
 
-### Primary vs additive triggers
-
-- **`persona_id_filter`** is a **primary** trigger. A persona with user files IS explicit
-  knowledge, so `persona_id_filter` alone can start a knowledge scope. Note: this is
-  NOT the raw ID of the persona being used — it is only set when the persona's
-  user files overflowed the LLM context window.
-- **`project_id_filter`** is **additive**. It widens an existing scope to include project
-  files but never restricts on its own — a chat inside a project should still search
-  team knowledge when no other knowledge is attached.
-
 ### No explicit knowledge attached
 
-When `document_set`, `attached_document_ids`, `hierarchy_node_ids`, and `persona_id_filter` are all empty/None:
+When `document_set`, `user_file_ids`, `attached_document_ids`, and `hierarchy_node_ids` are all empty/None:
 
 - **No knowledge scope filter is applied.** The assistant can see everything (subject to ACL).
-- `project_id_filter` is ignored — it never restricts on its own.
+- `project_id` and `persona_id` are ignored — they never restrict on their own.
 
 ### One explicit knowledge type
 
@@ -54,40 +44,39 @@ When `document_set`, `attached_document_ids`, `hierarchy_node_ids`, and `persona
 -- Only document sets
 AND (document_sets contains "Engineering" OR document_sets contains "Legal")
 
--- Only persona user files (overflowed context)
-AND (personas contains 42)
+-- Only user files
+AND (document_id = "uuid-1" OR document_id = "uuid-2")
 ```
 
 ### Multiple explicit knowledge types (OR'd)
 
 ```
--- Document sets + persona user files
+-- Document sets + user files
+AND (
+    document_sets contains "Engineering"
+    OR document_id = "uuid-1"
+)
+```
+
+### Explicit knowledge + overflowing user files
+
+When an explicit knowledge restriction is in effect **and** `project_id` or `persona_id` is set (user files overflowed the LLM context window), the additive scopes widen the filter:
+
+```
+-- Document sets + persona user files overflowed
 AND (
     document_sets contains "Engineering"
     OR personas contains 42
 )
-```
 
-### Explicit knowledge + overflowing project files
-
-When an explicit knowledge restriction is in effect **and** `project_id_filter` is set (project files overflowed the LLM context window), `project_id_filter` widens the filter:
-
-```
--- Document sets + project files overflowed
+-- User files + project files overflowed
 AND (
-    document_sets contains "Engineering"
-    OR user_project contains 7
-)
-
--- Persona user files + project files (won't happen in practice;
--- custom personas ignore project files per the precedence rule)
-AND (
-    personas contains 42
+    document_id = "uuid-1"
     OR user_project contains 7
 )
 ```
 
-### Only project_id_filter (no explicit knowledge)
+### Only project_id or persona_id (no explicit knowledge)
 
 No knowledge scope filter. The assistant searches everything.
 
@@ -102,10 +91,11 @@ AND (acl contains ...)
 | Filter field | Vespa field | Vespa type | Purpose |
 |---|---|---|---|
 | `document_set` | `document_sets` | `weightedset<string>` | Connector doc sets attached to assistant |
+| `user_file_ids` | `document_id` | `string` | User files uploaded to assistant |
 | `attached_document_ids` | `document_id` | `string` | Documents explicitly attached (OpenSearch only) |
 | `hierarchy_node_ids` | `ancestor_hierarchy_node_ids` | `array<int>` | Folder/space nodes (OpenSearch only) |
-| `persona_id_filter` | `personas` | `array<int>` | Persona tag for overflowing user files (**primary** trigger) |
-| `project_id_filter` | `user_project` | `array<int>` | Project tag for overflowing project files (**additive** only) |
+| `project_id` | `user_project` | `array<int>` | Project tag for overflowing user files |
+| `persona_id` | `personas` | `array<int>` | Persona tag for overflowing user files |
 | `access_control_list` | `access_control_list` | `weightedset<string>` | ACL entries for the requesting user |
 | `source_type` | `source_type` | `string` | Connector source type (e.g. `web`, `jira`) |
 | `tags` | `metadata_list` | `array<string>` | Document metadata tags |

backend/onyx/document_index/opensearch/client.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import time
 from contextlib import AbstractContextManager
@@ -19,7 +18,6 @@ from onyx.configs.app_configs import OPENSEARCH_HOST
 from onyx.configs.app_configs import OPENSEARCH_REST_API_PORT
 from onyx.document_index.interfaces_new import TenantState
 from onyx.document_index.opensearch.schema import DocumentChunk
-from onyx.document_index.opensearch.schema import DocumentChunkWithoutVectors
 from onyx.document_index.opensearch.schema import get_opensearch_doc_chunk_id
 from onyx.document_index.opensearch.search import DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW
 from onyx.utils.logger import setup_logger
@@ -58,8 +56,8 @@ class SearchHit(BaseModel, Generic[SchemaDocumentModel]):
     # Maps schema property name to a list of highlighted snippets with match
     # terms wrapped in tags (e.g. "something <hi>keyword</hi> other thing").
     match_highlights: dict[str, list[str]] = {}
-    # Score explanation from OpenSearch when "explain": true is set in the
-    # query. Contains detailed breakdown of how the score was calculated.
+    # Score explanation from OpenSearch when "explain": true is set in the query.
+    # Contains detailed breakdown of how the score was calculated.
     explanation: dict[str, Any] | None = None
 
 
@@ -835,13 +833,9 @@ class OpenSearchIndexClient(OpenSearchClient):
     @log_function_time(print_only=True, debug_only=True)
     def search(
         self, body: dict[str, Any], search_pipeline_id: str | None
-    ) -> list[SearchHit[DocumentChunkWithoutVectors]]:
+    ) -> list[SearchHit[DocumentChunk]]:
         """Searches the index.
 
-        NOTE: Does not return vector fields. In order to take advantage of
-        performance benefits, the search body should exclude the schema's vector
-        fields.
-
         TODO(andrei): Ideally we could check that every field in the body is
         present in the index, to avoid a class of runtime bugs that could easily
         be caught during development. Or change the function signature to accept
@@ -889,7 +883,7 @@ class OpenSearchIndexClient(OpenSearchClient):
             raise_on_timeout=True,
         )
 
-        search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = []
+        search_hits: list[SearchHit[DocumentChunk]] = []
         for hit in hits:
             document_chunk_source: dict[str, Any] | None = hit.get("_source")
             if not document_chunk_source:
@@ -899,10 +893,8 @@ class OpenSearchIndexClient(OpenSearchClient):
             document_chunk_score = hit.get("_score", None)
             match_highlights: dict[str, list[str]] = hit.get("highlight", {})
             explanation: dict[str, Any] | None = hit.get("_explanation", None)
-            search_hit = SearchHit[DocumentChunkWithoutVectors](
-                document_chunk=DocumentChunkWithoutVectors.model_validate(
-                    document_chunk_source
-                ),
+            search_hit = SearchHit[DocumentChunk](
+                document_chunk=DocumentChunk.model_validate(document_chunk_source),
                 score=document_chunk_score,
                 match_highlights=match_highlights,
                 explanation=explanation,
@@ -1063,7 +1055,7 @@ class OpenSearchIndexClient(OpenSearchClient):
                 f"Body: {get_new_body_without_vectors(body)}\n"
                 f"Search pipeline ID: {search_pipeline_id}\n"
                 f"Phase took: {phase_took}\n"
-                f"Profile: {json.dumps(profile, indent=2)}\n"
+                f"Profile: {profile}\n"
             )
         if timed_out:
             error_str = f"OpenSearch client error: Search timed out for index {self._index_name}."

backend/onyx/document_index/opensearch/constants.py

@@ -1,23 +1,12 @@
 # Default value for the maximum number of tokens a chunk can hold, if none is
 # specified when creating an index.
-import os
-from enum import Enum
+from onyx.configs.app_configs import (
+    OPENSEARCH_OVERRIDE_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
+)
 
 
 DEFAULT_MAX_CHUNK_SIZE = 512
 
-
-# By default OpenSearch will only return a maximum of this many results in a
-# given search. This value is configurable in the index settings.
-DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW = 10_000
-
-
-# For documents which do not have a value for LAST_UPDATED_FIELD_NAME, we assume
-# that the document was last updated this many days ago for the purpose of time
-# cutoff filtering during retrieval.
-ASSUMED_DOCUMENT_AGE_DAYS = 90
-
-
 # Size of the dynamic list used to consider elements during kNN graph creation.
 # Higher values improve search quality but increase indexing time. Values
 # typically range between 100 - 512.
@@ -37,10 +26,10 @@ M = 32  # Set relatively high for better accuracy.
 # we have a much higher chance of all 10 of the final desired docs showing up
 # and getting scored. In worse situations, the final 10 docs don't even show up
 # as the final 10 (worse than just a miss at the reranking step).
-# Defaults to 100 for now. Initially this defaulted to 750 but we were seeing
-# poor search performance.
-DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES = int(
-    os.environ.get("DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES", 100)
+DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES = (
+    OPENSEARCH_OVERRIDE_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES
+    if OPENSEARCH_OVERRIDE_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES > 0
+    else 750
 )
 
 # Number of vectors to examine to decide the top k neighbors for the HNSW
@@ -50,43 +39,23 @@ DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES = int(
 # larger than k, you can provide the size parameter to limit the final number of
 # results to k." from
 # https://docs.opensearch.org/latest/query-dsl/specialized/k-nn/index/#ef_search
-EF_SEARCH = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES
+EF_SEARCH = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES
 
+# Since the titles are included in the contents, the embedding matches are
+# heavily downweighted as they act as a boost rather than an independent scoring
+# component.
+SEARCH_TITLE_VECTOR_WEIGHT = 0.1
+SEARCH_CONTENT_VECTOR_WEIGHT = 0.45
+# Single keyword weight for both title and content (merged from former title
+# keyword + content keyword).
+SEARCH_KEYWORD_WEIGHT = 0.45
 
-class HybridSearchSubqueryConfiguration(Enum):
-    TITLE_VECTOR_CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD = 1
-    # Current default.
-    CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD = 2
+# NOTE: It is critical that the order of these weights matches the order of the
+# sub-queries in the hybrid search.
+HYBRID_SEARCH_NORMALIZATION_WEIGHTS = [
+    SEARCH_TITLE_VECTOR_WEIGHT,
+    SEARCH_CONTENT_VECTOR_WEIGHT,
+    SEARCH_KEYWORD_WEIGHT,
+]
 
-
-# Will raise and block application start if HYBRID_SEARCH_SUBQUERY_CONFIGURATION
-# is set but not a valid value. If not set, defaults to
-# CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD.
-HYBRID_SEARCH_SUBQUERY_CONFIGURATION: HybridSearchSubqueryConfiguration = (
-    HybridSearchSubqueryConfiguration(
-        int(os.environ["HYBRID_SEARCH_SUBQUERY_CONFIGURATION"])
-    )
-    if os.environ.get("HYBRID_SEARCH_SUBQUERY_CONFIGURATION", None) is not None
-    else HybridSearchSubqueryConfiguration.CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
-)
-
-
-class HybridSearchNormalizationPipeline(Enum):
-    # Current default.
-    MIN_MAX = 1
-    # NOTE: Using z-score normalization is better for hybrid search from a
-    # theoretical standpoint. Empirically on a small dataset of up to 10K docs,
-    # it's not very different. Likely more impactful at scale.
-    # https://opensearch.org/blog/introducing-the-z-score-normalization-technique-for-hybrid-search/
-    ZSCORE = 2
-
-
-# Will raise and block application start if HYBRID_SEARCH_NORMALIZATION_PIPELINE
-# is set but not a valid value. If not set, defaults to MIN_MAX.
-HYBRID_SEARCH_NORMALIZATION_PIPELINE: HybridSearchNormalizationPipeline = (
-    HybridSearchNormalizationPipeline(
-        int(os.environ["HYBRID_SEARCH_NORMALIZATION_PIPELINE"])
-    )
-    if os.environ.get("HYBRID_SEARCH_NORMALIZATION_PIPELINE", None) is not None
-    else HybridSearchNormalizationPipeline.MIN_MAX
-)
+assert sum(HYBRID_SEARCH_NORMALIZATION_WEIGHTS) == 1.0

backend/onyx/document_index/opensearch/opensearch_document_index.py

@@ -47,7 +47,6 @@ from onyx.document_index.opensearch.schema import ACCESS_CONTROL_LIST_FIELD_NAME
 from onyx.document_index.opensearch.schema import CONTENT_FIELD_NAME
 from onyx.document_index.opensearch.schema import DOCUMENT_SETS_FIELD_NAME
 from onyx.document_index.opensearch.schema import DocumentChunk
-from onyx.document_index.opensearch.schema import DocumentChunkWithoutVectors
 from onyx.document_index.opensearch.schema import DocumentSchema
 from onyx.document_index.opensearch.schema import get_opensearch_doc_chunk_id
 from onyx.document_index.opensearch.schema import GLOBAL_BOOST_FIELD_NAME
@@ -56,13 +55,16 @@ from onyx.document_index.opensearch.schema import PERSONAS_FIELD_NAME
 from onyx.document_index.opensearch.schema import USER_PROJECTS_FIELD_NAME
 from onyx.document_index.opensearch.search import DocumentQuery
 from onyx.document_index.opensearch.search import (
-    get_min_max_normalization_pipeline_name_and_config,
+    MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
 )
 from onyx.document_index.opensearch.search import (
-    get_normalization_pipeline_name_and_config,
+    MIN_MAX_NORMALIZATION_PIPELINE_NAME,
 )
 from onyx.document_index.opensearch.search import (
-    get_zscore_normalization_pipeline_name_and_config,
+    ZSCORE_NORMALIZATION_PIPELINE_CONFIG,
+)
+from onyx.document_index.opensearch.search import (
+    ZSCORE_NORMALIZATION_PIPELINE_NAME,
 )
 from onyx.indexing.models import DocMetadataAwareIndexChunk
 from onyx.indexing.models import Document
@@ -101,24 +103,18 @@ def set_cluster_state(client: OpenSearchClient) -> None:
             "is not the first time running Onyx against this instance of OpenSearch, these "
             "settings have likely already been set. Not taking any further action..."
         )
-    min_max_normalization_pipeline_name, min_max_normalization_pipeline_config = (
-        get_min_max_normalization_pipeline_name_and_config()
-    )
-    zscore_normalization_pipeline_name, zscore_normalization_pipeline_config = (
-        get_zscore_normalization_pipeline_name_and_config()
+    client.create_search_pipeline(
+        pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME,
+        pipeline_body=MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
     )
     client.create_search_pipeline(
-        pipeline_id=min_max_normalization_pipeline_name,
-        pipeline_body=min_max_normalization_pipeline_config,
-    )
-    client.create_search_pipeline(
-        pipeline_id=zscore_normalization_pipeline_name,
-        pipeline_body=zscore_normalization_pipeline_config,
+        pipeline_id=ZSCORE_NORMALIZATION_PIPELINE_NAME,
+        pipeline_body=ZSCORE_NORMALIZATION_PIPELINE_CONFIG,
     )
 
 
 def _convert_retrieved_opensearch_chunk_to_inference_chunk_uncleaned(
-    chunk: DocumentChunkWithoutVectors,
+    chunk: DocumentChunk,
     score: float | None,
     highlights: dict[str, list[str]],
 ) -> InferenceChunkUncleaned:
@@ -881,7 +877,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
         )
         results: list[InferenceChunk] = []
         for chunk_request in chunk_requests:
-            search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = []
+            search_hits: list[SearchHit[DocumentChunk]] = []
             query_body = DocumentQuery.get_from_document_id_query(
                 document_id=chunk_request.document_id,
                 tenant_state=self._tenant_state,
@@ -944,92 +940,17 @@ class OpenSearchDocumentIndex(DocumentIndex):
             index_filters=filters,
             include_hidden=False,
         )
-        normalization_pipeline_name, _ = get_normalization_pipeline_name_and_config()
-        search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = self._client.search(
+        # NOTE: Using z-score normalization here because it's better for hybrid
+        # search from a theoretical standpoint. Empirically on a small dataset
+        # of up to 10K docs, it's not very different. Likely more impactful at
+        # scale.
+        # https://opensearch.org/blog/introducing-the-z-score-normalization-technique-for-hybrid-search/
+        search_hits: list[SearchHit[DocumentChunk]] = self._client.search(
             body=query_body,
-            search_pipeline_id=normalization_pipeline_name,
-        )
-
-        # Good place for a breakpoint to inspect the search hits if you have
-        # "explain" enabled.
-        inference_chunks_uncleaned: list[InferenceChunkUncleaned] = [
-            _convert_retrieved_opensearch_chunk_to_inference_chunk_uncleaned(
-                search_hit.document_chunk, search_hit.score, search_hit.match_highlights
-            )
-            for search_hit in search_hits
-        ]
-        inference_chunks: list[InferenceChunk] = cleanup_content_for_chunks(
-            inference_chunks_uncleaned
-        )
-
-        return inference_chunks
-
-    def keyword_retrieval(
-        self,
-        query: str,
-        filters: IndexFilters,
-        num_to_retrieve: int,
-    ) -> list[InferenceChunk]:
-        logger.debug(
-            f"[OpenSearchDocumentIndex] Keyword retrieving {num_to_retrieve} chunks for index {self._index_name}."
-        )
-        query_body = DocumentQuery.get_keyword_search_query(
-            query_text=query,
-            num_hits=num_to_retrieve,
-            tenant_state=self._tenant_state,
-            # NOTE: Index filters includes metadata tags which were filtered
-            # for invalid unicode at indexing time. In theory it would be
-            # ideal to do filtering here as well, in practice we never did
-            # that in the Vespa codepath and have not seen issues in
-            # production, so we deliberately conform to the existing logic
-            # in order to not unknowningly introduce a possible bug.
-            index_filters=filters,
-            include_hidden=False,
-        )
-        search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = self._client.search(
-            body=query_body,
-            search_pipeline_id=None,
-        )
-
-        inference_chunks_uncleaned: list[InferenceChunkUncleaned] = [
-            _convert_retrieved_opensearch_chunk_to_inference_chunk_uncleaned(
-                search_hit.document_chunk, search_hit.score, search_hit.match_highlights
-            )
-            for search_hit in search_hits
-        ]
-        inference_chunks: list[InferenceChunk] = cleanup_content_for_chunks(
-            inference_chunks_uncleaned
-        )
-
-        return inference_chunks
-
-    def semantic_retrieval(
-        self,
-        query_embedding: Embedding,
-        filters: IndexFilters,
-        num_to_retrieve: int,
-    ) -> list[InferenceChunk]:
-        logger.debug(
-            f"[OpenSearchDocumentIndex] Semantic retrieving {num_to_retrieve} chunks for index {self._index_name}."
-        )
-        query_body = DocumentQuery.get_semantic_search_query(
-            query_embedding=query_embedding,
-            num_hits=num_to_retrieve,
-            tenant_state=self._tenant_state,
-            # NOTE: Index filters includes metadata tags which were filtered
-            # for invalid unicode at indexing time. In theory it would be
-            # ideal to do filtering here as well, in practice we never did
-            # that in the Vespa codepath and have not seen issues in
-            # production, so we deliberately conform to the existing logic
-            # in order to not unknowningly introduce a possible bug.
-            index_filters=filters,
-            include_hidden=False,
-        )
-        search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = self._client.search(
-            body=query_body,
-            search_pipeline_id=None,
+            search_pipeline_id=ZSCORE_NORMALIZATION_PIPELINE_NAME,
         )
 
+        # Good place for a breakpoint to inspect the search hits if you have "explain" enabled.
         inference_chunks_uncleaned: list[InferenceChunkUncleaned] = [
             _convert_retrieved_opensearch_chunk_to_inference_chunk_uncleaned(
                 search_hit.document_chunk, search_hit.score, search_hit.match_highlights
@@ -1056,7 +977,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
             index_filters=filters,
             num_to_retrieve=num_to_retrieve,
         )
-        search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = self._client.search(
+        search_hits: list[SearchHit[DocumentChunk]] = self._client.search(
             body=query_body,
             search_pipeline_id=None,
         )

backend/onyx/document_index/opensearch/schema.py

@@ -11,8 +11,6 @@ from pydantic import model_serializer
 from pydantic import model_validator
 from pydantic import SerializerFunctionWrapHandler
 
-from onyx.configs.app_configs import OPENSEARCH_INDEX_NUM_REPLICAS
-from onyx.configs.app_configs import OPENSEARCH_INDEX_NUM_SHARDS
 from onyx.configs.app_configs import OPENSEARCH_TEXT_ANALYZER
 from onyx.configs.app_configs import USING_AWS_MANAGED_OPENSEARCH
 from onyx.document_index.interfaces_new import TenantState
@@ -102,9 +100,9 @@ def set_or_convert_timezone_to_utc(value: datetime) -> datetime:
     return value
 
 
-class DocumentChunkWithoutVectors(BaseModel):
+class DocumentChunk(BaseModel):
     """
-    Represents a chunk of a document in the OpenSearch index without vectors.
+    Represents a chunk of a document in the OpenSearch index.
 
     The names of these fields are based on the OpenSearch schema. Changes to the
     schema require changes here. See get_document_schema.
@@ -126,7 +124,9 @@ class DocumentChunkWithoutVectors(BaseModel):
 
     # Either both should be None or both should be non-None.
     title: str | None = None
+    title_vector: list[float] | None = None
     content: str
+    content_vector: list[float]
 
     source_type: str
     # A list of key-value pairs separated by INDEX_SEPARATOR. See
@@ -176,9 +176,19 @@ class DocumentChunkWithoutVectors(BaseModel):
     def __str__(self) -> str:
         return (
             f"DocumentChunk(document_id={self.document_id}, chunk_index={self.chunk_index}, "
-            f"content length={len(self.content)}, tenant_id={self.tenant_id.tenant_id})."
+            f"content length={len(self.content)}, content vector length={len(self.content_vector)}, "
+            f"tenant_id={self.tenant_id.tenant_id})"
         )
 
+    @model_validator(mode="after")
+    def check_title_and_title_vector_are_consistent(self) -> Self:
+        # title and title_vector should both either be None or not.
+        if self.title is not None and self.title_vector is None:
+            raise ValueError("Bug: Title vector must not be None if title is not None.")
+        if self.title_vector is not None and self.title is None:
+            raise ValueError("Bug: Title must not be None if title vector is not None.")
+        return self
+
     @model_serializer(mode="wrap")
     def serialize_model(
         self, handler: SerializerFunctionWrapHandler
@@ -295,35 +305,6 @@ class DocumentChunkWithoutVectors(BaseModel):
             return TenantState(tenant_id=value, multitenant=MULTI_TENANT)
 
 
-class DocumentChunk(DocumentChunkWithoutVectors):
-    """Represents a chunk of a document in the OpenSearch index.
-
-    The names of these fields are based on the OpenSearch schema. Changes to the
-    schema require changes here. See get_document_schema.
-    """
-
-    model_config = {"frozen": True}
-
-    title_vector: list[float] | None = None
-    content_vector: list[float]
-
-    def __str__(self) -> str:
-        return (
-            f"DocumentChunk(document_id={self.document_id}, chunk_index={self.chunk_index}, "
-            f"content length={len(self.content)}, content vector length={len(self.content_vector)}, "
-            f"tenant_id={self.tenant_id.tenant_id})"
-        )
-
-    @model_validator(mode="after")
-    def check_title_and_title_vector_are_consistent(self) -> Self:
-        # title and title_vector should both either be None or not.
-        if self.title is not None and self.title_vector is None:
-            raise ValueError("Bug: Title vector must not be None if title is not None.")
-        if self.title_vector is not None and self.title is None:
-            raise ValueError("Bug: Title must not be None if title vector is not None.")
-        return self
-
-
 class DocumentSchema:
     """
     Represents the schema and indexing strategies of the OpenSearch index.
@@ -536,34 +517,77 @@ class DocumentSchema:
         return schema
 
     @staticmethod
-    def get_index_settings_based_on_environment() -> dict[str, Any]:
+    def get_index_settings() -> dict[str, Any]:
         """
-        Returns the index settings based on the environment.
+        Standard settings for reasonable local index and search performance.
         """
-        if USING_AWS_MANAGED_OPENSEARCH:
-            # NOTE: The number of data copies, including the primary (not a
-            # replica) copy, must be divisible by the number of AZs.
-            if MULTI_TENANT:
-                number_of_shards = 324
-                number_of_replicas = 2
-            else:
-                number_of_shards = 3
-                number_of_replicas = 2
-        else:
-            number_of_shards = 1
-            number_of_replicas = 1
-
-        if OPENSEARCH_INDEX_NUM_SHARDS is not None:
-            number_of_shards = OPENSEARCH_INDEX_NUM_SHARDS
-        if OPENSEARCH_INDEX_NUM_REPLICAS is not None:
-            number_of_replicas = OPENSEARCH_INDEX_NUM_REPLICAS
-
         return {
             "index": {
-                "number_of_shards": number_of_shards,
-                "number_of_replicas": number_of_replicas,
+                "number_of_shards": 1,
+                "number_of_replicas": 1,
                 # Required for vector search.
                 "knn": True,
                 "knn.algo_param.ef_search": EF_SEARCH,
             }
         }
+
+    @staticmethod
+    def get_index_settings_for_aws_managed_opensearch_st_dev() -> dict[str, Any]:
+        """
+        Settings for AWS-managed OpenSearch.
+
+        Our AWS-managed OpenSearch cluster has 3 data nodes in 3 availability
+        zones.
+          - We use 3 shards to distribute load across all data nodes.
+          - We use 2 replicas to ensure each shard has a copy in each
+            availability zone. This is a hard requirement from AWS. The number
+            of data copies, including the primary (not a replica) copy, must be
+            divisible by the number of AZs.
+        """
+        return {
+            "index": {
+                "number_of_shards": 3,
+                "number_of_replicas": 2,
+                # Required for vector search.
+                "knn": True,
+                "knn.algo_param.ef_search": EF_SEARCH,
+            }
+        }
+
+    @staticmethod
+    def get_index_settings_for_aws_managed_opensearch_mt_cloud() -> dict[str, Any]:
+        """
+        Settings for AWS-managed OpenSearch in multi-tenant cloud.
+
+        324 shards very roughly targets a storage load of ~30Gb per shard, which
+        according to AWS OpenSearch documentation is within a good target range.
+
+        As documented above we need 2 replicas for a total of 3 copies of the
+        data because the cluster is configured with 3-AZ awareness.
+        """
+        return {
+            "index": {
+                "number_of_shards": 324,
+                "number_of_replicas": 2,
+                # Required for vector search.
+                "knn": True,
+                "knn.algo_param.ef_search": EF_SEARCH,
+            }
+        }
+
+    @staticmethod
+    def get_index_settings_based_on_environment() -> dict[str, Any]:
+        """
+        Returns the index settings based on the environment.
+        """
+        if USING_AWS_MANAGED_OPENSEARCH:
+            if MULTI_TENANT:
+                return (
+                    DocumentSchema.get_index_settings_for_aws_managed_opensearch_mt_cloud()
+                )
+            else:
+                return (
+                    DocumentSchema.get_index_settings_for_aws_managed_opensearch_st_dev()
+                )
+        else:
+            return DocumentSchema.get_index_settings()

backend/onyx/document_index/opensearch/search.py

@@ -3,31 +3,20 @@ from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
 from typing import Any
+from uuid import UUID
 
 from onyx.configs.app_configs import DEFAULT_OPENSEARCH_QUERY_TIMEOUT_S
 from onyx.configs.app_configs import OPENSEARCH_EXPLAIN_ENABLED
-from onyx.configs.app_configs import OPENSEARCH_MATCH_HIGHLIGHTS_DISABLED
 from onyx.configs.app_configs import OPENSEARCH_PROFILING_DISABLED
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import INDEX_SEPARATOR
 from onyx.context.search.models import IndexFilters
 from onyx.context.search.models import Tag
 from onyx.document_index.interfaces_new import TenantState
-from onyx.document_index.opensearch.constants import ASSUMED_DOCUMENT_AGE_DAYS
 from onyx.document_index.opensearch.constants import (
-    DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES,
+    DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
 )
-from onyx.document_index.opensearch.constants import (
-    DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW,
-)
-from onyx.document_index.opensearch.constants import (
-    HYBRID_SEARCH_NORMALIZATION_PIPELINE,
-)
-from onyx.document_index.opensearch.constants import (
-    HYBRID_SEARCH_SUBQUERY_CONFIGURATION,
-)
-from onyx.document_index.opensearch.constants import HybridSearchNormalizationPipeline
-from onyx.document_index.opensearch.constants import HybridSearchSubqueryConfiguration
+from onyx.document_index.opensearch.constants import HYBRID_SEARCH_NORMALIZATION_WEIGHTS
 from onyx.document_index.opensearch.schema import ACCESS_CONTROL_LIST_FIELD_NAME
 from onyx.document_index.opensearch.schema import ANCESTOR_HIERARCHY_NODE_IDS_FIELD_NAME
 from onyx.document_index.opensearch.schema import CHUNK_INDEX_FIELD_NAME
@@ -54,113 +43,49 @@ from onyx.document_index.opensearch.schema import USER_PROJECTS_FIELD_NAME
 
 # TODO(andrei): Turn all magic dictionaries to pydantic models.
 
-
-def _get_hybrid_search_normalization_weights() -> list[float]:
-    if (
-        HYBRID_SEARCH_SUBQUERY_CONFIGURATION
-        is HybridSearchSubqueryConfiguration.TITLE_VECTOR_CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
-    ):
-        # Since the titles are included in the contents, the embedding matches
-        # are heavily downweighted as they act as a boost rather than an
-        # independent scoring component.
-        search_title_vector_weight = 0.1
-        search_content_vector_weight = 0.45
-        # Single keyword weight for both title and content (merged from former
-        # title keyword + content keyword).
-        search_keyword_weight = 0.45
-
-        # NOTE: It is critical that the order of these weights matches the order
-        # of the sub-queries in the hybrid search.
-        hybrid_search_normalization_weights = [
-            search_title_vector_weight,
-            search_content_vector_weight,
-            search_keyword_weight,
-        ]
-    elif (
-        HYBRID_SEARCH_SUBQUERY_CONFIGURATION
-        is HybridSearchSubqueryConfiguration.CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
-    ):
-        search_content_vector_weight = 0.5
-        # Single keyword weight for both title and content (merged from former
-        # title keyword + content keyword).
-        search_keyword_weight = 0.5
-
-        # NOTE: It is critical that the order of these weights matches the order
-        # of the sub-queries in the hybrid search.
-        hybrid_search_normalization_weights = [
-            search_content_vector_weight,
-            search_keyword_weight,
-        ]
-    else:
-        raise ValueError(
-            f"Bug: Unhandled hybrid search subquery configuration: {HYBRID_SEARCH_SUBQUERY_CONFIGURATION}."
-        )
-
-    assert (
-        sum(hybrid_search_normalization_weights) == 1.0
-    ), "Bug: Hybrid search normalization weights do not sum to 1.0."
-
-    return hybrid_search_normalization_weights
-
-
-def get_min_max_normalization_pipeline_name_and_config() -> tuple[str, dict[str, Any]]:
-    min_max_normalization_pipeline_name = "normalization_pipeline_min_max"
-    min_max_normalization_pipeline_config: dict[str, Any] = {
-        "description": "Normalization for keyword and vector scores using min-max",
-        "phase_results_processors": [
-            {
-                # https://docs.opensearch.org/latest/search-plugins/search-pipelines/normalization-processor/
-                "normalization-processor": {
-                    "normalization": {"technique": "min_max"},
-                    "combination": {
-                        "technique": "arithmetic_mean",
-                        "parameters": {
-                            "weights": _get_hybrid_search_normalization_weights()
-                        },
-                    },
-                }
+MIN_MAX_NORMALIZATION_PIPELINE_NAME = "normalization_pipeline_min_max"
+MIN_MAX_NORMALIZATION_PIPELINE_CONFIG: dict[str, Any] = {
+    "description": "Normalization for keyword and vector scores using min-max",
+    "phase_results_processors": [
+        {
+            # https://docs.opensearch.org/latest/search-plugins/search-pipelines/normalization-processor/
+            "normalization-processor": {
+                "normalization": {"technique": "min_max"},
+                "combination": {
+                    "technique": "arithmetic_mean",
+                    "parameters": {"weights": HYBRID_SEARCH_NORMALIZATION_WEIGHTS},
+                },
             }
-        ],
-    }
-    return min_max_normalization_pipeline_name, min_max_normalization_pipeline_config
+        }
+    ],
+}
 
-
-def get_zscore_normalization_pipeline_name_and_config() -> tuple[str, dict[str, Any]]:
-    zscore_normalization_pipeline_name = "normalization_pipeline_zscore"
-    zscore_normalization_pipeline_config: dict[str, Any] = {
-        "description": "Normalization for keyword and vector scores using z-score",
-        "phase_results_processors": [
-            {
-                # https://docs.opensearch.org/latest/search-plugins/search-pipelines/normalization-processor/
-                "normalization-processor": {
-                    "normalization": {"technique": "z_score"},
-                    "combination": {
-                        "technique": "arithmetic_mean",
-                        "parameters": {
-                            "weights": _get_hybrid_search_normalization_weights()
-                        },
-                    },
-                }
+ZSCORE_NORMALIZATION_PIPELINE_NAME = "normalization_pipeline_zscore"
+ZSCORE_NORMALIZATION_PIPELINE_CONFIG: dict[str, Any] = {
+    "description": "Normalization for keyword and vector scores using z-score",
+    "phase_results_processors": [
+        {
+            # https://docs.opensearch.org/latest/search-plugins/search-pipelines/normalization-processor/
+            "normalization-processor": {
+                "normalization": {"technique": "z_score"},
+                "combination": {
+                    "technique": "arithmetic_mean",
+                    "parameters": {"weights": HYBRID_SEARCH_NORMALIZATION_WEIGHTS},
+                },
             }
-        ],
-    }
-    return zscore_normalization_pipeline_name, zscore_normalization_pipeline_config
+        }
+    ],
+}
 
 
-def get_normalization_pipeline_name_and_config() -> tuple[str, dict[str, Any]]:
-    if (
-        HYBRID_SEARCH_NORMALIZATION_PIPELINE
-        is HybridSearchNormalizationPipeline.MIN_MAX
-    ):
-        return get_min_max_normalization_pipeline_name_and_config()
-    elif (
-        HYBRID_SEARCH_NORMALIZATION_PIPELINE is HybridSearchNormalizationPipeline.ZSCORE
-    ):
-        return get_zscore_normalization_pipeline_name_and_config()
-    else:
-        raise ValueError(
-            f"Bug: Unhandled hybrid search normalization pipeline: {HYBRID_SEARCH_NORMALIZATION_PIPELINE}."
-        )
+# By default OpenSearch will only return a maximum of this many results in a
+# given search. This value is configurable in the index settings.
+DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW = 10_000
+
+# For documents which do not have a value for LAST_UPDATED_FIELD_NAME, we assume
+# that the document was last updated this many days ago for the purpose of time
+# cutoff filtering during retrieval.
+ASSUMED_DOCUMENT_AGE_DAYS = 90
 
 
 class DocumentQuery:
@@ -218,8 +143,9 @@ class DocumentQuery:
             source_types=index_filters.source_type or [],
             tags=index_filters.tags or [],
             document_sets=index_filters.document_set or [],
-            project_id_filter=index_filters.project_id_filter,
-            persona_id_filter=index_filters.persona_id_filter,
+            user_file_ids=index_filters.user_file_ids or [],
+            project_id=index_filters.project_id,
+            persona_id=index_filters.persona_id,
             time_cutoff=index_filters.time_cutoff,
             min_chunk_index=min_chunk_index,
             max_chunk_index=max_chunk_index,
@@ -234,17 +160,9 @@ class DocumentQuery:
             # returning some number of results less than the index max allowed
             # return size.
             "size": DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW,
-            # By default exclude retrieving the vector fields in order to save
-            # on retrieval cost as we don't need them upstream.
-            "_source": {
-                "excludes": [TITLE_VECTOR_FIELD_NAME, CONTENT_VECTOR_FIELD_NAME]
-            },
+            "_source": get_full_document,
             "timeout": f"{DEFAULT_OPENSEARCH_QUERY_TIMEOUT_S}s",
         }
-        if not get_full_document:
-            # If we explicitly do not want the underlying document, we will only
-            # retrieve IDs.
-            final_get_ids_query["_source"] = False
         if not OPENSEARCH_PROFILING_DISABLED:
             final_get_ids_query["profile"] = True
 
@@ -284,8 +202,9 @@ class DocumentQuery:
             source_types=[],
             tags=[],
             document_sets=[],
-            project_id_filter=None,
-            persona_id_filter=None,
+            user_file_ids=[],
+            project_id=None,
+            persona_id=None,
             time_cutoff=None,
             min_chunk_index=None,
             max_chunk_index=None,
@@ -338,7 +257,7 @@ class DocumentQuery:
 
         # TODO(andrei, yuhong): We can tune this more dynamically based on
         # num_hits.
-        max_results_per_subquery = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES
+        max_results_per_subquery = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES
 
         hybrid_search_subqueries = DocumentQuery._get_hybrid_search_subqueries(
             query_text, query_vector, vector_candidates=max_results_per_subquery
@@ -353,14 +272,18 @@ class DocumentQuery:
             source_types=index_filters.source_type or [],
             tags=index_filters.tags or [],
             document_sets=index_filters.document_set or [],
-            project_id_filter=index_filters.project_id_filter,
-            persona_id_filter=index_filters.persona_id_filter,
+            user_file_ids=index_filters.user_file_ids or [],
+            project_id=index_filters.project_id,
+            persona_id=index_filters.persona_id,
             time_cutoff=index_filters.time_cutoff,
             min_chunk_index=None,
             max_chunk_index=None,
             attached_document_ids=index_filters.attached_document_ids,
             hierarchy_node_ids=index_filters.hierarchy_node_ids,
         )
+        match_highlights_configuration = (
+            DocumentQuery._get_match_highlights_configuration()
+        )
 
         # See https://docs.opensearch.org/latest/query-dsl/compound/hybrid/
         hybrid_search_query: dict[str, Any] = {
@@ -387,181 +310,16 @@ class DocumentQuery:
         final_hybrid_search_body: dict[str, Any] = {
             "query": hybrid_search_query,
             "size": num_hits,
+            "highlight": match_highlights_configuration,
             "timeout": f"{DEFAULT_OPENSEARCH_QUERY_TIMEOUT_S}s",
-            # Exclude retrieving the vector fields in order to save on
-            # retrieval cost as we don't need them upstream.
-            "_source": {
-                "excludes": [TITLE_VECTOR_FIELD_NAME, CONTENT_VECTOR_FIELD_NAME]
-            },
         }
 
-        if not OPENSEARCH_MATCH_HIGHLIGHTS_DISABLED:
-            final_hybrid_search_body["highlight"] = (
-                DocumentQuery._get_match_highlights_configuration()
-            )
-
-        # Explain is for scoring breakdowns. Setting this significantly
-        # increases query latency.
+        # Explain is for scoring breakdowns.
         if OPENSEARCH_EXPLAIN_ENABLED:
             final_hybrid_search_body["explain"] = True
 
         return final_hybrid_search_body
 
-    @staticmethod
-    def get_keyword_search_query(
-        query_text: str,
-        num_hits: int,
-        tenant_state: TenantState,
-        index_filters: IndexFilters,
-        include_hidden: bool,
-    ) -> dict[str, Any]:
-        """Returns a final keyword search query.
-
-        This query can be directly supplied to the OpenSearch client.
-
-        Args:
-            query_text: The text to query for.
-            num_hits: The final number of hits to return.
-            tenant_state: Tenant state containing the tenant ID.
-            index_filters: Filters for the keyword search query.
-            include_hidden: Whether to include hidden documents.
-
-        Returns:
-            A dictionary representing the final keyword search query.
-        """
-        if num_hits > DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW:
-            raise ValueError(
-                f"Bug: num_hits ({num_hits}) is greater than the current maximum allowed "
-                f"result window ({DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW})."
-            )
-
-        keyword_search_filters = DocumentQuery._get_search_filters(
-            tenant_state=tenant_state,
-            include_hidden=include_hidden,
-            # TODO(andrei): We've done no filtering for PUBLIC_DOC_PAT up to
-            # now. This should not cause any issues but it can introduce
-            # redundant filters in queries that may affect performance.
-            access_control_list=index_filters.access_control_list,
-            source_types=index_filters.source_type or [],
-            tags=index_filters.tags or [],
-            document_sets=index_filters.document_set or [],
-            project_id_filter=index_filters.project_id_filter,
-            persona_id_filter=index_filters.persona_id_filter,
-            time_cutoff=index_filters.time_cutoff,
-            min_chunk_index=None,
-            max_chunk_index=None,
-            attached_document_ids=index_filters.attached_document_ids,
-            hierarchy_node_ids=index_filters.hierarchy_node_ids,
-        )
-
-        keyword_search_query = (
-            DocumentQuery._get_title_content_combined_keyword_search_query(
-                query_text, search_filters=keyword_search_filters
-            )
-        )
-
-        final_keyword_search_query: dict[str, Any] = {
-            "query": keyword_search_query,
-            "size": num_hits,
-            "timeout": f"{DEFAULT_OPENSEARCH_QUERY_TIMEOUT_S}s",
-            # Exclude retrieving the vector fields in order to save on
-            # retrieval cost as we don't need them upstream.
-            "_source": {
-                "excludes": [TITLE_VECTOR_FIELD_NAME, CONTENT_VECTOR_FIELD_NAME]
-            },
-        }
-
-        if not OPENSEARCH_MATCH_HIGHLIGHTS_DISABLED:
-            final_keyword_search_query["highlight"] = (
-                DocumentQuery._get_match_highlights_configuration()
-            )
-
-        if not OPENSEARCH_PROFILING_DISABLED:
-            final_keyword_search_query["profile"] = True
-
-        # Explain is for scoring breakdowns. Setting this significantly
-        # increases query latency.
-        if OPENSEARCH_EXPLAIN_ENABLED:
-            final_keyword_search_query["explain"] = True
-
-        return final_keyword_search_query
-
-    @staticmethod
-    def get_semantic_search_query(
-        query_embedding: list[float],
-        num_hits: int,
-        tenant_state: TenantState,
-        index_filters: IndexFilters,
-        include_hidden: bool,
-    ) -> dict[str, Any]:
-        """Returns a final semantic search query.
-
-        This query can be directly supplied to the OpenSearch client.
-
-        Args:
-            query_embedding: The vector embedding of the text to query for.
-            num_hits: The final number of hits to return.
-            tenant_state: Tenant state containing the tenant ID.
-            index_filters: Filters for the semantic search query.
-            include_hidden: Whether to include hidden documents.
-
-        Returns:
-            A dictionary representing the final semantic search query.
-        """
-        if num_hits > DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW:
-            raise ValueError(
-                f"Bug: num_hits ({num_hits}) is greater than the current maximum allowed "
-                f"result window ({DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW})."
-            )
-
-        semantic_search_filters = DocumentQuery._get_search_filters(
-            tenant_state=tenant_state,
-            include_hidden=include_hidden,
-            # TODO(andrei): We've done no filtering for PUBLIC_DOC_PAT up to
-            # now. This should not cause any issues but it can introduce
-            # redundant filters in queries that may affect performance.
-            access_control_list=index_filters.access_control_list,
-            source_types=index_filters.source_type or [],
-            tags=index_filters.tags or [],
-            document_sets=index_filters.document_set or [],
-            project_id_filter=index_filters.project_id_filter,
-            persona_id_filter=index_filters.persona_id_filter,
-            time_cutoff=index_filters.time_cutoff,
-            min_chunk_index=None,
-            max_chunk_index=None,
-            attached_document_ids=index_filters.attached_document_ids,
-            hierarchy_node_ids=index_filters.hierarchy_node_ids,
-        )
-
-        semantic_search_query = (
-            DocumentQuery._get_content_vector_similarity_search_query(
-                query_embedding,
-                vector_candidates=num_hits,
-                search_filters=semantic_search_filters,
-            )
-        )
-
-        final_semantic_search_query: dict[str, Any] = {
-            "query": semantic_search_query,
-            "size": num_hits,
-            "timeout": f"{DEFAULT_OPENSEARCH_QUERY_TIMEOUT_S}s",
-            # Exclude retrieving the vector fields in order to save on
-            # retrieval cost as we don't need them upstream.
-            "_source": {
-                "excludes": [TITLE_VECTOR_FIELD_NAME, CONTENT_VECTOR_FIELD_NAME]
-            },
-        }
-
-        if not OPENSEARCH_PROFILING_DISABLED:
-            final_semantic_search_query["profile"] = True
-
-        # Explain is for scoring breakdowns. Setting this significantly
-        # increases query latency.
-        if OPENSEARCH_EXPLAIN_ENABLED:
-            final_semantic_search_query["explain"] = True
-
-        return final_semantic_search_query
-
     @staticmethod
     def get_random_search_query(
         tenant_state: TenantState,
@@ -585,8 +343,9 @@ class DocumentQuery:
             source_types=index_filters.source_type or [],
             tags=index_filters.tags or [],
             document_sets=index_filters.document_set or [],
-            project_id_filter=index_filters.project_id_filter,
-            persona_id_filter=index_filters.persona_id_filter,
+            user_file_ids=index_filters.user_file_ids or [],
+            project_id=index_filters.project_id,
+            persona_id=index_filters.persona_id,
             time_cutoff=index_filters.time_cutoff,
             min_chunk_index=None,
             max_chunk_index=None,
@@ -612,11 +371,6 @@ class DocumentQuery:
             },
             "size": num_to_retrieve,
             "timeout": f"{DEFAULT_OPENSEARCH_QUERY_TIMEOUT_S}s",
-            # Exclude retrieving the vector fields in order to save on
-            # retrieval cost as we don't need them upstream.
-            "_source": {
-                "excludes": [TITLE_VECTOR_FIELD_NAME, CONTENT_VECTOR_FIELD_NAME]
-            },
         }
         if not OPENSEARCH_PROFILING_DISABLED:
             final_random_search_query["profile"] = True
@@ -631,7 +385,7 @@ class DocumentQuery:
         # search. This is higher than the number of results because the scoring
         # is hybrid. For a detailed breakdown, see where the default value is
         # set.
-        vector_candidates: int = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES,
+        vector_candidates: int = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
     ) -> list[dict[str, Any]]:
         """Returns subqueries for hybrid search.
 
@@ -641,18 +395,20 @@ class DocumentQuery:
         The return of this function is not sufficient to be directly supplied to
         the OpenSearch client. See get_hybrid_search_query.
 
+        Matches:
+          - Title vector
+          - Content vector
+          - Keyword (title + content, match and phrase)
+
         Normalization is not performed here.
         The weights of each of these subqueries should be configured in a search
         pipeline.
 
-        The exact subqueries executed depend on the
-        HYBRID_SEARCH_SUBQUERY_CONFIGURATION setting.
-
         NOTE: For OpenSearch, 5 is the maximum number of query clauses allowed
         in a single hybrid query. Source:
         https://docs.opensearch.org/latest/query-dsl/compound/hybrid/
 
-        NOTE: Each query is independent during the search phase; there is no
+        NOTE: Each query is independent during the search phase, there is no
         backfilling of scores for missing query components. What this means is
         that if a document was a good vector match but did not show up for
         keyword, it gets a score of 0 for the keyword component of the hybrid
@@ -681,133 +437,74 @@ class DocumentQuery:
                 similarity search.
         """
         # Build sub-queries for hybrid search. Order must match normalization
-        # pipeline weights.
-        if (
-            HYBRID_SEARCH_SUBQUERY_CONFIGURATION
-            is HybridSearchSubqueryConfiguration.TITLE_VECTOR_CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
-        ):
-            return [
-                DocumentQuery._get_title_vector_similarity_search_query(
-                    query_vector, vector_candidates
-                ),
-                DocumentQuery._get_content_vector_similarity_search_query(
-                    query_vector, vector_candidates
-                ),
-                DocumentQuery._get_title_content_combined_keyword_search_query(
-                    query_text
-                ),
-            ]
-        elif (
-            HYBRID_SEARCH_SUBQUERY_CONFIGURATION
-            is HybridSearchSubqueryConfiguration.CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
-        ):
-            return [
-                DocumentQuery._get_content_vector_similarity_search_query(
-                    query_vector, vector_candidates
-                ),
-                DocumentQuery._get_title_content_combined_keyword_search_query(
-                    query_text
-                ),
-            ]
-        else:
-            raise ValueError(
-                f"Bug: Unhandled hybrid search subquery configuration: {HYBRID_SEARCH_SUBQUERY_CONFIGURATION}"
-            )
-
-    @staticmethod
-    def _get_title_vector_similarity_search_query(
-        query_vector: list[float],
-        vector_candidates: int = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES,
-    ) -> dict[str, Any]:
-        return {
-            "knn": {
-                TITLE_VECTOR_FIELD_NAME: {
-                    "vector": query_vector,
-                    "k": vector_candidates,
+        # pipeline weights: title vector, content vector, keyword (title + content).
+        hybrid_search_queries: list[dict[str, Any]] = [
+            # 1. Title vector search
+            {
+                "knn": {
+                    TITLE_VECTOR_FIELD_NAME: {
+                        "vector": query_vector,
+                        "k": vector_candidates,
+                    }
                 }
-            }
-        }
-
-    @staticmethod
-    def _get_content_vector_similarity_search_query(
-        query_vector: list[float],
-        vector_candidates: int = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES,
-        search_filters: list[dict[str, Any]] | None = None,
-    ) -> dict[str, Any]:
-        query = {
-            "knn": {
-                CONTENT_VECTOR_FIELD_NAME: {
-                    "vector": query_vector,
-                    "k": vector_candidates,
+            },
+            # 2. Content vector search
+            {
+                "knn": {
+                    CONTENT_VECTOR_FIELD_NAME: {
+                        "vector": query_vector,
+                        "k": vector_candidates,
+                    }
                 }
-            }
-        }
-
-        if search_filters is not None:
-            query["knn"][CONTENT_VECTOR_FIELD_NAME]["filter"] = {
-                "bool": {"filter": search_filters}
-            }
-
-        return query
-
-    @staticmethod
-    def _get_title_content_combined_keyword_search_query(
-        query_text: str,
-        search_filters: list[dict[str, Any]] | None = None,
-    ) -> dict[str, Any]:
-        query = {
-            "bool": {
-                "should": [
-                    {
-                        "match": {
-                            TITLE_FIELD_NAME: {
-                                "query": query_text,
-                                "operator": "or",
-                                # The title fields are strongly discounted as they are included in the content.
-                                # It just acts as a minor boost
-                                "boost": 0.1,
+            },
+            # 3. Keyword (title + content) match and phrase search.
+            {
+                "bool": {
+                    "should": [
+                        {
+                            "match": {
+                                TITLE_FIELD_NAME: {
+                                    "query": query_text,
+                                    "operator": "or",
+                                    # The title fields are strongly discounted as they are included in the content.
+                                    # It just acts as a minor boost
+                                    "boost": 0.1,
+                                }
                             }
-                        }
-                    },
-                    {
-                        "match_phrase": {
-                            TITLE_FIELD_NAME: {
-                                "query": query_text,
-                                "slop": 1,
-                                "boost": 0.2,
+                        },
+                        {
+                            "match_phrase": {
+                                TITLE_FIELD_NAME: {
+                                    "query": query_text,
+                                    "slop": 1,
+                                    "boost": 0.2,
+                                }
                             }
-                        }
-                    },
-                    {
-                        "match": {
-                            CONTENT_FIELD_NAME: {
-                                "query": query_text,
-                                "operator": "or",
-                                "boost": 1.0,
+                        },
+                        {
+                            "match": {
+                                CONTENT_FIELD_NAME: {
+                                    "query": query_text,
+                                    "operator": "or",
+                                    "boost": 1.0,
+                                }
                             }
-                        }
-                    },
-                    {
-                        "match_phrase": {
-                            CONTENT_FIELD_NAME: {
-                                "query": query_text,
-                                "slop": 1,
-                                "boost": 1.5,
+                        },
+                        {
+                            "match_phrase": {
+                                CONTENT_FIELD_NAME: {
+                                    "query": query_text,
+                                    "slop": 1,
+                                    "boost": 1.5,
+                                }
                             }
-                        }
-                    },
-                ],
-                # Ensure at least one term from the query is present in the
-                # document. This defaults to 1, unless a filter or must clause
-                # is supplied, in which case it defaults to 0.
-                "minimum_should_match": 1,
-            }
-        }
+                        },
+                    ]
+                }
+            },
+        ]
 
-        if search_filters is not None:
-            query["bool"]["filter"] = search_filters
-
-        return query
+        return hybrid_search_queries
 
     @staticmethod
     def _get_search_filters(
@@ -817,8 +514,9 @@ class DocumentQuery:
         source_types: list[DocumentSource],
         tags: list[Tag],
         document_sets: list[str],
-        project_id_filter: int | None,
-        persona_id_filter: int | None,
+        user_file_ids: list[UUID],
+        project_id: int | None,
+        persona_id: int | None,
         time_cutoff: datetime | None,
         min_chunk_index: int | None,
         max_chunk_index: int | None,
@@ -849,12 +547,12 @@ class DocumentQuery:
                 list corresponding to a tag will be retrieved.
             document_sets: If supplied, only documents with at least one
                 document set ID from this list will be retrieved.
-            project_id_filter: If not None, only documents with this project ID
-                in user projects will be retrieved. Additive — only applied
-                when a knowledge scope already exists.
-            persona_id_filter: If not None, only documents whose personas array
-                contains this persona ID will be retrieved. Primary — creates
-                a knowledge scope on its own.
+            user_file_ids: If supplied, only document IDs in this list will be
+                retrieved.
+            project_id: If not None, only documents with this project ID in user
+                projects will be retrieved.
+            persona_id: If not None, only documents whose personas array
+                contains this persona ID will be retrieved.
             time_cutoff: Time cutoff for the documents to retrieve. If not None,
                 Documents which were last updated before this date will not be
                 returned. For documents which do not have a value for their last
@@ -871,6 +569,10 @@ class DocumentQuery:
                 NOTE: See DocumentChunk.max_chunk_size.
             document_id: The document ID to retrieve. If None, no filter will be
                 applied for this. Defaults to None.
+                WARNING: This filters on the same property as user_file_ids.
+                Although it would never make sense to supply both, note that if
+                user_file_ids is supplied and does not contain document_id, no
+                matches will be retrieved.
             attached_document_ids: Document IDs explicitly attached to the
                 assistant. If provided along with hierarchy_node_ids, documents
                 matching EITHER criteria will be retrieved (OR logic).
@@ -931,6 +633,15 @@ class DocumentQuery:
                 )
             return document_set_filter
 
+        def _get_user_file_id_filter(user_file_ids: list[UUID]) -> dict[str, Any]:
+            # Logical OR operator on its elements.
+            user_file_id_filter: dict[str, Any] = {"bool": {"should": []}}
+            for user_file_id in user_file_ids:
+                user_file_id_filter["bool"]["should"].append(
+                    {"term": {DOCUMENT_ID_FIELD_NAME: {"value": str(user_file_id)}}}
+                )
+            return user_file_id_filter
+
         def _get_user_project_filter(project_id: int) -> dict[str, Any]:
             # Logical OR operator on its elements.
             user_project_filter: dict[str, Any] = {"bool": {"should": []}}
@@ -1031,17 +742,14 @@ class DocumentQuery:
         # assistant can see. When none are set the assistant searches
         # everything.
         #
-        # persona_id_filter is a primary trigger — a persona with user files IS
-        # explicit knowledge, so it can start a knowledge scope on its own.
-        #
-        # project_id_filter is additive — it widens the scope to also cover
-        # overflowing project files but never restricts on its own (a chat
-        # inside a project should still search team knowledge).
+        # project_id / persona_id are additive: they make overflowing user files
+        # findable but must NOT trigger the restriction on their own (an agent
+        # with no explicit knowledge should search everything).
         has_knowledge_scope = (
             attached_document_ids
             or hierarchy_node_ids
+            or user_file_ids
             or document_sets
-            or persona_id_filter is not None
         )
 
         if has_knowledge_scope:
@@ -1056,17 +764,23 @@ class DocumentQuery:
                 knowledge_filter["bool"]["should"].append(
                     _get_hierarchy_node_filter(hierarchy_node_ids)
                 )
+            if user_file_ids:
+                knowledge_filter["bool"]["should"].append(
+                    _get_user_file_id_filter(user_file_ids)
+                )
             if document_sets:
                 knowledge_filter["bool"]["should"].append(
                     _get_document_set_filter(document_sets)
                 )
-            if persona_id_filter is not None:
+            # Additive: widen scope to also cover overflowing user files, but
+            # only when an explicit restriction is already in effect.
+            if project_id is not None:
                 knowledge_filter["bool"]["should"].append(
-                    _get_persona_filter(persona_id_filter)
+                    _get_user_project_filter(project_id)
                 )
-            if project_id_filter is not None:
+            if persona_id is not None:
                 knowledge_filter["bool"]["should"].append(
-                    _get_user_project_filter(project_id_filter)
+                    _get_persona_filter(persona_id)
                 )
             filter_clauses.append(knowledge_filter)
 
@@ -1084,6 +798,8 @@ class DocumentQuery:
             )
 
         if document_id is not None:
+            # WARNING: If user_file_ids has elements and if none of them are
+            # document_id, no matches will be retrieved.
             filter_clauses.append(
                 {"term": {DOCUMENT_ID_FIELD_NAME: {"value": document_id}}}
             )

backend/onyx/document_index/vespa/chunk_retrieval.py

@@ -501,31 +501,20 @@ def query_vespa(
             response = http_client.post(SEARCH_ENDPOINT, json=params)
             response.raise_for_status()
     except httpx.HTTPError as e:
-        response_text = (
-            e.response.text if isinstance(e, httpx.HTTPStatusError) else None
-        )
-        status_code = (
-            e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None
-        )
-        yql_value = params.get("yql", "")
-        yql_length = len(str(yql_value))
-
-        # Log each detail on its own line so log collectors capture them
-        # as separate entries rather than truncating a single multiline msg
+        error_base = "Failed to query Vespa"
         logger.error(
-            f"Failed to query Vespa | "
-            f"status={status_code} | "
-            f"yql_length={yql_length} | "
-            f"exception={str(e)}"
+            f"{error_base}:\n"
+            f"Request URL: {e.request.url}\n"
+            f"Request Headers: {e.request.headers}\n"
+            f"Request Payload: {params}\n"
+            f"Exception: {str(e)}"
+            + (
+                f"\nResponse: {e.response.text}"
+                if isinstance(e, httpx.HTTPStatusError)
+                else ""
+            )
         )
-        if response_text:
-            logger.error(f"Vespa error response: {response_text[:1000]}")
-        logger.error(f"Vespa request URL: {e.request.url}")
-
-        # Re-raise with diagnostics so callers see what actually went wrong
-        raise httpx.HTTPError(
-            f"Failed to query Vespa (status={status_code}, " f"yql_length={yql_length})"
-        ) from e
+        raise httpx.HTTPError(error_base) from e
 
     response_json: dict[str, Any] = response.json()
 

backend/onyx/document_index/vespa/shared_utils/vespa_request_builders.py

@@ -43,22 +43,6 @@ def build_vespa_filters(
             return ""
         return f"({' or '.join(eq_elems)})"
 
-    def _build_weighted_set_filter(key: str, vals: list[str] | None) -> str:
-        """Build a Vespa weightedSet filter for large value lists.
-
-        Uses Vespa's native weightedSet() operator instead of OR-chained
-        'contains' clauses.  This is critical for fields like
-        access_control_list where a single user may have tens of thousands
-        of ACL entries — OR clauses at that scale cause Vespa to reject
-        the query with HTTP 400."""
-        if not key or not vals:
-            return ""
-        filtered = [val for val in vals if val]
-        if not filtered:
-            return ""
-        items = ", ".join(f'"{val}":1' for val in filtered)
-        return f"weightedSet({key}, {{{items}}})"
-
     def _build_int_or_filters(key: str, vals: list[int] | None) -> str:
         """For an integer field filter.
         Returns a bare clause or ""."""
@@ -173,16 +157,11 @@ def build_vespa_filters(
     if filters.tenant_id and MULTI_TENANT:
         filter_parts.append(build_tenant_id_filter(filters.tenant_id))
 
-    # ACL filters — use weightedSet for efficient matching against the
-    # access_control_list weightedset<string> field.  OR-chaining thousands
-    # of 'contains' clauses causes Vespa to reject the query (HTTP 400)
-    # for users with large numbers of external permission groups.
+    # ACL filters
     if filters.access_control_list is not None:
         _append(
             filter_parts,
-            _build_weighted_set_filter(
-                ACCESS_CONTROL_LIST, filters.access_control_list
-            ),
+            _build_or_filters(ACCESS_CONTROL_LIST, filters.access_control_list),
         )
 
     # Source type filters
@@ -199,29 +178,31 @@ def build_vespa_filters(
         ]
     _append(filter_parts, _build_or_filters(METADATA_LIST, tag_attributes))
 
-    # Knowledge scope: explicit knowledge attachments restrict what an
-    # assistant can see.  When none are set, the assistant can see
-    # everything.
+    # Knowledge scope: explicit knowledge attachments (document_sets,
+    # user_file_ids) restrict what an assistant can see.  When none are
+    # set, the assistant can see everything.
     #
-    # persona_id_filter is a primary trigger — a persona with user files IS
-    # explicit knowledge, so it can start a knowledge scope on its own.
-    #
-    # project_id_filter is additive — it widens the scope to also cover
-    # overflowing project files but never restricts on its own (a chat
-    # inside a project should still search team knowledge).
+    # project_id / persona_id are additive: they make overflowing user
+    # files findable in Vespa but must NOT trigger the restriction on
+    # their own (an agent with no explicit knowledge should search
+    # everything).
     knowledge_scope_parts: list[str] = []
 
     _append(
         knowledge_scope_parts, _build_or_filters(DOCUMENT_SETS, filters.document_set)
     )
-    _append(knowledge_scope_parts, _build_persona_filter(filters.persona_id_filter))
 
-    # project_id_filter only widens an existing scope.
+    user_file_ids_str = (
+        [str(uuid) for uuid in filters.user_file_ids] if filters.user_file_ids else None
+    )
+    _append(knowledge_scope_parts, _build_or_filters(DOCUMENT_ID, user_file_ids_str))
+
+    # Only include project/persona scopes when an explicit knowledge
+    # restriction is already in effect — they widen the scope to also
+    # cover overflowing user files but never restrict on their own.
     if knowledge_scope_parts:
-        _append(
-            knowledge_scope_parts,
-            _build_user_project_filter(filters.project_id_filter),
-        )
+        _append(knowledge_scope_parts, _build_user_project_filter(filters.project_id))
+        _append(knowledge_scope_parts, _build_persona_filter(filters.persona_id))
 
     if len(knowledge_scope_parts) > 1:
         filter_parts.append("(" + " or ".join(knowledge_scope_parts) + ")")

backend/onyx/error_handling/error_codes.py

@@ -35,8 +35,6 @@ class OnyxErrorCode(Enum):
     INSUFFICIENT_PERMISSIONS = ("INSUFFICIENT_PERMISSIONS", 403)
     ADMIN_ONLY = ("ADMIN_ONLY", 403)
     EE_REQUIRED = ("EE_REQUIRED", 403)
-    SINGLE_TENANT_ONLY = ("SINGLE_TENANT_ONLY", 403)
-    ENV_VAR_GATED = ("ENV_VAR_GATED", 403)
 
     # ------------------------------------------------------------------
     # Validation / Bad Request (400)
@@ -44,7 +42,6 @@ class OnyxErrorCode(Enum):
     VALIDATION_ERROR = ("VALIDATION_ERROR", 400)
     INVALID_INPUT = ("INVALID_INPUT", 400)
     MISSING_REQUIRED_FIELD = ("MISSING_REQUIRED_FIELD", 400)
-    QUERY_REJECTED = ("QUERY_REJECTED", 400)
 
     # ------------------------------------------------------------------
     # Not Found (404)
@@ -89,7 +86,6 @@ class OnyxErrorCode(Enum):
     SERVICE_UNAVAILABLE = ("SERVICE_UNAVAILABLE", 503)
     BAD_GATEWAY = ("BAD_GATEWAY", 502)
     LLM_PROVIDER_ERROR = ("LLM_PROVIDER_ERROR", 502)
-    HOOK_EXECUTION_FAILED = ("HOOK_EXECUTION_FAILED", 502)
     GATEWAY_TIMEOUT = ("GATEWAY_TIMEOUT", 504)
 
     def __init__(self, code: str, status_code: int) -> None:

backend/onyx/federated_connectors/federated_retrieval.py

@@ -38,7 +38,17 @@ def get_federated_retrieval_functions(
     source_types: list[DocumentSource] | None,
     document_set_names: list[str] | None,
     slack_context: SlackContext | None = None,
+    user_file_ids: list[UUID] | None = None,
 ) -> list[FederatedRetrievalInfo]:
+    # When User Knowledge (user files) is the only knowledge source enabled,
+    # skip federated connectors entirely. User Knowledge mode means the agent
+    # should ONLY use uploaded files, not team connectors like Slack.
+    if user_file_ids and not document_set_names:
+        logger.debug(
+            "Skipping all federated connectors: User Knowledge mode enabled "
+            f"with {len(user_file_ids)} user files and no document sets"
+        )
+        return []
 
     # Check for Slack bot context first (regardless of user_id)
     if slack_context:

backend/onyx/file_processing/image_summarization.py

@@ -88,13 +88,9 @@ def summarize_image_with_error_handling(
     try:
         return summarize_image_pipeline(llm, image_data, user_prompt, system_prompt)
     except UnsupportedImageFormatError:
-        magic_hex = image_data[:8].hex() if image_data else "empty"
         logger.info(
-            "Skipping image summarization due to unsupported MIME type "
-            "for %s (magic_bytes=%s, size=%d bytes)",
+            "Skipping image summarization due to unsupported MIME type for %s",
             context_name,
-            magic_hex,
-            len(image_data),
         )
         return None
 
@@ -138,23 +134,9 @@ def _summarize_image(
         return summary
 
     except Exception as e:
-        # Extract structured details from LiteLLM exceptions when available,
-        # rather than dumping the full messages payload (which contains base64
-        # image data and produces enormous, unreadable error logs).
-        str_e = str(e)
-        if len(str_e) > 512:
-            str_e = str_e[:512] + "... (truncated)"
-        parts = [f"Summarization failed: {type(e).__name__}: {str_e}"]
-        status_code = getattr(e, "status_code", None)
-        llm_provider = getattr(e, "llm_provider", None)
-        model = getattr(e, "model", None)
-        if status_code is not None:
-            parts.append(f"status_code={status_code}")
-        if llm_provider is not None:
-            parts.append(f"llm_provider={llm_provider}")
-        if model is not None:
-            parts.append(f"model={model}")
-        raise ValueError(" | ".join(parts)) from e
+        error_msg = f"Summarization failed. Messages: {messages}"
+        error_msg = error_msg[:1024]
+        raise ValueError(error_msg) from e
 
 
 def _encode_image_for_llm_prompt(image_data: bytes) -> str:

backend/onyx/file_store/utils.py

@@ -23,55 +23,45 @@ from onyx.utils.timing import log_function_time
 logger = setup_logger()
 
 
-def plaintext_file_name_for_id(file_id: str) -> str:
-    """Generate a consistent file name for storing plaintext content of a file."""
-    return f"plaintext_{file_id}"
+def user_file_id_to_plaintext_file_name(user_file_id: UUID) -> str:
+    """Generate a consistent file name for storing plaintext content of a user file."""
+    return f"plaintext_{user_file_id}"
 
 
-def store_plaintext(file_id: str, plaintext_content: str) -> bool:
+def store_user_file_plaintext(user_file_id: UUID, plaintext_content: str) -> bool:
     """
-    Store plaintext content for a file in the file store.
+    Store plaintext content for a user file in the file store.
 
     Args:
-        file_id: The ID of the file (user_file or artifact_file)
+        user_file_id: The ID of the user file
         plaintext_content: The plaintext content to store
 
     Returns:
         bool: True if storage was successful, False otherwise
     """
+    # Skip empty content
     if not plaintext_content:
         return False
 
-    plaintext_file_name = plaintext_file_name_for_id(file_id)
+    # Get plaintext file name
+    plaintext_file_name = user_file_id_to_plaintext_file_name(user_file_id)
+
     try:
         file_store = get_default_file_store()
         file_content = BytesIO(plaintext_content.encode("utf-8"))
         file_store.save_file(
             content=file_content,
-            display_name=f"Plaintext for {file_id}",
+            display_name=f"Plaintext for user file {user_file_id}",
             file_origin=FileOrigin.PLAINTEXT_CACHE,
             file_type="text/plain",
             file_id=plaintext_file_name,
         )
         return True
     except Exception as e:
-        logger.warning(f"Failed to store plaintext for {file_id}: {e}")
+        logger.warning(f"Failed to store plaintext for user file {user_file_id}: {e}")
         return False
 
 
-# --- Convenience wrappers for callers that use user-file UUIDs ---
-
-
-def user_file_id_to_plaintext_file_name(user_file_id: UUID) -> str:
-    """Generate a consistent file name for storing plaintext content of a user file."""
-    return plaintext_file_name_for_id(str(user_file_id))
-
-
-def store_user_file_plaintext(user_file_id: UUID, plaintext_content: str) -> bool:
-    """Store plaintext content for a user file (delegates to :func:`store_plaintext`)."""
-    return store_plaintext(str(user_file_id), plaintext_content)
-
-
 def load_chat_file_by_id(file_id: str) -> InMemoryChatFile:
     """Load a file directly from the file store using its file_record ID.
 

backend/onyx/hooks/api_dependencies.py

@@ -1,26 +0,0 @@
-from onyx.configs.app_configs import HOOK_ENABLED
-from onyx.error_handling.error_codes import OnyxErrorCode
-from onyx.error_handling.exceptions import OnyxError
-from shared_configs.configs import MULTI_TENANT
-
-
-def require_hook_enabled() -> None:
-    """FastAPI dependency that gates all hook management endpoints.
-
-    Hooks are only available in single-tenant / self-hosted deployments with
-    HOOK_ENABLED=true explicitly set. Two layers of protection:
-      1. MULTI_TENANT check — rejects even if HOOK_ENABLED is accidentally set true
-      2. HOOK_ENABLED flag — explicit opt-in by the operator
-
-    Use as: Depends(require_hook_enabled)
-    """
-    if MULTI_TENANT:
-        raise OnyxError(
-            OnyxErrorCode.SINGLE_TENANT_ONLY,
-            "Hooks are not available in multi-tenant deployments",
-        )
-    if not HOOK_ENABLED:
-        raise OnyxError(
-            OnyxErrorCode.ENV_VAR_GATED,
-            "Hooks are not enabled. Set HOOK_ENABLED=true to enable.",
-        )

backend/onyx/hooks/executor.py

@@ -1,391 +0,0 @@
-"""Hook executor — calls a customer's external HTTP endpoint for a given hook point.
-
-Usage (Celery tasks and FastAPI handlers):
-    result = execute_hook(
-        db_session=db_session,
-        hook_point=HookPoint.QUERY_PROCESSING,
-        payload={"query": "...", "user_email": "...", "chat_session_id": "..."},
-        response_type=QueryProcessingResponse,
-    )
-
-    if isinstance(result, HookSkipped):
-        # no active hook configured — continue with original behavior
-        ...
-    elif isinstance(result, HookSoftFailed):
-        # hook failed but fail strategy is SOFT — continue with original behavior
-        ...
-    else:
-        # result is a validated Pydantic model instance (spec.response_model)
-        ...
-
-is_reachable update policy
---------------------------
-``is_reachable`` on the Hook row is updated selectively — only when the outcome
-carries meaningful signal about physical reachability:
-
-  NetworkError (DNS, connection refused)  → False  (cannot reach the server)
-  HTTP 401 / 403                          → False  (api_key revoked or invalid)
-  TimeoutException                        → None   (server may be slow, skip write)
-  Other HTTP errors (4xx / 5xx)           → None   (server responded, skip write)
-  Unknown exception                       → None   (no signal, skip write)
-  Non-JSON / non-dict response            → None   (server responded, skip write)
-  Success (2xx, valid dict)               → True   (confirmed reachable)
-
-None means "leave the current value unchanged" — no DB round-trip is made.
-
-DB session design
------------------
-The executor uses three sessions:
-
-  1. Caller's session (db_session) — used only for the hook lookup read. All
-     needed fields are extracted from the Hook object before the HTTP call, so
-     the caller's session is not held open during the external HTTP request.
-
-  2. Log session — a separate short-lived session opened after the HTTP call
-     completes to write the HookExecutionLog row on failure. Success runs are
-     not recorded. Committed independently of everything else.
-
-  3. Reachable session — a second short-lived session to update is_reachable on
-     the Hook. Kept separate from the log session so a concurrent hook deletion
-     (which causes update_hook__no_commit to raise OnyxError(NOT_FOUND)) cannot
-     prevent the execution log from being written. This update is best-effort.
-"""
-
-import json
-import time
-from typing import Any
-from typing import TypeVar
-
-import httpx
-from pydantic import BaseModel
-from pydantic import ValidationError
-from sqlalchemy.orm import Session
-
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
-from onyx.db.enums import HookFailStrategy
-from onyx.db.enums import HookPoint
-from onyx.db.hook import create_hook_execution_log__no_commit
-from onyx.db.hook import get_non_deleted_hook_by_hook_point
-from onyx.db.hook import update_hook__no_commit
-from onyx.db.models import Hook
-from onyx.error_handling.error_codes import OnyxErrorCode
-from onyx.error_handling.exceptions import OnyxError
-from onyx.hooks.utils import HOOKS_AVAILABLE
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-class HookSkipped:
-    """No active hook configured for this hook point."""
-
-
-class HookSoftFailed:
-    """Hook was called but failed with SOFT fail strategy — continuing."""
-
-
-T = TypeVar("T", bound=BaseModel)
-
-
-# ---------------------------------------------------------------------------
-# Private helpers
-# ---------------------------------------------------------------------------
-
-
-class _HttpOutcome(BaseModel):
-    """Structured result of an HTTP hook call, returned by _process_response."""
-
-    is_success: bool
-    updated_is_reachable: (
-        bool | None
-    )  # True/False = write to DB, None = unchanged (skip write)
-    status_code: int | None
-    error_message: str | None
-    response_payload: dict[str, Any] | None
-
-
-def _lookup_hook(
-    db_session: Session,
-    hook_point: HookPoint,
-) -> Hook | HookSkipped:
-    """Return the active Hook or HookSkipped if hooks are unavailable/unconfigured.
-
-    No HTTP call is made and no DB writes are performed for any HookSkipped path.
-    There is nothing to log and no reachability information to update.
-    """
-    if not HOOKS_AVAILABLE:
-        return HookSkipped()
-    hook = get_non_deleted_hook_by_hook_point(
-        db_session=db_session, hook_point=hook_point
-    )
-    if hook is None or not hook.is_active:
-        return HookSkipped()
-    if not hook.endpoint_url:
-        return HookSkipped()
-    return hook
-
-
-def _process_response(
-    *,
-    response: httpx.Response | None,
-    exc: Exception | None,
-    timeout: float,
-) -> _HttpOutcome:
-    """Process the result of an HTTP call and return a structured outcome.
-
-    Called after the client.post() try/except. If post() raised, exc is set and
-    response is None. Otherwise response is set and exc is None. Handles
-    raise_for_status(), JSON decoding, and the dict shape check.
-    """
-    if exc is not None:
-        if isinstance(exc, httpx.NetworkError):
-            msg = f"Hook network error (endpoint unreachable): {exc}"
-            logger.warning(msg, exc_info=exc)
-            return _HttpOutcome(
-                is_success=False,
-                updated_is_reachable=False,
-                status_code=None,
-                error_message=msg,
-                response_payload=None,
-            )
-        if isinstance(exc, httpx.TimeoutException):
-            msg = f"Hook timed out after {timeout}s: {exc}"
-            logger.warning(msg, exc_info=exc)
-            return _HttpOutcome(
-                is_success=False,
-                updated_is_reachable=None,  # timeout doesn't indicate unreachability
-                status_code=None,
-                error_message=msg,
-                response_payload=None,
-            )
-        msg = f"Hook call failed: {exc}"
-        logger.exception(msg, exc_info=exc)
-        return _HttpOutcome(
-            is_success=False,
-            updated_is_reachable=None,  # unknown error — don't make assumptions
-            status_code=None,
-            error_message=msg,
-            response_payload=None,
-        )
-
-    if response is None:
-        raise ValueError(
-            "exactly one of response or exc must be non-None; both are None"
-        )
-    status_code = response.status_code
-
-    try:
-        response.raise_for_status()
-    except httpx.HTTPStatusError as e:
-        msg = f"Hook returned HTTP {e.response.status_code}: {e.response.text}"
-        logger.warning(msg, exc_info=e)
-        # 401/403 means the api_key has been revoked or is invalid — mark unreachable
-        # so the operator knows to update it. All other HTTP errors keep is_reachable
-        # as-is (server is up, the request just failed for application reasons).
-        auth_failed = e.response.status_code in (401, 403)
-        return _HttpOutcome(
-            is_success=False,
-            updated_is_reachable=False if auth_failed else None,
-            status_code=status_code,
-            error_message=msg,
-            response_payload=None,
-        )
-
-    try:
-        response_payload = response.json()
-    except (json.JSONDecodeError, httpx.DecodingError) as e:
-        msg = f"Hook returned non-JSON response: {e}"
-        logger.warning(msg, exc_info=e)
-        return _HttpOutcome(
-            is_success=False,
-            updated_is_reachable=None,  # server responded — reachability unchanged
-            status_code=status_code,
-            error_message=msg,
-            response_payload=None,
-        )
-
-    if not isinstance(response_payload, dict):
-        msg = f"Hook returned non-dict JSON (got {type(response_payload).__name__})"
-        logger.warning(msg)
-        return _HttpOutcome(
-            is_success=False,
-            updated_is_reachable=None,  # server responded — reachability unchanged
-            status_code=status_code,
-            error_message=msg,
-            response_payload=None,
-        )
-
-    return _HttpOutcome(
-        is_success=True,
-        updated_is_reachable=True,
-        status_code=status_code,
-        error_message=None,
-        response_payload=response_payload,
-    )
-
-
-def _persist_result(
-    *,
-    hook_id: int,
-    outcome: _HttpOutcome,
-    duration_ms: int,
-) -> None:
-    """Write the execution log on failure and optionally update is_reachable, each
-    in its own session so a failure in one does not affect the other."""
-    # Only write the execution log on failure — success runs are not recorded.
-    # Must not be skipped if the is_reachable update fails (e.g. hook concurrently
-    # deleted between the initial lookup and here).
-    if not outcome.is_success:
-        try:
-            with get_session_with_current_tenant() as log_session:
-                create_hook_execution_log__no_commit(
-                    db_session=log_session,
-                    hook_id=hook_id,
-                    is_success=False,
-                    error_message=outcome.error_message,
-                    status_code=outcome.status_code,
-                    duration_ms=duration_ms,
-                )
-                log_session.commit()
-        except Exception:
-            logger.exception(
-                f"Failed to persist hook execution log for hook_id={hook_id}"
-            )
-
-    # Update is_reachable separately — best-effort, non-critical.
-    # None means the value is unchanged (set by the caller to skip the no-op write).
-    # update_hook__no_commit can raise OnyxError(NOT_FOUND) if the hook was
-    # concurrently deleted, so keep this isolated from the log write above.
-    if outcome.updated_is_reachable is not None:
-        try:
-            with get_session_with_current_tenant() as reachable_session:
-                update_hook__no_commit(
-                    db_session=reachable_session,
-                    hook_id=hook_id,
-                    is_reachable=outcome.updated_is_reachable,
-                )
-                reachable_session.commit()
-        except Exception:
-            logger.warning(f"Failed to update is_reachable for hook_id={hook_id}")
-
-
-# ---------------------------------------------------------------------------
-# Public API
-# ---------------------------------------------------------------------------
-
-
-def _execute_hook_inner(
-    hook: Hook,
-    payload: dict[str, Any],
-    response_type: type[T],
-) -> T | HookSoftFailed:
-    """Make the HTTP call, validate the response, and return a typed model.
-
-    Raises OnyxError on HARD failure. Returns HookSoftFailed on SOFT failure.
-    """
-    timeout = hook.timeout_seconds
-    hook_id = hook.id
-    fail_strategy = hook.fail_strategy
-    endpoint_url = hook.endpoint_url
-    current_is_reachable: bool | None = hook.is_reachable
-
-    if not endpoint_url:
-        raise ValueError(
-            f"hook_id={hook_id} is active but has no endpoint_url — "
-            "active hooks without an endpoint_url must be rejected by _lookup_hook"
-        )
-
-    start = time.monotonic()
-    response: httpx.Response | None = None
-    exc: Exception | None = None
-    try:
-        api_key: str | None = (
-            hook.api_key.get_value(apply_mask=False) if hook.api_key else None
-        )
-        headers: dict[str, str] = {"Content-Type": "application/json"}
-        if api_key:
-            headers["Authorization"] = f"Bearer {api_key}"
-        with httpx.Client(
-            timeout=timeout, follow_redirects=False
-        ) as client:  # SSRF guard: never follow redirects
-            response = client.post(endpoint_url, json=payload, headers=headers)
-    except Exception as e:
-        exc = e
-    duration_ms = int((time.monotonic() - start) * 1000)
-
-    outcome = _process_response(response=response, exc=exc, timeout=timeout)
-
-    # Validate the response payload against response_type.
-    # A validation failure downgrades the outcome to a failure so it is logged,
-    # is_reachable is left unchanged (server responded — just a bad payload),
-    # and fail_strategy is respected below.
-    validated_model: T | None = None
-    if outcome.is_success and outcome.response_payload is not None:
-        try:
-            validated_model = response_type.model_validate(outcome.response_payload)
-        except ValidationError as e:
-            msg = (
-                f"Hook response failed validation against {response_type.__name__}: {e}"
-            )
-            outcome = _HttpOutcome(
-                is_success=False,
-                updated_is_reachable=None,  # server responded — reachability unchanged
-                status_code=outcome.status_code,
-                error_message=msg,
-                response_payload=None,
-            )
-
-    # Skip the is_reachable write when the value would not change — avoids a
-    # no-op DB round-trip on every call when the hook is already in the expected state.
-    if outcome.updated_is_reachable == current_is_reachable:
-        outcome = outcome.model_copy(update={"updated_is_reachable": None})
-    _persist_result(hook_id=hook_id, outcome=outcome, duration_ms=duration_ms)
-
-    if not outcome.is_success:
-        if fail_strategy == HookFailStrategy.HARD:
-            raise OnyxError(
-                OnyxErrorCode.HOOK_EXECUTION_FAILED,
-                outcome.error_message or "Hook execution failed.",
-            )
-        logger.warning(
-            f"Hook execution failed (soft fail) for hook_id={hook_id}: {outcome.error_message}"
-        )
-        return HookSoftFailed()
-
-    if validated_model is None:
-        raise OnyxError(
-            OnyxErrorCode.INTERNAL_ERROR,
-            f"validated_model is None for successful hook call (hook_id={hook_id})",
-        )
-    return validated_model
-
-
-def execute_hook(
-    *,
-    db_session: Session,
-    hook_point: HookPoint,
-    payload: dict[str, Any],
-    response_type: type[T],
-) -> T | HookSkipped | HookSoftFailed:
-    """Execute the hook for the given hook point synchronously.
-
-    Returns HookSkipped if no active hook is configured, HookSoftFailed if the
-    hook failed with SOFT fail strategy, or a validated response model on success.
-    Raises OnyxError on HARD failure or if the hook is misconfigured.
-    """
-    hook = _lookup_hook(db_session, hook_point)
-    if isinstance(hook, HookSkipped):
-        return hook
-
-    fail_strategy = hook.fail_strategy
-    hook_id = hook.id
-
-    try:
-        return _execute_hook_inner(hook, payload, response_type)
-    except Exception:
-        if fail_strategy == HookFailStrategy.SOFT:
-            logger.exception(
-                f"Unexpected error in hook execution (soft fail) for hook_id={hook_id}"
-            )
-            return HookSoftFailed()
-        raise

backend/onyx/hooks/models.py

@@ -1,123 +0,0 @@
-from datetime import datetime
-from enum import Enum
-from typing import Annotated
-from typing import Any
-
-from pydantic import BaseModel
-from pydantic import Field
-from pydantic import field_validator
-from pydantic import model_validator
-from pydantic import SecretStr
-
-from onyx.db.enums import HookFailStrategy
-from onyx.db.enums import HookPoint
-
-NonEmptySecretStr = Annotated[SecretStr, Field(min_length=1)]
-
-
-# ---------------------------------------------------------------------------
-# Request models
-# ---------------------------------------------------------------------------
-
-
-class HookCreateRequest(BaseModel):
-    name: str = Field(min_length=1)
-    hook_point: HookPoint
-    endpoint_url: str = Field(min_length=1)
-    api_key: NonEmptySecretStr | None = None
-    fail_strategy: HookFailStrategy | None = None  # if None, uses HookPointSpec default
-    timeout_seconds: float | None = Field(
-        default=None, gt=0
-    )  # if None, uses HookPointSpec default
-
-    @field_validator("name", "endpoint_url")
-    @classmethod
-    def no_whitespace_only(cls, v: str) -> str:
-        if not v.strip():
-            raise ValueError("cannot be whitespace-only.")
-        return v
-
-
-class HookUpdateRequest(BaseModel):
-    name: str | None = None
-    endpoint_url: str | None = None
-    api_key: NonEmptySecretStr | None = None
-    fail_strategy: HookFailStrategy | None = None
-    timeout_seconds: float | None = Field(default=None, gt=0)
-
-    @model_validator(mode="after")
-    def require_at_least_one_field(self) -> "HookUpdateRequest":
-        if not self.model_fields_set:
-            raise ValueError("At least one field must be provided for an update.")
-        if "name" in self.model_fields_set and not (self.name or "").strip():
-            raise ValueError("name cannot be cleared.")
-        if (
-            "endpoint_url" in self.model_fields_set
-            and not (self.endpoint_url or "").strip()
-        ):
-            raise ValueError("endpoint_url cannot be cleared.")
-        if "fail_strategy" in self.model_fields_set and self.fail_strategy is None:
-            raise ValueError(
-                "fail_strategy cannot be null; omit the field to leave it unchanged."
-            )
-        if "timeout_seconds" in self.model_fields_set and self.timeout_seconds is None:
-            raise ValueError(
-                "timeout_seconds cannot be null; omit the field to leave it unchanged."
-            )
-        return self
-
-
-# ---------------------------------------------------------------------------
-# Response models
-# ---------------------------------------------------------------------------
-
-
-class HookPointMetaResponse(BaseModel):
-    hook_point: HookPoint
-    display_name: str
-    description: str
-    docs_url: str | None
-    input_schema: dict[str, Any]
-    output_schema: dict[str, Any]
-    default_timeout_seconds: float
-    default_fail_strategy: HookFailStrategy
-    fail_hard_description: str
-
-
-class HookResponse(BaseModel):
-    id: int
-    name: str
-    hook_point: HookPoint
-    # Nullable to match the DB column — endpoint_url is required on creation but
-    # future hook point types may not use an external endpoint (e.g. built-in handlers).
-    endpoint_url: str | None
-    # Partially-masked API key (e.g. "abcd••••••••wxyz"), or None if no key is set.
-    api_key_masked: str | None
-    fail_strategy: HookFailStrategy
-    timeout_seconds: float  # always resolved — None from request is replaced with spec default before DB write
-    is_active: bool
-    is_reachable: bool | None
-    creator_email: str | None
-    created_at: datetime
-    updated_at: datetime
-
-
-class HookValidateStatus(str, Enum):
-    passed = "passed"  # server responded (any status except 401/403)
-    auth_failed = "auth_failed"  # server responded with 401 or 403
-    timeout = (
-        "timeout"  # TCP connected, but read/write timed out (server exists but slow)
-    )
-    cannot_connect = "cannot_connect"  # could not connect to the server
-
-
-class HookValidateResponse(BaseModel):
-    status: HookValidateStatus
-    error_message: str | None = None
-
-
-class HookExecutionRecord(BaseModel):
-    error_message: str | None = None
-    status_code: int | None = None
-    duration_ms: int | None = None
-    created_at: datetime

backend/onyx/hooks/points/base.py

@@ -1,74 +0,0 @@
-from typing import Any
-from typing import ClassVar
-
-from pydantic import BaseModel
-
-from onyx.db.enums import HookFailStrategy
-from onyx.db.enums import HookPoint
-
-
-_REQUIRED_ATTRS = (
-    "hook_point",
-    "display_name",
-    "description",
-    "default_timeout_seconds",
-    "fail_hard_description",
-    "default_fail_strategy",
-    "payload_model",
-    "response_model",
-)
-
-
-class HookPointSpec:
-    """Static metadata and contract for a pipeline hook point.
-
-    Each concrete subclass represents exactly one hook point and is instantiated
-    once at startup, registered in onyx.hooks.registry._REGISTRY. Prefer
-    get_hook_point_spec() or get_all_specs() from the registry over direct
-    instantiation.
-
-    Each hook point is a concrete subclass of this class. Onyx engineers
-    own these definitions — customers never touch this code.
-
-    Subclasses must define all attributes as class-level constants.
-    payload_model and response_model must be Pydantic BaseModel subclasses;
-    input_schema and output_schema are derived from them automatically.
-    """
-
-    hook_point: HookPoint
-    display_name: str
-    description: str
-    default_timeout_seconds: float
-    fail_hard_description: str
-    default_fail_strategy: HookFailStrategy
-    docs_url: str | None = None
-
-    payload_model: ClassVar[type[BaseModel]]
-    response_model: ClassVar[type[BaseModel]]
-
-    # Computed once at class definition time from payload_model / response_model.
-    input_schema: ClassVar[dict[str, Any]]
-    output_schema: ClassVar[dict[str, Any]]
-
-    def __init_subclass__(cls, **kwargs: object) -> None:
-        """Enforce that every subclass declares all required class attributes.
-
-        Called automatically by Python whenever a class inherits from HookPointSpec.
-        Raises TypeError at import time if any required attribute is missing or if
-        payload_model / response_model are not Pydantic BaseModel subclasses.
-        input_schema and output_schema are derived automatically from the models.
-        """
-        super().__init_subclass__(**kwargs)
-        missing = [attr for attr in _REQUIRED_ATTRS if not hasattr(cls, attr)]
-        if missing:
-            raise TypeError(f"{cls.__name__} must define class attributes: {missing}")
-        for attr in ("payload_model", "response_model"):
-            val = getattr(cls, attr, None)
-            if val is None or not (
-                isinstance(val, type) and issubclass(val, BaseModel)
-            ):
-                raise TypeError(
-                    f"{cls.__name__}.{attr} must be a Pydantic BaseModel subclass, got {val!r}"
-                )
-        cls.input_schema = cls.payload_model.model_json_schema()
-        cls.output_schema = cls.response_model.model_json_schema()

backend/onyx/hooks/points/document_ingestion.py

@@ -1,33 +0,0 @@
-from pydantic import BaseModel
-
-from onyx.db.enums import HookFailStrategy
-from onyx.db.enums import HookPoint
-from onyx.hooks.points.base import HookPointSpec
-
-
-# TODO(@Bo-Onyx): define payload and response fields
-class DocumentIngestionPayload(BaseModel):
-    pass
-
-
-class DocumentIngestionResponse(BaseModel):
-    pass
-
-
-class DocumentIngestionSpec(HookPointSpec):
-    """Hook point that runs during document ingestion.
-
-    # TODO(@Bo-Onyx): define call site, input/output schema, and timeout budget.
-    """
-
-    hook_point = HookPoint.DOCUMENT_INGESTION
-    display_name = "Document Ingestion"
-    description = "Runs during document ingestion. Allows filtering or transforming documents before indexing."
-    default_timeout_seconds = 30.0
-    fail_hard_description = "The document will not be indexed."
-    default_fail_strategy = HookFailStrategy.HARD
-    # TODO(Bo-Onyx): update later
-    docs_url = "https://docs.google.com/document/d/1pGhB8Wcnhhj8rS4baEJL6CX05yFhuIDNk1gbBRiWu94/edit?tab=t.ue263ual5vdi"
-
-    payload_model = DocumentIngestionPayload
-    response_model = DocumentIngestionResponse

backend/onyx/hooks/points/query_processing.py

@@ -1,72 +0,0 @@
-from pydantic import BaseModel
-from pydantic import ConfigDict
-from pydantic import Field
-
-from onyx.db.enums import HookFailStrategy
-from onyx.db.enums import HookPoint
-from onyx.hooks.points.base import HookPointSpec
-
-
-class QueryProcessingPayload(BaseModel):
-    model_config = ConfigDict(extra="forbid")
-
-    query: str = Field(description="The raw query string exactly as the user typed it.")
-    user_email: str | None = Field(
-        description="Email of the user submitting the query, or null if unauthenticated."
-    )
-    chat_session_id: str = Field(
-        description="UUID of the chat session, formatted as a hyphenated lowercase string (e.g. '550e8400-e29b-41d4-a716-446655440000'). Always present — the session is guaranteed to exist by the time this hook fires."
-    )
-
-
-class QueryProcessingResponse(BaseModel):
-    # Intentionally permissive — customer endpoints may return extra fields.
-    query: str | None = Field(
-        default=None,
-        description=(
-            "The query to use in the pipeline. "
-            "Null, empty string, whitespace-only, or absent = reject the query."
-        ),
-    )
-    rejection_message: str | None = Field(
-        default=None,
-        description="Message shown to the user when the query is rejected. Falls back to a generic message if not provided.",
-    )
-
-
-class QueryProcessingSpec(HookPointSpec):
-    """Hook point that runs on every user query before it enters the pipeline.
-
-    Call site: inside handle_stream_message_objects() in
-    backend/onyx/chat/process_message.py, immediately after message_text is
-    assigned from the request and before create_new_chat_message() saves it.
-
-    This is the earliest possible point in the query pipeline:
-    - Raw query — unmodified, exactly as the user typed it
-    - No side effects yet — message has not been saved to DB
-    - User identity is available for user-specific logic
-
-    Supported use cases:
-    - Query rejection: block queries based on content or user context
-    - Query rewriting: normalize, expand, or modify the query
-    - PII removal: scrub sensitive data before the LLM sees it
-    - Access control: reject queries from certain users or groups
-    - Query auditing: log or track queries based on business rules
-    """
-
-    hook_point = HookPoint.QUERY_PROCESSING
-    display_name = "Query Processing"
-    description = (
-        "Runs on every user query before it enters the pipeline. "
-        "Allows rewriting, filtering, or rejecting queries."
-    )
-    default_timeout_seconds = 5.0  # user is actively waiting — keep tight
-    fail_hard_description = (
-        "The query will be blocked and the user will see an error message."
-    )
-    default_fail_strategy = HookFailStrategy.HARD
-    # TODO(Bo-Onyx): update later
-    docs_url = "https://docs.google.com/document/d/1pGhB8Wcnhhj8rS4baEJL6CX05yFhuIDNk1gbBRiWu94/edit?tab=t.g2r1a1699u87"
-
-    payload_model = QueryProcessingPayload
-    response_model = QueryProcessingResponse

backend/onyx/hooks/registry.py

@@ -1,45 +0,0 @@
-from onyx.db.enums import HookPoint
-from onyx.hooks.points.base import HookPointSpec
-from onyx.hooks.points.document_ingestion import DocumentIngestionSpec
-from onyx.hooks.points.query_processing import QueryProcessingSpec
-
-# Internal: use `monkeypatch.setattr(registry_module, "_REGISTRY", {...})` to override in tests.
-_REGISTRY: dict[HookPoint, HookPointSpec] = {
-    HookPoint.DOCUMENT_INGESTION: DocumentIngestionSpec(),
-    HookPoint.QUERY_PROCESSING: QueryProcessingSpec(),
-}
-
-
-def validate_registry() -> None:
-    """Assert that every HookPoint enum value has a registered spec.
-
-    Call once at application startup (e.g. from the FastAPI lifespan hook).
-    Raises RuntimeError if any hook point is missing a spec.
-    """
-    missing = set(HookPoint) - set(_REGISTRY)
-    if missing:
-        raise RuntimeError(
-            f"Hook point(s) have no registered spec: {missing}. "
-            "Add an entry to onyx.hooks.registry._REGISTRY."
-        )
-
-
-def get_hook_point_spec(hook_point: HookPoint) -> HookPointSpec:
-    """Returns the spec for a given hook point.
-
-    Raises ValueError if the hook point has no registered spec — this is a
-    programmer error; every HookPoint enum value must have a corresponding spec
-    in _REGISTRY.
-    """
-    try:
-        return _REGISTRY[hook_point]
-    except KeyError:
-        raise ValueError(
-            f"No spec registered for hook point {hook_point!r}. "
-            "Add an entry to onyx.hooks.registry._REGISTRY."
-        )
-
-
-def get_all_specs() -> list[HookPointSpec]:
-    """Returns the specs for all registered hook points."""
-    return list(_REGISTRY.values())

backend/onyx/hooks/utils.py

@@ -1,5 +0,0 @@
-from onyx.configs.app_configs import HOOK_ENABLED
-from shared_configs.configs import MULTI_TENANT
-
-# True only when hooks are available: single-tenant deployment with HOOK_ENABLED=true.
-HOOKS_AVAILABLE: bool = HOOK_ENABLED and not MULTI_TENANT

backend/onyx/indexing/indexing_pipeline.py

@@ -395,12 +395,6 @@ def process_image_sections(documents: list[Document]) -> list[IndexingDocument]:
         llm = get_default_llm_with_vision()
 
     if not llm:
-        if get_image_extraction_and_analysis_enabled():
-            logger.warning(
-                "Image analysis is enabled but no vision-capable LLM is "
-                "available — images will not be summarized. Configure a "
-                "vision model in the admin LLM settings."
-            )
         # Even without LLM, we still convert to IndexingDocument with base Sections
         return [
             IndexingDocument(

backend/onyx/llm/factory.py

@@ -168,23 +168,10 @@ def get_default_llm_with_vision(
             if model_supports_image_input(
                 default_model.name, default_model.llm_provider.provider
             ):
-                logger.info(
-                    "Using default vision model: %s (provider=%s)",
-                    default_model.name,
-                    default_model.llm_provider.provider,
-                )
                 return create_vision_llm(
                     LLMProviderView.from_model(default_model.llm_provider),
                     default_model.name,
                 )
-            else:
-                logger.warning(
-                    "Default vision model %s (provider=%s) does not support "
-                    "image input — falling back to searching all providers",
-                    default_model.name,
-                    default_model.llm_provider.provider,
-                )
-
         # Fall back to searching all providers
         models = fetch_existing_models(
             db_session=db_session,
@@ -192,10 +179,6 @@ def get_default_llm_with_vision(
         )
 
         if not models:
-            logger.warning(
-                "No LLM models with VISION or CHAT flow type found — "
-                "image summarization will be disabled"
-            )
             return None
 
         for model in models:
@@ -217,25 +200,11 @@ def get_default_llm_with_vision(
 
     for model in sorted_models:
         if model_supports_image_input(model.name, model.llm_provider.provider):
-            logger.info(
-                "Using fallback vision model: %s (provider=%s)",
-                model.name,
-                model.llm_provider.provider,
-            )
             return create_vision_llm(
                 provider_map[model.llm_provider_id],
                 model.name,
             )
 
-    checked_models = [
-        f"{m.name} (provider={m.llm_provider.provider})" for m in sorted_models
-    ]
-    logger.warning(
-        "No vision-capable model found among %d candidates: %s — "
-        "image summarization will be disabled",
-        len(sorted_models),
-        ", ".join(checked_models),
-    )
     return None
 
 

backend/onyx/llm/multi_llm.py

@@ -530,11 +530,6 @@ class LitellmLLM(LLM):
                 ):
                     messages = _strip_tool_content_from_messages(messages)
 
-                # Only pass tool_choice when tools are present — some providers (e.g. Fireworks)
-                # reject requests where tool_choice is explicitly null.
-                if tools and tool_choice is not None:
-                    optional_kwargs["tool_choice"] = tool_choice
-
                 response = litellm.completion(
                     mock_response=get_llm_mock_response() or MOCK_LLM_RESPONSE,
                     model=model,
@@ -543,6 +538,7 @@ class LitellmLLM(LLM):
                     custom_llm_provider=self._custom_llm_provider or None,
                     messages=messages,
                     tools=tools,
+                    tool_choice=tool_choice,
                     stream=stream,
                     temperature=temperature,
                     timeout=timeout_override or self._timeout,

backend/onyx/llm/utils.py

@@ -219,26 +219,13 @@ def litellm_exception_to_error_msg(
             "ratelimiterror"
         ):
             upstream_detail = upstream_detail.split(":", 1)[1].strip()
-        upstream_detail_lower = upstream_detail.lower()
-        if (
-            "insufficient_quota" in upstream_detail_lower
-            or "exceeded your current quota" in upstream_detail_lower
-        ):
-            error_msg = (
-                f"{provider_name} quota exceeded: {upstream_detail}"
-                if upstream_detail
-                else f"{provider_name} quota exceeded: Verify billing and quota for this API key."
-            )
-            error_code = "BUDGET_EXCEEDED"
-            is_retryable = False
-        else:
-            error_msg = (
-                f"{provider_name} rate limit: {upstream_detail}"
-                if upstream_detail
-                else f"{provider_name} rate limit exceeded: Please slow down your requests and try again later."
-            )
-            error_code = "RATE_LIMIT"
-            is_retryable = True
+        error_msg = (
+            f"{provider_name} rate limit: {upstream_detail}"
+            if upstream_detail
+            else f"{provider_name} rate limit exceeded: Please slow down your requests and try again later."
+        )
+        error_code = "RATE_LIMIT"
+        is_retryable = True
     elif isinstance(core_exception, ServiceUnavailableError):
         provider_name = (
             llm.config.model_provider

backend/onyx/main.py

@@ -62,7 +62,6 @@ from onyx.db.engine.sql_engine import get_session_with_current_tenant
 from onyx.db.engine.sql_engine import SqlEngine
 from onyx.error_handling.exceptions import register_onyx_exception_handlers
 from onyx.file_store.file_store import get_default_file_store
-from onyx.hooks.registry import validate_registry
 from onyx.server.api_key.api import router as api_key_router
 from onyx.server.auth_check import check_router_auth
 from onyx.server.documents.cc_pair import router as cc_pair_router
@@ -77,7 +76,6 @@ from onyx.server.features.default_assistant.api import (
 )
 from onyx.server.features.document_set.api import router as document_set_router
 from onyx.server.features.hierarchy.api import router as hierarchy_router
-from onyx.server.features.hooks.api import router as hook_router
 from onyx.server.features.input_prompt.api import (
     admin_router as admin_input_prompt_router,
 )
@@ -310,7 +308,6 @@ def validate_no_vector_db_settings() -> None:
 async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:  # noqa: ARG001
     validate_no_vector_db_settings()
     validate_cache_backend_settings()
-    validate_registry()
 
     # Set recursion limit
     if SYSTEM_RECURSION_LIMIT is not None:
@@ -454,7 +451,6 @@ def get_application(lifespan_override: Lifespan | None = None) -> FastAPI:
 
     register_onyx_exception_handlers(application)
 
-    include_router_with_global_prefix_prepended(application, hook_router)
     include_router_with_global_prefix_prepended(application, password_router)
     include_router_with_global_prefix_prepended(application, chat_router)
     include_router_with_global_prefix_prepended(application, query_router)

backend/onyx/server/documents/cc_pair.py

@@ -43,9 +43,6 @@ from onyx.db.index_attempt import count_index_attempt_errors_for_cc_pair
 from onyx.db.index_attempt import count_index_attempts_for_cc_pair
 from onyx.db.index_attempt import get_index_attempt_errors_for_cc_pair
 from onyx.db.index_attempt import get_latest_index_attempt_for_cc_pair_id
-from onyx.db.index_attempt import (
-    get_latest_successful_index_attempt_for_cc_pair_id,
-)
 from onyx.db.index_attempt import get_paginated_index_attempts_for_cc_pair_id
 from onyx.db.indexing_coordination import IndexingCoordination
 from onyx.db.models import IndexAttempt
@@ -193,11 +190,6 @@ def get_cc_pair_full_info(
         only_finished=False,
     )
 
-    latest_successful_attempt = get_latest_successful_index_attempt_for_cc_pair_id(
-        db_session=db_session,
-        connector_credential_pair_id=cc_pair_id,
-    )
-
     # Get latest permission sync attempt for status
     latest_permission_sync_attempt = None
     if cc_pair.access_type == AccessType.SYNC:
@@ -215,11 +207,6 @@ def get_cc_pair_full_info(
             cc_pair_id=cc_pair_id,
         ),
         last_index_attempt=latest_attempt,
-        last_successful_index_time=(
-            latest_successful_attempt.time_started
-            if latest_successful_attempt
-            else None
-        ),
         latest_deletion_attempt=get_deletion_attempt_snapshot(
             connector_id=cc_pair.connector_id,
             credential_id=cc_pair.credential_id,

backend/onyx/server/documents/connector.py

@@ -3,7 +3,6 @@ import math
 import mimetypes
 import os
 import zipfile
-from datetime import datetime
 from io import BytesIO
 from typing import Any
 from typing import cast
@@ -110,9 +109,6 @@ from onyx.db.federated import fetch_all_federated_connectors_parallel
 from onyx.db.index_attempt import get_index_attempts_for_cc_pair
 from onyx.db.index_attempt import get_latest_index_attempts_by_status
 from onyx.db.index_attempt import get_latest_index_attempts_parallel
-from onyx.db.index_attempt import (
-    get_latest_successful_index_attempts_parallel,
-)
 from onyx.db.models import ConnectorCredentialPair
 from onyx.db.models import FederatedConnector
 from onyx.db.models import IndexAttempt
@@ -483,9 +479,7 @@ def is_zip_file(file: UploadFile) -> bool:
 
 
 def upload_files(
-    files: list[UploadFile],
-    file_origin: FileOrigin = FileOrigin.CONNECTOR,
-    unzip: bool = True,
+    files: list[UploadFile], file_origin: FileOrigin = FileOrigin.CONNECTOR
 ) -> FileUploadResponse:
 
     # Skip directories and known macOS metadata entries
@@ -508,46 +502,31 @@ def upload_files(
                 if seen_zip:
                     raise HTTPException(status_code=400, detail=SEEN_ZIP_DETAIL)
                 seen_zip = True
-
-                # Validate the zip by opening it (catches corrupt/non-zip files)
                 with zipfile.ZipFile(file.file, "r") as zf:
-                    if unzip:
-                        zip_metadata_file_id = save_zip_metadata_to_file_store(
-                            zf, file_store
+                    zip_metadata_file_id = save_zip_metadata_to_file_store(
+                        zf, file_store
+                    )
+                    for file_info in zf.namelist():
+                        if zf.getinfo(file_info).is_dir():
+                            continue
+
+                        if not should_process_file(file_info):
+                            continue
+
+                        sub_file_bytes = zf.read(file_info)
+
+                        mime_type, __ = mimetypes.guess_type(file_info)
+                        if mime_type is None:
+                            mime_type = "application/octet-stream"
+
+                        file_id = file_store.save_file(
+                            content=BytesIO(sub_file_bytes),
+                            display_name=os.path.basename(file_info),
+                            file_origin=file_origin,
+                            file_type=mime_type,
                         )
-                        for file_info in zf.namelist():
-                            if zf.getinfo(file_info).is_dir():
-                                continue
-
-                            if not should_process_file(file_info):
-                                continue
-
-                            sub_file_bytes = zf.read(file_info)
-
-                            mime_type, __ = mimetypes.guess_type(file_info)
-                            if mime_type is None:
-                                mime_type = "application/octet-stream"
-
-                            file_id = file_store.save_file(
-                                content=BytesIO(sub_file_bytes),
-                                display_name=os.path.basename(file_info),
-                                file_origin=file_origin,
-                                file_type=mime_type,
-                            )
-                            deduped_file_paths.append(file_id)
-                            deduped_file_names.append(os.path.basename(file_info))
-                        continue
-
-                # Store the zip as-is (unzip=False)
-                file.file.seek(0)
-                file_id = file_store.save_file(
-                    content=file.file,
-                    display_name=file.filename,
-                    file_origin=file_origin,
-                    file_type=file.content_type or "application/zip",
-                )
-                deduped_file_paths.append(file_id)
-                deduped_file_names.append(file.filename)
+                        deduped_file_paths.append(file_id)
+                        deduped_file_names.append(os.path.basename(file_info))
                 continue
 
             # Since we can't render docx files in the UI,
@@ -634,10 +613,9 @@ def _fetch_and_check_file_connector_cc_pair_permissions(
 @router.post("/admin/connector/file/upload", tags=PUBLIC_API_TAGS)
 def upload_files_api(
     files: list[UploadFile],
-    unzip: bool = True,
     _: User = Depends(current_curator_or_admin_user),
 ) -> FileUploadResponse:
-    return upload_files(files, FileOrigin.OTHER, unzip=unzip)
+    return upload_files(files, FileOrigin.OTHER)
 
 
 @router.get("/admin/connector/{connector_id}/files", tags=PUBLIC_API_TAGS)
@@ -1162,26 +1140,21 @@ def get_connector_indexing_status(
             ),
             (),
         ),
-        # Get most recent successful index attempts
-        (
-            lambda: get_latest_successful_index_attempts_parallel(
-                request.secondary_index,
-            ),
-            (),
-        ),
     ]
 
     if user and user.role == UserRole.ADMIN:
+        # For Admin users, we already got all the cc pair in editable_cc_pairs
+        # its not needed to get them again
         (
             editable_cc_pairs,
             federated_connectors,
             latest_index_attempts,
             latest_finished_index_attempts,
-            latest_successful_index_attempts,
         ) = run_functions_tuples_in_parallel(parallel_functions)
         non_editable_cc_pairs = []
     else:
         parallel_functions.append(
+            # Get non-editable connector/credential pairs
             (
                 lambda: get_connector_credential_pairs_for_user_parallel(
                     user, False, None, True, True, False, True, request.source
@@ -1195,7 +1168,6 @@ def get_connector_indexing_status(
             federated_connectors,
             latest_index_attempts,
             latest_finished_index_attempts,
-            latest_successful_index_attempts,
             non_editable_cc_pairs,
         ) = run_functions_tuples_in_parallel(parallel_functions)
 
@@ -1207,9 +1179,6 @@ def get_connector_indexing_status(
     latest_finished_index_attempts = cast(
         list[IndexAttempt], latest_finished_index_attempts
     )
-    latest_successful_index_attempts = cast(
-        list[IndexAttempt], latest_successful_index_attempts
-    )
 
     document_count_info = get_document_counts_for_all_cc_pairs(db_session)
 
@@ -1219,48 +1188,42 @@ def get_connector_indexing_status(
         for connector_id, credential_id, cnt in document_count_info
     }
 
-    def _attempt_lookup(
-        attempts: list[IndexAttempt],
-    ) -> dict[int, IndexAttempt]:
-        return {attempt.connector_credential_pair_id: attempt for attempt in attempts}
+    cc_pair_to_latest_index_attempt: dict[tuple[int, int], IndexAttempt] = {
+        (
+            attempt.connector_credential_pair.connector_id,
+            attempt.connector_credential_pair.credential_id,
+        ): attempt
+        for attempt in latest_index_attempts
+    }
 
-    cc_pair_to_latest_index_attempt = _attempt_lookup(latest_index_attempts)
-    cc_pair_to_latest_finished_index_attempt = _attempt_lookup(
-        latest_finished_index_attempts
-    )
-    cc_pair_to_latest_successful_index_attempt = _attempt_lookup(
-        latest_successful_index_attempts
-    )
+    cc_pair_to_latest_finished_index_attempt: dict[tuple[int, int], IndexAttempt] = {
+        (
+            attempt.connector_credential_pair.connector_id,
+            attempt.connector_credential_pair.credential_id,
+        ): attempt
+        for attempt in latest_finished_index_attempts
+    }
 
     def build_connector_indexing_status(
         cc_pair: ConnectorCredentialPair,
         is_editable: bool,
     ) -> ConnectorIndexingStatusLite | None:
+        # TODO remove this to enable ingestion API
         if cc_pair.name == "DefaultCCPair":
             return None
 
-        latest_attempt = cc_pair_to_latest_index_attempt.get(cc_pair.id)
-        latest_finished_attempt = cc_pair_to_latest_finished_index_attempt.get(
-            cc_pair.id
+        latest_attempt = cc_pair_to_latest_index_attempt.get(
+            (cc_pair.connector_id, cc_pair.credential_id)
         )
-        latest_successful_attempt = cc_pair_to_latest_successful_index_attempt.get(
-            cc_pair.id
+        latest_finished_attempt = cc_pair_to_latest_finished_index_attempt.get(
+            (cc_pair.connector_id, cc_pair.credential_id)
         )
         doc_count = cc_pair_to_document_cnt.get(
             (cc_pair.connector_id, cc_pair.credential_id), 0
         )
 
         return _get_connector_indexing_status_lite(
-            cc_pair,
-            latest_attempt,
-            latest_finished_attempt,
-            (
-                latest_successful_attempt.time_started
-                if latest_successful_attempt
-                else None
-            ),
-            is_editable,
-            doc_count,
+            cc_pair, latest_attempt, latest_finished_attempt, is_editable, doc_count
         )
 
     # Process editable cc_pairs
@@ -1356,7 +1319,7 @@ def get_connector_indexing_status(
     # Track admin page visit for analytics
     mt_cloud_telemetry(
         tenant_id=tenant_id,
-        distinct_id=str(user.id),
+        distinct_id=user.email,
         event=MilestoneRecordType.VISITED_ADMIN_PAGE,
     )
 
@@ -1421,7 +1384,6 @@ def _get_connector_indexing_status_lite(
     cc_pair: ConnectorCredentialPair,
     latest_index_attempt: IndexAttempt | None,
     latest_finished_index_attempt: IndexAttempt | None,
-    last_successful_index_time: datetime | None,
     is_editable: bool,
     document_cnt: int,
 ) -> ConnectorIndexingStatusLite | None:
@@ -1455,7 +1417,7 @@ def _get_connector_indexing_status_lite(
             else None
         ),
         last_status=latest_index_attempt.status if latest_index_attempt else None,
-        last_success=last_successful_index_time,
+        last_success=cc_pair.last_successful_index_time,
         docs_indexed=document_cnt,
         latest_index_attempt_docs_indexed=(
             latest_index_attempt.total_docs_indexed if latest_index_attempt else None
@@ -1571,7 +1533,7 @@ def create_connector_from_model(
 
         mt_cloud_telemetry(
             tenant_id=tenant_id,
-            distinct_id=str(user.id),
+            distinct_id=user.email,
             event=MilestoneRecordType.CREATED_CONNECTOR,
         )
 
@@ -1649,7 +1611,7 @@ def create_connector_with_mock_credential(
 
         mt_cloud_telemetry(
             tenant_id=tenant_id,
-            distinct_id=str(user.id),
+            distinct_id=user.email,
             event=MilestoneRecordType.CREATED_CONNECTOR,
         )
         return response
@@ -1953,7 +1915,9 @@ def submit_connector_request(
     if not connector_name:
         raise HTTPException(status_code=400, detail="Connector name cannot be empty")
 
+    # Get user identifier for telemetry
     user_email = user.email
+    distinct_id = user_email or tenant_id
 
     # Track connector request via PostHog telemetry (Cloud only)
     from shared_configs.configs import MULTI_TENANT
@@ -1961,11 +1925,11 @@ def submit_connector_request(
     if MULTI_TENANT:
         mt_cloud_telemetry(
             tenant_id=tenant_id,
-            distinct_id=str(user.id),
+            distinct_id=distinct_id,
             event=MilestoneRecordType.REQUESTED_CONNECTOR,
             properties={
                 "connector_name": connector_name,
-                "user_email": user.email,
+                "user_email": user_email,
             },
         )
 

backend/onyx/server/documents/models.py

@@ -330,7 +330,6 @@ class CCPairFullInfo(BaseModel):
         num_docs_indexed: int,  # not ideal, but this must be computed separately
         is_editable_for_current_user: bool,
         indexing: bool,
-        last_successful_index_time: datetime | None = None,
         last_permission_sync_attempt_status: PermissionSyncStatus | None = None,
         permission_syncing: bool = False,
         last_permission_sync_attempt_finished: datetime | None = None,
@@ -383,7 +382,9 @@ class CCPairFullInfo(BaseModel):
             creator_email=(
                 cc_pair_model.creator.email if cc_pair_model.creator else None
             ),
-            last_indexed=last_successful_index_time,
+            last_indexed=(
+                last_index_attempt.time_started if last_index_attempt else None
+            ),
             last_pruned=cc_pair_model.last_pruned,
             last_full_permission_sync=cls._get_last_full_permission_sync(cc_pair_model),
             overall_indexing_speed=overall_indexing_speed,
@@ -407,7 +408,7 @@ class FailedConnectorIndexingStatus(BaseModel):
     """Simplified version of ConnectorIndexingStatus for failed indexing attempts"""
 
     cc_pair_id: int
-    name: str
+    name: str | None
     error_msg: str | None
     is_deletable: bool
     connector_id: int
@@ -421,7 +422,7 @@ class ConnectorStatus(BaseModel):
     """
 
     cc_pair_id: int
-    name: str
+    name: str | None
     connector: ConnectorSnapshot
     credential: CredentialSnapshot
     access_type: AccessType
@@ -452,7 +453,7 @@ class DocsCountOperator(str, Enum):
 
 class ConnectorIndexingStatusLite(BaseModel):
     cc_pair_id: int
-    name: str
+    name: str | None
     source: DocumentSource
     access_type: AccessType
     cc_pair_status: ConnectorCredentialPairStatus
@@ -487,7 +488,7 @@ class ConnectorCredentialPairIdentifier(BaseModel):
 
 
 class ConnectorCredentialPairMetadata(BaseModel):
-    name: str
+    name: str | None = None
     access_type: AccessType
     auto_sync_options: dict[str, Any] | None = None
     groups: list[int] = Field(default_factory=list)
@@ -500,7 +501,7 @@ class CCStatusUpdateRequest(BaseModel):
 
 class ConnectorCredentialPairDescriptor(BaseModel):
     id: int
-    name: str
+    name: str | None = None
     connector: ConnectorSnapshot
     credential: CredentialSnapshot
     access_type: AccessType
@@ -510,7 +511,7 @@ class CCPairSummary(BaseModel):
     """Simplified connector-credential pair information with just essential data"""
 
     id: int
-    name: str
+    name: str | None
     source: DocumentSource
     access_type: AccessType
 

backend/onyx/server/features/build/sandbox/kubernetes/docker/templates/outputs/web/package-lock.json

@@ -15,7 +15,7 @@
         "date-fns": "^4.1.0",
         "embla-carousel-react": "^8.6.0",
         "lucide-react": "^0.562.0",
-        "next": "16.1.7",
+        "next": "16.1.5",
         "next-themes": "^0.4.6",
         "radix-ui": "^1.4.3",
         "react": "19.2.3",
@@ -1711,9 +1711,9 @@
       }
     },
     "node_modules/@next/env": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.7.tgz",
-      "integrity": "sha512-rJJbIdJB/RQr2F1nylZr/PJzamvNNhfr3brdKP6s/GW850jbtR70QlSfFselvIBbcPUOlQwBakexjFzqLzF6pg==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.5.tgz",
+      "integrity": "sha512-CRSCPJiSZoi4Pn69RYBDI9R7YK2g59vLexPQFXY0eyw+ILevIenCywzg+DqmlBik9zszEnw2HLFOUlLAcJbL7g==",
       "license": "MIT"
     },
     "node_modules/@next/eslint-plugin-next": {
@@ -1727,9 +1727,9 @@
       }
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.1.7.tgz",
-      "integrity": "sha512-b2wWIE8sABdyafc4IM8r5Y/dS6kD80JRtOGrUiKTsACFQfWWgUQ2NwoUX1yjFMXVsAwcQeNpnucF2ZrujsBBPg==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.1.5.tgz",
+      "integrity": "sha512-eK7Wdm3Hjy/SCL7TevlH0C9chrpeOYWx2iR7guJDaz4zEQKWcS1IMVfMb9UKBFMg1XgzcPTYPIp1Vcpukkjg6Q==",
       "cpu": [
         "arm64"
       ],
@@ -1743,9 +1743,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.1.7.tgz",
-      "integrity": "sha512-zcnVaaZulS1WL0Ss38R5Q6D2gz7MtBu8GZLPfK+73D/hp4GFMrC2sudLky1QibfV7h6RJBJs/gOFvYP0X7UVlQ==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.1.5.tgz",
+      "integrity": "sha512-foQscSHD1dCuxBmGkbIr6ScAUF6pRoDZP6czajyvmXPAOFNnQUJu2Os1SGELODjKp/ULa4fulnBWoHV3XdPLfA==",
       "cpu": [
         "x64"
       ],
@@ -1759,9 +1759,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.1.7.tgz",
-      "integrity": "sha512-2ant89Lux/Q3VyC8vNVg7uBaFVP9SwoK2jJOOR0L8TQnX8CAYnh4uctAScy2Hwj2dgjVHqHLORQZJ2wH6VxhSQ==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.1.5.tgz",
+      "integrity": "sha512-qNIb42o3C02ccIeSeKjacF3HXotGsxh/FMk/rSRmCzOVMtoWH88odn2uZqF8RLsSUWHcAqTgYmPD3pZ03L9ZAA==",
       "cpu": [
         "arm64"
       ],
@@ -1775,9 +1775,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.1.7.tgz",
-      "integrity": "sha512-uufcze7LYv0FQg9GnNeZ3/whYfo+1Q3HnQpm16o6Uyi0OVzLlk2ZWoY7j07KADZFY8qwDbsmFnMQP3p3+Ftprw==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.1.5.tgz",
+      "integrity": "sha512-U+kBxGUY1xMAzDTXmuVMfhaWUZQAwzRaHJ/I6ihtR5SbTVUEaDRiEU9YMjy1obBWpdOBuk1bcm+tsmifYSygfw==",
       "cpu": [
         "arm64"
       ],
@@ -1791,9 +1791,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.1.7.tgz",
-      "integrity": "sha512-KWVf2gxYvHtvuT+c4MBOGxuse5TD7DsMFYSxVxRBnOzok/xryNeQSjXgxSv9QpIVlaGzEn/pIuI6Koosx8CGWA==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.1.5.tgz",
+      "integrity": "sha512-gq2UtoCpN7Ke/7tKaU7i/1L7eFLfhMbXjNghSv0MVGF1dmuoaPeEVDvkDuO/9LVa44h5gqpWeJ4mRRznjDv7LA==",
       "cpu": [
         "x64"
       ],
@@ -1807,9 +1807,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.1.7.tgz",
-      "integrity": "sha512-HguhaGwsGr1YAGs68uRKc4aGWxLET+NevJskOcCAwXbwj0fYX0RgZW2gsOCzr9S11CSQPIkxmoSbuVaBp4Z3dA==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.1.5.tgz",
+      "integrity": "sha512-bQWSE729PbXT6mMklWLf8dotislPle2L70E9q6iwETYEOt092GDn0c+TTNj26AjmeceSsC4ndyGsK5nKqHYXjQ==",
       "cpu": [
         "x64"
       ],
@@ -1823,9 +1823,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.1.7.tgz",
-      "integrity": "sha512-S0n3KrDJokKTeFyM/vGGGR8+pCmXYrjNTk2ZozOL1C/JFdfUIL9O1ATaJOl5r2POe56iRChbsszrjMAdWSv7kQ==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.1.5.tgz",
+      "integrity": "sha512-LZli0anutkIllMtTAWZlDqdfvjWX/ch8AFK5WgkNTvaqwlouiD1oHM+WW8RXMiL0+vAkAJyAGEzPPjO+hnrSNQ==",
       "cpu": [
         "arm64"
       ],
@@ -1839,9 +1839,9 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.1.7.tgz",
-      "integrity": "sha512-mwgtg8CNZGYm06LeEd+bNnOUfwOyNem/rOiP14Lsz+AnUY92Zq/LXwtebtUiaeVkhbroRCQ0c8GlR4UT1U+0yg==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.1.5.tgz",
+      "integrity": "sha512-7is37HJTNQGhjPpQbkKjKEboHYQnCgpVt/4rBrrln0D9nderNxZ8ZWs8w1fAtzUx7wEyYjQ+/13myFgFj6K2Ng==",
       "cpu": [
         "x64"
       ],
@@ -4971,15 +4971,12 @@
       "license": "MIT"
     },
     "node_modules/baseline-browser-mapping": {
-      "version": "2.10.8",
-      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.8.tgz",
-      "integrity": "sha512-PCLz/LXGBsNTErbtB6i5u4eLpHeMfi93aUv5duMmj6caNu6IphS4q6UevDnL36sZQv9lrP11dbPKGMaXPwMKfQ==",
+      "version": "2.9.17",
+      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.17.tgz",
+      "integrity": "sha512-agD0MgJFUP/4nvjqzIB29zRPUuCF7Ge6mEv9s8dHrtYD7QWXRcx75rOADE/d5ah1NI+0vkDl0yorDd5U852IQQ==",
       "license": "Apache-2.0",
       "bin": {
-        "baseline-browser-mapping": "dist/cli.cjs"
-      },
-      "engines": {
-        "node": ">=6.0.0"
+        "baseline-browser-mapping": "dist/cli.js"
       }
     },
     "node_modules/body-parser": {
@@ -6978,9 +6975,9 @@
       }
     },
     "node_modules/flatted": {
-      "version": "3.4.2",
-      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz",
-      "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==",
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz",
+      "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==",
       "dev": true,
       "license": "ISC"
     },
@@ -8978,14 +8975,14 @@
       }
     },
     "node_modules/next": {
-      "version": "16.1.7",
-      "resolved": "https://registry.npmjs.org/next/-/next-16.1.7.tgz",
-      "integrity": "sha512-WM0L7WrSvKwoLegLYr6V+mz+RIofqQgVAfHhMp9a88ms0cFX8iX9ew+snpWlSBwpkURJOUdvCEt3uLl3NNzvWg==",
+      "version": "16.1.5",
+      "resolved": "https://registry.npmjs.org/next/-/next-16.1.5.tgz",
+      "integrity": "sha512-f+wE+NSbiQgh3DSAlTaw2FwY5yGdVViAtp8TotNQj4kk4Q8Bh1sC/aL9aH+Rg1YAVn18OYXsRDT7U/079jgP7w==",
       "license": "MIT",
       "dependencies": {
-        "@next/env": "16.1.7",
+        "@next/env": "16.1.5",
         "@swc/helpers": "0.5.15",
-        "baseline-browser-mapping": "^2.9.19",
+        "baseline-browser-mapping": "^2.8.3",
         "caniuse-lite": "^1.0.30001579",
         "postcss": "8.4.31",
         "styled-jsx": "5.1.6"
@@ -8997,14 +8994,14 @@
         "node": ">=20.9.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "16.1.7",
-        "@next/swc-darwin-x64": "16.1.7",
-        "@next/swc-linux-arm64-gnu": "16.1.7",
-        "@next/swc-linux-arm64-musl": "16.1.7",
-        "@next/swc-linux-x64-gnu": "16.1.7",
-        "@next/swc-linux-x64-musl": "16.1.7",
-        "@next/swc-win32-arm64-msvc": "16.1.7",
-        "@next/swc-win32-x64-msvc": "16.1.7",
+        "@next/swc-darwin-arm64": "16.1.5",
+        "@next/swc-darwin-x64": "16.1.5",
+        "@next/swc-linux-arm64-gnu": "16.1.5",
+        "@next/swc-linux-arm64-musl": "16.1.5",
+        "@next/swc-linux-x64-gnu": "16.1.5",
+        "@next/swc-linux-x64-musl": "16.1.5",
+        "@next/swc-win32-arm64-msvc": "16.1.5",
+        "@next/swc-win32-x64-msvc": "16.1.5",
         "sharp": "^0.34.4"
       },
       "peerDependencies": {

backend/onyx/server/features/build/sandbox/kubernetes/docker/templates/outputs/web/package.json

@@ -16,7 +16,7 @@
     "date-fns": "^4.1.0",
     "embla-carousel-react": "^8.6.0",
     "lucide-react": "^0.562.0",
-    "next": "16.1.7",
+    "next": "16.1.5",
     "next-themes": "^0.4.6",
     "radix-ui": "^1.4.3",
     "react": "19.2.3",