nit

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jamison Lahman <jamison@lahman.dev>

.github/workflows/deployment.yml

@@ -615,6 +615,7 @@ jobs:
           tags: |
             type=raw,value=${{ needs.determine-builds.outputs.is-test-run == 'true' && format('web-{0}', needs.determine-builds.outputs.sanitized-tag) || github.ref_name }}
             type=raw,value=${{ needs.determine-builds.outputs.is-test-run != 'true' && needs.determine-builds.outputs.is-latest == 'true' && 'latest' || '' }}
+            type=raw,value=${{ needs.determine-builds.outputs.is-test-run != 'true' && needs.determine-builds.outputs.is-latest == 'true' && 'craft-latest' || '' }}
             type=raw,value=${{ needs.determine-builds.outputs.is-test-run != 'true' && env.EDGE_TAG == 'true' && 'edge' || '' }}
             type=raw,value=${{ needs.determine-builds.outputs.is-test-run != 'true' && needs.determine-builds.outputs.is-beta == 'true' && 'beta' || '' }}
 
@@ -703,6 +704,9 @@ jobs:
             NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=true
             NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
             NODE_OPTIONS=--max-old-space-size=8192
+            SENTRY_RELEASE=${{ github.sha }}
+          secrets: |
+            sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }}
           cache-from: |
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-amd64
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
@@ -785,6 +789,9 @@ jobs:
             NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=true
             NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
             NODE_OPTIONS=--max-old-space-size=8192
+            SENTRY_RELEASE=${{ github.sha }}
+          secrets: |
+            sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }}
           cache-from: |
             type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-arm64
             type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
@@ -1263,8 +1270,6 @@ jobs:
             latest=false
           tags: |
             type=raw,value=craft-latest
-            # TODO: Consider aligning craft-latest tags with regular backend builds (e.g., latest, edge, beta)
-            # to keep tagging strategy consistent across all backend images
 
       - name: Create and push manifest
         env:
@@ -1488,6 +1493,7 @@ jobs:
           tags: |
             type=raw,value=${{ needs.determine-builds.outputs.is-test-run == 'true' && format('model-server-{0}', needs.determine-builds.outputs.sanitized-tag) || github.ref_name }}
             type=raw,value=${{ needs.determine-builds.outputs.is-test-run != 'true' && needs.determine-builds.outputs.is-latest == 'true' && 'latest' || '' }}
+            type=raw,value=${{ needs.determine-builds.outputs.is-test-run != 'true' && needs.determine-builds.outputs.is-latest == 'true' && 'craft-latest' || '' }}
             type=raw,value=${{ needs.determine-builds.outputs.is-test-run != 'true' && env.EDGE_TAG == 'true' && 'edge' || '' }}
             type=raw,value=${{ needs.determine-builds.outputs.is-test-run != 'true' && needs.determine-builds.outputs.is-beta-standalone == 'true' && 'beta' || '' }}
 
@@ -1503,232 +1509,105 @@ jobs:
             $(printf '%s\n' "${META_TAGS}" | xargs -I {} echo -t {}) \
             $IMAGES
 
-  trivy-scan-web:
+  trivy-scan:
     needs:
       - determine-builds
       - merge-web
-    if: needs.merge-web.result == 'success'
-    runs-on:
-      - runs-on
-      - runner=2cpu-linux-arm64
-      - run-id=${{ github.run_id }}-trivy-scan-web
-      - extras=ecr-cache
-    timeout-minutes: 90
-    environment: release
-    env:
-      REGISTRY_IMAGE: onyxdotapp/onyx-web-server
-    steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
-        with:
-          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
-          aws-region: us-east-2
-
-      - name: Get AWS Secrets
-        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
-        with:
-          secret-ids: |
-            DOCKER_USERNAME, deploy/docker-username
-            DOCKER_TOKEN, deploy/docker-token
-          parse-json-secrets: true
-
-      - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            if [ "${{ needs.determine-builds.outputs.is-test-run }}" == "true" ]; then
-              SCAN_IMAGE="${{ env.RUNS_ON_ECR_CACHE }}:web-${{ needs.determine-builds.outputs.sanitized-tag }}"
-            else
-              SCAN_IMAGE="docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}"
-            fi
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ env.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ env.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              ${SCAN_IMAGE}
-
-  trivy-scan-web-cloud:
-    needs:
-      - determine-builds
       - merge-web-cloud
-    if: needs.merge-web-cloud.result == 'success'
-    runs-on:
-      - runs-on
-      - runner=2cpu-linux-arm64
-      - run-id=${{ github.run_id }}-trivy-scan-web-cloud
-      - extras=ecr-cache
-    timeout-minutes: 90
-    environment: release
-    env:
-      REGISTRY_IMAGE: onyxdotapp/onyx-web-server-cloud
-    steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
-        with:
-          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
-          aws-region: us-east-2
-
-      - name: Get AWS Secrets
-        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
-        with:
-          secret-ids: |
-            DOCKER_USERNAME, deploy/docker-username
-            DOCKER_TOKEN, deploy/docker-token
-          parse-json-secrets: true
-
-      - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            if [ "${{ needs.determine-builds.outputs.is-test-run }}" == "true" ]; then
-              SCAN_IMAGE="${{ env.RUNS_ON_ECR_CACHE }}:web-cloud-${{ needs.determine-builds.outputs.sanitized-tag }}"
-            else
-              SCAN_IMAGE="docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}"
-            fi
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ env.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ env.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              ${SCAN_IMAGE}
-
-  trivy-scan-backend:
-    needs:
-      - determine-builds
       - merge-backend
-    if: needs.merge-backend.result == 'success'
+      - merge-model-server
+    if: >-
+      always() && !cancelled() &&
+      (needs.merge-web.result == 'success' ||
+       needs.merge-web-cloud.result == 'success' ||
+       needs.merge-backend.result == 'success' ||
+       needs.merge-model-server.result == 'success')
     runs-on:
       - runs-on
       - runner=2cpu-linux-arm64
-      - run-id=${{ github.run_id }}-trivy-scan-backend
+      - run-id=${{ github.run_id }}-trivy-scan-${{ matrix.component }}
       - extras=ecr-cache
-    timeout-minutes: 90
-    environment: release
-    env:
-      REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-backend-cloud' || 'onyxdotapp/onyx-backend' }}
+    permissions:
+      security-events: write # needed for SARIF uploads
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - component: web
+            registry-image: onyxdotapp/onyx-web-server
+          - component: web-cloud
+            registry-image: onyxdotapp/onyx-web-server-cloud
+          - component: backend
+            registry-image: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-backend-cloud' || 'onyxdotapp/onyx-backend' }}
+            trivyignore: backend/.trivyignore
+          - component: model-server
+            registry-image: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-model-server-cloud' || 'onyxdotapp/onyx-model-server' }}
     steps:
+      - name: Check if this scan should run
+        id: should-run
+        run: |
+          case "$COMPONENT" in
+            web) RESULT="$MERGE_WEB" ;;
+            web-cloud) RESULT="$MERGE_WEB_CLOUD" ;;
+            backend) RESULT="$MERGE_BACKEND" ;;
+            model-server) RESULT="$MERGE_MODEL_SERVER" ;;
+          esac
+          if [ "$RESULT" == "success" ]; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+        env:
+          COMPONENT: ${{ matrix.component }}
+          MERGE_WEB: ${{ needs.merge-web.result }}
+          MERGE_WEB_CLOUD: ${{ needs.merge-web-cloud.result }}
+          MERGE_BACKEND: ${{ needs.merge-backend.result }}
+          MERGE_MODEL_SERVER: ${{ needs.merge-model-server.result }}
+
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
+        if: steps.should-run.outputs.run == 'true'
 
       - name: Checkout
+        if: steps.should-run.outputs.run == 'true' && matrix.trivyignore != ''
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
         with:
           persist-credentials: false
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
-        with:
-          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
-          aws-region: us-east-2
-
-      - name: Get AWS Secrets
-        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
-        with:
-          secret-ids: |
-            DOCKER_USERNAME, deploy/docker-username
-            DOCKER_TOKEN, deploy/docker-token
-          parse-json-secrets: true
+      - name: Determine scan image
+        if: steps.should-run.outputs.run == 'true'
+        id: scan-image
+        run: |
+          if [ "$IS_TEST_RUN" == "true" ]; then
+            echo "image=${RUNS_ON_ECR_CACHE}:${TAG_PREFIX}-${SANITIZED_TAG}" >> "$GITHUB_OUTPUT"
+          else
+            echo "image=docker.io/${REGISTRY_IMAGE}:${REF_NAME}" >> "$GITHUB_OUTPUT"
+          fi
+        env:
+          IS_TEST_RUN: ${{ needs.determine-builds.outputs.is-test-run }}
+          TAG_PREFIX: ${{ matrix.component }}
+          SANITIZED_TAG: ${{ needs.determine-builds.outputs.sanitized-tag }}
+          REGISTRY_IMAGE: ${{ matrix.registry-image }}
+          REF_NAME: ${{ github.ref_name }}
 
       - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
+        if: steps.should-run.outputs.run == 'true'
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # ratchet:aquasecurity/trivy-action@v0.35.0
         with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            if [ "${{ needs.determine-builds.outputs.is-test-run }}" == "true" ]; then
-              SCAN_IMAGE="${{ env.RUNS_ON_ECR_CACHE }}:backend-${{ needs.determine-builds.outputs.sanitized-tag }}"
-            else
-              SCAN_IMAGE="docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}"
-            fi
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -v ${{ github.workspace }}/backend/.trivyignore:/tmp/.trivyignore:ro \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ env.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ env.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              --ignorefile /tmp/.trivyignore \
-              ${SCAN_IMAGE}
+          image-ref: ${{ steps.scan-image.outputs.image }}
+          severity: CRITICAL,HIGH
+          format: "sarif"
+          output: "trivy-results.sarif"
+          trivyignores: ${{ matrix.trivyignore }}
+        env:
+          TRIVY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          TRIVY_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
 
-  trivy-scan-model-server:
-    needs:
-      - determine-builds
-      - merge-model-server
-    if: needs.merge-model-server.result == 'success'
-    runs-on:
-      - runs-on
-      - runner=2cpu-linux-arm64
-      - run-id=${{ github.run_id }}-trivy-scan-model-server
-      - extras=ecr-cache
-    timeout-minutes: 90
-    environment: release
-    env:
-      REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-model-server-cloud' || 'onyxdotapp/onyx-model-server' }}
-    steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
+      - name: Upload Trivy scan results to GitHub Security tab
+        if: steps.should-run.outputs.run == 'true'
+        uses: github/codeql-action/upload-sarif@ba454b8ab46733eb6145342877cd148270bb77ab
         with:
-          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
-          aws-region: us-east-2
-
-      - name: Get AWS Secrets
-        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
-        with:
-          secret-ids: |
-            DOCKER_USERNAME, deploy/docker-username
-            DOCKER_TOKEN, deploy/docker-token
-          parse-json-secrets: true
-
-      - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            if [ "${{ needs.determine-builds.outputs.is-test-run }}" == "true" ]; then
-              SCAN_IMAGE="${{ env.RUNS_ON_ECR_CACHE }}:model-server-${{ needs.determine-builds.outputs.sanitized-tag }}"
-            else
-              SCAN_IMAGE="docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}"
-            fi
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ env.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ env.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              ${SCAN_IMAGE}
+          sarif_file: "trivy-results.sarif"
 
   notify-slack-on-failure:
     needs:

.github/workflows/helm-chart-releases.yml

@@ -47,7 +47,8 @@ jobs:
           done
 
       - name: Publish Helm charts to gh-pages
-        uses: stefanprodan/helm-gh-pages@0ad2bb377311d61ac04ad9eb6f252fb68e207260 # ratchet:stefanprodan/helm-gh-pages@v1.7.0
+        # NOTE: HEAD of https://github.com/stefanprodan/helm-gh-pages/pull/43
+        uses: stefanprodan/helm-gh-pages@ad32ad3b8720abfeaac83532fd1e9bdfca5bbe27 # zizmor: ignore[impostor-commit]
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           charts_dir: deployment/helm/charts

.github/workflows/nightly-llm-provider-chat.yml

@@ -35,6 +35,7 @@ jobs:
     needs: [provider-chat-test]
     if: failure() && github.event_name == 'schedule'
     runs-on: ubuntu-slim
+    environment: ci-protected
     timeout-minutes: 5
     steps:
       - name: Checkout

.github/workflows/post-merge-beta-cherry-pick.yml

@@ -183,6 +183,7 @@ jobs:
       - cherry-pick-to-latest-release
     if: needs.resolve-cherry-pick-request.outputs.should_cherrypick == 'true' && needs.resolve-cherry-pick-request.result == 'success' && needs.cherry-pick-to-latest-release.result == 'success'
     runs-on: ubuntu-slim
+    environment: ci-protected
     timeout-minutes: 10
     steps:
       - name: Checkout
@@ -232,6 +233,7 @@ jobs:
       - cherry-pick-to-latest-release
     if: always() && needs.resolve-cherry-pick-request.outputs.should_cherrypick == 'true' && (needs.resolve-cherry-pick-request.result == 'failure' || needs.cherry-pick-to-latest-release.result == 'failure')
     runs-on: ubuntu-slim
+    environment: ci-protected
     timeout-minutes: 10
     steps:
       - name: Checkout

.github/workflows/pr-desktop-build.yml

@@ -63,7 +63,7 @@ jobs:
           targets: ${{ matrix.target }}
 
       - name: Cache Cargo registry and build
-        uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # zizmor: ignore[cache-poisoning]
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # zizmor: ignore[cache-poisoning]
         with:
           path: |
             ~/.cargo/bin/

.github/workflows/pr-helm-chart-testing.yml

@@ -41,7 +41,7 @@ jobs:
           version: v3.19.0
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@b5eebdd9998021f29756c53432f48dab66394810
+        uses: helm/chart-testing-action@2e2940618cb426dce2999631d543b53cdcfc8527
         with:
           uv_version: "0.9.9"
 

.github/workflows/pr-playwright-tests.yml

@@ -284,7 +284,7 @@ jobs:
 
       - name: Cache playwright cache
         # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
-        uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
+        uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed # ratchet:runs-on/cache@v4
         with:
           path: ~/.cache/ms-playwright
           key: ${{ runner.os }}-playwright-npm-${{ hashFiles('web/package-lock.json') }}
@@ -626,7 +626,7 @@ jobs:
 
       - name: Cache playwright cache
         # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
-        uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
+        uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed # ratchet:runs-on/cache@v4
         with:
           path: ~/.cache/ms-playwright
           key: ${{ runner.os }}-playwright-npm-${{ hashFiles('web/package-lock.json') }}

.github/workflows/pr-python-checks.yml

@@ -56,7 +56,7 @@ jobs:
 
       - name: Cache mypy cache
         if: ${{ vars.DISABLE_MYPY_CACHE != 'true' }}
-        uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
+        uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed # ratchet:runs-on/cache@v4
         with:
           path: .mypy_cache
           key: mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-${{ hashFiles('**/*.py', '**/*.pyi', 'pyproject.toml') }}

.github/workflows/pr-python-connector-tests.yml

@@ -22,132 +22,40 @@ on:
     - cron: "0 16 * * *"
 
 permissions:
+  id-token: write # Required for OIDC-based AWS credential exchange
   contents: read
 
 env:
-  # AWS
-  AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }}
-  AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }}
-
-  # Cloudflare R2
+  PYTHONPATH: ./backend
+  DISABLE_TELEMETRY: "true"
   R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS: ${{ vars.R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS }}
-  R2_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.R2_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }}
-  R2_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.R2_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }}
-
-  # Google Cloud Storage
-  GCS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.GCS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }}
-  GCS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.GCS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }}
-
-  # Confluence
   CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_TEST_SPACE: ${{ vars.CONFLUENCE_TEST_SPACE }}
-  CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }}
   CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }}
-  CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
-  CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }}
-
-  # Jira
-  JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
-  JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
-  JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
-  JIRA_API_TOKEN_SCOPED: ${{ secrets.JIRA_API_TOKEN_SCOPED }}
-
-  # Gong
-  GONG_ACCESS_KEY: ${{ secrets.GONG_ACCESS_KEY }}
-  GONG_ACCESS_KEY_SECRET: ${{ secrets.GONG_ACCESS_KEY_SECRET }}
-
-  # Google
-  GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR }}
-  GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR_TEST_USER_1: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR_TEST_USER_1 }}
-  GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR }}
-  GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR }}
-  GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR }}
-
-  # Slab
-  SLAB_BOT_TOKEN: ${{ secrets.SLAB_BOT_TOKEN }}
-
-  # Zendesk
-  ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }}
-  ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }}
-  ZENDESK_TOKEN: ${{ secrets.ZENDESK_TOKEN }}
-
-  # Salesforce
   SF_USERNAME: ${{ vars.SF_USERNAME }}
-  SF_PASSWORD: ${{ secrets.SF_PASSWORD }}
-  SF_SECURITY_TOKEN: ${{ secrets.SF_SECURITY_TOKEN }}
-
-  # Hubspot
-  HUBSPOT_ACCESS_TOKEN: ${{ secrets.HUBSPOT_ACCESS_TOKEN }}
-
-  # IMAP
   IMAP_HOST: ${{ vars.IMAP_HOST }}
   IMAP_USERNAME: ${{ vars.IMAP_USERNAME }}
-  IMAP_PASSWORD: ${{ secrets.IMAP_PASSWORD }}
   IMAP_MAILBOXES: ${{ vars.IMAP_MAILBOXES }}
-
-  # Airtable
   AIRTABLE_TEST_BASE_ID: ${{ vars.AIRTABLE_TEST_BASE_ID }}
   AIRTABLE_TEST_TABLE_ID: ${{ vars.AIRTABLE_TEST_TABLE_ID }}
   AIRTABLE_TEST_TABLE_NAME: ${{ vars.AIRTABLE_TEST_TABLE_NAME }}
-  AIRTABLE_ACCESS_TOKEN: ${{ secrets.AIRTABLE_ACCESS_TOKEN }}
-
-  # Sharepoint
   SHAREPOINT_CLIENT_ID: ${{ vars.SHAREPOINT_CLIENT_ID }}
-  SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
   SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ vars.SHAREPOINT_CLIENT_DIRECTORY_ID }}
   SHAREPOINT_SITE: ${{ vars.SHAREPOINT_SITE }}
-  PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
-  PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
-  PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
-  PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
-
-  # Github
-  ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}
-
-  # Gitlab
-  GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }}
-
-  # Gitbook
-  GITBOOK_SPACE_ID: ${{ secrets.GITBOOK_SPACE_ID }}
-  GITBOOK_API_KEY: ${{ secrets.GITBOOK_API_KEY }}
-
-  # Notion
-  NOTION_INTEGRATION_TOKEN: ${{ secrets.NOTION_INTEGRATION_TOKEN }}
-
-  # Highspot
-  HIGHSPOT_KEY: ${{ secrets.HIGHSPOT_KEY }}
-  HIGHSPOT_SECRET: ${{ secrets.HIGHSPOT_SECRET }}
-
-  # Slack
-  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-
-  # Discord
-  DISCORD_CONNECTOR_BOT_TOKEN: ${{ secrets.DISCORD_CONNECTOR_BOT_TOKEN }}
-
-  # Teams
-  TEAMS_APPLICATION_ID: ${{ secrets.TEAMS_APPLICATION_ID }}
-  TEAMS_DIRECTORY_ID: ${{ secrets.TEAMS_DIRECTORY_ID }}
-  TEAMS_SECRET: ${{ secrets.TEAMS_SECRET }}
-
-  # Bitbucket
-  BITBUCKET_WORKSPACE: ${{ secrets.BITBUCKET_WORKSPACE }}
-  BITBUCKET_REPOSITORIES: ${{ secrets.BITBUCKET_REPOSITORIES }}
-  BITBUCKET_PROJECTS: ${{ secrets.BITBUCKET_PROJECTS }}
   BITBUCKET_EMAIL: ${{ vars.BITBUCKET_EMAIL }}
-  BITBUCKET_API_TOKEN: ${{ secrets.BITBUCKET_API_TOKEN }}
-
-  # Fireflies
-  FIREFLIES_API_KEY: ${{ secrets.FIREFLIES_API_KEY }}
 
 jobs:
   connectors-check:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-connectors-check", "extras=s3-cache"]
+    runs-on:
+      [
+        runs-on,
+        runner=8cpu-linux-x64,
+        "run-id=${{ github.run_id }}-connectors-check",
+        "extras=s3-cache",
+      ]
     timeout-minutes: 45
-
-    env:
-      PYTHONPATH: ./backend
-      DISABLE_TELEMETRY: "true"
+    environment: ci-protected
 
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
@@ -188,6 +96,66 @@ jobs:
               - 'backend/onyx/file_processing/**'
               - 'uv.lock'
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # ratchet:aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+
+      - name: Get connector test secrets from AWS Secrets Manager
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802 # ratchet:aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+          parse-json-secrets: false
+          secret-ids: |
+            AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS, test/aws-access-key-id
+            AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS, test/aws-secret-access-key
+            R2_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS, test/r2-access-key-id
+            R2_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS, test/r2-secret-access-key
+            GCS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS, test/gcs-access-key-id
+            GCS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS, test/gcs-secret-access-key
+            CONFLUENCE_ACCESS_TOKEN, test/confluence-access-token
+            CONFLUENCE_ACCESS_TOKEN_SCOPED, test/confluence-access-token-scoped
+            JIRA_BASE_URL, test/jira-base-url
+            JIRA_USER_EMAIL, test/jira-user-email
+            JIRA_API_TOKEN, test/jira-api-token
+            JIRA_API_TOKEN_SCOPED, test/jira-api-token-scoped
+            GONG_ACCESS_KEY, test/gong-access-key
+            GONG_ACCESS_KEY_SECRET, test/gong-access-key-secret
+            GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR, test/google-drive-service-account-json
+            GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR_TEST_USER_1, test/google-drive-oauth-creds-test-user-1
+            GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR, test/google-drive-oauth-creds
+            GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR, test/google-gmail-service-account-json
+            GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR, test/google-gmail-oauth-creds
+            SLAB_BOT_TOKEN, test/slab-bot-token
+            ZENDESK_SUBDOMAIN, test/zendesk-subdomain
+            ZENDESK_EMAIL, test/zendesk-email
+            ZENDESK_TOKEN, test/zendesk-token
+            SF_PASSWORD, test/sf-password
+            SF_SECURITY_TOKEN, test/sf-security-token
+            HUBSPOT_ACCESS_TOKEN, test/hubspot-access-token
+            IMAP_PASSWORD, test/imap-password
+            AIRTABLE_ACCESS_TOKEN, test/airtable-access-token
+            SHAREPOINT_CLIENT_SECRET, test/sharepoint-client-secret
+            PERM_SYNC_SHAREPOINT_CLIENT_ID, test/perm-sync-sharepoint-client-id
+            PERM_SYNC_SHAREPOINT_PRIVATE_KEY, test/perm-sync-sharepoint-private-key
+            PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD, test/perm-sync-sharepoint-cert-password
+            PERM_SYNC_SHAREPOINT_DIRECTORY_ID, test/perm-sync-sharepoint-directory-id
+            ACCESS_TOKEN_GITHUB, test/github-access-token
+            GITLAB_ACCESS_TOKEN, test/gitlab-access-token
+            GITBOOK_SPACE_ID, test/gitbook-space-id
+            GITBOOK_API_KEY, test/gitbook-api-key
+            NOTION_INTEGRATION_TOKEN, test/notion-integration-token
+            HIGHSPOT_KEY, test/highspot-key
+            HIGHSPOT_SECRET, test/highspot-secret
+            SLACK_BOT_TOKEN, test/slack-bot-token
+            DISCORD_CONNECTOR_BOT_TOKEN, test/discord-bot-token
+            TEAMS_APPLICATION_ID, test/teams-application-id
+            TEAMS_DIRECTORY_ID, test/teams-directory-id
+            TEAMS_SECRET, test/teams-secret
+            BITBUCKET_WORKSPACE, test/bitbucket-workspace
+            BITBUCKET_API_TOKEN, test/bitbucket-api-token
+            FIREFLIES_API_KEY, test/fireflies-api-key
+
       - name: Run Tests (excluding HubSpot, Salesforce, GitHub, and Coda)
         shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
         run: |

.github/workflows/pr-python-model-tests.yml

@@ -31,6 +31,7 @@ jobs:
       - runner=4cpu-linux-arm64
       - "run-id=${{ github.run_id }}-model-check"
       - "extras=ecr-cache"
+    environment: ci-protected
     timeout-minutes: 45
 
     env:

.github/workflows/release-cli.yml

@@ -13,15 +13,6 @@ jobs:
     permissions:
       id-token: write
     timeout-minutes: 10
-    strategy:
-      matrix:
-        os-arch:
-          - { goos: "linux", goarch: "amd64" }
-          - { goos: "linux", goarch: "arm64" }
-          - { goos: "windows", goarch: "amd64" }
-          - { goos: "windows", goarch: "arm64" }
-          - { goos: "darwin", goarch: "amd64" }
-          - { goos: "darwin", goarch: "arm64" }
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
         with:
@@ -31,9 +22,11 @@ jobs:
           enable-cache: false
           version: "0.9.9"
       - run: |
-          GOOS="${{ matrix.os-arch.goos }}" \
-          GOARCH="${{ matrix.os-arch.goarch }}" \
-          uv build --wheel
+          for goos in linux windows darwin; do
+            for goarch in amd64 arm64; do
+              GOOS="$goos" GOARCH="$goarch" uv build --wheel
+            done
+          done
         working-directory: cli
       - run: uv publish
         working-directory: cli

.github/workflows/storybook-deploy.yml

@@ -25,6 +25,7 @@ permissions:
 jobs:
   Deploy-Storybook:
     runs-on: ubuntu-latest
+    environment: ci-protected
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v4
@@ -54,6 +55,7 @@ jobs:
     needs: Deploy-Storybook
     if: always() && needs.Deploy-Storybook.result == 'failure'
     runs-on: ubuntu-latest
+    environment: ci-protected
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v4

.github/workflows/sync_foss.yml

@@ -9,6 +9,7 @@ on:
 jobs:
   sync-foss:
     runs-on: ubuntu-latest
+    environment: ci-protected
     timeout-minutes: 45
     permissions:
       contents: read

.github/workflows/tag-nightly.yml

@@ -11,6 +11,7 @@ permissions:
 jobs:
   create-and-push-tag:
     runs-on: ubuntu-slim
+    environment: ci-protected
     timeout-minutes: 45
 
     steps:

.greptile/config.json

@@ -0,0 +1,64 @@
+{
+    "labels": [],
+    "comment": "",
+    "fixWithAI": true,
+    "hideFooter": false,
+    "strictness": 3,
+    "statusCheck": true,
+    "commentTypes": [
+      "logic",
+      "syntax",
+      "style"
+    ],
+    "instructions": "",
+    "disabledLabels": [],
+    "excludeAuthors": [
+      "dependabot[bot]",
+      "renovate[bot]"
+    ],
+    "ignoreKeywords": "",
+    "ignorePatterns": "",
+    "includeAuthors": [],
+    "summarySection": {
+      "included": true,
+      "collapsible": false,
+      "defaultOpen": false
+    },
+    "excludeBranches": [],
+    "fileChangeLimit": 300,
+    "includeBranches": [],
+    "includeKeywords": "",
+    "triggerOnUpdates": true,
+    "updateExistingSummaryComment": true,
+    "updateSummaryOnly": false,
+    "issuesTableSection": {
+      "included": true,
+      "collapsible": false,
+      "defaultOpen": false
+    },
+    "statusCommentsEnabled": true,
+    "confidenceScoreSection": {
+      "included": true,
+      "collapsible": false
+    },
+    "sequenceDiagramSection": {
+      "included": true,
+      "collapsible": false,
+      "defaultOpen": false
+    },
+    "shouldUpdateDescription": false,
+    "rules": [
+      {
+        "scope": ["web/**"],
+        "rule": "In Onyx's Next.js app, the `app/ee/admin/` directory is a filesystem convention for Enterprise Edition route overrides — it does NOT add an `/ee/` prefix to the URL. Both `app/admin/groups/page.tsx` and `app/ee/admin/groups/page.tsx` serve the same URL `/admin/groups`. Hardcoded `/admin/...` paths in router.push() calls are correct and do NOT break EE deployments. Do not flag hardcoded admin paths as bugs."
+      },
+      {
+        "scope": ["web/**"],
+        "rule": "In Onyx, each API key creates a unique user row in the database with a unique `user_id` (UUID). There is a 1:1 mapping between API keys and their backing user records. Multiple API keys do NOT share the same `user_id`. Do not flag potential duplicate row IDs when using `user_id` from API key descriptors."
+      },
+      {
+        "scope": ["backend/**/*.py"],
+        "rule": "Never raise HTTPException directly in business code. Use `raise OnyxError(OnyxErrorCode.XXX, \"message\")` from `onyx.error_handling.exceptions`. A global FastAPI exception handler converts OnyxError into structured JSON responses with {\"error_code\": \"...\", \"detail\": \"...\"}. Error codes are defined in `onyx.error_handling.error_codes.OnyxErrorCode`. For upstream errors with dynamic HTTP status codes, use `status_code_override`: `raise OnyxError(OnyxErrorCode.BAD_GATEWAY, detail, status_code_override=upstream_status)`."
+      }
+    ]
+}

.greptile/files.json

@@ -0,0 +1,57 @@
+[
+  {
+    "scope": [],
+    "path": "contributing_guides/best_practices.md",
+    "description": "Best practices for contributing to the codebase"
+  },
+  {
+    "scope": ["web/**"],
+    "path": "web/AGENTS.md",
+    "description": "Frontend coding standards for the web directory"
+  },
+  {
+    "scope": ["web/**"],
+    "path": "web/tests/README.md",
+    "description": "Frontend testing guide and conventions"
+  },
+  {
+    "scope": ["web/**"],
+    "path": "web/CLAUDE.md",
+    "description": "Single source of truth for frontend coding standards"
+  },
+  {
+    "scope": ["web/**"],
+    "path": "web/lib/opal/README.md",
+    "description": "Opal component library usage guide"
+  },
+  {
+    "scope": ["backend/**"],
+    "path": "backend/tests/README.md",
+    "description": "Backend testing guide covering all 4 test types, fixtures, and conventions"
+  },
+  {
+    "scope": ["backend/onyx/connectors/**"],
+    "path": "backend/onyx/connectors/README.md",
+    "description": "Connector development guide covering design, interfaces, and required changes"
+  },
+  {
+    "scope": [],
+    "path": "CLAUDE.md",
+    "description": "Project instructions and coding standards"
+  },
+  {
+    "scope": [],
+    "path": "backend/alembic/README.md",
+    "description": "Migration guidance, including multi-tenant migration behavior"
+  },
+  {
+    "scope": [],
+    "path": "deployment/helm/charts/onyx/values-lite.yaml",
+    "description": "Lite deployment Helm values and service assumptions"
+  },
+  {
+    "scope": [],
+    "path": "deployment/docker_compose/docker-compose.onyx-lite.yml",
+    "description": "Lite deployment Docker Compose overlay and disabled service behavior"
+  }
+]

.greptile/rules.md

@@ -0,0 +1,44 @@
+# Greptile Review Rules
+
+## Type Annotations
+
+Use explicit type annotations for variables to enhance code clarity, especially when moving type hints around in the code.
+
+## Best Practices
+
+Use the "Engineering Best Practices" section of `CONTRIBUTING.md` as core review context. Prefer consistency with existing patterns, fix issues in code you touch, avoid tacking new features onto muddy interfaces, fail loudly instead of silently swallowing errors, keep code strictly typed, preserve clear state boundaries, remove duplicate or dead logic, break up overly long functions, avoid hidden import-time side effects, respect module boundaries, and favor correctness-by-construction over relying on callers to use an API correctly.
+
+## TODOs
+
+Whenever a TODO is added, there must always be an associated name or ticket with that TODO in the style of `TODO(name): ...` or `TODO(1234): ...`
+
+## Debugging Code
+
+Remove temporary debugging code before merging to production, especially tenant-specific debugging logs.
+
+## Hardcoded Booleans
+
+When hardcoding a boolean variable to a constant value, remove the variable entirely and clean up all places where it's used rather than just setting it to a constant.
+
+## Multi-tenant vs Single-tenant
+
+Code changes must consider both multi-tenant and single-tenant deployments. In multi-tenant mode, preserve tenant isolation, ensure tenant context is propagated correctly, and avoid assumptions that only hold for a single shared schema or globally shared state. In single-tenant mode, avoid introducing unnecessary tenant-specific requirements or cloud-only control-plane dependencies.
+
+## Nginx Routing — New Backend Routes
+
+Whenever a new backend route is added that does NOT start with `/api`, it must also be explicitly added to ALL nginx configs:
+
+- `deployment/helm/charts/onyx/templates/nginx-conf.yaml` (Helm/k8s)
+- `deployment/data/nginx/app.conf.template` (docker-compose dev)
+- `deployment/data/nginx/app.conf.template.prod` (docker-compose prod)
+- `deployment/data/nginx/app.conf.template.no-letsencrypt` (docker-compose no-letsencrypt)
+
+Routes not starting with `/api` are not caught by the existing `^/(api|openapi\.json)` location block and will fall through to `location /`, which proxies to the Next.js web server and returns an HTML 404. The new location block must be placed before the `/api` block. Examples of routes that need this treatment: `/scim`, `/mcp`.
+
+## Full vs Lite Deployments
+
+Code changes must consider both regular Onyx deployments and Onyx lite deployments. Lite deployments disable the vector DB, Redis, model servers, and background workers by default, use PostgreSQL-backed cache/auth/file storage, and rely on the API server to handle background work. Do not assume those services are available unless the code path is explicitly limited to full deployments.
+
+## SWR Cache Keys — Always Use SWR_KEYS Registry
+
+All `useSWR()` calls and `mutate()` calls in the frontend must reference the centralized `SWR_KEYS` registry in `web/src/lib/swr-keys.ts` instead of inline endpoint strings or local string constants. Never write `useSWR("/api/some/endpoint", ...)` or `mutate("/api/some/endpoint")` — always use the corresponding `SWR_KEYS.someEndpoint` constant. If the endpoint does not yet exist in the registry, add it there first. This applies to all variants of an endpoint (e.g. query-string variants like `?get_editable=true` must also be registered as their own key).

.pre-commit-config.yaml

@@ -122,7 +122,7 @@ repos:
     rev: 5d1e709b7be35cb2025444e19de266b056b7b7ee # frozen: v2.10.1
     hooks:
       - id: golangci-lint
-        language_version: "1.26.0"
+        language_version: "1.26.1"
         entry: bash -c "find . -name go.mod -not -path './.venv/*' -print0 | xargs -0 -I{} bash -c 'cd \"$(dirname {})\" && golangci-lint run ./...'"
 
   - repo: https://github.com/astral-sh/ruff-pre-commit

.vscode/launch.json

@@ -117,7 +117,8 @@
       "presentation": {
         "group": "2"
       },
-      "consoleTitle": "API Server Console"
+      "consoleTitle": "API Server Console",
+      "justMyCode": false
     },
     {
       "name": "Slack Bot",
@@ -268,7 +269,8 @@
       "presentation": {
         "group": "2"
       },
-      "consoleTitle": "Celery heavy Console"
+      "consoleTitle": "Celery heavy Console",
+      "justMyCode": false
     },
     {
       "name": "Celery kg_processing",
@@ -355,7 +357,8 @@
       "presentation": {
         "group": "2"
       },
-      "consoleTitle": "Celery user_file_processing Console"
+      "consoleTitle": "Celery user_file_processing Console",
+      "justMyCode": false
     },
     {
       "name": "Celery docfetching",
@@ -413,7 +416,8 @@
       "presentation": {
         "group": "2"
       },
-      "consoleTitle": "Celery docprocessing Console"
+      "consoleTitle": "Celery docprocessing Console",
+      "justMyCode": false
     },
     {
       "name": "Celery beat",

AGENTS.md

@@ -357,5 +357,5 @@ raise OnyxError(OnyxErrorCode.BAD_GATEWAY, detail, status_code_override=e.respon
 ## Best Practices
 
 In addition to the other content in this file, best practices for contributing
-to the codebase can be found at `contributing_guides/best_practices.md`.
-Understand its contents and follow them.
+to the codebase can be found in the "Engineering Best Practices" section of
+`CONTRIBUTING.md`. Understand its contents and follow them.

CONTRIBUTING.md

@@ -1,32 +1,487 @@
 # Contributing to Onyx
+
 Hey there! We are so excited that you're interested in Onyx.
 
+## Table of Contents
+
+- [Contribution Opportunities](#contribution-opportunities)
+- [Contribution Process](#contribution-process)
+- [Development Setup](#development-setup)
+  - [Prerequisites](#prerequisites)
+  - [Backend: Python Requirements](#backend-python-requirements)
+  - [Frontend: Node Dependencies](#frontend-node-dependencies)
+  - [Formatting and Linting](#formatting-and-linting)
+- [Running the Application](#running-the-application)
+  - [VSCode Debugger (Recommended)](#vscode-debugger-recommended)
+  - [Manually Running for Development](#manually-running-for-development)
+  - [Running in Docker](#running-in-docker)
+- [macOS-Specific Notes](#macos-specific-notes)
+- [Engineering Best Practices](#engineering-best-practices)
+  - [Principles and Collaboration](#principles-and-collaboration)
+  - [Style and Maintainability](#style-and-maintainability)
+  - [Performance and Correctness](#performance-and-correctness)
+  - [Repository Conventions](#repository-conventions)
+- [Release Process](#release-process)
+- [Getting Help](#getting-help)
+- [Enterprise Edition Contributions](#enterprise-edition-contributions)
+
+---
 
 ## Contribution Opportunities
+
 The [GitHub Issues](https://github.com/onyx-dot-app/onyx/issues) page is a great place to look for and share contribution ideas.
 
-If you have your own feature that you would like to build please create an issue and community members can provide feedback and
-thumb it up if they feel a common need. 
+If you have your own feature that you would like to build, please create an issue and community members can provide feedback and upvote if they feel a common need.
 
+---
 
-## Contributing Code
-Please reference the documents in contributing_guides folder to ensure that the code base is kept to a high standard.
-1. dev_setup.md (start here): gives you a guide to setting up a local development environment.
-2. contribution_process.md: how to ensure you are building valuable features that will get reviewed and merged.
-3. best_practices.md: before asking for reviews, ensure your changes meet the repo code quality standards.
+## Contribution Process
 
 To contribute, please follow the
 ["fork and pull request"](https://docs.github.com/en/get-started/quickstart/contributing-to-projects) workflow.
 
+### 1. Get the feature or enhancement approved
+
+Create a GitHub issue and see if there are upvotes. If you feel the feature is sufficiently value-additive and you would like approval to contribute it to the repo, tag [Yuhong](https://github.com/yuhongsun96) to review.
+
+If you do not get a response within a week, feel free to email yuhong@onyx.app and include the issue in the message.
+
+Not all small features and enhancements will be accepted as there is a balance between feature richness and bloat. We strive to provide the best user experience possible so we have to be intentional about what we include in the app.
+
+### 2. Get the design approved
+
+The Onyx team will either provide a design doc and PRD for the feature or request one from you, the contributor. The scope and detail of the design will depend on the individual feature.
+
+### 3. IP attribution for EE contributions
+
+If you are contributing features to Onyx Enterprise Edition, you are required to sign the [IP Assignment Agreement](contributor_ip_assignment/EE_Contributor_IP_Assignment_Agreement.md).
+
+### 4. Review and testing
+
+Your features must pass all tests and all comments must be addressed prior to merging.
+
+### Implicit agreements
+
+If we approve an issue, we are promising you the following:
+- Your work will receive timely attention and we will put aside other important items to ensure you are not blocked.
+- You will receive necessary coaching on eng quality, system design, etc. to ensure the feature is completed well.
+- The Onyx team will pull resources and bandwidth from design, PM, and engineering to ensure that you have all the resources to build the feature to the quality required for merging.
+
+Because this is a large investment from our team, we ask that you:
+- Thoroughly read all the requirements of the design docs, engineering best practices, and try to minimize overhead for the Onyx team.
+- Complete the feature in a timely manner to reduce context switching and an ongoing resource pull from the Onyx team.
+
+---
+
+## Development Setup
+
+Onyx being a fully functional app, relies on some external software, specifically:
+
+- [Postgres](https://www.postgresql.org/) (Relational DB)
+- [OpenSearch](https://opensearch.org/) (Vector DB/Search Engine)
+- [Redis](https://redis.io/) (Cache)
+- [MinIO](https://min.io/) (File Store)
+- [Nginx](https://nginx.org/) (Not needed for development flows generally)
+
+> **Note:**
+> This guide provides instructions to build and run Onyx locally from source with Docker containers providing the above external software.
+> We believe this combination is easier for development purposes. If you prefer to use pre-built container images, see [Running in Docker](#running-in-docker) below.
+
+### Prerequisites
+
+- **Python 3.11** — If using a lower version, modifications will have to be made to the code. Higher versions may have library compatibility issues.
+- **Docker** — Required for running external services (Postgres, OpenSearch, Redis, MinIO).
+- **Node.js v22** — We recommend using [nvm](https://github.com/nvm-sh/nvm) to manage Node installations.
+
+### Backend: Python Requirements
+
+We use [uv](https://docs.astral.sh/uv/) and recommend creating a [virtual environment](https://docs.astral.sh/uv/pip/environments/#using-a-virtual-environment).
+
+```bash
+uv venv .venv --python 3.11
+source .venv/bin/activate
+```
+
+_For Windows, activate the virtual environment using Command Prompt:_
+
+```bash
+.venv\Scripts\activate
+```
+
+If using PowerShell, the command slightly differs:
+
+```powershell
+.venv\Scripts\Activate.ps1
+```
+
+Install the required Python dependencies:
+
+```bash
+uv sync --all-extras
+```
+
+Install Playwright for Python (headless browser required by the Web Connector):
+
+```bash
+uv run playwright install
+```
+
+### Frontend: Node Dependencies
+
+```bash
+nvm install 22 && nvm use 22
+node -v # verify your active version
+```
+
+Navigate to `onyx/web` and run:
+
+```bash
+npm i
+```
+
+### Formatting and Linting
+
+#### Backend
+
+Set up pre-commit hooks (black / reorder-python-imports):
+
+```bash
+uv run pre-commit install
+```
+
+We also use `mypy` for static type checking. Onyx is fully type-annotated, and we want to keep it that way! To run the mypy checks manually:
+
+```bash
+uv run mypy .  # from onyx/backend
+```
+
+#### Frontend
+
+We use `prettier` for formatting. The desired version will be installed via `npm i` from the `onyx/web` directory. To run the formatter:
+
+```bash
+npx prettier --write .  # from onyx/web
+```
+
+Pre-commit will also run prettier automatically on files you've recently touched. If re-formatted, your commit will fail. Re-stage your changes and commit again.
+
+---
+
+## Running the Application
+
+### VSCode Debugger (Recommended)
+
+We highly recommend using VSCode's debugger for development.
+
+#### Initial Setup
+
+1. Copy `.vscode/env_template.txt` to `.vscode/.env`
+2. Fill in the necessary environment variables in `.vscode/.env`
+
+#### Using the Debugger
+
+Before starting, make sure the Docker Daemon is running.
+
+1. Open the Debug view in VSCode (Cmd+Shift+D on macOS)
+2. From the dropdown at the top, select "Clear and Restart External Volumes and Containers" and press the green play button
+3. From the dropdown at the top, select "Run All Onyx Services" and press the green play button
+4. Navigate to http://localhost:3000 in your browser to start using the app
+5. Set breakpoints by clicking to the left of line numbers to help debug while the app is running
+6. Use the debug toolbar to step through code, inspect variables, etc.
+
+> **Note:** "Clear and Restart External Volumes and Containers" will reset your Postgres and OpenSearch (relational-db and index). Only run this if you are okay with wiping your data.
+
+**Features:**
+- Hot reload is enabled for the web server and API servers
+- Python debugging is configured with debugpy
+- Environment variables are loaded from `.vscode/.env`
+- Console output is organized in the integrated terminal with labeled tabs
+
+### Manually Running for Development
+
+#### Docker containers for external software
+
+You will need Docker installed to run these containers.
+
+Navigate to `onyx/deployment/docker_compose`, then start up Postgres/OpenSearch/Redis/MinIO with:
+
+```bash
+docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d index relational_db cache minio
+```
+
+(index refers to OpenSearch, relational_db refers to Postgres, and cache refers to Redis)
+
+#### Running Onyx locally
+
+To start the frontend, navigate to `onyx/web` and run:
+
+```bash
+npm run dev
+```
+
+Next, start the model server which runs the local NLP models. Navigate to `onyx/backend` and run:
+
+```bash
+uvicorn model_server.main:app --reload --port 9000
+```
+
+_For Windows (for compatibility with both PowerShell and Command Prompt):_
+
+```bash
+powershell -Command "uvicorn model_server.main:app --reload --port 9000"
+```
+
+The first time running Onyx, you will need to run the DB migrations for Postgres. After the first time, this is no longer required unless the DB models change.
+
+Navigate to `onyx/backend` and with the venv active, run:
+
+```bash
+alembic upgrade head
+```
+
+Next, start the task queue which orchestrates the background jobs. Still in `onyx/backend`, run:
+
+```bash
+python ./scripts/dev_run_background_jobs.py
+```
+
+To run the backend API server, navigate back to `onyx/backend` and run:
+
+```bash
+AUTH_TYPE=basic uvicorn onyx.main:app --reload --port 8080
+```
+
+_For Windows (for compatibility with both PowerShell and Command Prompt):_
+
+```bash
+powershell -Command "
+    $env:AUTH_TYPE='basic'
+    uvicorn onyx.main:app --reload --port 8080
+"
+```
+
+> **Note:** If you need finer logging, add the additional environment variable `LOG_LEVEL=DEBUG` to the relevant services.
+
+#### Wrapping up
+
+You should now have 4 servers running:
+
+- Web server
+- Backend API
+- Model server
+- Background jobs
+
+Now, visit http://localhost:3000 in your browser. You should see the Onyx onboarding wizard where you can connect your external LLM provider to Onyx.
+
+You've successfully set up a local Onyx instance!
+
+### Running in Docker
+
+You can run the full Onyx application stack from pre-built images including all external software dependencies.
+
+Navigate to `onyx/deployment/docker_compose` and run:
+
+```bash
+docker compose up -d
+```
+
+After Docker pulls and starts these containers, navigate to http://localhost:3000 to use Onyx.
+
+If you want to make changes to Onyx and run those changes in Docker, you can also build a local version of the Onyx container images that incorporates your changes:
+
+```bash
+docker compose up -d --build
+```
+
+---
+
+## macOS-Specific Notes
+
+### Setting up Python
+
+Ensure [Homebrew](https://brew.sh/) is already set up, then install Python 3.11:
+
+```bash
+brew install python@3.11
+```
+
+Add Python 3.11 to your path by adding the following line to `~/.zshrc`:
+
+```
+export PATH="$(brew --prefix)/opt/python@3.11/libexec/bin:$PATH"
+```
+
+> **Note:** You will need to open a new terminal for the path change above to take effect.
+
+### Setting up Docker
+
+On macOS, you will need to install [Docker Desktop](https://www.docker.com/products/docker-desktop/) and ensure it is running before continuing with the docker commands.
+
+### Formatting and Linting
+
+macOS will likely require you to remove some quarantine attributes on some of the hooks for them to execute properly. After installing pre-commit, run the following command:
+
+```bash
+sudo xattr -r -d com.apple.quarantine ~/.cache/pre-commit
+```
+
+---
+
+## Engineering Best Practices
+
+> These are also what we adhere to as a team internally, we love to build in the open and to uplevel our community and each other through being transparent.
+
+### Principles and Collaboration
+
+- **Use 1-way vs 2-way doors.** For 2-way doors, move faster and iterate. For 1-way doors, be more deliberate.
+- **Consistency > being "right."** Prefer consistent patterns across the codebase. If something is truly bad, fix it everywhere.
+- **Fix what you touch (selectively).**
+  - Don't feel obligated to fix every best-practice issue you notice.
+  - Don't introduce new bad practices.
+  - If your change touches code that violates best practices, fix it as part of the change.
+- **Don't tack features on.** When adding functionality, restructure logically as needed to avoid muddying interfaces and accumulating tech debt.
+
+### Style and Maintainability
+
+#### Comments and readability
+Add clear comments:
+- At logical boundaries (e.g., interfaces) so the reader doesn't need to dig 10 layers deeper.
+- Wherever assumptions are made or something non-obvious/unexpected is done.
+- For complicated flows/functions.
+- Wherever it saves time (e.g., nontrivial regex patterns).
+
+#### Errors and exceptions
+- **Fail loudly** rather than silently skipping work.
+  - Example: raise and let exceptions propagate instead of silently dropping a document.
+- **Don't overuse `try/except`.**
+  - Put `try/except` at the correct logical level.
+  - Do not mask exceptions unless it is clearly appropriate.
+
+#### Typing
+- Everything should be **as strictly typed as possible**.
+- Use `cast` for annoying/loose-typed interfaces (e.g., results of `run_functions_tuples_in_parallel`).
+  - Only `cast` when the type checker sees `Any` or types are too loose.
+- Prefer types that are easy to read.
+  - Avoid dense types like `dict[tuple[str, str], list[list[float]]]`.
+  - Prefer domain models, e.g.:
+    - `EmbeddingModel(provider_name, model_name)` as a Pydantic model
+    - `dict[EmbeddingModel, list[EmbeddingVector]]`
+
+#### State, objects, and boundaries
+- Keep **clear logical boundaries** for state containers and objects.
+- A **config** object should never contain things like a `db_session`.
+- Avoid state containers that are overly nested, or huge + flat (use judgment).
+- Prefer **composition and functional style** over inheritance/OOP.
+- Prefer **no mutation** unless there's a strong reason.
+- State objects should be **intentional and explicit**, ideally nonmutating.
+- Use interfaces/objects to create clear separation of responsibility.
+- Prefer simplicity when there's no clear gain.
+  - Avoid overcomplicated mechanisms like semaphores.
+  - Prefer **hash maps (dicts)** over tree structures unless there's a strong reason.
+
+#### Naming
+- Name variables carefully and intentionally.
+- Prefer long, explicit names when undecided.
+- Avoid single-character variables except for small, self-contained utilities (or not at all).
+- Keep the same object/name consistent through the call stack and within functions when reasonable.
+  - Good: `for token in tokens:`
+  - Bad: `for msg in tokens:` (if iterating tokens)
+- Function names should bias toward **long + descriptive** for codebase search.
+  - IntelliSense can miss call sites; search works best with unique names.
+
+#### Correctness by construction
+- Prefer self-contained correctness — don't rely on callers to "use it right" if you can make misuse hard.
+- Avoid redundancies: if a function takes an arg, it shouldn't also take a state object that contains that same arg.
+- No dead code (unless there's a very good reason).
+- No commented-out code in main or feature branches (unless there's a very good reason).
+- No duplicate logic:
+  - Don't copy/paste into branches when shared logic can live above the conditional.
+  - If you're afraid to touch the original, you don't understand it well enough.
+  - LLMs often create subtle duplicate logic — review carefully and remove it.
+  - Avoid "nearly identical" objects that confuse when to use which.
+- Avoid extremely long functions with chained logic:
+  - Encapsulate steps into helpers for readability, even if not reused.
+  - "Pythonic" multi-step expressions are OK in moderation; don't trade clarity for cleverness.
+
+### Performance and Correctness
+
+- Avoid holding resources for extended periods (DB sessions, locks/semaphores).
+- Validate objects on creation and right before use.
+- Connector code (data to Onyx documents):
+  - Any in-memory structure that can grow without bound based on input must be periodically size-checked.
+  - If a connector is OOMing (often shows up as "missing celery tasks"), this is a top thing to check retroactively.
+- Async and event loops:
+  - Never introduce new async/event loop Python code, and try to make existing async code synchronous when possible if it makes sense.
+  - Writing async code without 100% understanding the code and having a concrete reason to do so is likely to introduce bugs and not add any meaningful performance gains.
+
+### Repository Conventions
+
+#### Where code lives
+- Pydantic + data models: `models.py` files.
+- DB interface functions (excluding lazy loading): `db/` directory.
+- LLM prompts: `prompts/` directory, roughly mirroring the code layout that uses them.
+- API routes: `server/` directory.
+
+#### Pydantic and modeling
+- Prefer **Pydantic** over dataclasses.
+- If absolutely required, use `allow_arbitrary_types`.
+
+#### Data conventions
+- Prefer explicit `None` over sentinel empty strings (usually; depends on intent).
+- Prefer explicit identifiers: use string enums instead of integer codes.
+- Avoid magic numbers (co-location is good when necessary). **Always avoid magic strings.**
+
+#### Logging
+- Log messages where they are created.
+- Don't propagate log messages around just to log them elsewhere.
+
+#### Encapsulation
+- Don't use private attributes/methods/properties from other classes/modules.
+- "Private" is private — respect that boundary.
+
+#### SQLAlchemy guidance
+- Lazy loading is often bad at scale, especially across multiple list relationships.
+- Be careful when accessing SQLAlchemy object attributes:
+  - It can help avoid redundant DB queries,
+  - but it can also fail if accessed outside an active session,
+  - and lazy loading can add hidden DB dependencies to otherwise "simple" functions.
+- Reference: https://www.reddit.com/r/SQLAlchemy/comments/138f248/joinedload_vs_selectinload/
+
+#### Trunk-based development and feature flags
+- **PRs should contain no more than 500 lines of real change.**
+- **Merge to main frequently.** Avoid long-lived feature branches — they create merge conflicts and integration pain.
+- **Use feature flags for incremental rollout.**
+  - Large features should be merged in small, shippable increments behind a flag.
+  - This allows continuous integration without exposing incomplete functionality.
+- **Keep flags short-lived.** Once a feature is fully rolled out, remove the flag and dead code paths promptly.
+- **Flag at the right level.** Prefer flagging at API/UI entry points rather than deep in business logic.
+- **Test both flag states.** Ensure the codebase works correctly with the flag on and off.
+
+#### Miscellaneous
+- Any TODOs you add in the code must be accompanied by either the name/username of the owner of that TODO, or an issue number for an issue referencing that piece of work.
+- Avoid module-level logic that runs on import, which leads to import-time side effects. Essentially every piece of meaningful logic should exist within some function that has to be explicitly invoked. Acceptable exceptions may include loading environment variables or setting up loggers.
+  - If you find yourself needing something like this, you may want that logic to exist in a file dedicated for manual execution (contains `if __name__ == "__main__":`) which should not be imported by anything else.
+- Do not conflate Python scripts you intend to run from the command line (contains `if __name__ == "__main__":`) with modules you intend to import from elsewhere. If for some unlikely reason they have to be the same file, any logic specific to executing the file (including imports) should be contained in the `if __name__ == "__main__":` block.
+  - Generally these executable files exist in `backend/scripts/`.
+
+---
+
+## Release Process
+
+Onyx loosely follows the SemVer versioning standard.
+A set of Docker containers will be pushed automatically to DockerHub with every tag.
+You can see the containers [here](https://hub.docker.com/search?q=onyx%2F).
+
+---
+
+## Getting Help
 
-## Getting Help 🙋
 We have support channels and generally interesting discussions on our [Discord](https://discord.gg/4NA5SbzrWb).
 
 See you there!
 
+---
 
-## Release Process
-Onyx loosely follows the SemVer versioning standard.
-Major changes are released with a "minor" version bump. Currently we use patch release versions to indicate small feature changes.
-A set of Docker containers will be pushed automatically to DockerHub with every tag.
-You can see the containers [here](https://hub.docker.com/search?q=onyx%2F).
+## Enterprise Edition Contributions
+
+If you are contributing features to Onyx Enterprise Edition (code under any `ee/` directory), you are required to sign the [IP Assignment Agreement](contributor_ip_assignment/EE_Contributor_IP_Assignment_Agreement.md) ([PDF version](contributor_ip_assignment/EE_Contributor_IP_Assignment_Agreement.pdf)).

README.md

@@ -4,8 +4,6 @@
     <a href="https://www.onyx.app/?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme"> <img width="50%" src="https://github.com/onyx-dot-app/onyx/blob/logo/OnyxLogoCropped.jpg?raw=true" /></a>
 </h2>
 
-<p align="center">Open Source AI Platform</p>
-
 <p align="center">
     <a href="https://discord.gg/TDJ59cGV2X" target="_blank">
         <img src="https://img.shields.io/badge/discord-join-blue.svg?logo=discord&logoColor=white" alt="Discord" />
@@ -27,82 +25,94 @@
   </a>
 </p>
 
+# Onyx - The Open Source AI Platform
 
-**[Onyx](https://www.onyx.app/?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme)** is a feature-rich, self-hostable Chat UI that works with any LLM. It is easy to deploy and can run in a completely airgapped environment.
+**[Onyx](https://www.onyx.app/?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme)** is the application layer for LLMs - bringing a feature-rich interface that can be easily hosted by anyone.
+Onyx enables LLMs through advanced capabilities like RAG, web search, code execution, file creation, deep research and more.
 
-Onyx comes loaded with advanced features like Agents, Web Search, RAG, MCP, Deep Research, Connectors to 40+ knowledge sources, and more.
+Connect your applications with over 50+ indexing based connectors provided out of the box or via MCP.
 
 > [!TIP]
-> Run Onyx with one command (or see deployment section below):
+> Deploy with a single command:
 > ```
-> curl -fsSL https://raw.githubusercontent.com/onyx-dot-app/onyx/main/deployment/docker_compose/install.sh > install.sh && chmod +x install.sh && ./install.sh
+> curl -fsSL https://onyx.app/install_onyx.sh | bash
 > ```
 
-****
-
-![Onyx Chat Silent Demo](https://github.com/onyx-dot-app/onyx/releases/download/v0.21.1/OnyxChatSilentDemo.gif)
-
+![Onyx Chat Silent Demo](https://github.com/onyx-dot-app/onyx/releases/download/v3.0.0/Onyx.gif)
 
+---
 
 ## ⭐ Features
-- **🤖 Custom Agents:** Build AI Agents with unique instructions, knowledge and actions.
-- **🌍 Web Search:** Browse the web with Google PSE, Exa, and Serper as well as an in-house scraper or Firecrawl.
-- **🔍 RAG:** Best in class hybrid-search + knowledge graph for uploaded files and ingested documents from connectors. 
-- **🔄 Connectors:** Pull knowledge, metadata, and access information from over 40 applications.
-- **🔬 Deep Research:** Get in depth answers with an agentic multi-step search.
-- **▶️ Actions & MCP:** Give AI Agents the ability to interact with external systems.
-- **💻 Code Interpreter:** Execute code to analyze data, render graphs and create files.
+
+- **🔍 Agentic RAG:** Get best in class search and answer quality based on hybrid index + AI Agents for information retrieval
+  - Benchmark to release soon!
+- **🔬 Deep Research:** Get in depth reports with a multi-step research flow.
+  - Top of [leaderboard](https://github.com/onyx-dot-app/onyx_deep_research_bench) as of Feb 2026.
+- **🤖 Custom Agents:** Build AI Agents with unique instructions, knowledge, and actions.
+- **🌍 Web Search:** Browse the web to get up to date information.
+  - Supports Serper, Google PSE, Brave, SearXNG, and others.
+  - Comes with an in house web crawler and support for Firecrawl/Exa.
+- **📄 Artifacts:** Generate documents, graphics, and other downloadable artifacts.
+- **▶️ Actions & MCP:** Let Onyx agents interact with external applications, comes with flexible Auth options.
+- **💻 Code Execution:** Execute code in a sandbox to analyze data, render graphs, or modify files.
+- **🎙️ Voice Mode:** Chat with Onyx via text-to-speech and speech-to-text.
 - **🎨 Image Generation:** Generate images based on user prompts.
-- **👥 Collaboration:** Chat sharing, feedback gathering, user management, usage analytics, and more.
 
-Onyx works with all LLMs (like OpenAI, Anthropic, Gemini, etc.) and self-hosted LLMs (like Ollama, vLLM, etc.)
+Onyx supports all major LLM providers, both self-hosted (like Ollama, LiteLLM, vLLM, etc.) and proprietary (like Anthropic, OpenAI, Gemini, etc.).
 
-To learn more about the features, check out our [documentation](https://docs.onyx.app/welcome?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme)!
+To learn more - check out our [docs](https://docs.onyx.app/welcome?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme)!
 
+---
 
+## 🚀 Deployment Modes
 
-## 🚀 Deployment
-Onyx supports deployments in Docker, Kubernetes, Terraform, along with guides for major cloud providers.
+> Onyx supports deployments in Docker, Kubernetes, Helm/Terraform and provides guides for major cloud providers.
+> Detailed deployment guides found [here](https://docs.onyx.app/deployment/overview).
 
-See guides below:
-- [Docker](https://docs.onyx.app/deployment/local/docker?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme) or [Quickstart](https://docs.onyx.app/deployment/getting_started/quickstart?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme) (best for most users)
-- [Kubernetes](https://docs.onyx.app/deployment/local/kubernetes?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme) (best for large teams)
-- [Terraform](https://docs.onyx.app/deployment/local/terraform?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme) (best for teams already using Terraform)
-- Cloud specific guides (best if specifically using [AWS EKS](https://docs.onyx.app/deployment/cloud/aws/eks?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme), [Azure VMs](https://docs.onyx.app/deployment/cloud/azure?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme), etc.)
+Onyx supports two separate deployment options: standard and lite.
+
+#### Onyx Lite
+
+The Lite mode can be thought of as a lightweight Chat UI. It requires less resources (under 1GB memory) and runs a less complex stack.
+It is great for users who want to test out Onyx quickly or for teams who are only interested in the Chat UI and Agents functionalities.
+
+#### Standard Onyx
+
+The complete feature set of Onyx which is recommended for serious users and larger teams. Additional components not included in Lite mode:
+- Vector + Keyword index for RAG.
+- Background containers to run job queues and workers for syncing knowledge from connectors.
+- AI model inference servers to run deep learning models used during indexing and inference.
+- Performance optimizations for large scale use via in memory cache (Redis) and blob store (MinIO).
 
 > [!TIP]  
-> **To try Onyx for free without deploying, check out [Onyx Cloud](https://cloud.onyx.app/signup?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme)**.
+> **To try Onyx for free without deploying, visit [Onyx Cloud](https://cloud.onyx.app/signup?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme)**.
 
+---
 
+## 🏢 Onyx for Enterprise
 
-## 🔍 Other Notable Benefits
-Onyx is built for teams of all sizes, from individual users to the largest global enterprises.
-
-- **Enterprise Search**: far more than simple RAG, Onyx has custom indexing and retrieval that remains performant and accurate for scales of up to tens of millions of documents.
-- **Security**: SSO (OIDC/SAML/OAuth2), RBAC, encryption of credentials, etc.
-- **Management UI**: different user roles such as basic, curator, and admin.
-- **Document Permissioning**: mirrors user access from external apps for RAG use cases.
-
-
-
-## 🚧 Roadmap
-To see ongoing and upcoming projects, check out our [roadmap](https://github.com/orgs/onyx-dot-app/projects/2)!
-
-
+Onyx is built for teams of all sizes, from individual users to the largest global enterprises:
+- 👥 Collaboration: Share chats and agents with other members of your organization.
+- 🔐 Single Sign On: SSO via Google OAuth, OIDC, or SAML. Group syncing and user provisioning via SCIM.
+- 🛡️ Role Based Access Control: RBAC for sensitive resources like access to agents, actions, etc.
+- 📊 Analytics: Usage graphs broken down by teams, LLMs, or agents.
+- 🕵️ Query History: Audit usage to ensure safe adoption of AI in your organization.
+- 💻 Custom code: Run custom code to remove PII, reject sensitive queries, or to run custom analysis.
+- 🎨 Whitelabeling: Customize the look and feel of Onyx with custom naming, icons, banners, and more.
 
 ## 📚 Licensing
+
 There are two editions of Onyx:
 
-- Onyx Community Edition (CE) is available freely under the MIT license.
+- Onyx Community Edition (CE) is available freely under the MIT license and covers all of the core features for Chat, RAG, Agents, and Actions.
 - Onyx Enterprise Edition (EE) includes extra features that are primarily useful for larger organizations.
+
 For feature details, check out [our website](https://www.onyx.app/pricing?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme).
 
-
-
 ## 👪 Community
+
 Join our open source community on **[Discord](https://discord.gg/TDJ59cGV2X)**!
 
-
-
 ## 💡 Contributing
+
 Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.

backend/alembic/versions/1d78c0ca7853_remove_voice_provider_deleted_column.py

@@ -0,0 +1,35 @@
+"""remove voice_provider deleted column
+
+Revision ID: 1d78c0ca7853
+Revises: a3f8b2c1d4e5
+Create Date: 2026-03-26 11:30:53.883127
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "1d78c0ca7853"
+down_revision = "a3f8b2c1d4e5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Hard-delete any soft-deleted rows before dropping the column
+    op.execute("DELETE FROM voice_provider WHERE deleted = true")
+    op.drop_column("voice_provider", "deleted")
+
+
+def downgrade() -> None:
+    op.add_column(
+        "voice_provider",
+        sa.Column(
+            "deleted",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+    )

backend/alembic/versions/25a5501dc766_group_permissions_phase1.py

@@ -0,0 +1,109 @@
+"""group_permissions_phase1
+
+Revision ID: 25a5501dc766
+Revises: b728689f45b1
+Create Date: 2026-03-23 11:41:25.557442
+
+"""
+
+from alembic import op
+import fastapi_users_db_sqlalchemy
+import sqlalchemy as sa
+
+from onyx.db.enums import AccountType
+from onyx.db.enums import GrantSource
+from onyx.db.enums import Permission
+
+
+# revision identifiers, used by Alembic.
+revision = "25a5501dc766"
+down_revision = "b728689f45b1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # 1. Add account_type column to user table (nullable for now).
+    #    TODO(subash): backfill account_type for existing rows and add NOT NULL.
+    op.add_column(
+        "user",
+        sa.Column(
+            "account_type",
+            sa.Enum(AccountType, native_enum=False),
+            nullable=True,
+        ),
+    )
+
+    # 2. Add is_default column to user_group table
+    op.add_column(
+        "user_group",
+        sa.Column(
+            "is_default",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+
+    # 3. Create permission_grant table
+    op.create_table(
+        "permission_grant",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("group_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "permission",
+            sa.Enum(Permission, native_enum=False),
+            nullable=False,
+        ),
+        sa.Column(
+            "grant_source",
+            sa.Enum(GrantSource, native_enum=False),
+            nullable=False,
+        ),
+        sa.Column(
+            "granted_by",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column(
+            "granted_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+        sa.Column(
+            "is_deleted",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.ForeignKeyConstraint(
+            ["group_id"],
+            ["user_group.id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["granted_by"],
+            ["user.id"],
+            ondelete="SET NULL",
+        ),
+        sa.UniqueConstraint(
+            "group_id", "permission", name="uq_permission_grant_group_permission"
+        ),
+    )
+
+    # 4. Index on user__user_group(user_id) — existing composite PK
+    #    has user_group_id as leading column; user-filtered queries need this
+    op.create_index(
+        "ix_user__user_group_user_id",
+        "user__user_group",
+        ["user_id"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_user__user_group_user_id", table_name="user__user_group")
+    op.drop_table("permission_grant")
+    op.drop_column("user_group", "is_default")
+    op.drop_column("user", "account_type")

backend/alembic/versions/8188861f4e92_csv_to_tabular_chat_file_type.py

@@ -0,0 +1,54 @@
+"""csv to tabular chat file type
+
+Revision ID: 8188861f4e92
+Revises: d8cdfee5df80
+Create Date: 2026-03-31 19:23:05.753184
+
+"""
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "8188861f4e92"
+down_revision = "d8cdfee5df80"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        UPDATE chat_message
+        SET files = (
+            SELECT jsonb_agg(
+                CASE
+                    WHEN elem->>'type' = 'csv'
+                    THEN jsonb_set(elem, '{type}', '"tabular"')
+                    ELSE elem
+                END
+            )
+            FROM jsonb_array_elements(files) AS elem
+        )
+        WHERE files::text LIKE '%"type": "csv"%'
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        """
+        UPDATE chat_message
+        SET files = (
+            SELECT jsonb_agg(
+                CASE
+                    WHEN elem->>'type' = 'tabular'
+                    THEN jsonb_set(elem, '{type}', '"csv"')
+                    ELSE elem
+                END
+            )
+            FROM jsonb_array_elements(files) AS elem
+        )
+        WHERE files::text LIKE '%"type": "tabular"%'
+        """
+    )

backend/alembic/versions/a3f8b2c1d4e5_add_preferred_response_id_to_chat_message.py

@@ -0,0 +1,36 @@
+"""add preferred_response_id and model_display_name to chat_message
+
+Revision ID: a3f8b2c1d4e5
+Create Date: 2026-03-22
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "a3f8b2c1d4e5"
+down_revision = "25a5501dc766"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_message",
+        sa.Column(
+            "preferred_response_id",
+            sa.Integer(),
+            sa.ForeignKey("chat_message.id", ondelete="SET NULL"),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "chat_message",
+        sa.Column("model_display_name", sa.String(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_message", "model_display_name")
+    op.drop_column("chat_message", "preferred_response_id")

backend/alembic/versions/d8cdfee5df80_add_skipped_to_userfilestatus.py

@@ -0,0 +1,55 @@
+"""add skipped to userfilestatus
+
+Revision ID: d8cdfee5df80
+Revises: 1d78c0ca7853
+Create Date: 2026-04-01 10:47:12.593950
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "d8cdfee5df80"
+down_revision = "1d78c0ca7853"
+branch_labels = None
+depends_on = None
+
+
+TABLE = "user_file"
+COLUMN = "status"
+CONSTRAINT_NAME = "ck_user_file_status"
+
+OLD_VALUES = ("PROCESSING", "INDEXING", "COMPLETED", "FAILED", "CANCELED", "DELETING")
+NEW_VALUES = (
+    "PROCESSING",
+    "INDEXING",
+    "COMPLETED",
+    "SKIPPED",
+    "FAILED",
+    "CANCELED",
+    "DELETING",
+)
+
+
+def _drop_status_check_constraint() -> None:
+    inspector = sa.inspect(op.get_bind())
+    for constraint in inspector.get_check_constraints(TABLE):
+        if COLUMN in constraint.get("sqltext", ""):
+            constraint_name = constraint["name"]
+            if constraint_name is not None:
+                op.drop_constraint(constraint_name, TABLE, type_="check")
+
+
+def upgrade() -> None:
+    _drop_status_check_constraint()
+    in_clause = ", ".join(f"'{v}'" for v in NEW_VALUES)
+    op.create_check_constraint(CONSTRAINT_NAME, TABLE, f"{COLUMN} IN ({in_clause})")
+
+
+def downgrade() -> None:
+    op.execute(f"UPDATE {TABLE} SET {COLUMN} = 'COMPLETED' WHERE {COLUMN} = 'SKIPPED'")
+    _drop_status_check_constraint()
+    in_clause = ", ".join(f"'{v}'" for v in OLD_VALUES)
+    op.create_check_constraint(CONSTRAINT_NAME, TABLE, f"{COLUMN} IN ({in_clause})")

backend/ee/onyx/background/celery/apps/primary.py

@@ -5,6 +5,7 @@ from onyx.background.celery.apps.primary import celery_app
 celery_app.autodiscover_tasks(
     app_base.filter_task_modules(
         [
+            "ee.onyx.background.celery.tasks.hooks",
             "ee.onyx.background.celery.tasks.doc_permission_syncing",
             "ee.onyx.background.celery.tasks.external_group_syncing",
             "ee.onyx.background.celery.tasks.cloud",

backend/ee/onyx/background/celery/tasks/beat_schedule.py

@@ -55,6 +55,15 @@ ee_tasks_to_schedule: list[dict] = []
 
 if not MULTI_TENANT:
     ee_tasks_to_schedule = [
+        {
+            "name": "hook-execution-log-cleanup",
+            "task": OnyxCeleryTask.HOOK_EXECUTION_LOG_CLEANUP_TASK,
+            "schedule": timedelta(days=1),
+            "options": {
+                "priority": OnyxCeleryPriority.LOW,
+                "expires": BEAT_EXPIRES_DEFAULT,
+            },
+        },
         {
             "name": "autogenerate-usage-report",
             "task": OnyxCeleryTask.GENERATE_USAGE_REPORT_TASK,

backend/ee/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -28,6 +28,7 @@ from onyx.access.models import DocExternalAccess
 from onyx.access.models import ElementExternalAccess
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_find_task
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
@@ -187,7 +188,6 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str) -> bool | None
     # (which lives on a different db number)
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_CONNECTOR_DOC_PERMISSIONS_SYNC_BEAT_LOCK,
@@ -227,6 +227,7 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str) -> bool | None
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
             try:
+                r_celery = celery_get_broker_client(self.app)
                 validate_permission_sync_fences(
                     tenant_id, r, r_replica, r_celery, lock_beat
                 )
@@ -473,6 +474,8 @@ def connector_permission_sync_generator_task(
             cc_pair = get_connector_credential_pair_from_id(
                 db_session=db_session,
                 cc_pair_id=cc_pair_id,
+                eager_load_connector=True,
+                eager_load_credential=True,
             )
             if cc_pair is None:
                 raise ValueError(

backend/ee/onyx/background/celery/tasks/external_group_syncing/tasks.py

@@ -29,6 +29,7 @@ from ee.onyx.external_permissions.sync_params import (
 from ee.onyx.external_permissions.sync_params import get_source_perm_sync_config
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_find_task
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
 from onyx.background.celery.tasks.beat_schedule import CLOUD_BEAT_MULTIPLIER_DEFAULT
 from onyx.background.error_logging import emit_background_error
@@ -162,7 +163,6 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str) -> bool | None:
     # (which lives on a different db number)
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_CONNECTOR_EXTERNAL_GROUP_SYNC_BEAT_LOCK,
@@ -221,6 +221,7 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str) -> bool | None:
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
             try:
+                r_celery = celery_get_broker_client(self.app)
                 validate_external_group_sync_fences(
                     tenant_id, self.app, r, r_replica, r_celery, lock_beat
                 )

backend/ee/onyx/background/celery/tasks/tenant_provisioning/tasks.py

@@ -13,6 +13,7 @@ from redis.lock import Lock as RedisLock
 from ee.onyx.server.tenants.provisioning import setup_tenant
 from ee.onyx.server.tenants.schema_management import create_schema_if_not_exists
 from ee.onyx.server.tenants.schema_management import get_current_alembic_version
+from ee.onyx.server.tenants.schema_management import run_alembic_migrations
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.configs.app_configs import TARGET_AVAILABLE_TENANTS
 from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
@@ -29,9 +30,10 @@ from shared_configs.configs import TENANT_ID_PREFIX
 # Each tenant takes ~80s (alembic migrations), so 5 tenants ≈ 7 minutes.
 _MAX_TENANTS_PER_RUN = 5
 
-# Time limits sized for worst-case batch: _MAX_TENANTS_PER_RUN × ~90s + buffer.
-_TENANT_PROVISIONING_SOFT_TIME_LIMIT = 60 * 10  # 10 minutes
-_TENANT_PROVISIONING_TIME_LIMIT = 60 * 15  # 15 minutes
+# Time limits sized for worst-case: provisioning up to _MAX_TENANTS_PER_RUN new tenants
+# (~90s each) plus migrating up to TARGET_AVAILABLE_TENANTS pool tenants (~90s each).
+_TENANT_PROVISIONING_SOFT_TIME_LIMIT = 60 * 20  # 20 minutes
+_TENANT_PROVISIONING_TIME_LIMIT = 60 * 25  # 25 minutes
 
 
 @shared_task(
@@ -91,8 +93,7 @@ def check_available_tenants(self: Task) -> None:  # noqa: ARG001
         batch_size = min(tenants_to_provision, _MAX_TENANTS_PER_RUN)
         if batch_size < tenants_to_provision:
             task_logger.info(
-                f"Capping batch to {batch_size} "
-                f"(need {tenants_to_provision}, will catch up next cycle)"
+                f"Capping batch to {batch_size} (need {tenants_to_provision}, will catch up next cycle)"
             )
 
         provisioned = 0
@@ -103,12 +104,14 @@ def check_available_tenants(self: Task) -> None:  # noqa: ARG001
                     provisioned += 1
             except Exception:
                 task_logger.exception(
-                    f"Failed to provision tenant {i + 1}/{batch_size}, "
-                    "continuing with remaining tenants"
+                    f"Failed to provision tenant {i + 1}/{batch_size}, continuing with remaining tenants"
                 )
 
         task_logger.info(f"Provisioning complete: {provisioned}/{batch_size} succeeded")
 
+        # Migrate any pool tenants that were provisioned before a new migration was deployed
+        _migrate_stale_pool_tenants()
+
     except Exception:
         task_logger.exception("Error in check_available_tenants task")
 
@@ -121,6 +124,46 @@ def check_available_tenants(self: Task) -> None:  # noqa: ARG001
             )
 
 
+def _migrate_stale_pool_tenants() -> None:
+    """
+    Run alembic upgrade head on all pool tenants. Since alembic upgrade head is
+    idempotent, tenants already at head are a fast no-op. This ensures pool
+    tenants are always current so that signup doesn't hit schema mismatches
+    (e.g. missing columns added after the tenant was pre-provisioned).
+    """
+    with get_session_with_shared_schema() as db_session:
+        pool_tenants = db_session.query(AvailableTenant).all()
+        tenant_ids = [t.tenant_id for t in pool_tenants]
+
+    if not tenant_ids:
+        return
+
+    task_logger.info(
+        f"Checking {len(tenant_ids)} pool tenant(s) for pending migrations"
+    )
+
+    for tenant_id in tenant_ids:
+        try:
+            run_alembic_migrations(tenant_id)
+            new_version = get_current_alembic_version(tenant_id)
+            with get_session_with_shared_schema() as db_session:
+                tenant = (
+                    db_session.query(AvailableTenant)
+                    .filter_by(tenant_id=tenant_id)
+                    .first()
+                )
+                if tenant and tenant.alembic_version != new_version:
+                    task_logger.info(
+                        f"Migrated pool tenant {tenant_id}: {tenant.alembic_version} -> {new_version}"
+                    )
+                    tenant.alembic_version = new_version
+                    db_session.commit()
+        except Exception:
+            task_logger.exception(
+                f"Failed to migrate pool tenant {tenant_id}, skipping"
+            )
+
+
 def pre_provision_tenant() -> bool:
     """
     Pre-provision a new tenant and store it in the NewAvailableTenant table.

backend/ee/onyx/configs/license_enforcement_config.py

@@ -69,5 +69,7 @@ EE_ONLY_PATH_PREFIXES: frozenset[str] = frozenset(
         "/admin/token-rate-limits",
         # Evals
         "/evals",
+        # Hook extensions
+        "/admin/hooks",
     }
 )

backend/ee/onyx/external_permissions/sharepoint/permission_utils.py

@@ -250,20 +250,24 @@ def _get_sharepoint_list_item_id(drive_item: DriveItem) -> str | None:
         raise e
 
 
-def _is_public_item(drive_item: DriveItem) -> bool:
-    is_public = False
+def _is_public_item(
+    drive_item: DriveItem,
+    treat_sharing_link_as_public: bool = False,
+) -> bool:
+    if not treat_sharing_link_as_public:
+        return False
+
     try:
         permissions = sleep_and_retry(
             drive_item.permissions.get_all(page_loaded=lambda _: None), "is_public_item"
         )
         for permission in permissions:
-            if permission.link and (
-                permission.link.scope == "anonymous"
-                or permission.link.scope == "organization"
+            if permission.link and permission.link.scope in (
+                "anonymous",
+                "organization",
             ):
-                is_public = True
-                break
-        return is_public
+                return True
+        return False
     except Exception as e:
         logger.error(f"Failed to check if item {drive_item.id} is public: {e}")
         return False
@@ -504,6 +508,7 @@ def get_external_access_from_sharepoint(
     drive_item: DriveItem | None,
     site_page: dict[str, Any] | None,
     add_prefix: bool = False,
+    treat_sharing_link_as_public: bool = False,
 ) -> ExternalAccess:
     """
     Get external access information from SharePoint.
@@ -563,8 +568,7 @@ def get_external_access_from_sharepoint(
                     )
 
     if drive_item and drive_name:
-        # Here we check if the item have have any public links, if so we return early
-        is_public = _is_public_item(drive_item)
+        is_public = _is_public_item(drive_item, treat_sharing_link_as_public)
         if is_public:
             logger.info(f"Item {drive_item.id} is public")
             return ExternalAccess(

backend/ee/onyx/external_permissions/slack/doc_sync.py

@@ -8,6 +8,7 @@ from ee.onyx.external_permissions.slack.utils import fetch_user_id_to_email_map
 from onyx.access.models import DocExternalAccess
 from onyx.access.models import ExternalAccess
 from onyx.connectors.credentials_provider import OnyxDBCredentialsProvider
+from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import HierarchyNode
 from onyx.connectors.slack.connector import get_channels
 from onyx.connectors.slack.connector import make_paginated_slack_api_call
@@ -105,9 +106,11 @@ def _get_slack_document_access(
     slack_connector: SlackConnector,
     channel_permissions: dict[str, ExternalAccess],  # noqa: ARG001
     callback: IndexingHeartbeatInterface | None,
+    indexing_start: SecondsSinceUnixEpoch | None = None,
 ) -> Generator[DocExternalAccess, None, None]:
     slim_doc_generator = slack_connector.retrieve_all_slim_docs_perm_sync(
-        callback=callback
+        callback=callback,
+        start=indexing_start,
     )
 
     for doc_metadata_batch in slim_doc_generator:
@@ -180,9 +183,15 @@ def slack_doc_sync(
 
     slack_connector = SlackConnector(**cc_pair.connector.connector_specific_config)
     slack_connector.set_credentials_provider(provider)
+    indexing_start_ts: SecondsSinceUnixEpoch | None = (
+        cc_pair.connector.indexing_start.timestamp()
+        if cc_pair.connector.indexing_start is not None
+        else None
+    )
 
     yield from _get_slack_document_access(
-        slack_connector,
+        slack_connector=slack_connector,
         channel_permissions=channel_permissions,
         callback=callback,
+        indexing_start=indexing_start_ts,
     )

backend/ee/onyx/external_permissions/utils.py

@@ -6,6 +6,7 @@ from onyx.access.models import ElementExternalAccess
 from onyx.access.models import ExternalAccess
 from onyx.access.models import NodeExternalAccess
 from onyx.configs.constants import DocumentSource
+from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnectorWithPermSync
 from onyx.connectors.models import HierarchyNode
 from onyx.db.models import ConnectorCredentialPair
@@ -40,10 +41,19 @@ def generic_doc_sync(
 
     logger.info(f"Starting {doc_source} doc sync for CC Pair ID: {cc_pair.id}")
 
+    indexing_start: SecondsSinceUnixEpoch | None = (
+        cc_pair.connector.indexing_start.timestamp()
+        if cc_pair.connector.indexing_start is not None
+        else None
+    )
+
     newly_fetched_doc_ids: set[str] = set()
 
     logger.info(f"Fetching all slim documents from {doc_source}")
-    for doc_batch in slim_connector.retrieve_all_slim_docs_perm_sync(callback=callback):
+    for doc_batch in slim_connector.retrieve_all_slim_docs_perm_sync(
+        start=indexing_start,
+        callback=callback,
+    ):
         logger.info(f"Got {len(doc_batch)} slim documents from {doc_source}")
 
         if callback:

backend/ee/onyx/hooks/executor.py

@@ -0,0 +1,385 @@
+"""Hook executor — calls a customer's external HTTP endpoint for a given hook point.
+
+Usage (Celery tasks and FastAPI handlers):
+    result = execute_hook(
+        db_session=db_session,
+        hook_point=HookPoint.QUERY_PROCESSING,
+        payload={"query": "...", "user_email": "...", "chat_session_id": "..."},
+        response_type=QueryProcessingResponse,
+    )
+
+    if isinstance(result, HookSkipped):
+        # no active hook configured — continue with original behavior
+        ...
+    elif isinstance(result, HookSoftFailed):
+        # hook failed but fail strategy is SOFT — continue with original behavior
+        ...
+    else:
+        # result is a validated Pydantic model instance (response_type)
+        ...
+
+is_reachable update policy
+--------------------------
+``is_reachable`` on the Hook row is updated selectively — only when the outcome
+carries meaningful signal about physical reachability:
+
+  NetworkError (DNS, connection refused)  → False  (cannot reach the server)
+  HTTP 401 / 403                          → False  (api_key revoked or invalid)
+  TimeoutException                        → None   (server may be slow, skip write)
+  Other HTTP errors (4xx / 5xx)           → None   (server responded, skip write)
+  Unknown exception                       → None   (no signal, skip write)
+  Non-JSON / non-dict response            → None   (server responded, skip write)
+  Success (2xx, valid dict)               → True   (confirmed reachable)
+
+None means "leave the current value unchanged" — no DB round-trip is made.
+
+DB session design
+-----------------
+The executor uses three sessions:
+
+  1. Caller's session (db_session) — used only for the hook lookup read. All
+     needed fields are extracted from the Hook object before the HTTP call, so
+     the caller's session is not held open during the external HTTP request.
+
+  2. Log session — a separate short-lived session opened after the HTTP call
+     completes to write the HookExecutionLog row on failure. Success runs are
+     not recorded. Committed independently of everything else.
+
+  3. Reachable session — a second short-lived session to update is_reachable on
+     the Hook. Kept separate from the log session so a concurrent hook deletion
+     (which causes update_hook__no_commit to raise OnyxError(NOT_FOUND)) cannot
+     prevent the execution log from being written. This update is best-effort.
+"""
+
+import json
+import time
+from typing import Any
+from typing import TypeVar
+
+import httpx
+from pydantic import BaseModel
+from pydantic import ValidationError
+from sqlalchemy.orm import Session
+
+from onyx.db.engine.sql_engine import get_session_with_current_tenant
+from onyx.db.enums import HookFailStrategy
+from onyx.db.enums import HookPoint
+from onyx.db.hook import create_hook_execution_log__no_commit
+from onyx.db.hook import get_non_deleted_hook_by_hook_point
+from onyx.db.hook import update_hook__no_commit
+from onyx.db.models import Hook
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
+from onyx.hooks.executor import HookSkipped
+from onyx.hooks.executor import HookSoftFailed
+from onyx.utils.logger import setup_logger
+from shared_configs.configs import MULTI_TENANT
+
+logger = setup_logger()
+
+
+T = TypeVar("T", bound=BaseModel)
+
+
+# ---------------------------------------------------------------------------
+# Private helpers
+# ---------------------------------------------------------------------------
+
+
+class _HttpOutcome(BaseModel):
+    """Structured result of an HTTP hook call, returned by _process_response."""
+
+    is_success: bool
+    updated_is_reachable: (
+        bool | None
+    )  # True/False = write to DB, None = unchanged (skip write)
+    status_code: int | None
+    error_message: str | None
+    response_payload: dict[str, Any] | None
+
+
+def _lookup_hook(
+    db_session: Session,
+    hook_point: HookPoint,
+) -> Hook | HookSkipped:
+    """Return the active Hook or HookSkipped if hooks are unavailable/unconfigured.
+
+    No HTTP call is made and no DB writes are performed for any HookSkipped path.
+    There is nothing to log and no reachability information to update.
+    """
+    if MULTI_TENANT:
+        return HookSkipped()
+    hook = get_non_deleted_hook_by_hook_point(
+        db_session=db_session, hook_point=hook_point
+    )
+    if hook is None or not hook.is_active:
+        return HookSkipped()
+    if not hook.endpoint_url:
+        return HookSkipped()
+    return hook
+
+
+def _process_response(
+    *,
+    response: httpx.Response | None,
+    exc: Exception | None,
+    timeout: float,
+) -> _HttpOutcome:
+    """Process the result of an HTTP call and return a structured outcome.
+
+    Called after the client.post() try/except. If post() raised, exc is set and
+    response is None. Otherwise response is set and exc is None. Handles
+    raise_for_status(), JSON decoding, and the dict shape check.
+    """
+    if exc is not None:
+        if isinstance(exc, httpx.NetworkError):
+            msg = f"Hook network error (endpoint unreachable): {exc}"
+            logger.warning(msg, exc_info=exc)
+            return _HttpOutcome(
+                is_success=False,
+                updated_is_reachable=False,
+                status_code=None,
+                error_message=msg,
+                response_payload=None,
+            )
+        if isinstance(exc, httpx.TimeoutException):
+            msg = f"Hook timed out after {timeout}s: {exc}"
+            logger.warning(msg, exc_info=exc)
+            return _HttpOutcome(
+                is_success=False,
+                updated_is_reachable=None,  # timeout doesn't indicate unreachability
+                status_code=None,
+                error_message=msg,
+                response_payload=None,
+            )
+        msg = f"Hook call failed: {exc}"
+        logger.exception(msg, exc_info=exc)
+        return _HttpOutcome(
+            is_success=False,
+            updated_is_reachable=None,  # unknown error — don't make assumptions
+            status_code=None,
+            error_message=msg,
+            response_payload=None,
+        )
+
+    if response is None:
+        raise ValueError(
+            "exactly one of response or exc must be non-None; both are None"
+        )
+    status_code = response.status_code
+
+    try:
+        response.raise_for_status()
+    except httpx.HTTPStatusError as e:
+        msg = f"Hook returned HTTP {e.response.status_code}: {e.response.text}"
+        logger.warning(msg, exc_info=e)
+        # 401/403 means the api_key has been revoked or is invalid — mark unreachable
+        # so the operator knows to update it. All other HTTP errors keep is_reachable
+        # as-is (server is up, the request just failed for application reasons).
+        auth_failed = e.response.status_code in (401, 403)
+        return _HttpOutcome(
+            is_success=False,
+            updated_is_reachable=False if auth_failed else None,
+            status_code=status_code,
+            error_message=msg,
+            response_payload=None,
+        )
+
+    try:
+        response_payload = response.json()
+    except (json.JSONDecodeError, httpx.DecodingError) as e:
+        msg = f"Hook returned non-JSON response: {e}"
+        logger.warning(msg, exc_info=e)
+        return _HttpOutcome(
+            is_success=False,
+            updated_is_reachable=None,  # server responded — reachability unchanged
+            status_code=status_code,
+            error_message=msg,
+            response_payload=None,
+        )
+
+    if not isinstance(response_payload, dict):
+        msg = f"Hook returned non-dict JSON (got {type(response_payload).__name__})"
+        logger.warning(msg)
+        return _HttpOutcome(
+            is_success=False,
+            updated_is_reachable=None,  # server responded — reachability unchanged
+            status_code=status_code,
+            error_message=msg,
+            response_payload=None,
+        )
+
+    return _HttpOutcome(
+        is_success=True,
+        updated_is_reachable=True,
+        status_code=status_code,
+        error_message=None,
+        response_payload=response_payload,
+    )
+
+
+def _persist_result(
+    *,
+    hook_id: int,
+    outcome: _HttpOutcome,
+    duration_ms: int,
+) -> None:
+    """Write the execution log on failure and optionally update is_reachable, each
+    in its own session so a failure in one does not affect the other."""
+    # Only write the execution log on failure — success runs are not recorded.
+    # Must not be skipped if the is_reachable update fails (e.g. hook concurrently
+    # deleted between the initial lookup and here).
+    if not outcome.is_success:
+        try:
+            with get_session_with_current_tenant() as log_session:
+                create_hook_execution_log__no_commit(
+                    db_session=log_session,
+                    hook_id=hook_id,
+                    is_success=False,
+                    error_message=outcome.error_message,
+                    status_code=outcome.status_code,
+                    duration_ms=duration_ms,
+                )
+                log_session.commit()
+        except Exception:
+            logger.exception(
+                f"Failed to persist hook execution log for hook_id={hook_id}"
+            )
+
+    # Update is_reachable separately — best-effort, non-critical.
+    # None means the value is unchanged (set by the caller to skip the no-op write).
+    # update_hook__no_commit can raise OnyxError(NOT_FOUND) if the hook was
+    # concurrently deleted, so keep this isolated from the log write above.
+    if outcome.updated_is_reachable is not None:
+        try:
+            with get_session_with_current_tenant() as reachable_session:
+                update_hook__no_commit(
+                    db_session=reachable_session,
+                    hook_id=hook_id,
+                    is_reachable=outcome.updated_is_reachable,
+                )
+                reachable_session.commit()
+        except Exception:
+            logger.warning(f"Failed to update is_reachable for hook_id={hook_id}")
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def _execute_hook_inner(
+    hook: Hook,
+    payload: dict[str, Any],
+    response_type: type[T],
+) -> T | HookSoftFailed:
+    """Make the HTTP call, validate the response, and return a typed model.
+
+    Raises OnyxError on HARD failure. Returns HookSoftFailed on SOFT failure.
+    """
+    timeout = hook.timeout_seconds
+    hook_id = hook.id
+    fail_strategy = hook.fail_strategy
+    endpoint_url = hook.endpoint_url
+    current_is_reachable: bool | None = hook.is_reachable
+
+    if not endpoint_url:
+        raise ValueError(
+            f"hook_id={hook_id} is active but has no endpoint_url — "
+            "active hooks without an endpoint_url must be rejected by _lookup_hook"
+        )
+
+    start = time.monotonic()
+    response: httpx.Response | None = None
+    exc: Exception | None = None
+    try:
+        api_key: str | None = (
+            hook.api_key.get_value(apply_mask=False) if hook.api_key else None
+        )
+        headers: dict[str, str] = {"Content-Type": "application/json"}
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+        with httpx.Client(
+            timeout=timeout, follow_redirects=False
+        ) as client:  # SSRF guard: never follow redirects
+            response = client.post(endpoint_url, json=payload, headers=headers)
+    except Exception as e:
+        exc = e
+    duration_ms = int((time.monotonic() - start) * 1000)
+
+    outcome = _process_response(response=response, exc=exc, timeout=timeout)
+
+    # Validate the response payload against response_type.
+    # A validation failure downgrades the outcome to a failure so it is logged,
+    # is_reachable is left unchanged (server responded — just a bad payload),
+    # and fail_strategy is respected below.
+    validated_model: T | None = None
+    if outcome.is_success and outcome.response_payload is not None:
+        try:
+            validated_model = response_type.model_validate(outcome.response_payload)
+        except ValidationError as e:
+            msg = (
+                f"Hook response failed validation against {response_type.__name__}: {e}"
+            )
+            outcome = _HttpOutcome(
+                is_success=False,
+                updated_is_reachable=None,  # server responded — reachability unchanged
+                status_code=outcome.status_code,
+                error_message=msg,
+                response_payload=None,
+            )
+
+    # Skip the is_reachable write when the value would not change — avoids a
+    # no-op DB round-trip on every call when the hook is already in the expected state.
+    if outcome.updated_is_reachable == current_is_reachable:
+        outcome = outcome.model_copy(update={"updated_is_reachable": None})
+    _persist_result(hook_id=hook_id, outcome=outcome, duration_ms=duration_ms)
+
+    if not outcome.is_success:
+        if fail_strategy == HookFailStrategy.HARD:
+            raise OnyxError(
+                OnyxErrorCode.HOOK_EXECUTION_FAILED,
+                outcome.error_message or "Hook execution failed.",
+            )
+        logger.warning(
+            f"Hook execution failed (soft fail) for hook_id={hook_id}: {outcome.error_message}"
+        )
+        return HookSoftFailed()
+
+    if validated_model is None:
+        raise OnyxError(
+            OnyxErrorCode.INTERNAL_ERROR,
+            f"validated_model is None for successful hook call (hook_id={hook_id})",
+        )
+    return validated_model
+
+
+def _execute_hook_impl(
+    *,
+    db_session: Session,
+    hook_point: HookPoint,
+    payload: dict[str, Any],
+    response_type: type[T],
+) -> T | HookSkipped | HookSoftFailed:
+    """EE implementation — loaded by CE's execute_hook via fetch_versioned_implementation.
+
+    Returns HookSkipped if no active hook is configured, HookSoftFailed if the
+    hook failed with SOFT fail strategy, or a validated response model on success.
+    Raises OnyxError on HARD failure or if the hook is misconfigured.
+    """
+    hook = _lookup_hook(db_session, hook_point)
+    if isinstance(hook, HookSkipped):
+        return hook
+
+    fail_strategy = hook.fail_strategy
+    hook_id = hook.id
+
+    try:
+        return _execute_hook_inner(hook, payload, response_type)
+    except Exception:
+        if fail_strategy == HookFailStrategy.SOFT:
+            logger.exception(
+                f"Unexpected error in hook execution (soft fail) for hook_id={hook_id}"
+            )
+            return HookSoftFailed()
+        raise

backend/ee/onyx/main.py

@@ -15,6 +15,7 @@ from ee.onyx.server.enterprise_settings.api import (
     basic_router as enterprise_settings_router,
 )
 from ee.onyx.server.evals.api import router as evals_router
+from ee.onyx.server.features.hooks.api import router as hook_router
 from ee.onyx.server.license.api import router as license_router
 from ee.onyx.server.manage.standard_answer import router as standard_answer_router
 from ee.onyx.server.middleware.license_enforcement import (
@@ -138,6 +139,7 @@ def get_application() -> FastAPI:
     include_router_with_global_prefix_prepended(application, ee_oauth_router)
     include_router_with_global_prefix_prepended(application, ee_document_cc_pair_router)
     include_router_with_global_prefix_prepended(application, evals_router)
+    include_router_with_global_prefix_prepended(application, hook_router)
 
     # Enterprise-only global settings
     include_router_with_global_prefix_prepended(

backend/ee/onyx/search/process_search_query.py

@@ -44,19 +44,21 @@ def _run_single_search(
     user: User,
     db_session: Session,
     num_hits: int | None = None,
+    hybrid_alpha: float | None = None,
 ) -> list[InferenceChunk]:
     """Execute a single search query and return chunks."""
     chunk_search_request = ChunkSearchRequest(
         query=query,
         user_selected_filters=filters,
         limit=num_hits,
+        hybrid_alpha=hybrid_alpha,
     )
 
     return search_pipeline(
         chunk_search_request=chunk_search_request,
         document_index=document_index,
         user=user,
-        persona=None,  # No persona for direct search
+        persona_search_info=None,
         db_session=db_session,
     )
 
@@ -74,7 +76,7 @@ def stream_search_query(
     Core search function that yields streaming packets.
     Used by both streaming and non-streaming endpoints.
     """
-    # Get document index
+    # Get document index.
     search_settings = get_current_search_settings(db_session)
     # This flow is for search so we do not get all indices.
     document_index = get_default_document_index(search_settings, None, db_session)
@@ -119,6 +121,7 @@ def stream_search_query(
             user=user,
             db_session=db_session,
             num_hits=request.num_hits,
+            hybrid_alpha=request.hybrid_alpha,
         )
     else:
         # Multiple queries - run in parallel and merge with RRF
@@ -133,6 +136,7 @@ def stream_search_query(
                     user,
                     db_session,
                     request.num_hits,
+                    request.hybrid_alpha,
                 ),
             )
             for query in all_executed_queries

backend/ee/onyx/server/features/hooks/api.py

@@ -44,11 +44,12 @@ def _check_ssrf_safety(endpoint_url: str) -> None:
     """Raise OnyxError if endpoint_url could be used for SSRF.
 
     Delegates to validate_outbound_http_url with https_only=True.
+    Uses BAD_GATEWAY so the frontend maps the error to the Endpoint URL field.
     """
     try:
         validate_outbound_http_url(endpoint_url, https_only=True)
     except (SSRFException, ValueError) as e:
-        raise OnyxError(OnyxErrorCode.INVALID_INPUT, str(e))
+        raise OnyxError(OnyxErrorCode.BAD_GATEWAY, str(e))
 
 
 # ---------------------------------------------------------------------------
@@ -62,6 +63,9 @@ def _hook_to_response(hook: Hook, creator_email: str | None = None) -> HookRespo
         name=hook.name,
         hook_point=hook.hook_point,
         endpoint_url=hook.endpoint_url,
+        api_key_masked=(
+            hook.api_key.get_value(apply_mask=True) if hook.api_key else None
+        ),
         fail_strategy=hook.fail_strategy,
         timeout_seconds=hook.timeout_seconds,
         is_active=hook.is_active,
@@ -119,9 +123,8 @@ def _validate_endpoint(
     (not reachable — indicates the api_key is invalid).
 
     Timeout handling:
-    - ConnectTimeout: TCP handshake never completed → cannot_connect.
-    - ReadTimeout / WriteTimeout: TCP was established, server responded slowly → timeout
-      (operator should consider increasing timeout_seconds).
+    - Any httpx.TimeoutException (ConnectTimeout, ReadTimeout, WriteTimeout, PoolTimeout) →
+      timeout (operator should consider increasing timeout_seconds).
     - All other exceptions → cannot_connect.
     """
     _check_ssrf_safety(endpoint_url)
@@ -138,19 +141,11 @@ def _validate_endpoint(
             )
         return HookValidateResponse(status=HookValidateStatus.passed)
     except httpx.TimeoutException as exc:
-        # ConnectTimeout: TCP handshake never completed → cannot_connect.
-        # ReadTimeout / WriteTimeout: TCP was established, server just responded slowly → timeout.
-        if isinstance(exc, httpx.ConnectTimeout):
-            logger.warning(
-                "Hook endpoint validation: connect timeout for %s",
-                endpoint_url,
-                exc_info=exc,
-            )
-            return HookValidateResponse(
-                status=HookValidateStatus.cannot_connect, error_message=str(exc)
-            )
+        # Any timeout (connect, read, or write) means the configured timeout_seconds
+        # is too low for this endpoint. Report as timeout so the UI directs the user
+        # to increase the timeout setting.
         logger.warning(
-            "Hook endpoint validation: read/write timeout for %s",
+            "Hook endpoint validation: timeout for %s",
             endpoint_url,
             exc_info=exc,
         )
@@ -220,8 +215,8 @@ def create_hook(
     db_session: Session = Depends(get_session),
 ) -> HookResponse:
     """Create a new hook. The endpoint is validated before persisting — creation fails if
-    the endpoint cannot be reached or the api_key is invalid. Hooks are created inactive;
-    use POST /{hook_id}/activate once ready to receive traffic."""
+    the endpoint cannot be reached or the api_key is invalid. Hooks are created active.
+    """
     spec = get_hook_point_spec(req.hook_point)
     api_key = req.api_key.get_secret_value() if req.api_key else None
     validation = _validate_endpoint(
@@ -240,9 +235,10 @@ def create_hook(
         api_key=api_key,
         fail_strategy=req.fail_strategy or spec.default_fail_strategy,
         timeout_seconds=req.timeout_seconds or spec.default_timeout_seconds,
+        is_active=True,
+        is_reachable=True,
         creator_id=user.id,
     )
-    hook.is_reachable = True
     db_session.commit()
     return _hook_to_response(hook, creator_email=user.email)
 

backend/ee/onyx/server/query_and_chat/models.py

@@ -27,15 +27,17 @@ class SearchFlowClassificationResponse(BaseModel):
     is_search_flow: bool
 
 
-# NOTE: This model is used for the core flow of the Onyx application, any changes to it should be reviewed and approved by an
-# experienced team member. It is very important to 1. avoid bloat and 2. that this remains backwards compatible across versions.
+# NOTE: This model is used for the core flow of the Onyx application, any
+# changes to it should be reviewed and approved by an experienced team member.
+# It is very important to 1. avoid bloat and 2. that this remains backwards
+# compatible across versions.
 class SendSearchQueryRequest(BaseModel):
     search_query: str
     filters: BaseFilters | None = None
     num_docs_fed_to_llm_selection: int | None = None
     run_query_expansion: bool = False
     num_hits: int = 30
-
+    hybrid_alpha: float | None = None
     include_content: bool = False
     stream: bool = False
 

backend/ee/onyx/server/query_and_chat/search_backend.py

@@ -20,6 +20,7 @@ from ee.onyx.server.query_and_chat.models import SearchQueryResponse
 from ee.onyx.server.query_and_chat.models import SendSearchQueryRequest
 from ee.onyx.server.query_and_chat.streaming_models import SearchErrorPacket
 from onyx.auth.users import current_user
+from onyx.configs.app_configs import ONYX_SEARCH_UI_USES_OPENSEARCH_KEYWORD_SEARCH
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
 from onyx.db.models import User
@@ -67,8 +68,10 @@ def search_flow_classification(
     return SearchFlowClassificationResponse(is_search_flow=is_search_flow)
 
 
-# NOTE: This endpoint is used for the core flow of the Onyx application, any changes to it should be reviewed and approved by an
-# experienced team member. It is very important to 1. avoid bloat and 2. that this remains backwards compatible across versions.
+# NOTE: This endpoint is used for the core flow of the Onyx application, any
+# changes to it should be reviewed and approved by an experienced team member.
+# It is very important to 1. avoid bloat and 2. that this remains backwards
+# compatible across versions.
 @router.post(
     "/send-search-message",
     response_model=None,
@@ -80,13 +83,19 @@ def handle_send_search_message(
     db_session: Session = Depends(get_session),
 ) -> StreamingResponse | SearchFullResponse:
     """
-    Execute a search query with optional streaming.
+    Executes a search query with optional streaming.
 
-    When stream=True: Returns StreamingResponse with SSE
-    When stream=False: Returns SearchFullResponse
+    If hybrid_alpha is unset and ONYX_SEARCH_UI_USES_OPENSEARCH_KEYWORD_SEARCH
+    is True, executes pure keyword search.
+
+    Returns:
+        StreamingResponse with SSE if stream=True, otherwise SearchFullResponse.
     """
     logger.debug(f"Received search query: {request.search_query}")
 
+    if request.hybrid_alpha is None and ONYX_SEARCH_UI_USES_OPENSEARCH_KEYWORD_SEARCH:
+        request.hybrid_alpha = 0.0
+
     # Non-streaming path
     if not request.stream:
         try:

backend/ee/onyx/server/tenants/provisioning.py

@@ -99,6 +99,26 @@ async def get_or_provision_tenant(
         tenant_id = await get_available_tenant()
 
         if tenant_id:
+            # Run migrations to ensure the pre-provisioned tenant schema is current.
+            # Pool tenants may have been created before a new migration was deployed.
+            # Capture as a non-optional local so mypy can type the lambda correctly.
+            _tenant_id: str = tenant_id
+            loop = asyncio.get_running_loop()
+            try:
+                await loop.run_in_executor(
+                    None, lambda: run_alembic_migrations(_tenant_id)
+                )
+            except Exception:
+                # The tenant was already dequeued from the pool — roll it back so
+                # it doesn't end up orphaned (schema exists, but not assigned to anyone).
+                logger.exception(
+                    f"Migration failed for pre-provisioned tenant {_tenant_id}; rolling back"
+                )
+                try:
+                    await rollback_tenant_provisioning(_tenant_id)
+                except Exception:
+                    logger.exception(f"Failed to rollback orphaned tenant {_tenant_id}")
+                raise
             # If we have a pre-provisioned tenant, assign it to the user
             await assign_tenant_to_user(tenant_id, email, referral_source)
             logger.info(f"Assigned pre-provisioned tenant {tenant_id} to user {email}")

backend/model_server/main.py

@@ -100,6 +100,7 @@ def get_model_app() -> FastAPI:
             dsn=SENTRY_DSN,
             integrations=[StarletteIntegration(), FastApiIntegration()],
             traces_sample_rate=0.1,
+            release=__version__,
         )
         logger.info("Sentry initialized")
     else:

backend/onyx/background/celery/apps/app_base.py

@@ -20,6 +20,7 @@ from sentry_sdk.integrations.celery import CeleryIntegration
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
+from onyx import __version__
 from onyx.background.celery.apps.task_formatters import CeleryTaskColoredFormatter
 from onyx.background.celery.apps.task_formatters import CeleryTaskPlainFormatter
 from onyx.background.celery.celery_utils import celery_is_worker_primary
@@ -65,6 +66,7 @@ if SENTRY_DSN:
         dsn=SENTRY_DSN,
         integrations=[CeleryIntegration()],
         traces_sample_rate=0.1,
+        release=__version__,
     )
     logger.info("Sentry initialized")
 else:
@@ -515,7 +517,8 @@ def reset_tenant_id(
 
 
 def wait_for_vespa_or_shutdown(
-    sender: Any, **kwargs: Any  # noqa: ARG001
+    sender: Any,  # noqa: ARG001
+    **kwargs: Any,  # noqa: ARG001
 ) -> None:  # noqa: ARG001
     """Waits for Vespa to become ready subject to a timeout.
     Raises WorkerShutdown if the timeout is reached."""

backend/onyx/background/celery/apps/docfetching.py

@@ -13,6 +13,14 @@ from celery.signals import worker_shutdown
 import onyx.background.celery.apps.app_base as app_base
 from onyx.configs.constants import POSTGRES_CELERY_WORKER_DOCFETCHING_APP_NAME
 from onyx.db.engine.sql_engine import SqlEngine
+from onyx.server.metrics.celery_task_metrics import on_celery_task_postrun
+from onyx.server.metrics.celery_task_metrics import on_celery_task_prerun
+from onyx.server.metrics.celery_task_metrics import on_celery_task_rejected
+from onyx.server.metrics.celery_task_metrics import on_celery_task_retry
+from onyx.server.metrics.celery_task_metrics import on_celery_task_revoked
+from onyx.server.metrics.indexing_task_metrics import on_indexing_task_postrun
+from onyx.server.metrics.indexing_task_metrics import on_indexing_task_prerun
+from onyx.server.metrics.metrics_server import start_metrics_server
 from onyx.utils.logger import setup_logger
 from shared_configs.configs import MULTI_TENANT
 
@@ -34,6 +42,8 @@ def on_task_prerun(
     **kwds: Any,
 ) -> None:
     app_base.on_task_prerun(sender, task_id, task, args, kwargs, **kwds)
+    on_celery_task_prerun(task_id, task)
+    on_indexing_task_prerun(task_id, task, kwargs)
 
 
 @signals.task_postrun.connect
@@ -48,6 +58,36 @@ def on_task_postrun(
     **kwds: Any,
 ) -> None:
     app_base.on_task_postrun(sender, task_id, task, args, kwargs, retval, state, **kwds)
+    on_celery_task_postrun(task_id, task, state)
+    on_indexing_task_postrun(task_id, task, kwargs, state)
+
+
+@signals.task_retry.connect
+def on_task_retry(sender: Any | None = None, **kwargs: Any) -> None:  # noqa: ARG001
+    # task_retry signal doesn't pass task_id in kwargs; get it from
+    # the sender (the task instance) via sender.request.id.
+    task_id = getattr(getattr(sender, "request", None), "id", None)
+    on_celery_task_retry(task_id, sender)
+
+
+@signals.task_revoked.connect
+def on_task_revoked(sender: Any | None = None, **kwargs: Any) -> None:
+    task_name = getattr(sender, "name", None) or str(sender)
+    on_celery_task_revoked(kwargs.get("task_id"), task_name)
+
+
+@signals.task_rejected.connect
+def on_task_rejected(sender: Any | None = None, **kwargs: Any) -> None:  # noqa: ARG001
+    # task_rejected sends the Consumer as sender, not the task instance.
+    # The task name must be extracted from the Celery message headers.
+    message = kwargs.get("message")
+    task_name: str | None = None
+    if message is not None:
+        headers = getattr(message, "headers", None) or {}
+        task_name = headers.get("task")
+    if task_name is None:
+        task_name = "unknown"
+    on_celery_task_rejected(None, task_name)
 
 
 @celeryd_init.connect
@@ -76,6 +116,7 @@ def on_worker_init(sender: Worker, **kwargs: Any) -> None:
 
 @worker_ready.connect
 def on_worker_ready(sender: Any, **kwargs: Any) -> None:
+    start_metrics_server("docfetching")
     app_base.on_worker_ready(sender, **kwargs)
 
 

backend/onyx/background/celery/apps/docprocessing.py

@@ -14,6 +14,14 @@ from celery.signals import worker_shutdown
 import onyx.background.celery.apps.app_base as app_base
 from onyx.configs.constants import POSTGRES_CELERY_WORKER_DOCPROCESSING_APP_NAME
 from onyx.db.engine.sql_engine import SqlEngine
+from onyx.server.metrics.celery_task_metrics import on_celery_task_postrun
+from onyx.server.metrics.celery_task_metrics import on_celery_task_prerun
+from onyx.server.metrics.celery_task_metrics import on_celery_task_rejected
+from onyx.server.metrics.celery_task_metrics import on_celery_task_retry
+from onyx.server.metrics.celery_task_metrics import on_celery_task_revoked
+from onyx.server.metrics.indexing_task_metrics import on_indexing_task_postrun
+from onyx.server.metrics.indexing_task_metrics import on_indexing_task_prerun
+from onyx.server.metrics.metrics_server import start_metrics_server
 from onyx.utils.logger import setup_logger
 from shared_configs.configs import MULTI_TENANT
 
@@ -35,6 +43,8 @@ def on_task_prerun(
     **kwds: Any,
 ) -> None:
     app_base.on_task_prerun(sender, task_id, task, args, kwargs, **kwds)
+    on_celery_task_prerun(task_id, task)
+    on_indexing_task_prerun(task_id, task, kwargs)
 
 
 @signals.task_postrun.connect
@@ -49,6 +59,36 @@ def on_task_postrun(
     **kwds: Any,
 ) -> None:
     app_base.on_task_postrun(sender, task_id, task, args, kwargs, retval, state, **kwds)
+    on_celery_task_postrun(task_id, task, state)
+    on_indexing_task_postrun(task_id, task, kwargs, state)
+
+
+@signals.task_retry.connect
+def on_task_retry(sender: Any | None = None, **kwargs: Any) -> None:  # noqa: ARG001
+    # task_retry signal doesn't pass task_id in kwargs; get it from
+    # the sender (the task instance) via sender.request.id.
+    task_id = getattr(getattr(sender, "request", None), "id", None)
+    on_celery_task_retry(task_id, sender)
+
+
+@signals.task_revoked.connect
+def on_task_revoked(sender: Any | None = None, **kwargs: Any) -> None:
+    task_name = getattr(sender, "name", None) or str(sender)
+    on_celery_task_revoked(kwargs.get("task_id"), task_name)
+
+
+@signals.task_rejected.connect
+def on_task_rejected(sender: Any | None = None, **kwargs: Any) -> None:  # noqa: ARG001
+    # task_rejected sends the Consumer as sender, not the task instance.
+    # The task name must be extracted from the Celery message headers.
+    message = kwargs.get("message")
+    task_name: str | None = None
+    if message is not None:
+        headers = getattr(message, "headers", None) or {}
+        task_name = headers.get("task")
+    if task_name is None:
+        task_name = "unknown"
+    on_celery_task_rejected(None, task_name)
 
 
 @celeryd_init.connect
@@ -82,6 +122,7 @@ def on_worker_init(sender: Worker, **kwargs: Any) -> None:
 
 @worker_ready.connect
 def on_worker_ready(sender: Any, **kwargs: Any) -> None:
+    start_metrics_server("docprocessing")
     app_base.on_worker_ready(sender, **kwargs)
 
 
@@ -90,6 +131,12 @@ def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
     app_base.on_worker_shutdown(sender, **kwargs)
 
 
+# Note: worker_process_init only fires in prefork pool mode. Docprocessing uses
+# worker_pool="threads" (see configs/docprocessing.py), so this handler is
+# effectively a no-op in normal operation. It remains as a safety net in case
+# the pool type is ever changed to prefork. Prometheus metrics are safe in
+# thread-pool mode since all threads share the same process memory and can
+# update the same Counter/Gauge/Histogram objects directly.
 @worker_process_init.connect
 def init_worker(**kwargs: Any) -> None:  # noqa: ARG001
     SqlEngine.reset_engine()

backend/onyx/background/celery/apps/monitoring.py

@@ -54,8 +54,14 @@ def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None
     app_base.on_celeryd_init(sender, conf, **kwargs)
 
 
+# Set by on_worker_init so on_worker_ready knows whether to start the server.
+_prometheus_collectors_ok: bool = False
+
+
 @worker_init.connect
 def on_worker_init(sender: Any, **kwargs: Any) -> None:
+    global _prometheus_collectors_ok
+
     logger.info("worker_init signal received.")
     logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
 
@@ -65,6 +71,8 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
     app_base.wait_for_redis(sender, **kwargs)
     app_base.wait_for_db(sender, **kwargs)
 
+    _prometheus_collectors_ok = _setup_prometheus_collectors(sender)
+
     # Less startup checks in multi-tenant case
     if MULTI_TENANT:
         return
@@ -72,8 +80,37 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
     app_base.on_secondary_worker_init(sender, **kwargs)
 
 
+def _setup_prometheus_collectors(sender: Any) -> bool:
+    """Register Prometheus collectors that need Redis/DB access.
+
+    Passes the Celery app so the queue depth collector can obtain a fresh
+    broker Redis client on each scrape (rather than holding a stale reference).
+
+    Returns True if registration succeeded, False otherwise.
+    """
+    try:
+        from onyx.server.metrics.indexing_pipeline_setup import (
+            setup_indexing_pipeline_metrics,
+        )
+
+        setup_indexing_pipeline_metrics(sender.app)
+        logger.info("Prometheus indexing pipeline collectors registered")
+        return True
+    except Exception:
+        logger.exception("Failed to register Prometheus indexing pipeline collectors")
+        return False
+
+
 @worker_ready.connect
 def on_worker_ready(sender: Any, **kwargs: Any) -> None:
+    if _prometheus_collectors_ok:
+        from onyx.server.metrics.metrics_server import start_metrics_server
+
+        start_metrics_server("monitoring")
+    else:
+        logger.warning(
+            "Skipping Prometheus metrics server — collector registration failed"
+        )
     app_base.on_worker_ready(sender, **kwargs)
 
 

backend/onyx/background/celery/apps/primary.py

@@ -317,7 +317,6 @@ celery_app.autodiscover_tasks(
             "onyx.background.celery.tasks.docprocessing",
             "onyx.background.celery.tasks.evals",
             "onyx.background.celery.tasks.hierarchyfetching",
-            "onyx.background.celery.tasks.hooks",
             "onyx.background.celery.tasks.periodic",
             "onyx.background.celery.tasks.pruning",
             "onyx.background.celery.tasks.shared",

backend/onyx/background/celery/celery_redis.py

@@ -1,5 +1,6 @@
 # These are helper objects for tracking the keys we need to write in redis
 import json
+import threading
 from typing import Any
 from typing import cast
 
@@ -7,7 +8,59 @@ from celery import Celery
 from redis import Redis
 
 from onyx.background.celery.configs.base import CELERY_SEPARATOR
+from onyx.configs.app_configs import REDIS_HEALTH_CHECK_INTERVAL
 from onyx.configs.constants import OnyxCeleryPriority
+from onyx.configs.constants import REDIS_SOCKET_KEEPALIVE_OPTIONS
+
+
+_broker_client: Redis | None = None
+_broker_url: str | None = None
+_broker_client_lock = threading.Lock()
+
+
+def celery_get_broker_client(app: Celery) -> Redis:
+    """Return a shared Redis client connected to the Celery broker DB.
+
+    Uses a module-level singleton so all tasks on a worker share one
+    connection instead of creating a new one per call. The client
+    connects directly to the broker Redis DB (parsed from the broker URL).
+
+    Thread-safe via lock — safe for use in Celery thread-pool workers.
+
+    Usage:
+        r_celery = celery_get_broker_client(self.app)
+        length = celery_get_queue_length(queue, r_celery)
+    """
+    global _broker_client, _broker_url
+    with _broker_client_lock:
+        url = app.conf.broker_url
+        if _broker_client is not None and _broker_url == url:
+            try:
+                _broker_client.ping()
+                return _broker_client
+            except Exception:
+                try:
+                    _broker_client.close()
+                except Exception:
+                    pass
+                _broker_client = None
+        elif _broker_client is not None:
+            try:
+                _broker_client.close()
+            except Exception:
+                pass
+            _broker_client = None
+
+        _broker_url = url
+        _broker_client = Redis.from_url(
+            url,
+            decode_responses=False,
+            health_check_interval=REDIS_HEALTH_CHECK_INTERVAL,
+            socket_keepalive=True,
+            socket_keepalive_options=REDIS_SOCKET_KEEPALIVE_OPTIONS,
+            retry_on_timeout=True,
+        )
+        return _broker_client
 
 
 def celery_get_unacked_length(r: Redis) -> int:

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -14,7 +14,6 @@ from onyx.configs.constants import ONYX_CLOUD_CELERY_TASK_PREFIX
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.hooks.utils import HOOKS_AVAILABLE
 from shared_configs.configs import MULTI_TENANT
 
 # choosing 15 minutes because it roughly gives us enough time to process many tasks
@@ -362,19 +361,6 @@ if not MULTI_TENANT:
 
     tasks_to_schedule.extend(beat_task_templates)
 
-if HOOKS_AVAILABLE:
-    tasks_to_schedule.append(
-        {
-            "name": "hook-execution-log-cleanup",
-            "task": OnyxCeleryTask.HOOK_EXECUTION_LOG_CLEANUP_TASK,
-            "schedule": timedelta(days=1),
-            "options": {
-                "priority": OnyxCeleryPriority.LOW,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        }
-    )
-
 
 def generate_cloud_tasks(
     beat_tasks: list[dict], beat_templates: list[dict], beat_multiplier: float

backend/onyx/background/celery/tasks/connector_deletion/tasks.py

@@ -14,6 +14,7 @@ from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.configs.app_configs import JOB_TIMEOUT
@@ -132,7 +133,6 @@ def revoke_tasks_blocking_deletion(
 def check_for_connector_deletion_task(self: Task, *, tenant_id: str) -> bool | None:
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_CONNECTOR_DELETION_BEAT_LOCK,
@@ -149,6 +149,7 @@ def check_for_connector_deletion_task(self: Task, *, tenant_id: str) -> bool | N
         if not r.exists(OnyxRedisSignals.BLOCK_VALIDATE_CONNECTOR_DELETION_FENCES):
             # clear fences that don't have associated celery tasks in progress
             try:
+                r_celery = celery_get_broker_client(self.app)
                 validate_connector_deletion_fences(
                     tenant_id, r, r_replica, r_celery, lock_beat
                 )

backend/onyx/background/celery/tasks/docfetching/tasks.py

@@ -9,6 +9,7 @@ from celery import Celery
 from celery import shared_task
 from celery import Task
 
+from onyx import __version__
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.memory_monitoring import emit_process_memory
 from onyx.background.celery.tasks.docprocessing.heartbeat import start_heartbeat
@@ -137,6 +138,7 @@ def _docfetching_task(
         sentry_sdk.init(
             dsn=SENTRY_DSN,
             traces_sample_rate=0.1,
+            release=__version__,
         )
         logger.info("Sentry initialized")
     else:

backend/onyx/background/celery/tasks/docprocessing/tasks.py

@@ -22,6 +22,7 @@ from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_find_task
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
 from onyx.background.celery.celery_utils import httpx_init_vespa_pool
 from onyx.background.celery.memory_monitoring import emit_process_memory
@@ -318,6 +319,11 @@ def monitor_indexing_attempt_progress(
     )
 
     current_db_time = get_db_current_time(db_session)
+    total_batches: int | str = (
+        coordination_status.total_batches
+        if coordination_status.total_batches is not None
+        else "?"
+    )
     if coordination_status.found:
         task_logger.info(
             f"Indexing attempt progress: "
@@ -325,7 +331,7 @@ def monitor_indexing_attempt_progress(
             f"cc_pair={attempt.connector_credential_pair_id} "
             f"search_settings={attempt.search_settings_id} "
             f"completed_batches={coordination_status.completed_batches} "
-            f"total_batches={coordination_status.total_batches or '?'} "
+            f"total_batches={total_batches} "
             f"total_docs={coordination_status.total_docs} "
             f"total_failures={coordination_status.total_failures}"
             f"elapsed={(current_db_time - attempt.time_created).seconds}"
@@ -409,7 +415,7 @@ def check_indexing_completion(
     logger.info(
         f"Indexing status: "
         f"indexing_completed={indexing_completed} "
-        f"batches_processed={batches_processed}/{batches_total or '?'} "
+        f"batches_processed={batches_processed}/{batches_total if batches_total is not None else '?'} "
         f"total_docs={coordination_status.total_docs} "
         f"total_chunks={coordination_status.total_chunks} "
         f"total_failures={coordination_status.total_failures}"
@@ -449,7 +455,7 @@ def check_indexing_completion(
             ):
                 # Check if the task exists in the celery queue
                 # This handles the case where Redis dies after task creation but before task execution
-                redis_celery = task.app.broker_connection().channel().client  # type: ignore
+                redis_celery = celery_get_broker_client(task.app)
                 task_exists = celery_find_task(
                     attempt.celery_task_id,
                     OnyxCeleryQueues.CONNECTOR_DOC_FETCHING,

backend/onyx/background/celery/tasks/monitoring/tasks.py

@@ -1,6 +1,5 @@
 import json
 import time
-from collections.abc import Callable
 from datetime import timedelta
 from itertools import islice
 from typing import Any
@@ -19,6 +18,7 @@ from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
 from onyx.background.celery.memory_monitoring import emit_process_memory
@@ -698,31 +698,27 @@ def monitor_background_processes(self: Task, *, tenant_id: str) -> None:
         return None
 
     try:
-        # Get Redis client for Celery broker
-        redis_celery = self.app.broker_connection().channel().client  # type: ignore
         redis_std = get_redis_client()
 
-        # Define metric collection functions and their dependencies
-        metric_functions: list[Callable[[], list[Metric]]] = [
-            lambda: _collect_queue_metrics(redis_celery),
-            lambda: _collect_connector_metrics(db_session, redis_std),
-            lambda: _collect_sync_metrics(db_session, redis_std),
-        ]
+        # Collect queue metrics with broker connection
+        r_celery = celery_get_broker_client(self.app)
+        queue_metrics = _collect_queue_metrics(r_celery)
 
-        # Collect and log each metric
+        # Collect remaining metrics (no broker connection needed)
         with get_session_with_current_tenant() as db_session:
-            for metric_fn in metric_functions:
-                metrics = metric_fn()
-                for metric in metrics:
-                    # double check to make sure we aren't double-emitting metrics
-                    if metric.key is None or not _has_metric_been_emitted(
-                        redis_std, metric.key
-                    ):
-                        metric.log()
-                        metric.emit(tenant_id)
+            all_metrics: list[Metric] = queue_metrics
+            all_metrics.extend(_collect_connector_metrics(db_session, redis_std))
+            all_metrics.extend(_collect_sync_metrics(db_session, redis_std))
 
-                    if metric.key is not None:
-                        _mark_metric_as_emitted(redis_std, metric.key)
+            for metric in all_metrics:
+                if metric.key is None or not _has_metric_been_emitted(
+                    redis_std, metric.key
+                ):
+                    metric.log()
+                    metric.emit(tenant_id)
+
+                if metric.key is not None:
+                    _mark_metric_as_emitted(redis_std, metric.key)
 
         task_logger.info("Successfully collected background metrics")
     except SoftTimeLimitExceeded:
@@ -890,7 +886,7 @@ def monitor_celery_queues_helper(
 ) -> None:
     """A task to monitor all celery queue lengths."""
 
-    r_celery = task.app.broker_connection().channel().client  # type: ignore
+    r_celery = celery_get_broker_client(task.app)
     n_celery = celery_get_queue_length(OnyxCeleryQueues.PRIMARY, r_celery)
     n_docfetching = celery_get_queue_length(
         OnyxCeleryQueues.CONNECTOR_DOC_FETCHING, r_celery
@@ -1080,7 +1076,7 @@ def cloud_monitor_celery_pidbox(
     num_deleted = 0
 
     MAX_PIDBOX_IDLE = 24 * 3600  # 1 day in seconds
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
+    r_celery = celery_get_broker_client(self.app)
     for key in r_celery.scan_iter("*.reply.celery.pidbox"):
         key_bytes = cast(bytes, key)
         key_str = key_bytes.decode("utf-8")

backend/onyx/background/celery/tasks/pruning/tasks.py

@@ -17,6 +17,7 @@ from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_find_task
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
@@ -203,7 +204,6 @@ def _is_pruning_due(cc_pair: ConnectorCredentialPair) -> bool:
 def check_for_pruning(self: Task, *, tenant_id: str) -> bool | None:
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_PRUNE_BEAT_LOCK,
@@ -261,6 +261,7 @@ def check_for_pruning(self: Task, *, tenant_id: str) -> bool | None:
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
             try:
+                r_celery = celery_get_broker_client(self.app)
                 validate_pruning_fences(tenant_id, r, r_replica, r_celery, lock_beat)
             except Exception:
                 task_logger.exception("Exception while validating pruning fences")

backend/onyx/background/celery/tasks/user_file_processing/tasks.py

@@ -16,6 +16,7 @@ from sqlalchemy.orm import Session
 
 from onyx.access.access import build_access_for_user_files
 from onyx.background.celery.apps.app_base import task_logger
+from onyx.background.celery.celery_redis import celery_get_broker_client
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_utils import httpx_init_vespa_pool
 from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
@@ -105,7 +106,7 @@ def _user_file_delete_queued_key(user_file_id: str | UUID) -> str:
 
 
 def get_user_file_project_sync_queue_depth(celery_app: Celery) -> int:
-    redis_celery: Redis = celery_app.broker_connection().channel().client  # type: ignore
+    redis_celery = celery_get_broker_client(celery_app)
     return celery_get_queue_length(
         OnyxCeleryQueues.USER_FILE_PROJECT_SYNC, redis_celery
     )
@@ -238,7 +239,7 @@ def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
     skipped_guard = 0
     try:
         # --- Protection 1: queue depth backpressure ---
-        r_celery = self.app.broker_connection().channel().client  # type: ignore
+        r_celery = celery_get_broker_client(self.app)
         queue_len = celery_get_queue_length(
             OnyxCeleryQueues.USER_FILE_PROCESSING, r_celery
         )
@@ -591,7 +592,7 @@ def check_for_user_file_delete(self: Task, *, tenant_id: str) -> None:
         # --- Protection 1: queue depth backpressure ---
         # NOTE: must use the broker's Redis client (not redis_client) because
         # Celery queues live on a separate Redis DB with CELERY_SEPARATOR keys.
-        r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
+        r_celery = celery_get_broker_client(self.app)
         queue_len = celery_get_queue_length(OnyxCeleryQueues.USER_FILE_DELETE, r_celery)
         if queue_len > USER_FILE_DELETE_MAX_QUEUE_DEPTH:
             task_logger.warning(

backend/onyx/chat/chat_utils.py

@@ -5,6 +5,7 @@ from typing import cast
 from uuid import UUID
 
 from fastapi.datastructures import Headers
+from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
 from onyx.chat.models import ChatHistoryResult
@@ -51,6 +52,60 @@ logger = setup_logger()
 IMAGE_GENERATION_TOOL_NAME = "generate_image"
 
 
+class FileContextResult(BaseModel):
+    """Result of building a file's LLM context representation."""
+
+    message: ChatMessageSimple
+    tool_metadata: FileToolMetadata
+
+
+def build_file_context(
+    tool_file_id: str,
+    filename: str,
+    file_type: ChatFileType,
+    content_text: str | None = None,
+    token_count: int = 0,
+    approx_char_count: int | None = None,
+) -> FileContextResult:
+    """Build the LLM context representation for a single file.
+
+    Centralises how files should appear in the LLM prompt
+    — the ID that FileReaderTool accepts (``UserFile.id`` for user files).
+    """
+    if file_type.use_metadata_only():
+        message_text = (
+            f"File: {filename} (id={tool_file_id})\n"
+            "Use the file_reader or python tools to access "
+            "this file's contents."
+        )
+        message = ChatMessageSimple(
+            message=message_text,
+            token_count=max(1, len(message_text) // 4),
+            message_type=MessageType.USER,
+            file_id=tool_file_id,
+        )
+    else:
+        message_text = f"File: {filename}\n{content_text or ''}\nEnd of File"
+        message = ChatMessageSimple(
+            message=message_text,
+            token_count=token_count,
+            message_type=MessageType.USER,
+            file_id=tool_file_id,
+        )
+
+    metadata = FileToolMetadata(
+        file_id=tool_file_id,
+        filename=filename,
+        approx_char_count=(
+            approx_char_count
+            if approx_char_count is not None
+            else len(content_text or "")
+        ),
+    )
+
+    return FileContextResult(message=message, tool_metadata=metadata)
+
+
 def create_chat_session_from_request(
     chat_session_request: ChatSessionCreationRequest,
     user_id: UUID | None,
@@ -538,7 +593,7 @@ def convert_chat_history(
     for idx, chat_message in enumerate(chat_history):
         if chat_message.message_type == MessageType.USER:
             # Process files attached to this message
-            text_files: list[ChatLoadedFile] = []
+            text_files: list[tuple[ChatLoadedFile, FileDescriptor]] = []
             image_files: list[ChatLoadedFile] = []
 
             if chat_message.files:
@@ -549,34 +604,26 @@ def convert_chat_history(
                         if loaded_file.file_type == ChatFileType.IMAGE:
                             image_files.append(loaded_file)
                         else:
-                            # Text files (DOC, PLAIN_TEXT, CSV) are added as separate messages
-                            text_files.append(loaded_file)
+                            # Text files (DOC, PLAIN_TEXT, TABULAR) are added as separate messages
+                            text_files.append((loaded_file, file_descriptor))
 
             # Add text files as separate messages before the user message.
             # Each message is tagged with ``file_id`` so that forgotten files
             # can be detected after context-window truncation.
-            for text_file in text_files:
-                file_text = text_file.content_text or ""
-                filename = text_file.filename
-                message = (
-                    f"File: {filename}\n{file_text}\nEnd of File"
-                    if filename
-                    else file_text
-                )
-                simple_messages.append(
-                    ChatMessageSimple(
-                        message=message,
-                        token_count=text_file.token_count,
-                        message_type=MessageType.USER,
-                        image_files=None,
-                        file_id=text_file.file_id,
-                    )
-                )
-                all_injected_file_metadata[text_file.file_id] = FileToolMetadata(
-                    file_id=text_file.file_id,
-                    filename=filename or "unknown",
-                    approx_char_count=len(file_text),
+            for text_file, fd in text_files:
+                # Use user_file_id as the FileReaderTool accepts that.
+                # Fall back to the file-store path id.
+                tool_id = fd.get("user_file_id") or text_file.file_id
+                filename = text_file.filename or "unknown"
+                ctx = build_file_context(
+                    tool_file_id=tool_id,
+                    filename=filename,
+                    file_type=text_file.file_type,
+                    content_text=text_file.content_text,
+                    token_count=text_file.token_count,
                 )
+                simple_messages.append(ctx.message)
+                all_injected_file_metadata[tool_id] = ctx.tool_metadata
 
             # Sum token counts from image files (excluding project image files)
             image_token_count = (

backend/onyx/chat/models.py

@@ -8,6 +8,7 @@ from onyx.configs.constants import MessageType
 from onyx.context.search.models import SearchDoc
 from onyx.file_store.models import InMemoryChatFile
 from onyx.server.query_and_chat.models import MessageResponseIDInfo
+from onyx.server.query_and_chat.models import MultiModelMessageResponseIDInfo
 from onyx.server.query_and_chat.streaming_models import CitationInfo
 from onyx.server.query_and_chat.streaming_models import GeneratedImage
 from onyx.server.query_and_chat.streaming_models import Packet
@@ -35,7 +36,13 @@ class CreateChatSessionID(BaseModel):
     chat_session_id: UUID
 
 
-AnswerStreamPart = Packet | MessageResponseIDInfo | StreamingError | CreateChatSessionID
+AnswerStreamPart = (
+    Packet
+    | MessageResponseIDInfo
+    | MultiModelMessageResponseIDInfo
+    | StreamingError
+    | CreateChatSessionID
+)
 
 AnswerStream = Iterator[AnswerStreamPart]
 

backend/onyx/chat/process_message.py

@@ -18,6 +18,7 @@ from onyx.cache.interface import CacheBackend
 from onyx.chat.chat_processing_checker import set_processing_status
 from onyx.chat.chat_state import ChatStateContainer
 from onyx.chat.chat_state import run_chat_loop_with_state_containers
+from onyx.chat.chat_utils import build_file_context
 from onyx.chat.chat_utils import convert_chat_history
 from onyx.chat.chat_utils import create_chat_history_chain
 from onyx.chat.chat_utils import create_chat_session_from_request
@@ -90,6 +91,7 @@ from onyx.llm.request_context import reset_llm_mock_response
 from onyx.llm.request_context import set_llm_mock_response
 from onyx.llm.utils import litellm_exception_to_error_msg
 from onyx.onyxbot.slack.models import SlackContext
+from onyx.server.query_and_chat.chat_utils import mime_type_to_chat_file_type
 from onyx.server.query_and_chat.models import AUTO_PLACE_AFTER_LATEST_MESSAGE
 from onyx.server.query_and_chat.models import MessageResponseIDInfo
 from onyx.server.query_and_chat.models import SendMessageRequest
@@ -117,6 +119,8 @@ from shared_configs.contextvars import get_current_tenant_id
 logger = setup_logger()
 ERROR_TYPE_CANCELLED = "cancelled"
 
+APPROX_CHARS_PER_TOKEN = 4
+
 
 class _AvailableFiles(BaseModel):
     """Separated file IDs for the FileReaderTool so it knows which loader to use."""
@@ -301,16 +305,27 @@ def extract_context_files(
     if not user_files:
         return _empty_extracted_context_files()
 
-    aggregate_tokens = sum(uf.token_count or 0 for uf in user_files)
+    # Aggregate tokens for the file content that will be added
+    # Skip tokens for those with metadata only
+    aggregate_tokens = sum(
+        uf.token_count or 0
+        for uf in user_files
+        if not mime_type_to_chat_file_type(uf.file_type).use_metadata_only()
+    )
     max_actual_tokens = (
         llm_max_context_window - reserved_token_count
     ) * max_llm_context_percentage
 
     if aggregate_tokens >= max_actual_tokens:
-        tool_metadata = []
         use_as_search_filter = not DISABLE_VECTOR_DB
         if DISABLE_VECTOR_DB:
-            tool_metadata = _build_file_tool_metadata_for_user_files(user_files)
+            overflow_tool_metadata = [_build_tool_metadata(uf) for uf in user_files]
+        else:
+            overflow_tool_metadata = [
+                _build_tool_metadata(uf)
+                for uf in user_files
+                if mime_type_to_chat_file_type(uf.file_type).use_metadata_only()
+            ]
         return ExtractedContextFiles(
             file_texts=[],
             image_files=[],
@@ -318,11 +333,11 @@ def extract_context_files(
             total_token_count=0,
             file_metadata=[],
             uncapped_token_count=aggregate_tokens,
-            file_metadata_for_tool=tool_metadata,
+            file_metadata_for_tool=overflow_tool_metadata,
         )
 
     # Files fit — load them into context
-    user_file_map = {str(uf.id): uf for uf in user_files}
+    user_file_map = {uf.file_id: uf for uf in user_files}
     in_memory_files = load_in_memory_chat_files(
         user_file_ids=[uf.id for uf in user_files],
         db_session=db_session,
@@ -331,23 +346,38 @@ def extract_context_files(
     file_texts: list[str] = []
     image_files: list[ChatLoadedFile] = []
     file_metadata: list[ContextFileMetadata] = []
+    tool_metadata: list[FileToolMetadata] = []
     total_token_count = 0
 
     for f in in_memory_files:
         uf = user_file_map.get(str(f.file_id))
-        if f.file_type.is_text_file():
+        filename = f.filename or f"file_{f.file_id}"
+
+        if f.file_type.use_metadata_only():
+            # Metadata-only files are not injected as full text.
+            # Only the metadata is provided, with LLM using tools
+            if not uf:
+                logger.error(
+                    f"File with id={f.file_id} in metadata-only path with no associated user file"
+                )
+                continue
+            tool_metadata.append(_build_tool_metadata(uf))
+        elif f.file_type.is_text_file():
             text_content = _extract_text_from_in_memory_file(f)
             if not text_content:
                 continue
+            if not uf:
+                logger.warning(f"No user file for file_id={f.file_id}")
+                continue
             file_texts.append(text_content)
             file_metadata.append(
                 ContextFileMetadata(
-                    file_id=str(f.file_id),
-                    filename=f.filename or f"file_{f.file_id}",
+                    file_id=str(uf.id),
+                    filename=filename,
                     file_content=text_content,
                 )
             )
-            if uf and uf.token_count:
+            if uf.token_count:
                 total_token_count += uf.token_count
         elif f.file_type == ChatFileType.IMAGE:
             token_count = uf.token_count if uf and uf.token_count else 0
@@ -370,24 +400,22 @@ def extract_context_files(
         total_token_count=total_token_count,
         file_metadata=file_metadata,
         uncapped_token_count=aggregate_tokens,
+        file_metadata_for_tool=tool_metadata,
     )
 
 
-APPROX_CHARS_PER_TOKEN = 4
+def _build_tool_metadata(user_file: UserFile) -> FileToolMetadata:
+    """Build lightweight FileToolMetadata from a UserFile record.
 
-
-def _build_file_tool_metadata_for_user_files(
-    user_files: list[UserFile],
-) -> list[FileToolMetadata]:
-    """Build lightweight FileToolMetadata from a list of UserFile records."""
-    return [
-        FileToolMetadata(
-            file_id=str(uf.id),
-            filename=uf.name,
-            approx_char_count=(uf.token_count or 0) * APPROX_CHARS_PER_TOKEN,
-        )
-        for uf in user_files
-    ]
+    Delegates to ``build_file_context`` so that the file ID exposed to the
+    LLM is always consistent with what FileReaderTool expects.
+    """
+    return build_file_context(
+        tool_file_id=str(user_file.id),
+        filename=user_file.name,
+        file_type=mime_type_to_chat_file_type(user_file.file_type),
+        approx_char_count=(user_file.token_count or 0) * APPROX_CHARS_PER_TOKEN,
+    ).tool_metadata
 
 
 def determine_search_params(

backend/onyx/configs/app_configs.py

@@ -44,6 +44,31 @@ SEND_USER_METADATA_TO_LLM_PROVIDER = (
 # User Facing Features Configs
 #####
 BLURB_SIZE = 128  # Number Encoder Tokens included in the chunk blurb
+
+# Hard ceiling for the admin-configurable file upload size (in MB).
+# Self-hosted customers can raise or lower this via the environment variable.
+_raw_max_upload_size_mb = int(os.environ.get("MAX_ALLOWED_UPLOAD_SIZE_MB", "250"))
+if _raw_max_upload_size_mb < 0:
+    logger.warning(
+        "MAX_ALLOWED_UPLOAD_SIZE_MB=%d is negative; falling back to 250",
+        _raw_max_upload_size_mb,
+    )
+    _raw_max_upload_size_mb = 250
+MAX_ALLOWED_UPLOAD_SIZE_MB = _raw_max_upload_size_mb
+
+# Default fallback for the per-user file upload size limit (in MB) when no
+# admin-configured value exists.  Clamped to MAX_ALLOWED_UPLOAD_SIZE_MB at
+# runtime so this never silently exceeds the hard ceiling.
+_raw_default_upload_size_mb = int(
+    os.environ.get("DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB", "100")
+)
+if _raw_default_upload_size_mb < 0:
+    logger.warning(
+        "DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB=%d is negative; falling back to 100",
+        _raw_default_upload_size_mb,
+    )
+    _raw_default_upload_size_mb = 100
+DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB = _raw_default_upload_size_mb
 GENERATIVE_MODEL_ACCESS_CHECK_FREQ = int(
     os.environ.get("GENERATIVE_MODEL_ACCESS_CHECK_FREQ") or 86400
 )  # 1 day
@@ -61,17 +86,6 @@ CACHE_BACKEND = CacheBackendType(
     os.environ.get("CACHE_BACKEND", CacheBackendType.REDIS)
 )
 
-# Maximum token count for a single uploaded file. Files exceeding this are rejected.
-# Defaults to 100k tokens (or 10M when vector DB is disabled).
-_DEFAULT_FILE_TOKEN_LIMIT = 10_000_000 if DISABLE_VECTOR_DB else 100_000
-FILE_TOKEN_COUNT_THRESHOLD = int(
-    os.environ.get("FILE_TOKEN_COUNT_THRESHOLD", str(_DEFAULT_FILE_TOKEN_LIMIT))
-)
-
-# Maximum upload size for a single user file (chat/projects) in MB.
-USER_FILE_MAX_UPLOAD_SIZE_MB = int(os.environ.get("USER_FILE_MAX_UPLOAD_SIZE_MB") or 50)
-USER_FILE_MAX_UPLOAD_SIZE_BYTES = USER_FILE_MAX_UPLOAD_SIZE_MB * 1024 * 1024
-
 # If set to true, will show extra/uncommon connectors in the "Other" category
 SHOW_EXTRA_CONNECTORS = os.environ.get("SHOW_EXTRA_CONNECTORS", "").lower() == "true"
 
@@ -332,6 +346,10 @@ OPENSEARCH_INDEX_NUM_REPLICAS: int | None = (
     if os.environ.get("OPENSEARCH_INDEX_NUM_REPLICAS", None) is not None
     else None
 )
+ONYX_SEARCH_UI_USES_OPENSEARCH_KEYWORD_SEARCH = (
+    os.environ.get("ONYX_SEARCH_UI_USES_OPENSEARCH_KEYWORD_SEARCH", "").lower()
+    == "true"
+)
 
 VESPA_HOST = os.environ.get("VESPA_HOST") or "localhost"
 # NOTE: this is used if and only if the vespa config server is accessible via a
@@ -787,6 +805,10 @@ MINI_CHUNK_SIZE = 150
 # This is the number of regular chunks per large chunk
 LARGE_CHUNK_RATIO = 4
 
+# The maximum number of chunks that can be held for 1 document processing batch
+# The purpose of this is to set an upper bound on memory usage
+MAX_CHUNKS_PER_DOC_BATCH = int(os.environ.get("MAX_CHUNKS_PER_DOC_BATCH") or 1000)
+
 # Include the document level metadata in each chunk. If the metadata is too long, then it is thrown out
 # We don't want the metadata to overwhelm the actual contents of the chunk
 SKIP_METADATA_IN_CHUNK = os.environ.get("SKIP_METADATA_IN_CHUNK", "").lower() == "true"
@@ -1057,7 +1079,6 @@ POD_NAMESPACE = os.environ.get("POD_NAMESPACE")
 
 DEV_MODE = os.environ.get("DEV_MODE", "").lower() == "true"
 
-HOOK_ENABLED = os.environ.get("HOOK_ENABLED", "").lower() == "true"
 
 INTEGRATION_TESTS_MODE = os.environ.get("INTEGRATION_TESTS_MODE", "").lower() == "true"
 

backend/onyx/configs/chat_configs.py

@@ -24,11 +24,11 @@ CONTEXT_CHUNKS_BELOW = int(os.environ.get("CONTEXT_CHUNKS_BELOW") or 1)
 LLM_SOCKET_READ_TIMEOUT = int(
     os.environ.get("LLM_SOCKET_READ_TIMEOUT") or "60"
 )  # 60 seconds
-# Weighting factor between Vector and Keyword Search, 1 for completely vector search
+# Weighting factor between vector and keyword Search; 1 for completely vector
+# search, 0 for keyword. Enforces a valid range of [0, 1]. A supplied value from
+# the env outside of this range will be clipped to the respective end of the
+# range. Defaults to 0.5.
 HYBRID_ALPHA = max(0, min(1, float(os.environ.get("HYBRID_ALPHA") or 0.5)))
-HYBRID_ALPHA_KEYWORD = max(
-    0, min(1, float(os.environ.get("HYBRID_ALPHA_KEYWORD") or 0.4))
-)
 # Weighting factor between Title and Content of documents during search, 1 for completely
 # Title based. Default heavily favors Content because Title is also included at the top of
 # Content. This is to avoid cases where the Content is very relevant but it may not be clear

backend/onyx/configs/constants.py

@@ -212,6 +212,7 @@ class DocumentSource(str, Enum):
     PRODUCTBOARD = "productboard"
     FILE = "file"
     CODA = "coda"
+    CANVAS = "canvas"
     NOTION = "notion"
     ZULIP = "zulip"
     LINEAR = "linear"
@@ -672,6 +673,7 @@ DocumentSourceDescription: dict[DocumentSource, str] = {
     DocumentSource.SLAB: "slab data",
     DocumentSource.PRODUCTBOARD: "productboard data (boards, etc.)",
     DocumentSource.FILE: "files",
+    DocumentSource.CANVAS: "canvas lms - courses, pages, assignments, and announcements",
     DocumentSource.CODA: "coda - team workspace with docs, tables, and pages",
     DocumentSource.NOTION: "notion data - a workspace that combines note-taking, \
 project management, and collaboration tools into a single, customizable platform",

backend/onyx/connectors/canvas/access.py

@@ -0,0 +1,32 @@
+"""
+Permissioning / AccessControl logic for Canvas courses.
+
+CE stub — returns None (no permissions). The EE implementation is loaded
+at runtime via ``fetch_versioned_implementation``.
+"""
+
+from collections.abc import Callable
+from typing import cast
+
+from onyx.access.models import ExternalAccess
+from onyx.connectors.canvas.client import CanvasApiClient
+from onyx.utils.variable_functionality import fetch_versioned_implementation
+from onyx.utils.variable_functionality import global_version
+
+
+def get_course_permissions(
+    canvas_client: CanvasApiClient,
+    course_id: int,
+) -> ExternalAccess | None:
+    if not global_version.is_ee_version():
+        return None
+
+    ee_get_course_permissions = cast(
+        Callable[[CanvasApiClient, int], ExternalAccess | None],
+        fetch_versioned_implementation(
+            "onyx.external_permissions.canvas.access",
+            "get_course_permissions",
+        ),
+    )
+
+    return ee_get_course_permissions(canvas_client, course_id)

backend/onyx/connectors/canvas/client.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Iterator
+from typing import Any
+from urllib.parse import urlparse
+
+from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
+    rl_requests,
+)
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
+
+logger = logging.getLogger(__name__)
+
+# Requests timeout in seconds.
+_CANVAS_CALL_TIMEOUT: int = 30
+_CANVAS_API_VERSION: str = "/api/v1"
+# Matches the "next" URL in a Canvas Link header, e.g.:
+#   <https://canvas.example.com/api/v1/courses?page=2>; rel="next"
+# Captures the URL inside the angle brackets.
+_NEXT_LINK_PATTERN: re.Pattern[str] = re.compile(r'<([^>]+)>;\s*rel="next"')
+
+
+_STATUS_TO_ERROR_CODE: dict[int, OnyxErrorCode] = {
+    401: OnyxErrorCode.CREDENTIAL_EXPIRED,
+    403: OnyxErrorCode.INSUFFICIENT_PERMISSIONS,
+    404: OnyxErrorCode.BAD_GATEWAY,
+    429: OnyxErrorCode.RATE_LIMITED,
+}
+
+
+def _error_code_for_status(status_code: int) -> OnyxErrorCode:
+    """Map an HTTP status code to the appropriate OnyxErrorCode.
+
+    Expects a >= 400 status code. Known codes (401, 403, 404, 429) are
+    mapped to specific error codes; all other codes (unrecognised 4xx
+    and 5xx) map to BAD_GATEWAY as unexpected upstream errors.
+    """
+    if status_code in _STATUS_TO_ERROR_CODE:
+        return _STATUS_TO_ERROR_CODE[status_code]
+    return OnyxErrorCode.BAD_GATEWAY
+
+
+class CanvasApiClient:
+    def __init__(
+        self,
+        bearer_token: str,
+        canvas_base_url: str,
+    ) -> None:
+        parsed_base = urlparse(canvas_base_url)
+        if not parsed_base.hostname:
+            raise ValueError("canvas_base_url must include a valid host")
+        if parsed_base.scheme != "https":
+            raise ValueError("canvas_base_url must use https")
+
+        self._bearer_token = bearer_token
+        self.base_url = (
+            canvas_base_url.rstrip("/").removesuffix(_CANVAS_API_VERSION)
+            + _CANVAS_API_VERSION
+        )
+        # Hostname is already validated above; reuse parsed_base instead
+        # of re-parsing.  Used by _parse_next_link to validate pagination URLs.
+        self._expected_host: str = parsed_base.hostname
+
+    def get(
+        self,
+        endpoint: str = "",
+        params: dict[str, Any] | None = None,
+        full_url: str | None = None,
+    ) -> tuple[Any, str | None]:
+        """Make a GET request to the Canvas API.
+
+        Returns a tuple of (json_body, next_url).
+        next_url is parsed from the Link header and is None if there are no more pages.
+        If full_url is provided, it is used directly (for following pagination links).
+
+        Security note: full_url must only be set to values returned by
+        ``_parse_next_link``, which validates the host against the configured
+        Canvas base URL.  Passing an arbitrary URL would leak the bearer token.
+        """
+        # full_url is used when following pagination (Canvas returns the
+        # next-page URL in the Link header).  For the first request we build
+        # the URL from the endpoint name instead.
+        url = full_url if full_url else self._build_url(endpoint)
+        headers = self._build_headers()
+
+        response = rl_requests.get(
+            url,
+            headers=headers,
+            params=params if not full_url else None,
+            timeout=_CANVAS_CALL_TIMEOUT,
+        )
+
+        try:
+            response_json = response.json()
+        except ValueError as e:
+            if response.status_code < 300:
+                raise OnyxError(
+                    OnyxErrorCode.BAD_GATEWAY,
+                    detail=f"Invalid JSON in Canvas response: {e}",
+                )
+            logger.warning(
+                "Failed to parse JSON from Canvas error response (status=%d): %s",
+                response.status_code,
+                e,
+            )
+            response_json = {}
+
+        if response.status_code >= 400:
+            # Try to extract the most specific error message from the
+            # Canvas response body.  Canvas uses three different shapes
+            # depending on the endpoint and error type:
+            default_error: str = response.reason or f"HTTP {response.status_code}"
+            error = default_error
+            if isinstance(response_json, dict):
+                # Shape 1: {"error": {"message": "Not authorized"}}
+                error_field = response_json.get("error")
+                if isinstance(error_field, dict):
+                    response_error = error_field.get("message", "")
+                    if response_error:
+                        error = response_error
+                # Shape 2: {"error": "Invalid access token"}
+                elif isinstance(error_field, str):
+                    error = error_field
+                # Shape 3: {"errors": [{"message": "..."}]}
+                # Used for validation errors.  Only use as fallback if
+                # we didn't already find a more specific message above.
+                if error == default_error:
+                    errors_list = response_json.get("errors")
+                    if isinstance(errors_list, list) and errors_list:
+                        first_error = errors_list[0]
+                        if isinstance(first_error, dict):
+                            msg = first_error.get("message", "")
+                            if msg:
+                                error = msg
+            raise OnyxError(
+                _error_code_for_status(response.status_code),
+                detail=error,
+                status_code_override=response.status_code,
+            )
+
+        next_url = self._parse_next_link(response.headers.get("Link", ""))
+        return response_json, next_url
+
+    def _parse_next_link(self, link_header: str) -> str | None:
+        """Extract the 'next' URL from a Canvas Link header.
+
+        Only returns URLs whose host matches the configured Canvas base URL
+        to prevent leaking the bearer token to arbitrary hosts.
+        """
+        expected_host = self._expected_host
+        for match in _NEXT_LINK_PATTERN.finditer(link_header):
+            url = match.group(1)
+            parsed_url = urlparse(url)
+            if parsed_url.hostname != expected_host:
+                raise OnyxError(
+                    OnyxErrorCode.BAD_GATEWAY,
+                    detail=(
+                        "Canvas pagination returned an unexpected host "
+                        f"({parsed_url.hostname}); expected {expected_host}"
+                    ),
+                )
+            if parsed_url.scheme != "https":
+                raise OnyxError(
+                    OnyxErrorCode.BAD_GATEWAY,
+                    detail=(
+                        "Canvas pagination link must use https, "
+                        f"got {parsed_url.scheme!r}"
+                    ),
+                )
+            return url
+        return None
+
+    def _build_headers(self) -> dict[str, str]:
+        """Return the Authorization header with the bearer token."""
+        return {"Authorization": f"Bearer {self._bearer_token}"}
+
+    def _build_url(self, endpoint: str) -> str:
+        """Build a full Canvas API URL from an endpoint path.
+
+        Assumes endpoint is non-empty (e.g. ``"courses"``, ``"announcements"``).
+        Only called on a first request, endpoint must be set for first request.
+        Verify endpoint exists in case of future changes where endpoint might be optional.
+        Leading slashes are stripped to avoid double-slash in the result.
+        self.base_url is already normalized with no trailing slash.
+        """
+        final_url = self.base_url
+        clean_endpoint = endpoint.lstrip("/")
+        if clean_endpoint:
+            final_url += "/" + clean_endpoint
+        return final_url
+
+    def paginate(
+        self,
+        endpoint: str,
+        params: dict[str, Any] | None = None,
+    ) -> Iterator[list[Any]]:
+        """Yield each page of results, following Link-header pagination.
+
+        Makes the first request with endpoint + params, then follows
+        next_url from Link headers for subsequent pages.
+        """
+        response, next_url = self.get(endpoint, params=params)
+        while True:
+            if not response:
+                break
+            yield response
+            if not next_url:
+                break
+            response, next_url = self.get(full_url=next_url)

backend/onyx/connectors/canvas/connector.py

@@ -0,0 +1,458 @@
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+from typing import cast
+from typing import Literal
+from typing import NoReturn
+from typing import TypeAlias
+
+from pydantic import BaseModel
+from retry import retry
+from typing_extensions import override
+
+from onyx.access.models import ExternalAccess
+from onyx.configs.app_configs import INDEX_BATCH_SIZE
+from onyx.configs.constants import DocumentSource
+from onyx.connectors.canvas.access import get_course_permissions
+from onyx.connectors.canvas.client import CanvasApiClient
+from onyx.connectors.exceptions import ConnectorValidationError
+from onyx.connectors.exceptions import CredentialExpiredError
+from onyx.connectors.exceptions import InsufficientPermissionsError
+from onyx.connectors.exceptions import UnexpectedValidationError
+from onyx.connectors.interfaces import CheckpointedConnectorWithPermSync
+from onyx.connectors.interfaces import CheckpointOutput
+from onyx.connectors.interfaces import GenerateSlimDocumentOutput
+from onyx.connectors.interfaces import SecondsSinceUnixEpoch
+from onyx.connectors.interfaces import SlimConnectorWithPermSync
+from onyx.connectors.models import ConnectorCheckpoint
+from onyx.connectors.models import ConnectorMissingCredentialError
+from onyx.connectors.models import Document
+from onyx.connectors.models import ImageSection
+from onyx.connectors.models import TextSection
+from onyx.error_handling.exceptions import OnyxError
+from onyx.file_processing.html_utils import parse_html_page_basic
+from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def _handle_canvas_api_error(e: OnyxError) -> NoReturn:
+    """Map Canvas API errors to connector framework exceptions."""
+    if e.status_code == 401:
+        raise CredentialExpiredError(
+            "Canvas API token is invalid or expired (HTTP 401)."
+        )
+    elif e.status_code == 403:
+        raise InsufficientPermissionsError(
+            "Canvas API token does not have sufficient permissions (HTTP 403)."
+        )
+    elif e.status_code == 429:
+        raise ConnectorValidationError(
+            "Canvas rate-limit exceeded (HTTP 429). Please try again later."
+        )
+    elif e.status_code >= 500:
+        raise UnexpectedValidationError(
+            f"Unexpected Canvas HTTP error (status={e.status_code}): {e}"
+        )
+    else:
+        raise ConnectorValidationError(
+            f"Canvas API error (status={e.status_code}): {e}"
+        )
+
+
+class CanvasCourse(BaseModel):
+    id: int
+    name: str | None = None
+    course_code: str | None = None
+    created_at: str | None = None
+    workflow_state: str | None = None
+
+    @classmethod
+    def from_api(cls, payload: dict[str, Any]) -> "CanvasCourse":
+        return cls(
+            id=payload["id"],
+            name=payload.get("name"),
+            course_code=payload.get("course_code"),
+            created_at=payload.get("created_at"),
+            workflow_state=payload.get("workflow_state"),
+        )
+
+
+class CanvasPage(BaseModel):
+    page_id: int
+    url: str
+    title: str
+    body: str | None = None
+    created_at: str | None = None
+    updated_at: str | None = None
+    course_id: int
+
+    @classmethod
+    def from_api(cls, payload: dict[str, Any], course_id: int) -> "CanvasPage":
+        return cls(
+            page_id=payload["page_id"],
+            url=payload["url"],
+            title=payload["title"],
+            body=payload.get("body"),
+            created_at=payload.get("created_at"),
+            updated_at=payload.get("updated_at"),
+            course_id=course_id,
+        )
+
+
+class CanvasAssignment(BaseModel):
+    id: int
+    name: str
+    description: str | None = None
+    html_url: str
+    course_id: int
+    created_at: str | None = None
+    updated_at: str | None = None
+    due_at: str | None = None
+
+    @classmethod
+    def from_api(cls, payload: dict[str, Any], course_id: int) -> "CanvasAssignment":
+        return cls(
+            id=payload["id"],
+            name=payload["name"],
+            description=payload.get("description"),
+            html_url=payload["html_url"],
+            course_id=course_id,
+            created_at=payload.get("created_at"),
+            updated_at=payload.get("updated_at"),
+            due_at=payload.get("due_at"),
+        )
+
+
+class CanvasAnnouncement(BaseModel):
+    id: int
+    title: str
+    message: str | None = None
+    html_url: str
+    posted_at: str | None = None
+    course_id: int
+
+    @classmethod
+    def from_api(cls, payload: dict[str, Any], course_id: int) -> "CanvasAnnouncement":
+        return cls(
+            id=payload["id"],
+            title=payload["title"],
+            message=payload.get("message"),
+            html_url=payload["html_url"],
+            posted_at=payload.get("posted_at"),
+            course_id=course_id,
+        )
+
+
+CanvasStage: TypeAlias = Literal["pages", "assignments", "announcements"]
+
+
+class CanvasConnectorCheckpoint(ConnectorCheckpoint):
+    """Checkpoint state for resumable Canvas indexing.
+
+    Fields:
+        course_ids: Materialized list of course IDs to process.
+        current_course_index: Index into course_ids for current course.
+        stage: Which item type we're processing for the current course.
+        next_url: Pagination cursor within the current stage. None means
+            start from the first page; a URL means resume from that page.
+
+    Invariant:
+        If current_course_index is incremented, stage must be reset to
+        "pages" and next_url must be reset to None.
+    """
+
+    course_ids: list[int] = []
+    current_course_index: int = 0
+    stage: CanvasStage = "pages"
+    next_url: str | None = None
+
+    def advance_course(self) -> None:
+        """Move to the next course and reset within-course state."""
+        self.current_course_index += 1
+        self.stage = "pages"
+        self.next_url = None
+
+
+class CanvasConnector(
+    CheckpointedConnectorWithPermSync[CanvasConnectorCheckpoint],
+    SlimConnectorWithPermSync,
+):
+    def __init__(
+        self,
+        canvas_base_url: str,
+        batch_size: int = INDEX_BATCH_SIZE,
+    ) -> None:
+        self.canvas_base_url = canvas_base_url.rstrip("/").removesuffix("/api/v1")
+        self.batch_size = batch_size
+        self._canvas_client: CanvasApiClient | None = None
+        self._course_permissions_cache: dict[int, ExternalAccess | None] = {}
+
+    @property
+    def canvas_client(self) -> CanvasApiClient:
+        if self._canvas_client is None:
+            raise ConnectorMissingCredentialError("Canvas")
+        return self._canvas_client
+
+    def _get_course_permissions(self, course_id: int) -> ExternalAccess | None:
+        """Get course permissions with caching."""
+        if course_id not in self._course_permissions_cache:
+            self._course_permissions_cache[course_id] = get_course_permissions(
+                canvas_client=self.canvas_client,
+                course_id=course_id,
+            )
+        return self._course_permissions_cache[course_id]
+
+    @retry(tries=3, delay=1, backoff=2)
+    def _list_courses(self) -> list[CanvasCourse]:
+        """Fetch all courses accessible to the authenticated user."""
+        logger.debug("Fetching Canvas courses")
+
+        courses: list[CanvasCourse] = []
+        for page in self.canvas_client.paginate(
+            "courses", params={"per_page": "100", "state[]": "available"}
+        ):
+            courses.extend(CanvasCourse.from_api(c) for c in page)
+        return courses
+
+    @retry(tries=3, delay=1, backoff=2)
+    def _list_pages(self, course_id: int) -> list[CanvasPage]:
+        """Fetch all pages for a given course."""
+        logger.debug(f"Fetching pages for course {course_id}")
+
+        pages: list[CanvasPage] = []
+        for page in self.canvas_client.paginate(
+            f"courses/{course_id}/pages",
+            params={"per_page": "100", "include[]": "body", "published": "true"},
+        ):
+            pages.extend(CanvasPage.from_api(p, course_id=course_id) for p in page)
+        return pages
+
+    @retry(tries=3, delay=1, backoff=2)
+    def _list_assignments(self, course_id: int) -> list[CanvasAssignment]:
+        """Fetch all assignments for a given course."""
+        logger.debug(f"Fetching assignments for course {course_id}")
+
+        assignments: list[CanvasAssignment] = []
+        for page in self.canvas_client.paginate(
+            f"courses/{course_id}/assignments",
+            params={"per_page": "100", "published": "true"},
+        ):
+            assignments.extend(
+                CanvasAssignment.from_api(a, course_id=course_id) for a in page
+            )
+        return assignments
+
+    @retry(tries=3, delay=1, backoff=2)
+    def _list_announcements(self, course_id: int) -> list[CanvasAnnouncement]:
+        """Fetch all announcements for a given course."""
+        logger.debug(f"Fetching announcements for course {course_id}")
+
+        announcements: list[CanvasAnnouncement] = []
+        for page in self.canvas_client.paginate(
+            "announcements",
+            params={
+                "per_page": "100",
+                "context_codes[]": f"course_{course_id}",
+                "active_only": "true",
+            },
+        ):
+            announcements.extend(
+                CanvasAnnouncement.from_api(a, course_id=course_id) for a in page
+            )
+        return announcements
+
+    def _build_document(
+        self,
+        doc_id: str,
+        link: str,
+        text: str,
+        semantic_identifier: str,
+        doc_updated_at: datetime | None,
+        course_id: int,
+        doc_type: str,
+    ) -> Document:
+        """Build a Document with standard Canvas fields."""
+        return Document(
+            id=doc_id,
+            sections=cast(
+                list[TextSection | ImageSection],
+                [TextSection(link=link, text=text)],
+            ),
+            source=DocumentSource.CANVAS,
+            semantic_identifier=semantic_identifier,
+            doc_updated_at=doc_updated_at,
+            metadata={"course_id": str(course_id), "type": doc_type},
+        )
+
+    def _convert_page_to_document(self, page: CanvasPage) -> Document:
+        """Convert a Canvas page to a Document."""
+        link = f"{self.canvas_base_url}/courses/{page.course_id}/pages/{page.url}"
+
+        text_parts = [page.title]
+        body_text = parse_html_page_basic(page.body) if page.body else ""
+        if body_text:
+            text_parts.append(body_text)
+
+        doc_updated_at = (
+            datetime.fromisoformat(page.updated_at.replace("Z", "+00:00")).astimezone(
+                timezone.utc
+            )
+            if page.updated_at
+            else None
+        )
+
+        document = self._build_document(
+            doc_id=f"canvas-page-{page.course_id}-{page.page_id}",
+            link=link,
+            text="\n\n".join(text_parts),
+            semantic_identifier=page.title or f"Page {page.page_id}",
+            doc_updated_at=doc_updated_at,
+            course_id=page.course_id,
+            doc_type="page",
+        )
+        return document
+
+    def _convert_assignment_to_document(self, assignment: CanvasAssignment) -> Document:
+        """Convert a Canvas assignment to a Document."""
+        text_parts = [assignment.name]
+        desc_text = (
+            parse_html_page_basic(assignment.description)
+            if assignment.description
+            else ""
+        )
+        if desc_text:
+            text_parts.append(desc_text)
+        if assignment.due_at:
+            due_dt = datetime.fromisoformat(
+                assignment.due_at.replace("Z", "+00:00")
+            ).astimezone(timezone.utc)
+            text_parts.append(f"Due: {due_dt.strftime('%B %d, %Y %H:%M UTC')}")
+
+        doc_updated_at = (
+            datetime.fromisoformat(
+                assignment.updated_at.replace("Z", "+00:00")
+            ).astimezone(timezone.utc)
+            if assignment.updated_at
+            else None
+        )
+
+        document = self._build_document(
+            doc_id=f"canvas-assignment-{assignment.course_id}-{assignment.id}",
+            link=assignment.html_url,
+            text="\n\n".join(text_parts),
+            semantic_identifier=assignment.name or f"Assignment {assignment.id}",
+            doc_updated_at=doc_updated_at,
+            course_id=assignment.course_id,
+            doc_type="assignment",
+        )
+        return document
+
+    def _convert_announcement_to_document(
+        self, announcement: CanvasAnnouncement
+    ) -> Document:
+        """Convert a Canvas announcement to a Document."""
+        text_parts = [announcement.title]
+        msg_text = (
+            parse_html_page_basic(announcement.message) if announcement.message else ""
+        )
+        if msg_text:
+            text_parts.append(msg_text)
+
+        doc_updated_at = (
+            datetime.fromisoformat(
+                announcement.posted_at.replace("Z", "+00:00")
+            ).astimezone(timezone.utc)
+            if announcement.posted_at
+            else None
+        )
+
+        document = self._build_document(
+            doc_id=f"canvas-announcement-{announcement.course_id}-{announcement.id}",
+            link=announcement.html_url,
+            text="\n\n".join(text_parts),
+            semantic_identifier=announcement.title or f"Announcement {announcement.id}",
+            doc_updated_at=doc_updated_at,
+            course_id=announcement.course_id,
+            doc_type="announcement",
+        )
+        return document
+
+    @override
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
+        """Load and validate Canvas credentials."""
+        access_token = credentials.get("canvas_access_token")
+        if not access_token:
+            raise ConnectorMissingCredentialError("Canvas")
+
+        try:
+            client = CanvasApiClient(
+                bearer_token=access_token,
+                canvas_base_url=self.canvas_base_url,
+            )
+            client.get("courses", params={"per_page": "1"})
+        except ValueError as e:
+            raise ConnectorValidationError(f"Invalid Canvas base URL: {e}")
+        except OnyxError as e:
+            _handle_canvas_api_error(e)
+
+        self._canvas_client = client
+        return None
+
+    @override
+    def validate_connector_settings(self) -> None:
+        """Validate Canvas connector settings by testing API access."""
+        try:
+            self.canvas_client.get("courses", params={"per_page": "1"})
+            logger.info("Canvas connector settings validated successfully")
+        except OnyxError as e:
+            _handle_canvas_api_error(e)
+        except ConnectorMissingCredentialError:
+            raise
+        except Exception as exc:
+            raise UnexpectedValidationError(
+                f"Unexpected error during Canvas settings validation: {exc}"
+            )
+
+    @override
+    def load_from_checkpoint(
+        self,
+        start: SecondsSinceUnixEpoch,
+        end: SecondsSinceUnixEpoch,
+        checkpoint: CanvasConnectorCheckpoint,
+    ) -> CheckpointOutput[CanvasConnectorCheckpoint]:
+        # TODO(benwu408): implemented in PR3 (checkpoint)
+        raise NotImplementedError
+
+    @override
+    def load_from_checkpoint_with_perm_sync(
+        self,
+        start: SecondsSinceUnixEpoch,
+        end: SecondsSinceUnixEpoch,
+        checkpoint: CanvasConnectorCheckpoint,
+    ) -> CheckpointOutput[CanvasConnectorCheckpoint]:
+        # TODO(benwu408): implemented in PR3 (checkpoint)
+        raise NotImplementedError
+
+    @override
+    def build_dummy_checkpoint(self) -> CanvasConnectorCheckpoint:
+        # TODO(benwu408): implemented in PR3 (checkpoint)
+        raise NotImplementedError
+
+    @override
+    def validate_checkpoint_json(
+        self, checkpoint_json: str
+    ) -> CanvasConnectorCheckpoint:
+        # TODO(benwu408): implemented in PR3 (checkpoint)
+        raise NotImplementedError
+
+    @override
+    def retrieve_all_slim_docs_perm_sync(
+        self,
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
+        callback: IndexingHeartbeatInterface | None = None,
+    ) -> GenerateSlimDocumentOutput:
+        # TODO(benwu408): implemented in PR4 (perm sync)
+        raise NotImplementedError

backend/onyx/connectors/confluence/connector.py

@@ -890,8 +890,8 @@ class ConfluenceConnector(
 
     def _retrieve_all_slim_docs(
         self,
-        start: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
-        end: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
         include_permissions: bool = True,
     ) -> GenerateSlimDocumentOutput:
@@ -915,8 +915,8 @@ class ConfluenceConnector(
                 self.confluence_client, doc_id, restrictions, ancestors
             ) or space_level_access_info.get(page_space_key)
 
-        # Query pages
-        page_query = self.base_cql_page_query + self.cql_label_filter
+        # Query pages (with optional time filtering for indexing_start)
+        page_query = self._construct_page_cql_query(start, end)
         for page in self.confluence_client.cql_paginate_all_expansions(
             cql=page_query,
             expand=restrictions_expand,
@@ -950,7 +950,9 @@ class ConfluenceConnector(
 
             # Query attachments for each page
             page_hierarchy_node_yielded = False
-            attachment_query = self._construct_attachment_query(_get_page_id(page))
+            attachment_query = self._construct_attachment_query(
+                _get_page_id(page), start, end
+            )
             for attachment in self.confluence_client.cql_paginate_all_expansions(
                 cql=attachment_query,
                 expand=restrictions_expand,

backend/onyx/connectors/confluence/onyx_confluence.py

@@ -123,7 +123,7 @@ class OnyxConfluence:
 
         self.shared_base_kwargs: dict[str, str | int | bool] = {
             "api_version": "cloud" if is_cloud else "latest",
-            "backoff_and_retry": True,
+            "backoff_and_retry": False,
             "cloud": is_cloud,
         }
         if timeout:
@@ -456,7 +456,7 @@ class OnyxConfluence:
                         return attr(*args, **kwargs)
 
                 except HTTPError as e:
-                    delay_until = _handle_http_error(e, attempt)
+                    delay_until = _handle_http_error(e, attempt, MAX_RETRIES)
                     logger.warning(
                         f"HTTPError in confluence call. Retrying in {delay_until} seconds..."
                     )

backend/onyx/connectors/confluence/utils.py

@@ -363,7 +363,7 @@ def handle_confluence_rate_limit(confluence_call: F) -> F:
                 # and applying our own retries in a more specific set of circumstances
                 return confluence_call(*args, **kwargs)
             except requests.HTTPError as e:
-                delay_until = _handle_http_error(e, attempt)
+                delay_until = _handle_http_error(e, attempt, MAX_RETRIES)
                 logger.warning(
                     f"HTTPError in confluence call. Retrying in {delay_until} seconds..."
                 )
@@ -384,7 +384,7 @@ def handle_confluence_rate_limit(confluence_call: F) -> F:
     return cast(F, wrapped_call)
 
 
-def _handle_http_error(e: requests.HTTPError, attempt: int) -> int:
+def _handle_http_error(e: requests.HTTPError, attempt: int, max_retries: int) -> int:
     MIN_DELAY = 2
     MAX_DELAY = 60
     STARTING_DELAY = 5
@@ -408,6 +408,17 @@ def _handle_http_error(e: requests.HTTPError, attempt: int) -> int:
 
         raise e
 
+    if e.response.status_code >= 500:
+        if attempt >= max_retries - 1:
+            raise e
+
+        delay = min(STARTING_DELAY * (BACKOFF**attempt), MAX_DELAY)
+        logger.warning(
+            f"Server error {e.response.status_code}. "
+            f"Retrying in {delay} seconds (attempt {attempt + 1})..."
+        )
+        return math.ceil(time.monotonic() + delay)
+
     if (
         e.response.status_code != 429
         and RATE_LIMIT_MESSAGE_LOWERCASE not in e.response.text.lower()

backend/onyx/connectors/discord/connector.py

@@ -11,11 +11,13 @@ from discord import Client
 from discord.channel import TextChannel
 from discord.channel import Thread
 from discord.enums import MessageType
+from discord.errors import LoginFailure
 from discord.flags import Intents
 from discord.message import Message as DiscordMessage
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
+from onyx.connectors.exceptions import CredentialInvalidError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -209,8 +211,19 @@ def _manage_async_retrieval(
         intents = Intents.default()
         intents.message_content = True
         async with Client(intents=intents) as discord_client:
-            asyncio.create_task(discord_client.start(token))
-            await discord_client.wait_until_ready()
+            start_task = asyncio.create_task(discord_client.start(token))
+            ready_task = asyncio.create_task(discord_client.wait_until_ready())
+
+            done, _ = await asyncio.wait(
+                {start_task, ready_task},
+                return_when=asyncio.FIRST_COMPLETED,
+            )
+
+            # start() runs indefinitely once connected, so it only lands
+            # in `done` when login/connection failed — propagate the error.
+            if start_task in done:
+                ready_task.cancel()
+                start_task.result()
 
             filtered_channels: list[TextChannel] = await _fetch_filtered_channels(
                 discord_client=discord_client,
@@ -276,6 +289,19 @@ class DiscordConnector(PollConnector, LoadConnector):
         self._discord_bot_token = credentials["discord_bot_token"]
         return None
 
+    def validate_connector_settings(self) -> None:
+        loop = asyncio.new_event_loop()
+        try:
+            client = Client(intents=Intents.default())
+            try:
+                loop.run_until_complete(client.login(self.discord_bot_token))
+            except LoginFailure as e:
+                raise CredentialInvalidError(f"Invalid Discord bot token: {e}")
+            finally:
+                loop.run_until_complete(client.close())
+        finally:
+            loop.close()
+
     def _manage_doc_batching(
         self,
         start: datetime | None = None,

backend/onyx/connectors/jira/connector.py

@@ -10,6 +10,7 @@ from datetime import timedelta
 from datetime import timezone
 from typing import Any
 
+import requests
 from jira import JIRA
 from jira.exceptions import JIRAError
 from jira.resources import Issue
@@ -239,29 +240,53 @@ def enhanced_search_ids(
     )
 
 
-def bulk_fetch_issues(
-    jira_client: JIRA, issue_ids: list[str], fields: str | None = None
-) -> list[Issue]:
-    # TODO: move away from this jira library if they continue to not support
-    # the endpoints we need. Using private fields is not ideal, but
-    # is likely fine for now since we pin the library version
+def _bulk_fetch_request(
+    jira_client: JIRA, issue_ids: list[str], fields: str | None
+) -> list[dict[str, Any]]:
+    """Raw POST to the bulkfetch endpoint. Returns the list of raw issue dicts."""
     bulk_fetch_path = jira_client._get_url("issue/bulkfetch")
-
     # Prepare the payload according to Jira API v3 specification
     payload: dict[str, Any] = {"issueIdsOrKeys": issue_ids}
-
     # Only restrict fields if specified, might want to explicitly do this in the future
     # to avoid reading unnecessary data
     payload["fields"] = fields.split(",") if fields else ["*all"]
 
+    resp = jira_client._session.post(bulk_fetch_path, json=payload)
+    return resp.json()["issues"]
+
+
+def bulk_fetch_issues(
+    jira_client: JIRA, issue_ids: list[str], fields: str | None = None
+) -> list[Issue]:
+    # TODO(evan): move away from this jira library if they continue to not support
+    # the endpoints we need. Using private fields is not ideal, but
+    # is likely fine for now since we pin the library version
+
     try:
-        response = jira_client._session.post(bulk_fetch_path, json=payload).json()
+        raw_issues = _bulk_fetch_request(jira_client, issue_ids, fields)
+    except requests.exceptions.JSONDecodeError:
+        if len(issue_ids) <= 1:
+            logger.exception(
+                f"Jira bulk-fetch response for issue(s) {issue_ids} could not "
+                f"be decoded as JSON (response too large or truncated)."
+            )
+            raise
+
+        mid = len(issue_ids) // 2
+        logger.warning(
+            f"Jira bulk-fetch JSON decode failed for batch of {len(issue_ids)} issues. "
+            f"Splitting into sub-batches of {mid} and {len(issue_ids) - mid}."
+        )
+        left = bulk_fetch_issues(jira_client, issue_ids[:mid], fields)
+        right = bulk_fetch_issues(jira_client, issue_ids[mid:], fields)
+        return left + right
     except Exception as e:
         logger.error(f"Error fetching issues: {e}")
-        raise e
+        raise
+
     return [
         Issue(jira_client._options, jira_client._session, raw=issue)
-        for issue in response["issues"]
+        for issue in raw_issues
     ]
 
 

backend/onyx/connectors/notion/connector.py

@@ -53,7 +53,7 @@ class NotionPage(BaseModel):
     id: str
     created_time: str
     last_edited_time: str
-    archived: bool
+    in_trash: bool
     properties: dict[str, Any]
     url: str
 
@@ -63,6 +63,13 @@ class NotionPage(BaseModel):
     )
 
 
+class NotionDataSource(BaseModel):
+    """Represents a Notion Data Source within a database."""
+
+    id: str
+    name: str = ""
+
+
 class NotionBlock(BaseModel):
     """Represents a Notion Block object"""
 
@@ -107,7 +114,7 @@ class NotionConnector(LoadConnector, PollConnector):
         self.batch_size = batch_size
         self.headers = {
             "Content-Type": "application/json",
-            "Notion-Version": "2022-06-28",
+            "Notion-Version": "2026-03-11",
         }
         self.indexed_pages: set[str] = set()
         self.root_page_id = root_page_id
@@ -127,6 +134,9 @@ class NotionConnector(LoadConnector, PollConnector):
         # Maps child page IDs to their containing page ID (discovered in _read_blocks).
         # Used to resolve block_id parent types to the actual containing page.
         self._child_page_parent_map: dict[str, str] = {}
+        # Maps data_source_id -> database_id (populated in _read_pages_from_database).
+        # Used to resolve data_source_id parent types back to the database.
+        self._data_source_to_database_map: dict[str, str] = {}
 
     @classmethod
     @override
@@ -227,7 +237,11 @@ class NotionConnector(LoadConnector, PollConnector):
 
     @retry(tries=3, delay=1, backoff=2)
     def _fetch_database_as_page(self, database_id: str) -> NotionPage:
-        """Attempt to fetch a database as a page."""
+        """Attempt to fetch a database as a page.
+
+        Note: As of API 2025-09-03, database objects no longer include
+        `properties` (schema moved to individual data sources).
+        """
         logger.debug(f"Fetching database for ID '{database_id}' as a page")
         database_url = f"https://api.notion.com/v1/databases/{database_id}"
         res = rl_requests.get(
@@ -246,18 +260,52 @@ class NotionConnector(LoadConnector, PollConnector):
             database_name[0].get("text", {}).get("content") if database_name else None
         )
 
+        db_data.setdefault("properties", {})
+
         return NotionPage(**db_data, database_name=database_name)
 
     @retry(tries=3, delay=1, backoff=2)
-    def _fetch_database(
-        self, database_id: str, cursor: str | None = None
+    def _fetch_data_sources_for_database(
+        self, database_id: str
+    ) -> list[NotionDataSource]:
+        """Fetch the list of data sources for a database."""
+        logger.debug(f"Fetching data sources for database '{database_id}'")
+        res = rl_requests.get(
+            f"https://api.notion.com/v1/databases/{database_id}",
+            headers=self.headers,
+            timeout=_NOTION_CALL_TIMEOUT,
+        )
+        try:
+            res.raise_for_status()
+        except Exception as e:
+            if res.status_code in (403, 404):
+                logger.error(
+                    f"Unable to access database with ID '{database_id}'. "
+                    f"This is likely due to the database not being shared "
+                    f"with the Onyx integration. Exact exception:\n{e}"
+                )
+                return []
+            logger.exception(f"Error fetching database - {res.json()}")
+            raise e
+
+        db_data = res.json()
+        data_sources = db_data.get("data_sources", [])
+        return [
+            NotionDataSource(id=ds["id"], name=ds.get("name", ""))
+            for ds in data_sources
+            if ds.get("id")
+        ]
+
+    @retry(tries=3, delay=1, backoff=2)
+    def _fetch_data_source(
+        self, data_source_id: str, cursor: str | None = None
     ) -> dict[str, Any]:
-        """Fetch a database from it's ID via the Notion API."""
-        logger.debug(f"Fetching database for ID '{database_id}'")
-        block_url = f"https://api.notion.com/v1/databases/{database_id}/query"
+        """Query a data source via POST /v1/data_sources/{id}/query."""
+        logger.debug(f"Querying data source '{data_source_id}'")
+        url = f"https://api.notion.com/v1/data_sources/{data_source_id}/query"
         body = None if not cursor else {"start_cursor": cursor}
         res = rl_requests.post(
-            block_url,
+            url,
             headers=self.headers,
             json=body,
             timeout=_NOTION_CALL_TIMEOUT,
@@ -265,25 +313,14 @@ class NotionConnector(LoadConnector, PollConnector):
         try:
             res.raise_for_status()
         except Exception as e:
-            json_data = res.json()
-            code = json_data.get("code")
-            # Sep 3 2025 backend changed the error message for this case
-            # TODO: it is also now possible for there to be multiple data sources per database; at present we
-            # just don't handle that. We will need to upgrade the API to the current version + query the
-            # new data sources endpoint to handle that case correctly.
-            if code == "object_not_found" or (
-                code == "validation_error"
-                and "does not contain any data sources" in json_data.get("message", "")
-            ):
-                # this happens when a database is not shared with the integration
-                # in this case, we should just ignore the database
+            if res.status_code in (403, 404):
                 logger.error(
-                    f"Unable to access database with ID '{database_id}'. "
-                    f"This is likely due to the database not being shared "
+                    f"Unable to access data source with ID '{data_source_id}'. "
+                    f"This is likely due to it not being shared "
                     f"with the Onyx integration. Exact exception:\n{e}"
                 )
                 return {"results": [], "next_cursor": None}
-            logger.exception(f"Error fetching database - {res.json()}")
+            logger.exception(f"Error querying data source - {res.json()}")
             raise e
         return res.json()
 
@@ -348,8 +385,9 @@ class NotionConnector(LoadConnector, PollConnector):
             # Fallback to workspace if we don't know the parent
             return self.workspace_id
         elif parent_type == "data_source_id":
-            # Newer Notion API may use data_source_id for databases
-            return parent.get("database_id") or parent.get("data_source_id")
+            ds_id = parent.get("data_source_id")
+            if ds_id:
+                return self._data_source_to_database_map.get(ds_id, self.workspace_id)
         elif parent_type in ["page_id", "database_id"]:
             return parent.get(parent_type)
 
@@ -497,18 +535,32 @@ class NotionConnector(LoadConnector, PollConnector):
         if db_node:
             hierarchy_nodes.append(db_node)
 
-        cursor = None
-        while True:
-            data = self._fetch_database(database_id, cursor)
+        # Discover all data sources under this database, then query each one.
+        # Even legacy single-source databases have one entry in the array.
+        data_sources = self._fetch_data_sources_for_database(database_id)
+        if not data_sources:
+            logger.warning(
+                f"Database '{database_id}' returned zero data sources — "
+                f"no pages will be indexed from this database."
+            )
+        for ds in data_sources:
+            self._data_source_to_database_map[ds.id] = database_id
+            cursor = None
+            while True:
+                data = self._fetch_data_source(ds.id, cursor)
 
-            for result in data["results"]:
-                obj_id = result["id"]
-                obj_type = result["object"]
-                text = self._properties_to_str(result.get("properties", {}))
-                if text:
-                    result_blocks.append(NotionBlock(id=obj_id, text=text, prefix="\n"))
+                for result in data["results"]:
+                    obj_id = result["id"]
+                    obj_type = result["object"]
+                    text = self._properties_to_str(result.get("properties", {}))
+                    if text:
+                        result_blocks.append(
+                            NotionBlock(id=obj_id, text=text, prefix="\n")
+                        )
+
+                    if not self.recursive_index_enabled:
+                        continue
 
-                if self.recursive_index_enabled:
                     if obj_type == "page":
                         logger.debug(
                             f"Found page with ID '{obj_id}' in database '{database_id}'"
@@ -518,7 +570,6 @@ class NotionConnector(LoadConnector, PollConnector):
                         logger.debug(
                             f"Found database with ID '{obj_id}' in database '{database_id}'"
                         )
-                        # Get nested database name from properties if available
                         nested_db_title = result.get("title", [])
                         nested_db_name = None
                         if nested_db_title and len(nested_db_title) > 0:
@@ -533,10 +584,10 @@ class NotionConnector(LoadConnector, PollConnector):
                         result_pages.extend(nested_output.child_page_ids)
                         hierarchy_nodes.extend(nested_output.hierarchy_nodes)
 
-            if data["next_cursor"] is None:
-                break
+                if data["next_cursor"] is None:
+                    break
 
-            cursor = data["next_cursor"]
+                cursor = data["next_cursor"]
 
         return BlockReadOutput(
             blocks=result_blocks,
@@ -807,36 +858,55 @@ class NotionConnector(LoadConnector, PollConnector):
     def _yield_database_hierarchy_nodes(
         self,
     ) -> Generator[HierarchyNode | Document, None, None]:
-        """Search for all databases and yield hierarchy nodes for each.
+        """Search for all data sources and yield hierarchy nodes for their parent databases.
 
         This must be called BEFORE page indexing so that database hierarchy nodes
         exist when pages inside databases reference them as parents.
+
+        With the new API, search returns data source objects instead of databases.
+        Multiple data sources can share the same parent database, so we use
+        database_id as the hierarchy node key and deduplicate via
+        _maybe_yield_hierarchy_node.
         """
         query_dict: dict[str, Any] = {
-            "filter": {"property": "object", "value": "database"},
+            "filter": {"property": "object", "value": "data_source"},
             "page_size": _NOTION_PAGE_SIZE,
         }
         pages_seen = 0
         while pages_seen < _MAX_PAGES:
             db_res = self._search_notion(query_dict)
-            for db in db_res.results:
-                db_id = db["id"]
-                # Extract title from the title array
-                title_arr = db.get("title", [])
-                db_name = None
-                if title_arr:
-                    db_name = " ".join(
-                        t.get("plain_text", "") for t in title_arr
-                    ).strip()
-                if not db_name:
+            for ds in db_res.results:
+                # Extract the parent database_id from the data source's parent
+                ds_parent = ds.get("parent", {})
+                db_id = ds_parent.get("database_id")
+                if not db_id:
+                    continue
+
+                # Populate the mapping so _get_parent_raw_id can resolve later
+                ds_id = ds.get("id")
+                if not ds_id:
+                    continue
+                self._data_source_to_database_map[ds_id] = db_id
+
+                # Fetch the database to get its actual name and parent
+                try:
+                    db_page = self._fetch_database_as_page(db_id)
+                    db_name = db_page.database_name or f"Database {db_id}"
+                    parent_raw_id = self._get_parent_raw_id(db_page.parent)
+                    db_url = (
+                        db_page.url or f"https://notion.so/{db_id.replace('-', '')}"
+                    )
+                except requests.exceptions.RequestException as e:
+                    logger.warning(
+                        f"Could not fetch database '{db_id}', "
+                        f"defaulting to workspace root. Error: {e}"
+                    )
                     db_name = f"Database {db_id}"
+                    parent_raw_id = self.workspace_id
+                    db_url = f"https://notion.so/{db_id.replace('-', '')}"
 
-                # Get parent using existing helper
-                parent_raw_id = self._get_parent_raw_id(db.get("parent"))
-
-                # Notion URLs omit dashes from UUIDs
-                db_url = db.get("url") or f"https://notion.so/{db_id.replace('-', '')}"
-
+                # _maybe_yield_hierarchy_node deduplicates by raw_node_id,
+                # so multiple data sources under one database produce one node.
                 node = self._maybe_yield_hierarchy_node(
                     raw_node_id=db_id,
                     raw_parent_id=parent_raw_id or self.workspace_id,

backend/onyx/connectors/registry.py

@@ -72,6 +72,10 @@ CONNECTOR_CLASS_MAP = {
         module_path="onyx.connectors.coda.connector",
         class_name="CodaConnector",
     ),
+    DocumentSource.CANVAS: ConnectorMapping(
+        module_path="onyx.connectors.canvas.connector",
+        class_name="CanvasConnector",
+    ),
     DocumentSource.NOTION: ConnectorMapping(
         module_path="onyx.connectors.notion.connector",
         class_name="NotionConnector",

backend/onyx/connectors/sharepoint/connector.py

@@ -1,5 +1,6 @@
 import base64
 import copy
+import fnmatch
 import html
 import io
 import os
@@ -84,6 +85,44 @@ SHARED_DOCUMENTS_MAP_REVERSE = {v: k for k, v in SHARED_DOCUMENTS_MAP.items()}
 
 ASPX_EXTENSION = ".aspx"
 
+
+def _is_site_excluded(site_url: str, excluded_site_patterns: list[str]) -> bool:
+    """Check if a site URL matches any of the exclusion glob patterns."""
+    for pattern in excluded_site_patterns:
+        if fnmatch.fnmatch(site_url, pattern) or fnmatch.fnmatch(
+            site_url.rstrip("/"), pattern.rstrip("/")
+        ):
+            return True
+    return False
+
+
+def _is_path_excluded(item_path: str, excluded_path_patterns: list[str]) -> bool:
+    """Check if a drive item path matches any of the exclusion glob patterns.
+
+    item_path is the relative path within a drive, e.g. "Engineering/API/report.docx".
+    Matches are attempted against the full path and the filename alone so that
+    patterns like "*.tmp" match files at any depth.
+    """
+    filename = item_path.rsplit("/", 1)[-1] if "/" in item_path else item_path
+    for pattern in excluded_path_patterns:
+        if fnmatch.fnmatch(item_path, pattern) or fnmatch.fnmatch(filename, pattern):
+            return True
+    return False
+
+
+def _build_item_relative_path(parent_reference_path: str | None, item_name: str) -> str:
+    """Build the relative path of a drive item from its parentReference.path and name.
+
+    Example: parentReference.path="/drives/abc/root:/Eng/API", name="report.docx"
+    => "Eng/API/report.docx"
+    """
+    if parent_reference_path and "root:/" in parent_reference_path:
+        folder = unquote(parent_reference_path.split("root:/", 1)[1])
+        if folder:
+            return f"{folder}/{item_name}"
+    return item_name
+
+
 DEFAULT_AUTHORITY_HOST = "https://login.microsoftonline.com"
 DEFAULT_GRAPH_API_HOST = "https://graph.microsoft.com"
 DEFAULT_SHAREPOINT_DOMAIN_SUFFIX = "sharepoint.com"
@@ -478,6 +517,7 @@ def _convert_driveitem_to_document_with_permissions(
     include_permissions: bool = False,
     parent_hierarchy_raw_node_id: str | None = None,
     access_token: str | None = None,
+    treat_sharing_link_as_public: bool = False,
 ) -> Document | ConnectorFailure | None:
 
     if not driveitem.name or not driveitem.id:
@@ -610,6 +650,7 @@ def _convert_driveitem_to_document_with_permissions(
             drive_item=sdk_item,
             drive_name=drive_name,
             add_prefix=True,
+            treat_sharing_link_as_public=treat_sharing_link_as_public,
         )
     else:
         external_access = ExternalAccess.empty()
@@ -644,6 +685,7 @@ def _convert_sitepage_to_document(
     graph_client: GraphClient,
     include_permissions: bool = False,
     parent_hierarchy_raw_node_id: str | None = None,
+    treat_sharing_link_as_public: bool = False,
 ) -> Document:
     """Convert a SharePoint site page to a Document object."""
     # Extract text content from the site page
@@ -773,6 +815,7 @@ def _convert_sitepage_to_document(
             graph_client=graph_client,
             site_page=site_page,
             add_prefix=True,
+            treat_sharing_link_as_public=treat_sharing_link_as_public,
         )
     else:
         external_access = ExternalAccess.empty()
@@ -803,6 +846,7 @@ def _convert_driveitem_to_slim_document(
     ctx: ClientContext,
     graph_client: GraphClient,
     parent_hierarchy_raw_node_id: str | None = None,
+    treat_sharing_link_as_public: bool = False,
 ) -> SlimDocument:
     if driveitem.id is None:
         raise ValueError("DriveItem ID is required")
@@ -813,6 +857,7 @@ def _convert_driveitem_to_slim_document(
         graph_client=graph_client,
         drive_item=sdk_item,
         drive_name=drive_name,
+        treat_sharing_link_as_public=treat_sharing_link_as_public,
     )
 
     return SlimDocument(
@@ -827,6 +872,7 @@ def _convert_sitepage_to_slim_document(
     ctx: ClientContext | None,
     graph_client: GraphClient,
     parent_hierarchy_raw_node_id: str | None = None,
+    treat_sharing_link_as_public: bool = False,
 ) -> SlimDocument:
     """Convert a SharePoint site page to a SlimDocument object."""
     if site_page.get("id") is None:
@@ -836,6 +882,7 @@ def _convert_sitepage_to_slim_document(
         ctx=ctx,
         graph_client=graph_client,
         site_page=site_page,
+        treat_sharing_link_as_public=treat_sharing_link_as_public,
     )
     id = site_page.get("id")
     if id is None:
@@ -855,14 +902,20 @@ class SharepointConnector(
         self,
         batch_size: int = INDEX_BATCH_SIZE,
         sites: list[str] = [],
+        excluded_sites: list[str] = [],
+        excluded_paths: list[str] = [],
         include_site_pages: bool = True,
         include_site_documents: bool = True,
+        treat_sharing_link_as_public: bool = False,
         authority_host: str = DEFAULT_AUTHORITY_HOST,
         graph_api_host: str = DEFAULT_GRAPH_API_HOST,
         sharepoint_domain_suffix: str = DEFAULT_SHAREPOINT_DOMAIN_SUFFIX,
     ) -> None:
         self.batch_size = batch_size
         self.sites = list(sites)
+        self.excluded_sites = [s for p in excluded_sites if (s := p.strip())]
+        self.excluded_paths = [s for p in excluded_paths if (s := p.strip())]
+        self.treat_sharing_link_as_public = treat_sharing_link_as_public
         self.site_descriptors: list[SiteDescriptor] = self._extract_site_and_drive_info(
             sites
         )
@@ -1233,6 +1286,29 @@ class SharepointConnector(
                 break
             sites = sites._get_next().execute_query()
 
+    def _is_driveitem_excluded(self, driveitem: DriveItemData) -> bool:
+        """Check if a drive item should be excluded based on excluded_paths patterns."""
+        if not self.excluded_paths:
+            return False
+        relative_path = _build_item_relative_path(
+            driveitem.parent_reference_path, driveitem.name
+        )
+        return _is_path_excluded(relative_path, self.excluded_paths)
+
+    def _filter_excluded_sites(
+        self, site_descriptors: list[SiteDescriptor]
+    ) -> list[SiteDescriptor]:
+        """Remove sites matching any excluded_sites glob pattern."""
+        if not self.excluded_sites:
+            return site_descriptors
+        result = []
+        for sd in site_descriptors:
+            if _is_site_excluded(sd.url, self.excluded_sites):
+                logger.info(f"Excluding site by denylist: {sd.url}")
+                continue
+            result.append(sd)
+        return result
+
     def fetch_sites(self) -> list[SiteDescriptor]:
         sites = self.graph_client.sites.get_all_sites().execute_query()
 
@@ -1249,7 +1325,7 @@ class SharepointConnector(
             for site in self._handle_paginated_sites(sites)
             if "-my.sharepoint" not in site.web_url
         ]
-        return site_descriptors
+        return self._filter_excluded_sites(site_descriptors)
 
     def _fetch_site_pages(
         self,
@@ -1689,8 +1765,14 @@ class SharepointConnector(
         checkpoint.current_drive_delta_next_link = None
         checkpoint.seen_document_ids.clear()
 
-    def _fetch_slim_documents_from_sharepoint(self) -> GenerateSlimDocumentOutput:
-        site_descriptors = self.site_descriptors or self.fetch_sites()
+    def _fetch_slim_documents_from_sharepoint(
+        self,
+        start: datetime | None = None,
+        end: datetime | None = None,
+    ) -> GenerateSlimDocumentOutput:
+        site_descriptors = self._filter_excluded_sites(
+            self.site_descriptors or self.fetch_sites()
+        )
 
         # Create a temporary checkpoint for hierarchy node tracking
         temp_checkpoint = SharepointConnectorCheckpoint(has_more=True)
@@ -1708,8 +1790,14 @@ class SharepointConnector(
             # Process site documents if flag is True
             if self.include_site_documents:
                 for driveitem, drive_name, drive_web_url in self._fetch_driveitems(
-                    site_descriptor=site_descriptor
+                    site_descriptor=site_descriptor,
+                    start=start,
+                    end=end,
                 ):
+                    if self._is_driveitem_excluded(driveitem):
+                        logger.debug(f"Excluding by path denylist: {driveitem.web_url}")
+                        continue
+
                     if drive_web_url:
                         doc_batch.extend(
                             self._yield_drive_hierarchy_node(
@@ -1747,6 +1835,7 @@ class SharepointConnector(
                                 ctx,
                                 self.graph_client,
                                 parent_hierarchy_raw_node_id=parent_hierarchy_url,
+                                treat_sharing_link_as_public=self.treat_sharing_link_as_public,
                             )
                         )
                     except Exception as e:
@@ -1758,7 +1847,9 @@ class SharepointConnector(
 
             # Process site pages if flag is True
             if self.include_site_pages:
-                site_pages = self._fetch_site_pages(site_descriptor)
+                site_pages = self._fetch_site_pages(
+                    site_descriptor, start=start, end=end
+                )
                 for site_page in site_pages:
                     logger.debug(
                         f"Processing site page: {site_page.get('webUrl', site_page.get('name', 'Unknown'))}"
@@ -1770,6 +1861,7 @@ class SharepointConnector(
                             ctx,
                             self.graph_client,
                             parent_hierarchy_raw_node_id=site_descriptor.url,
+                            treat_sharing_link_as_public=self.treat_sharing_link_as_public,
                         )
                     )
                     if len(doc_batch) >= SLIM_BATCH_SIZE:
@@ -2043,7 +2135,9 @@ class SharepointConnector(
             and not checkpoint.process_site_pages
         ):
             logger.info("Initializing SharePoint sites for processing")
-            site_descs = self.site_descriptors or self.fetch_sites()
+            site_descs = self._filter_excluded_sites(
+                self.site_descriptors or self.fetch_sites()
+            )
             checkpoint.cached_site_descriptors = deque(site_descs)
 
             if not checkpoint.cached_site_descriptors:
@@ -2264,6 +2358,10 @@ class SharepointConnector(
             for driveitem in driveitems:
                 item_count += 1
 
+                if self._is_driveitem_excluded(driveitem):
+                    logger.debug(f"Excluding by path denylist: {driveitem.web_url}")
+                    continue
+
                 if driveitem.id and driveitem.id in checkpoint.seen_document_ids:
                     logger.debug(
                         f"Skipping duplicate document {driveitem.id} ({driveitem.name})"
@@ -2318,6 +2416,7 @@ class SharepointConnector(
                         parent_hierarchy_raw_node_id=parent_hierarchy_url,
                         graph_api_base=self.graph_api_base,
                         access_token=access_token,
+                        treat_sharing_link_as_public=self.treat_sharing_link_as_public,
                     )
 
                     if isinstance(doc_or_failure, Document):
@@ -2398,6 +2497,7 @@ class SharepointConnector(
                         include_permissions=include_permissions,
                         # Site pages have the site as their parent
                         parent_hierarchy_raw_node_id=site_descriptor.url,
+                        treat_sharing_link_as_public=self.treat_sharing_link_as_public,
                     )
                 )
             logger.info(
@@ -2473,12 +2573,22 @@ class SharepointConnector(
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
-        end: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,  # noqa: ARG002
     ) -> GenerateSlimDocumentOutput:
-
-        yield from self._fetch_slim_documents_from_sharepoint()
+        start_dt = (
+            datetime.fromtimestamp(start, tz=timezone.utc)
+            if start is not None
+            else None
+        )
+        end_dt = (
+            datetime.fromtimestamp(end, tz=timezone.utc) if end is not None else None
+        )
+        yield from self._fetch_slim_documents_from_sharepoint(
+            start=start_dt,
+            end=end_dt,
+        )
 
 
 if __name__ == "__main__":

backend/onyx/connectors/sharepoint/connector_utils.py

@@ -17,6 +17,7 @@ def get_sharepoint_external_access(
     drive_name: str | None = None,
     site_page: dict[str, Any] | None = None,
     add_prefix: bool = False,
+    treat_sharing_link_as_public: bool = False,
 ) -> ExternalAccess:
     if drive_item and drive_item.id is None:
         raise ValueError("DriveItem ID is required")
@@ -34,7 +35,13 @@ def get_sharepoint_external_access(
     )
 
     external_access = get_external_access_func(
-        ctx, graph_client, drive_name, drive_item, site_page, add_prefix
+        ctx,
+        graph_client,
+        drive_name,
+        drive_item,
+        site_page,
+        add_prefix,
+        treat_sharing_link_as_public,
     )
 
     return external_access

backend/onyx/connectors/slack/connector.py

@@ -516,6 +516,8 @@ def _get_all_doc_ids(
     ] = default_msg_filter,
     callback: IndexingHeartbeatInterface | None = None,
     workspace_url: str | None = None,
+    start: SecondsSinceUnixEpoch | None = None,
+    end: SecondsSinceUnixEpoch | None = None,
 ) -> GenerateSlimDocumentOutput:
     """
     Get all document ids in the workspace, channel by channel
@@ -546,6 +548,8 @@ def _get_all_doc_ids(
             client=client,
             channel=channel,
             callback=callback,
+            oldest=str(start) if start else None,  # 0.0 -> None intentionally
+            latest=str(end) if end is not None else None,
         )
 
         for message_batch in channel_message_batches:
@@ -847,8 +851,8 @@ class SlackConnector(
 
     def retrieve_all_slim_docs_perm_sync(
         self,
-        start: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
-        end: SecondsSinceUnixEpoch | None = None,  # noqa: ARG002
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         if self.client is None:
@@ -861,6 +865,8 @@ class SlackConnector(
             msg_filter_func=self.msg_filter_func,
             callback=callback,
             workspace_url=self._workspace_url,
+            start=start,
+            end=end,
         )
 
     def _load_from_checkpoint(

backend/onyx/context/search/models.py

@@ -401,3 +401,16 @@ class SavedSearchDocWithContent(SavedSearchDoc):
     section in addition to the match_highlights."""
 
     content: str
+
+
+class PersonaSearchInfo(BaseModel):
+    """Snapshot of persona data needed by the search pipeline.
+
+    Extracted from the ORM Persona before the DB session is released so that
+    SearchTool and search_pipeline never lazy-load relationships post-commit.
+    """
+
+    document_set_names: list[str]
+    search_start_date: datetime | None
+    attached_document_ids: list[str]
+    hierarchy_node_ids: list[int]

backend/onyx/context/search/pipeline.py

@@ -9,12 +9,12 @@ from onyx.context.search.models import ChunkSearchRequest
 from onyx.context.search.models import IndexFilters
 from onyx.context.search.models import InferenceChunk
 from onyx.context.search.models import InferenceSection
+from onyx.context.search.models import PersonaSearchInfo
 from onyx.context.search.preprocessing.access_filters import (
     build_access_filters_for_user,
 )
 from onyx.context.search.retrieval.search_runner import search_chunks
 from onyx.context.search.utils import inference_section_from_chunks
-from onyx.db.models import Persona
 from onyx.db.models import User
 from onyx.document_index.interfaces import DocumentIndex
 from onyx.federated_connectors.federated_retrieval import FederatedRetrievalInfo
@@ -247,8 +247,8 @@ def search_pipeline(
     document_index: DocumentIndex,
     # Used for ACLs and federated search, anonymous users only see public docs
     user: User,
-    # Used for default filters and settings
-    persona: Persona | None,
+    # Pre-extracted persona search configuration (None when no persona)
+    persona_search_info: PersonaSearchInfo | None,
     db_session: Session | None = None,
     auto_detect_filters: bool = False,
     llm: LLM | None = None,
@@ -263,24 +263,18 @@ def search_pipeline(
     prefetched_federated_retrieval_infos: list[FederatedRetrievalInfo] | None = None,
 ) -> list[InferenceChunk]:
     persona_document_sets: list[str] | None = (
-        [persona_document_set.name for persona_document_set in persona.document_sets]
-        if persona
-        else None
+        persona_search_info.document_set_names if persona_search_info else None
     )
     persona_time_cutoff: datetime | None = (
-        persona.search_start_date if persona else None
+        persona_search_info.search_start_date if persona_search_info else None
     )
-
-    # Extract assistant knowledge filters from persona
     attached_document_ids: list[str] | None = (
-        [doc.id for doc in persona.attached_documents]
-        if persona and persona.attached_documents
+        persona_search_info.attached_document_ids or None
+        if persona_search_info
         else None
     )
     hierarchy_node_ids: list[int] | None = (
-        [node.id for node in persona.hierarchy_nodes]
-        if persona and persona.hierarchy_nodes
-        else None
+        persona_search_info.hierarchy_node_ids or None if persona_search_info else None
     )
 
     filters = _build_index_filters(

backend/onyx/context/search/retrieval/search_runner.py

@@ -14,6 +14,10 @@ from onyx.context.search.utils import get_query_embedding
 from onyx.context.search.utils import inference_section_from_chunks
 from onyx.document_index.interfaces import DocumentIndex
 from onyx.document_index.interfaces import VespaChunkRequest
+from onyx.document_index.interfaces_new import DocumentIndex as NewDocumentIndex
+from onyx.document_index.opensearch.opensearch_document_index import (
+    OpenSearchOldDocumentIndex,
+)
 from onyx.federated_connectors.federated_retrieval import FederatedRetrievalInfo
 from onyx.federated_connectors.federated_retrieval import (
     get_federated_retrieval_functions,
@@ -49,7 +53,7 @@ def combine_retrieval_results(
     return sorted_chunks
 
 
-def _embed_and_search(
+def _embed_and_hybrid_search(
     query_request: ChunkIndexRequest,
     document_index: DocumentIndex,
     db_session: Session | None = None,
@@ -81,6 +85,17 @@ def _embed_and_search(
     return top_chunks
 
 
+def _keyword_search(
+    query_request: ChunkIndexRequest,
+    document_index: NewDocumentIndex,
+) -> list[InferenceChunk]:
+    return document_index.keyword_retrieval(
+        query=query_request.query,
+        filters=query_request.filters,
+        num_to_retrieve=query_request.limit or NUM_RETURNED_HITS,
+    )
+
+
 def search_chunks(
     query_request: ChunkIndexRequest,
     user_id: UUID | None,
@@ -128,21 +143,38 @@ def search_chunks(
     )
 
     if normal_search_enabled:
-        run_queries.append(
-            (
-                _embed_and_search,
-                (query_request, document_index, db_session, embedding_model),
+        if (
+            query_request.hybrid_alpha is not None
+            and query_request.hybrid_alpha == 0.0
+            and isinstance(document_index, OpenSearchOldDocumentIndex)
+        ):
+            # If hybrid alpha is explicitly set to keyword only, do pure keyword
+            # search without generating an embedding. This is currently only
+            # supported with OpenSearchDocumentIndex.
+            opensearch_new_document_index: NewDocumentIndex = document_index._real_index
+            run_queries.append(
+                (
+                    lambda: _keyword_search(
+                        query_request, opensearch_new_document_index
+                    ),
+                    (),
+                )
+            )
+        else:
+            run_queries.append(
+                (
+                    _embed_and_hybrid_search,
+                    (query_request, document_index, db_session, embedding_model),
+                )
             )
-        )
 
     parallel_search_results = run_functions_tuples_in_parallel(run_queries)
     top_chunks = combine_retrieval_results(parallel_search_results)
 
     if not top_chunks:
         logger.debug(
-            f"Hybrid search returned no results for query: {query_request.query}with filters: {query_request.filters}"
+            f"Search returned no results for query: {query_request.query} with filters: {query_request.filters}."
         )
-        return []
 
     return top_chunks
 

backend/onyx/db/chat.py

@@ -8,7 +8,6 @@ from uuid import UUID
 from fastapi import HTTPException
 from sqlalchemy import delete
 from sqlalchemy import desc
-from sqlalchemy import exists
 from sqlalchemy import func
 from sqlalchemy import nullsfirst
 from sqlalchemy import or_
@@ -64,6 +63,9 @@ def get_chat_session_by_id(
             joinedload(ChatSession.persona).options(
                 selectinload(Persona.tools),
                 selectinload(Persona.user_files),
+                selectinload(Persona.document_sets),
+                selectinload(Persona.attached_documents),
+                selectinload(Persona.hierarchy_nodes),
             ),
             joinedload(ChatSession.project),
         )
@@ -129,32 +131,47 @@ def get_chat_sessions_by_user(
     if before is not None:
         stmt = stmt.where(ChatSession.time_updated < before)
 
-    if limit:
-        stmt = stmt.limit(limit)
-
     if project_id is not None:
         stmt = stmt.where(ChatSession.project_id == project_id)
     elif only_non_project_chats:
         stmt = stmt.where(ChatSession.project_id.is_(None))
 
-    if not include_failed_chats:
-        non_system_message_exists_subq = (
-            exists()
-            .where(ChatMessage.chat_session_id == ChatSession.id)
-            .where(ChatMessage.message_type != MessageType.SYSTEM)
-            .correlate(ChatSession)
-        )
-
-        # Leeway for newly created chats that don't have messages yet
-        time = datetime.now(timezone.utc) - timedelta(minutes=5)
-        recently_created = ChatSession.time_created >= time
-
-        stmt = stmt.where(or_(non_system_message_exists_subq, recently_created))
+    # When filtering out failed chats, we apply the limit in Python after
+    # filtering rather than in SQL, since the post-filter may remove rows.
+    if limit and include_failed_chats:
+        stmt = stmt.limit(limit)
 
     result = db_session.execute(stmt)
-    chat_sessions = result.scalars().all()
+    chat_sessions = list(result.scalars().all())
 
-    return list(chat_sessions)
+    if not include_failed_chats and chat_sessions:
+        # Filter out "failed" sessions (those with only SYSTEM messages)
+        # using a separate efficient query instead of a correlated EXISTS
+        # subquery, which causes full sequential scans of chat_message.
+        leeway = datetime.now(timezone.utc) - timedelta(minutes=5)
+        session_ids = [cs.id for cs in chat_sessions if cs.time_created < leeway]
+
+        if session_ids:
+            valid_session_ids_stmt = (
+                select(ChatMessage.chat_session_id)
+                .where(ChatMessage.chat_session_id.in_(session_ids))
+                .where(ChatMessage.message_type != MessageType.SYSTEM)
+                .distinct()
+            )
+            valid_session_ids = set(
+                db_session.execute(valid_session_ids_stmt).scalars().all()
+            )
+
+            chat_sessions = [
+                cs
+                for cs in chat_sessions
+                if cs.time_created >= leeway or cs.id in valid_session_ids
+            ]
+
+        if limit:
+            chat_sessions = chat_sessions[:limit]
+
+    return chat_sessions
 
 
 def delete_orphaned_search_docs(db_session: Session) -> None:

backend/onyx/db/connector_credential_pair.py

@@ -750,3 +750,31 @@ def resync_cc_pair(
     )
 
     db_session.commit()
+
+
+# ── Metrics query helpers ──────────────────────────────────────────────
+
+
+def get_connector_health_for_metrics(
+    db_session: Session,
+) -> list:  # Returns list of Row tuples
+    """Return connector health data for Prometheus metrics.
+
+    Each row is (cc_pair_id, status, in_repeated_error_state,
+    last_successful_index_time, name, source).
+    """
+    return (
+        db_session.query(
+            ConnectorCredentialPair.id,
+            ConnectorCredentialPair.status,
+            ConnectorCredentialPair.in_repeated_error_state,
+            ConnectorCredentialPair.last_successful_index_time,
+            ConnectorCredentialPair.name,
+            Connector.source,
+        )
+        .join(
+            Connector,
+            ConnectorCredentialPair.connector_id == Connector.id,
+        )
+        .all()
+    )

backend/onyx/db/enums.py

@@ -1,4 +1,31 @@
+from __future__ import annotations
+
 from enum import Enum as PyEnum
+from typing import ClassVar
+
+
+class AccountType(str, PyEnum):
+    """
+    What kind of account this is — determines whether the user
+    enters the group-based permission system.
+
+    STANDARD + SERVICE_ACCOUNT → participate in group system
+    BOT, EXT_PERM_USER, ANONYMOUS → fixed behavior
+    """
+
+    STANDARD = "standard"
+    BOT = "bot"
+    EXT_PERM_USER = "ext_perm_user"
+    SERVICE_ACCOUNT = "service_account"
+    ANONYMOUS = "anonymous"
+
+
+class GrantSource(str, PyEnum):
+    """How a permission grant was created."""
+
+    USER = "user"
+    SCIM = "scim"
+    SYSTEM = "system"
 
 
 class IndexingStatus(str, PyEnum):
@@ -188,6 +215,7 @@ class UserFileStatus(str, PyEnum):
     PROCESSING = "PROCESSING"
     INDEXING = "INDEXING"
     COMPLETED = "COMPLETED"
+    SKIPPED = "SKIPPED"
     FAILED = "FAILED"
     CANCELED = "CANCELED"
     DELETING = "DELETING"
@@ -314,3 +342,54 @@ class HookPoint(str, PyEnum):
 class HookFailStrategy(str, PyEnum):
     HARD = "hard"  # exception propagates, pipeline aborts
     SOFT = "soft"  # log error, return original input, pipeline continues
+
+
+class Permission(str, PyEnum):
+    """
+    Permission tokens for group-based authorization.
+    19 tokens total. full_admin_panel_access is an override —
+    if present, any permission check passes.
+    """
+
+    # Basic (auto-granted to every new group)
+    BASIC_ACCESS = "basic"
+
+    # Read tokens — implied only, never granted directly
+    READ_CONNECTORS = "read:connectors"
+    READ_DOCUMENT_SETS = "read:document_sets"
+    READ_AGENTS = "read:agents"
+    READ_USERS = "read:users"
+
+    # Add / Manage pairs
+    ADD_AGENTS = "add:agents"
+    MANAGE_AGENTS = "manage:agents"
+    MANAGE_DOCUMENT_SETS = "manage:document_sets"
+    ADD_CONNECTORS = "add:connectors"
+    MANAGE_CONNECTORS = "manage:connectors"
+    MANAGE_LLMS = "manage:llms"
+
+    # Toggle tokens
+    READ_AGENT_ANALYTICS = "read:agent_analytics"
+    MANAGE_ACTIONS = "manage:actions"
+    READ_QUERY_HISTORY = "read:query_history"
+    MANAGE_USER_GROUPS = "manage:user_groups"
+    CREATE_USER_API_KEYS = "create:user_api_keys"
+    CREATE_SERVICE_ACCOUNT_API_KEYS = "create:service_account_api_keys"
+    CREATE_SLACK_DISCORD_BOTS = "create:slack_discord_bots"
+
+    # Override — any permission check passes
+    FULL_ADMIN_PANEL_ACCESS = "admin"
+
+    # Permissions that are implied by other grants and must never be stored
+    # directly in the permission_grant table.
+    IMPLIED: ClassVar[frozenset[Permission]]
+
+
+Permission.IMPLIED = frozenset(
+    {
+        Permission.READ_CONNECTORS,
+        Permission.READ_DOCUMENT_SETS,
+        Permission.READ_AGENTS,
+        Permission.READ_USERS,
+    }
+)

backend/onyx/db/hook.py

@@ -75,6 +75,7 @@ def create_hook__no_commit(
     fail_strategy: HookFailStrategy,
     timeout_seconds: float,
     is_active: bool = False,
+    is_reachable: bool | None = None,
     creator_id: UUID | None = None,
 ) -> Hook:
     """Create a new hook for the given hook point.
@@ -100,6 +101,7 @@ def create_hook__no_commit(
         fail_strategy=fail_strategy,
         timeout_seconds=timeout_seconds,
         is_active=is_active,
+        is_reachable=is_reachable,
         creator_id=creator_id,
     )
     # Use a savepoint so that a failed insert only rolls back this operation,

backend/onyx/db/index_attempt.py

@@ -2,6 +2,8 @@ from collections.abc import Sequence
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
+from typing import NamedTuple
+from typing import TYPE_CHECKING
 from typing import TypeVarTuple
 
 from sqlalchemy import and_
@@ -28,6 +30,9 @@ from onyx.utils.logger import setup_logger
 from onyx.utils.telemetry import optional_telemetry
 from onyx.utils.telemetry import RecordType
 
+if TYPE_CHECKING:
+    from onyx.configs.constants import DocumentSource
+
 # from sqlalchemy.sql.selectable import Select
 
 # Comment out unused imports that cause mypy errors
@@ -972,3 +977,106 @@ def get_index_attempt_errors_for_cc_pair(
         stmt = stmt.offset(page * page_size).limit(page_size)
 
     return list(db_session.scalars(stmt).all())
+
+
+# ── Metrics query helpers ──────────────────────────────────────────────
+
+
+class ActiveIndexAttemptMetric(NamedTuple):
+    """Row returned by get_active_index_attempts_for_metrics."""
+
+    status: IndexingStatus
+    source: "DocumentSource"
+    cc_pair_id: int
+    cc_pair_name: str | None
+    attempt_count: int
+
+
+def get_active_index_attempts_for_metrics(
+    db_session: Session,
+) -> list[ActiveIndexAttemptMetric]:
+    """Return non-terminal index attempts grouped by status, source, and connector.
+
+    Each row is (status, source, cc_pair_id, cc_pair_name, attempt_count).
+    """
+    from onyx.db.models import Connector
+
+    terminal_statuses = [s for s in IndexingStatus if s.is_terminal()]
+    rows = (
+        db_session.query(
+            IndexAttempt.status,
+            Connector.source,
+            ConnectorCredentialPair.id,
+            ConnectorCredentialPair.name,
+            func.count(),
+        )
+        .join(
+            ConnectorCredentialPair,
+            IndexAttempt.connector_credential_pair_id == ConnectorCredentialPair.id,
+        )
+        .join(
+            Connector,
+            ConnectorCredentialPair.connector_id == Connector.id,
+        )
+        .filter(IndexAttempt.status.notin_(terminal_statuses))
+        .group_by(
+            IndexAttempt.status,
+            Connector.source,
+            ConnectorCredentialPair.id,
+            ConnectorCredentialPair.name,
+        )
+        .all()
+    )
+    return [ActiveIndexAttemptMetric(*row) for row in rows]
+
+
+def get_failed_attempt_counts_by_cc_pair(
+    db_session: Session,
+    since: datetime | None = None,
+) -> dict[int, int]:
+    """Return {cc_pair_id: failed_attempt_count} for all connectors.
+
+    When ``since`` is provided, only attempts created after that timestamp
+    are counted. Defaults to the last 90 days to avoid unbounded historical
+    aggregation.
+    """
+    if since is None:
+        since = datetime.now(timezone.utc) - timedelta(days=90)
+
+    rows = (
+        db_session.query(
+            IndexAttempt.connector_credential_pair_id,
+            func.count(),
+        )
+        .filter(IndexAttempt.status == IndexingStatus.FAILED)
+        .filter(IndexAttempt.time_created >= since)
+        .group_by(IndexAttempt.connector_credential_pair_id)
+        .all()
+    )
+    return {cc_id: count for cc_id, count in rows}
+
+
+def get_docs_indexed_by_cc_pair(
+    db_session: Session,
+    since: datetime | None = None,
+) -> dict[int, int]:
+    """Return {cc_pair_id: total_new_docs_indexed} across successful attempts.
+
+    Only counts attempts with status SUCCESS to avoid inflating counts with
+    partial results from failed attempts. When ``since`` is provided, only
+    attempts created after that timestamp are included.
+    """
+    if since is None:
+        since = datetime.now(timezone.utc) - timedelta(days=90)
+
+    query = (
+        db_session.query(
+            IndexAttempt.connector_credential_pair_id,
+            func.sum(func.coalesce(IndexAttempt.new_docs_indexed, 0)),
+        )
+        .filter(IndexAttempt.status == IndexingStatus.SUCCESS)
+        .filter(IndexAttempt.time_created >= since)
+        .group_by(IndexAttempt.connector_credential_pair_id)
+    )
+    rows = query.all()
+    return {cc_id: int(total or 0) for cc_id, total in rows}

backend/onyx/db/models.py

@@ -48,6 +48,7 @@ from sqlalchemy.types import LargeBinary
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy import PrimaryKeyConstraint
 
+from onyx.db.enums import AccountType
 from onyx.auth.schemas import UserRole
 from onyx.configs.constants import (
     ANONYMOUS_USER_UUID,
@@ -78,6 +79,8 @@ from onyx.db.enums import (
     MCPAuthenticationPerformer,
     MCPTransport,
     MCPServerStatus,
+    Permission,
+    GrantSource,
     LLMModelFlowType,
     ThemePreference,
     DefaultAppMode,
@@ -302,6 +305,9 @@ class User(SQLAlchemyBaseUserTableUUID, Base):
     role: Mapped[UserRole] = mapped_column(
         Enum(UserRole, native_enum=False, default=UserRole.BASIC)
     )
+    account_type: Mapped[AccountType | None] = mapped_column(
+        Enum(AccountType, native_enum=False), nullable=True
+    )
 
     """
     Preferences probably should be in a separate table at some point, but for now
@@ -2645,6 +2651,15 @@ class ChatMessage(Base):
         nullable=True,
     )
 
+    # For multi-model turns: the user message points to which assistant response
+    # was selected as the preferred one to continue the conversation with.
+    preferred_response_id: Mapped[int | None] = mapped_column(
+        ForeignKey("chat_message.id", ondelete="SET NULL"), nullable=True
+    )
+
+    # The display name of the model that generated this assistant message
+    model_display_name: Mapped[str | None] = mapped_column(String, nullable=True)
+
     # What does this message contain
     reasoning_tokens: Mapped[str | None] = mapped_column(Text, nullable=True)
     message: Mapped[str] = mapped_column(Text)
@@ -2712,6 +2727,12 @@ class ChatMessage(Base):
         remote_side="ChatMessage.id",
     )
 
+    preferred_response: Mapped["ChatMessage | None"] = relationship(
+        "ChatMessage",
+        foreign_keys=[preferred_response_id],
+        remote_side="ChatMessage.id",
+    )
+
     # Chat messages only need to know their immediate tool call children
     # If there are nested tool calls, they are stored in the tool_call_children relationship.
     tool_calls: Mapped[list["ToolCall"] | None] = relationship(
@@ -3114,8 +3135,6 @@ class VoiceProvider(Base):
     is_default_stt: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     is_default_tts: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 
-    deleted: Mapped[bool] = mapped_column(Boolean, default=False)
-
     time_created: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True), server_default=func.now()
     )
@@ -3971,6 +3990,8 @@ class SamlAccount(Base):
 class User__UserGroup(Base):
     __tablename__ = "user__user_group"
 
+    __table_args__ = (Index("ix_user__user_group_user_id", "user_id"),)
+
     is_curator: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 
     user_group_id: Mapped[int] = mapped_column(
@@ -3981,6 +4002,48 @@ class User__UserGroup(Base):
     )
 
 
+class PermissionGrant(Base):
+    __tablename__ = "permission_grant"
+
+    __table_args__ = (
+        UniqueConstraint(
+            "group_id", "permission", name="uq_permission_grant_group_permission"
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    group_id: Mapped[int] = mapped_column(
+        ForeignKey("user_group.id", ondelete="CASCADE"), nullable=False
+    )
+    permission: Mapped[Permission] = mapped_column(
+        Enum(Permission, native_enum=False), nullable=False
+    )
+    grant_source: Mapped[GrantSource] = mapped_column(
+        Enum(GrantSource, native_enum=False), nullable=False
+    )
+    granted_by: Mapped[UUID | None] = mapped_column(
+        ForeignKey("user.id", ondelete="SET NULL"), nullable=True
+    )
+    granted_at: Mapped[datetime.datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+    is_deleted: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=False, server_default=text("false")
+    )
+
+    group: Mapped["UserGroup"] = relationship(
+        "UserGroup", back_populates="permission_grants"
+    )
+
+    @validates("permission")
+    def _validate_permission(self, _key: str, value: Permission) -> Permission:
+        if value in Permission.IMPLIED:
+            raise ValueError(
+                f"{value!r} is an implied permission and cannot be granted directly"
+            )
+        return value
+
+
 class UserGroup__ConnectorCredentialPair(Base):
     __tablename__ = "user_group__connector_credential_pair"
 
@@ -4075,6 +4138,8 @@ class UserGroup(Base):
     is_up_for_deletion: Mapped[bool] = mapped_column(
         Boolean, nullable=False, default=False
     )
+    # whether this is a default group (e.g. "Basic", "Admins") that cannot be deleted
+    is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 
     # Last time a user updated this user group
     time_last_modified_by_user: Mapped[datetime.datetime] = mapped_column(
@@ -4118,6 +4183,9 @@ class UserGroup(Base):
     accessible_mcp_servers: Mapped[list["MCPServer"]] = relationship(
         "MCPServer", secondary="mcp_server__user_group", back_populates="user_groups"
     )
+    permission_grants: Mapped[list["PermissionGrant"]] = relationship(
+        "PermissionGrant", back_populates="group", cascade="all, delete-orphan"
+    )
 
 
 """Tables related to Token Rate Limiting

backend/onyx/db/persona.py

@@ -50,8 +50,18 @@ from onyx.utils.variable_functionality import fetch_versioned_implementation
 logger = setup_logger()
 
 
-def get_default_behavior_persona(db_session: Session) -> Persona | None:
+def get_default_behavior_persona(
+    db_session: Session,
+    eager_load_for_tools: bool = False,
+) -> Persona | None:
     stmt = select(Persona).where(Persona.id == DEFAULT_PERSONA_ID)
+    if eager_load_for_tools:
+        stmt = stmt.options(
+            selectinload(Persona.tools),
+            selectinload(Persona.document_sets),
+            selectinload(Persona.attached_documents),
+            selectinload(Persona.hierarchy_nodes),
+        )
     return db_session.scalars(stmt).first()
 
 

backend/onyx/db/projects.py

@@ -7,6 +7,7 @@ from fastapi import HTTPException
 from fastapi import UploadFile
 from pydantic import BaseModel
 from pydantic import ConfigDict
+from pydantic import Field
 from sqlalchemy import func
 from sqlalchemy.orm import Session
 from starlette.background import BackgroundTasks
@@ -17,6 +18,7 @@ from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
+from onyx.db.enums import UserFileStatus
 from onyx.db.models import Project__UserFile
 from onyx.db.models import User
 from onyx.db.models import UserFile
@@ -34,9 +36,19 @@ class CategorizedFilesResult(BaseModel):
     user_files: list[UserFile]
     rejected_files: list[RejectedFile]
     id_to_temp_id: dict[str, str]
+    # Filenames that should be stored but not indexed.
+    skip_indexing_filenames: set[str] = Field(default_factory=set)
     # Allow SQLAlchemy ORM models inside this result container
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
+    @property
+    def indexable_files(self) -> list[UserFile]:
+        return [
+            uf
+            for uf in self.user_files
+            if (uf.name or "") not in self.skip_indexing_filenames
+        ]
+
 
 def build_hashed_file_key(file: UploadFile) -> str:
     name_prefix = (file.filename or "")[:50]
@@ -70,6 +82,7 @@ def create_user_files(
         )
         if new_temp_id is not None:
             id_to_temp_id[str(new_id)] = new_temp_id
+        should_skip = (file.filename or "") in categorized_files.skip_indexing
         new_file = UserFile(
             id=new_id,
             user_id=user.id,
@@ -81,6 +94,7 @@ def create_user_files(
             link_url=link_url,
             content_type=file.content_type,
             file_type=file.content_type,
+            status=UserFileStatus.SKIPPED if should_skip else UserFileStatus.PROCESSING,
             last_accessed_at=datetime.datetime.now(datetime.timezone.utc),
         )
         # Persist the UserFile first to satisfy FK constraints for association table
@@ -98,6 +112,7 @@ def create_user_files(
         user_files=user_files,
         rejected_files=rejected_files,
         id_to_temp_id=id_to_temp_id,
+        skip_indexing_filenames=categorized_files.skip_indexing,
     )
 
 
@@ -123,6 +138,7 @@ def upload_files_to_user_files_with_indexing(
     user_files = categorized_files_result.user_files
     rejected_files = categorized_files_result.rejected_files
     id_to_temp_id = categorized_files_result.id_to_temp_id
+    indexable_files = categorized_files_result.indexable_files
     # Trigger per-file processing immediately for the current tenant
     tenant_id = get_current_tenant_id()
     for rejected_file in rejected_files:
@@ -134,12 +150,12 @@ def upload_files_to_user_files_with_indexing(
         from onyx.background.task_utils import drain_processing_loop
 
         background_tasks.add_task(drain_processing_loop, tenant_id)
-        for user_file in user_files:
+        for user_file in indexable_files:
             logger.info(f"Queued in-process processing for user_file_id={user_file.id}")
     else:
         from onyx.background.celery.versioned_apps.client import app as client_app
 
-        for user_file in user_files:
+        for user_file in indexable_files:
             task = client_app.send_task(
                 OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
                 kwargs={"user_file_id": user_file.id, "tenant_id": tenant_id},
@@ -155,6 +171,7 @@ def upload_files_to_user_files_with_indexing(
         user_files=user_files,
         rejected_files=rejected_files,
         id_to_temp_id=id_to_temp_id,
+        skip_indexing_filenames=categorized_files_result.skip_indexing_filenames,
     )
 
 

backend/onyx/db/voice.py

@@ -17,39 +17,30 @@ MAX_VOICE_PLAYBACK_SPEED = 2.0
 def fetch_voice_providers(db_session: Session) -> list[VoiceProvider]:
     """Fetch all voice providers."""
     return list(
-        db_session.scalars(
-            select(VoiceProvider)
-            .where(VoiceProvider.deleted.is_(False))
-            .order_by(VoiceProvider.name)
-        ).all()
+        db_session.scalars(select(VoiceProvider).order_by(VoiceProvider.name)).all()
     )
 
 
 def fetch_voice_provider_by_id(
-    db_session: Session, provider_id: int, include_deleted: bool = False
+    db_session: Session, provider_id: int
 ) -> VoiceProvider | None:
     """Fetch a voice provider by ID."""
-    stmt = select(VoiceProvider).where(VoiceProvider.id == provider_id)
-    if not include_deleted:
-        stmt = stmt.where(VoiceProvider.deleted.is_(False))
-    return db_session.scalar(stmt)
+    return db_session.scalar(
+        select(VoiceProvider).where(VoiceProvider.id == provider_id)
+    )
 
 
 def fetch_default_stt_provider(db_session: Session) -> VoiceProvider | None:
     """Fetch the default STT provider."""
     return db_session.scalar(
-        select(VoiceProvider)
-        .where(VoiceProvider.is_default_stt.is_(True))
-        .where(VoiceProvider.deleted.is_(False))
+        select(VoiceProvider).where(VoiceProvider.is_default_stt.is_(True))
     )
 
 
 def fetch_default_tts_provider(db_session: Session) -> VoiceProvider | None:
     """Fetch the default TTS provider."""
     return db_session.scalar(
-        select(VoiceProvider)
-        .where(VoiceProvider.is_default_tts.is_(True))
-        .where(VoiceProvider.deleted.is_(False))
+        select(VoiceProvider).where(VoiceProvider.is_default_tts.is_(True))
     )
 
 
@@ -58,9 +49,7 @@ def fetch_voice_provider_by_type(
 ) -> VoiceProvider | None:
     """Fetch a voice provider by type."""
     return db_session.scalar(
-        select(VoiceProvider)
-        .where(VoiceProvider.provider_type == provider_type)
-        .where(VoiceProvider.deleted.is_(False))
+        select(VoiceProvider).where(VoiceProvider.provider_type == provider_type)
     )
 
 
@@ -119,10 +108,10 @@ def upsert_voice_provider(
 
 
 def delete_voice_provider(db_session: Session, provider_id: int) -> None:
-    """Soft-delete a voice provider by ID."""
+    """Delete a voice provider by ID."""
     provider = fetch_voice_provider_by_id(db_session, provider_id)
     if provider:
-        provider.deleted = True
+        db_session.delete(provider)
         db_session.flush()
 
 

backend/onyx/document_index/disabled.py

@@ -5,6 +5,7 @@ accidentally reaches the vector DB layer will fail loudly instead of timing
 out against a nonexistent Vespa/OpenSearch instance.
 """
 
+from collections.abc import Iterable
 from typing import Any
 
 from onyx.context.search.models import IndexFilters
@@ -66,7 +67,7 @@ class DisabledDocumentIndex(DocumentIndex):
     # ------------------------------------------------------------------
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],  # noqa: ARG002
+        chunks: Iterable[DocMetadataAwareIndexChunk],  # noqa: ARG002
         index_batch_params: IndexBatchParams,  # noqa: ARG002
     ) -> set[DocumentInsertionRecord]:
         raise RuntimeError(VECTOR_DB_DISABLED_ERROR)

backend/onyx/document_index/interfaces.py

@@ -1,4 +1,5 @@
 import abc
+from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import datetime
 from typing import Any
@@ -206,7 +207,7 @@ class Indexable(abc.ABC):
     @abc.abstractmethod
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
+        chunks: Iterable[DocMetadataAwareIndexChunk],
         index_batch_params: IndexBatchParams,
     ) -> set[DocumentInsertionRecord]:
         """
@@ -226,8 +227,8 @@ class Indexable(abc.ABC):
         it is done automatically outside of this code.
 
         Parameters:
-        - chunks: Document chunks with all of the information needed for indexing to the document
-                index.
+        - chunks: Document chunks with all of the information needed for
+                indexing to the document index.
         - tenant_id: The tenant id of the user whose chunks are being indexed
         - large_chunks_enabled: Whether large chunks are enabled
 

backend/onyx/document_index/interfaces_new.py

@@ -1,4 +1,5 @@
 import abc
+from collections.abc import Iterable
 from typing import Self
 
 from pydantic import BaseModel
@@ -209,10 +210,10 @@ class Indexable(abc.ABC):
     @abc.abstractmethod
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
+        chunks: Iterable[DocMetadataAwareIndexChunk],
         indexing_metadata: IndexingMetadata,
     ) -> list[DocumentInsertionRecord]:
-        """Indexes a list of document chunks into the document index.
+        """Indexes an iterable of document chunks into the document index.
 
         This is often a batch operation including chunks from multiple
         documents.
@@ -381,6 +382,47 @@ class HybridCapable(abc.ABC):
         """
         raise NotImplementedError
 
+    @abc.abstractmethod
+    def keyword_retrieval(
+        self,
+        query: str,
+        filters: IndexFilters,
+        num_to_retrieve: int,
+    ) -> list[InferenceChunk]:
+        """Runs keyword-only search and returns a list of inference chunks.
+
+        Args:
+            query: User query.
+            filters: Filters for things like permissions, source type, time,
+                etc.
+            num_to_retrieve: Number of highest matching chunks to return.
+
+        Returns:
+            Score-ranked (highest first) list of highest matching chunks.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def semantic_retrieval(
+        self,
+        query_embedding: Embedding,
+        filters: IndexFilters,
+        num_to_retrieve: int,
+    ) -> list[InferenceChunk]:
+        """Runs semantic-only search and returns a list of inference chunks.
+
+        Args:
+            query_embedding: Vector representation of the query. Must be of the
+                correct dimensionality for the primary index.
+            filters: Filters for things like permissions, source type, time,
+                etc.
+            num_to_retrieve: Number of highest matching chunks to return.
+
+        Returns:
+            Score-ranked (highest first) list of highest matching chunks.
+        """
+        raise NotImplementedError
+
 
 class RandomCapable(abc.ABC):
     """

backend/onyx/document_index/opensearch/client.py

@@ -18,10 +18,13 @@ from onyx.configs.app_configs import OPENSEARCH_ADMIN_USERNAME
 from onyx.configs.app_configs import OPENSEARCH_HOST
 from onyx.configs.app_configs import OPENSEARCH_REST_API_PORT
 from onyx.document_index.interfaces_new import TenantState
+from onyx.document_index.opensearch.constants import OpenSearchSearchType
 from onyx.document_index.opensearch.schema import DocumentChunk
 from onyx.document_index.opensearch.schema import DocumentChunkWithoutVectors
 from onyx.document_index.opensearch.schema import get_opensearch_doc_chunk_id
 from onyx.document_index.opensearch.search import DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW
+from onyx.server.metrics.opensearch_search import observe_opensearch_search
+from onyx.server.metrics.opensearch_search import track_opensearch_search_in_progress
 from onyx.utils.logger import setup_logger
 from onyx.utils.timing import log_function_time
 
@@ -256,7 +259,6 @@ class OpenSearchClient(AbstractContextManager):
         """
         return self._client.ping()
 
-    @log_function_time(print_only=True, debug_only=True)
     def close(self) -> None:
         """Closes the client.
 
@@ -304,6 +306,7 @@ class OpenSearchIndexClient(OpenSearchClient):
         verify_certs: bool = False,
         ssl_show_warn: bool = False,
         timeout: int = DEFAULT_OPENSEARCH_CLIENT_TIMEOUT_S,
+        emit_metrics: bool = True,
     ):
         super().__init__(
             host=host,
@@ -315,6 +318,7 @@ class OpenSearchIndexClient(OpenSearchClient):
             timeout=timeout,
         )
         self._index_name = index_name
+        self._emit_metrics = emit_metrics
         logger.debug(
             f"OpenSearch client created successfully for index {self._index_name}."
         )
@@ -834,7 +838,10 @@ class OpenSearchIndexClient(OpenSearchClient):
 
     @log_function_time(print_only=True, debug_only=True)
     def search(
-        self, body: dict[str, Any], search_pipeline_id: str | None
+        self,
+        body: dict[str, Any],
+        search_pipeline_id: str | None,
+        search_type: OpenSearchSearchType = OpenSearchSearchType.UNKNOWN,
     ) -> list[SearchHit[DocumentChunkWithoutVectors]]:
         """Searches the index.
 
@@ -852,6 +859,8 @@ class OpenSearchIndexClient(OpenSearchClient):
                 documentation for more information on search request bodies.
             search_pipeline_id: The ID of the search pipeline to use. If None,
                 the default search pipeline will be used.
+            search_type: Label for Prometheus metrics. Does not affect search
+                behavior.
 
         Raises:
             Exception: There was an error searching the index.
@@ -864,21 +873,27 @@ class OpenSearchIndexClient(OpenSearchClient):
         )
         result: dict[str, Any]
         params = {"phase_took": "true"}
-        if search_pipeline_id:
-            result = self._client.search(
-                index=self._index_name,
-                search_pipeline=search_pipeline_id,
-                body=body,
-                params=params,
-            )
-        else:
-            result = self._client.search(
-                index=self._index_name, body=body, params=params
-            )
+        ctx = self._get_emit_metrics_context_manager(search_type)
+        t0 = time.perf_counter()
+        with ctx:
+            if search_pipeline_id:
+                result = self._client.search(
+                    index=self._index_name,
+                    search_pipeline=search_pipeline_id,
+                    body=body,
+                    params=params,
+                )
+            else:
+                result = self._client.search(
+                    index=self._index_name, body=body, params=params
+                )
+        client_duration_s = time.perf_counter() - t0
 
         hits, time_took, timed_out, phase_took, profile = (
             self._get_hits_and_profile_from_search_result(result)
         )
+        if self._emit_metrics:
+            observe_opensearch_search(search_type, client_duration_s, time_took)
         self._log_search_result_perf(
             time_took=time_took,
             timed_out=timed_out,
@@ -914,7 +929,11 @@ class OpenSearchIndexClient(OpenSearchClient):
         return search_hits
 
     @log_function_time(print_only=True, debug_only=True)
-    def search_for_document_ids(self, body: dict[str, Any]) -> list[str]:
+    def search_for_document_ids(
+        self,
+        body: dict[str, Any],
+        search_type: OpenSearchSearchType = OpenSearchSearchType.UNKNOWN,
+    ) -> list[str]:
         """Searches the index and returns only document chunk IDs.
 
         In order to take advantage of the performance benefits of only returning
@@ -931,6 +950,8 @@ class OpenSearchIndexClient(OpenSearchClient):
                 documentation for more information on search request bodies.
                 TODO(andrei): Make this a more deep interface; callers shouldn't
                 need to know to set _source: False for example.
+            search_type: Label for Prometheus metrics. Does not affect search
+                behavior.
 
         Raises:
             Exception: There was an error searching the index.
@@ -948,13 +969,19 @@ class OpenSearchIndexClient(OpenSearchClient):
             )
 
         params = {"phase_took": "true"}
-        result: dict[str, Any] = self._client.search(
-            index=self._index_name, body=body, params=params
-        )
+        ctx = self._get_emit_metrics_context_manager(search_type)
+        t0 = time.perf_counter()
+        with ctx:
+            result: dict[str, Any] = self._client.search(
+                index=self._index_name, body=body, params=params
+            )
+        client_duration_s = time.perf_counter() - t0
 
         hits, time_took, timed_out, phase_took, profile = (
             self._get_hits_and_profile_from_search_result(result)
         )
+        if self._emit_metrics:
+            observe_opensearch_search(search_type, client_duration_s, time_took)
         self._log_search_result_perf(
             time_took=time_took,
             timed_out=timed_out,
@@ -1071,6 +1098,20 @@ class OpenSearchIndexClient(OpenSearchClient):
             if raise_on_timeout:
                 raise RuntimeError(error_str)
 
+    def _get_emit_metrics_context_manager(
+        self, search_type: OpenSearchSearchType
+    ) -> AbstractContextManager[None]:
+        """
+        Returns a context manager that tracks in-flight OpenSearch searches via
+        a Gauge if emit_metrics is True, otherwise returns a null context
+        manager.
+        """
+        return (
+            track_opensearch_search_in_progress(search_type)
+            if self._emit_metrics
+            else nullcontext()
+        )
+
 
 def wait_for_opensearch_with_timeout(
     wait_interval_s: int = 5,

backend/onyx/document_index/opensearch/constants.py

@@ -53,6 +53,17 @@ DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES = int(
 EF_SEARCH = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES
 
 
+class OpenSearchSearchType(str, Enum):
+    """Search type label used for Prometheus metrics."""
+
+    HYBRID = "hybrid"
+    KEYWORD = "keyword"
+    SEMANTIC = "semantic"
+    RANDOM = "random"
+    DOC_ID_RETRIEVAL = "doc_id_retrieval"
+    UNKNOWN = "unknown"
+
+
 class HybridSearchSubqueryConfiguration(Enum):
     TITLE_VECTOR_CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD = 1
     # Current default.

backend/onyx/document_index/opensearch/opensearch_document_index.py

@@ -1,11 +1,12 @@
 import json
-from collections import defaultdict
+from collections.abc import Iterable
 from typing import Any
 
 import httpx
 from opensearchpy import NotFoundError
 
 from onyx.access.models import DocumentAccess
+from onyx.configs.app_configs import MAX_CHUNKS_PER_DOC_BATCH
 from onyx.configs.app_configs import VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT
 from onyx.configs.chat_configs import NUM_RETURNED_HITS
 from onyx.configs.chat_configs import TITLE_CONTENT_RATIO
@@ -43,6 +44,7 @@ from onyx.document_index.opensearch.client import OpenSearchClient
 from onyx.document_index.opensearch.client import OpenSearchIndexClient
 from onyx.document_index.opensearch.client import SearchHit
 from onyx.document_index.opensearch.cluster_settings import OPENSEARCH_CLUSTER_SETTINGS
+from onyx.document_index.opensearch.constants import OpenSearchSearchType
 from onyx.document_index.opensearch.schema import ACCESS_CONTROL_LIST_FIELD_NAME
 from onyx.document_index.opensearch.schema import CONTENT_FIELD_NAME
 from onyx.document_index.opensearch.schema import DOCUMENT_SETS_FIELD_NAME
@@ -350,7 +352,7 @@ class OpenSearchOldDocumentIndex(OldDocumentIndex):
 
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
+        chunks: Iterable[DocMetadataAwareIndexChunk],
         index_batch_params: IndexBatchParams,
     ) -> set[OldDocumentInsertionRecord]:
         """
@@ -646,10 +648,10 @@ class OpenSearchDocumentIndex(DocumentIndex):
 
     def index(
         self,
-        chunks: list[DocMetadataAwareIndexChunk],
-        indexing_metadata: IndexingMetadata,  # noqa: ARG002
+        chunks: Iterable[DocMetadataAwareIndexChunk],
+        indexing_metadata: IndexingMetadata,
     ) -> list[DocumentInsertionRecord]:
-        """Indexes a list of document chunks into the document index.
+        """Indexes an iterable of document chunks into the document index.
 
         Groups chunks by document ID and for each document, deletes existing
         chunks and indexes the new chunks in bulk.
@@ -672,29 +674,34 @@ class OpenSearchDocumentIndex(DocumentIndex):
                 document is newly indexed or had already existed and was just
                 updated.
         """
-        # Group chunks by document ID.
-        doc_id_to_chunks: dict[str, list[DocMetadataAwareIndexChunk]] = defaultdict(
-            list
+        total_chunks = sum(
+            cc.new_chunk_cnt
+            for cc in indexing_metadata.doc_id_to_chunk_cnt_diff.values()
         )
-        for chunk in chunks:
-            doc_id_to_chunks[chunk.source_document.id].append(chunk)
         logger.debug(
-            f"[OpenSearchDocumentIndex] Indexing {len(chunks)} chunks from {len(doc_id_to_chunks)} "
+            f"[OpenSearchDocumentIndex] Indexing {total_chunks} chunks from {len(indexing_metadata.doc_id_to_chunk_cnt_diff)} "
             f"documents for index {self._index_name}."
         )
 
         document_indexing_results: list[DocumentInsertionRecord] = []
-        # Try to index per-document.
-        for _, chunks in doc_id_to_chunks.items():
+        deleted_doc_ids: set[str] = set()
+        # Buffer chunks per document as they arrive from the iterable.
+        # When the document ID changes flush the buffered chunks.
+        current_doc_id: str | None = None
+        current_chunks: list[DocMetadataAwareIndexChunk] = []
+
+        def _flush_chunks(doc_chunks: list[DocMetadataAwareIndexChunk]) -> None:
+            assert len(doc_chunks) > 0, "doc_chunks is empty"
+
             # Create a batch of OpenSearch-formatted chunks for bulk insertion.
-            # Do this before deleting existing chunks to reduce the amount of
-            # time the document index has no content for a given document, and
-            # to reduce the chance of entering a state where we delete chunks,
-            # then some error happens, and never successfully index new chunks.
+            # Since we are doing this in batches, an error occurring midway
+            # can result in a state where chunks are deleted and not all the
+            # new chunks have been indexed.
             chunk_batch: list[DocumentChunk] = [
-                _convert_onyx_chunk_to_opensearch_document(chunk) for chunk in chunks
+                _convert_onyx_chunk_to_opensearch_document(chunk)
+                for chunk in doc_chunks
             ]
-            onyx_document: Document = chunks[0].source_document
+            onyx_document: Document = doc_chunks[0].source_document
             # First delete the doc's chunks from the index. This is so that
             # there are no dangling chunks in the index, in the event that the
             # new document's content contains fewer chunks than the previous
@@ -703,22 +710,43 @@ class OpenSearchDocumentIndex(DocumentIndex):
             # if the chunk count has actually decreased. This assumes that
             # overlapping chunks are perfectly overwritten. If we can't
             # guarantee that then we need the code as-is.
-            num_chunks_deleted = self.delete(
-                onyx_document.id, onyx_document.chunk_count
-            )
-            # If we see that chunks were deleted we assume the doc already
-            # existed.
-            document_insertion_record = DocumentInsertionRecord(
-                document_id=onyx_document.id,
-                already_existed=num_chunks_deleted > 0,
-            )
+            if onyx_document.id not in deleted_doc_ids:
+                num_chunks_deleted = self.delete(
+                    onyx_document.id, onyx_document.chunk_count
+                )
+                deleted_doc_ids.add(onyx_document.id)
+                # If we see that chunks were deleted we assume the doc already
+                # existed. We record the result before bulk_index_documents
+                # runs. If indexing raises, this entire result list is discarded
+                # by the caller's retry logic, so early recording is safe.
+                document_indexing_results.append(
+                    DocumentInsertionRecord(
+                        document_id=onyx_document.id,
+                        already_existed=num_chunks_deleted > 0,
+                    )
+                )
             # Now index. This will raise if a chunk of the same ID exists, which
             # we do not expect because we should have deleted all chunks.
             self._client.bulk_index_documents(
                 documents=chunk_batch,
                 tenant_state=self._tenant_state,
             )
-            document_indexing_results.append(document_insertion_record)
+
+        for chunk in chunks:
+            doc_id = chunk.source_document.id
+            if doc_id != current_doc_id:
+                if current_chunks:
+                    _flush_chunks(current_chunks)
+                current_doc_id = doc_id
+                current_chunks = [chunk]
+            elif len(current_chunks) >= MAX_CHUNKS_PER_DOC_BATCH:
+                _flush_chunks(current_chunks)
+                current_chunks = [chunk]
+            else:
+                current_chunks.append(chunk)
+
+        if current_chunks:
+            _flush_chunks(current_chunks)
 
         return document_indexing_results
 
@@ -900,6 +928,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
             search_hits = self._client.search(
                 body=query_body,
                 search_pipeline_id=None,
+                search_type=OpenSearchSearchType.DOC_ID_RETRIEVAL,
             )
             inference_chunks_uncleaned: list[InferenceChunkUncleaned] = [
                 _convert_retrieved_opensearch_chunk_to_inference_chunk_uncleaned(
@@ -923,6 +952,8 @@ class OpenSearchDocumentIndex(DocumentIndex):
         filters: IndexFilters,
         num_to_retrieve: int,
     ) -> list[InferenceChunk]:
+        # TODO(andrei): There is some duplicated logic in this function with
+        # others in this file.
         logger.debug(
             f"[OpenSearchDocumentIndex] Hybrid retrieving {num_to_retrieve} chunks for index {self._index_name}."
         )
@@ -948,6 +979,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
         search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = self._client.search(
             body=query_body,
             search_pipeline_id=normalization_pipeline_name,
+            search_type=OpenSearchSearchType.HYBRID,
         )
 
         # Good place for a breakpoint to inspect the search hits if you have
@@ -970,6 +1002,8 @@ class OpenSearchDocumentIndex(DocumentIndex):
         filters: IndexFilters,
         num_to_retrieve: int,
     ) -> list[InferenceChunk]:
+        # TODO(andrei): There is some duplicated logic in this function with
+        # others in this file.
         logger.debug(
             f"[OpenSearchDocumentIndex] Keyword retrieving {num_to_retrieve} chunks for index {self._index_name}."
         )
@@ -989,6 +1023,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
         search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = self._client.search(
             body=query_body,
             search_pipeline_id=None,
+            search_type=OpenSearchSearchType.KEYWORD,
         )
 
         inference_chunks_uncleaned: list[InferenceChunkUncleaned] = [
@@ -1009,6 +1044,8 @@ class OpenSearchDocumentIndex(DocumentIndex):
         filters: IndexFilters,
         num_to_retrieve: int,
     ) -> list[InferenceChunk]:
+        # TODO(andrei): There is some duplicated logic in this function with
+        # others in this file.
         logger.debug(
             f"[OpenSearchDocumentIndex] Semantic retrieving {num_to_retrieve} chunks for index {self._index_name}."
         )
@@ -1028,6 +1065,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
         search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = self._client.search(
             body=query_body,
             search_pipeline_id=None,
+            search_type=OpenSearchSearchType.SEMANTIC,
         )
 
         inference_chunks_uncleaned: list[InferenceChunkUncleaned] = [
@@ -1059,6 +1097,7 @@ class OpenSearchDocumentIndex(DocumentIndex):
         search_hits: list[SearchHit[DocumentChunkWithoutVectors]] = self._client.search(
             body=query_body,
             search_pipeline_id=None,
+            search_type=OpenSearchSearchType.RANDOM,
         )
         inference_chunks_uncleaned: list[InferenceChunkUncleaned] = [
             _convert_retrieved_opensearch_chunk_to_inference_chunk_uncleaned(