Add DB_READONLY_PASSWORD option in helm chart

.github/CODEOWNERS

@@ -1,3 +1 @@
 * @onyx-dot-app/onyx-core-team
-# Helm charts Owners
-/helm/ @justin-tahara

.github/actions/custom-build-and-push/action.yml

@@ -35,16 +35,6 @@ inputs:
   cache-to:
     description: 'Cache destinations'
     required: false
-  outputs:
-    description: 'Output destinations'
-    required: false
-  provenance:
-    description: 'Generate provenance attestation'
-    required: false
-    default: 'false'
-  build-args:
-    description: 'Build arguments'
-    required: false
   retry-wait-time:
     description: 'Time to wait before attempt 2 in seconds'
     required: false
@@ -72,9 +62,6 @@ runs:
         no-cache: ${{ inputs.no-cache }}
         cache-from: ${{ inputs.cache-from }}
         cache-to: ${{ inputs.cache-to }}
-        outputs: ${{ inputs.outputs }}
-        provenance: ${{ inputs.provenance }}
-        build-args: ${{ inputs.build-args }}
 
     - name: Wait before attempt 2
       if: steps.buildx1.outcome != 'success'
@@ -98,9 +85,6 @@ runs:
         no-cache: ${{ inputs.no-cache }}
         cache-from: ${{ inputs.cache-from }}
         cache-to: ${{ inputs.cache-to }}
-        outputs: ${{ inputs.outputs }}
-        provenance: ${{ inputs.provenance }}
-        build-args: ${{ inputs.build-args }}
 
     - name: Wait before attempt 3
       if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success'
@@ -124,9 +108,6 @@ runs:
         no-cache: ${{ inputs.no-cache }}
         cache-from: ${{ inputs.cache-from }}
         cache-to: ${{ inputs.cache-to }}
-        outputs: ${{ inputs.outputs }}
-        provenance: ${{ inputs.provenance }}
-        build-args: ${{ inputs.build-args }}
 
     - name: Report failure
       if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success' && steps.buildx3.outcome != 'success'

.github/pull_request_template.md

@@ -6,6 +6,9 @@
 
 [Describe the tests you ran to verify your changes]
 
-## Additional Options
+## Backporting (check the box to trigger backport action)
 
+Note: You have to check that the action passes, otherwise resolve the conflicts manually and tag the patches.
+
+- [ ] This PR should be backported (make sure to check that the backport attempt succeeds)
 - [ ] [Optional] Override Linear Check

.github/workflows/check-lazy-imports.yml

@@ -1,24 +0,0 @@
-name: Check Lazy Imports
-
-on:
-  merge_group:
-  pull_request:
-    branches:
-      - main
-      - 'release/**'
-
-jobs:
-  check-lazy-imports:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.11'
-
-    - name: Check lazy imports
-      run: python3 backend/scripts/check_lazy_imports.py

.github/workflows/docker-build-push-backend-container-on-tag.yml

@@ -8,9 +8,9 @@ on:
 env:
   REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-backend-cloud' || 'onyxdotapp/onyx-backend' }}
   DEPLOYMENT: ${{ contains(github.ref_name, 'cloud') && 'cloud' || 'standalone' }}
-
-  # tag nightly builds with "edge"
-  EDGE_TAG: ${{ startsWith(github.ref_name, 'nightly-latest') }}
+  
+  # don't tag cloud images with "latest"
+  LATEST_TAG: ${{ contains(github.ref_name, 'latest') && !contains(github.ref_name, 'cloud') }}
 
 jobs:
   build-and-push:
@@ -33,16 +33,7 @@ jobs:
         run: |
           platform=${{ matrix.platform }}
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-
-      - name: Check if stable release version
-        id: check_version
-        run: |
-          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ "${{ github.ref_name }}" != *"cloud"* ]]; then
-            echo "is_stable=true" >> $GITHUB_OUTPUT
-          else
-            echo "is_stable=false" >> $GITHUB_OUTPUT
-          fi
-
+          
       - name: Checkout code
         uses: actions/checkout@v4
 
@@ -55,8 +46,7 @@ jobs:
             latest=false
           tags: |
             type=raw,value=${{ github.ref_name }}
-            type=raw,value=${{ steps.check_version.outputs.is_stable == 'true' && 'latest' || '' }}
-            type=raw,value=${{ env.EDGE_TAG == 'true' && 'edge' || '' }}
+            type=raw,value=${{ env.LATEST_TAG == 'true' && 'latest' || '' }}
             
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -129,8 +119,7 @@ jobs:
             latest=false
           tags: |
             type=raw,value=${{ github.ref_name }}
-            type=raw,value=${{ steps.check_version.outputs.is_stable == 'true' && 'latest' || '' }}
-            type=raw,value=${{ env.EDGE_TAG == 'true' && 'edge' || '' }}
+            type=raw,value=${{ env.LATEST_TAG == 'true' && 'latest' || '' }}
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3
@@ -153,25 +142,15 @@ jobs:
       # can re-enable when they figure it out
       # https://github.com/aquasecurity/trivy/discussions/7538
       # https://github.com/aquasecurity/trivy-action/issues/389
-      # Security: Using pinned digest (0.65.0@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436)
-      # Security: No Docker socket mount needed for remote registry scanning
       - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@v3
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
+          TRIVY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          TRIVY_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
         with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -v ${{ github.workspace }}/backend/.trivyignore:/tmp/.trivyignore:ro \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              --ignorefile /tmp/.trivyignore \
-              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          # To run locally: trivy image --severity HIGH,CRITICAL onyxdotapp/onyx-backend
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: "CRITICAL,HIGH"
+          trivyignores: ./backend/.trivyignore

.github/workflows/docker-build-push-cloud-web-container-on-tag.yml

@@ -139,20 +139,12 @@ jobs:
       # https://github.com/aquasecurity/trivy/discussions/7538
       # https://github.com/aquasecurity/trivy-action/issues/389
       - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@v3
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
+          TRIVY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          TRIVY_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
         with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: "CRITICAL,HIGH"

.github/workflows/docker-build-push-model-server-container-on-tag.yml

@@ -11,8 +11,8 @@ env:
   BUILDKIT_PROGRESS: plain
   DEPLOYMENT: ${{ contains(github.ref_name, 'cloud') && 'cloud' || 'standalone' }}
 
-  # tag nightly builds with "edge"
-  EDGE_TAG: ${{ startsWith(github.ref_name, 'nightly-latest') }}
+  # don't tag cloud images with "latest"
+  LATEST_TAG: ${{ contains(github.ref_name, 'latest') && !contains(github.ref_name, 'cloud') }}
   
 jobs:
 
@@ -99,7 +99,7 @@ jobs:
     needs: [check_model_server_changes]
     if: needs.check_model_server_changes.outputs.changed == 'true'
     runs-on:
-      [runs-on, runner=8cpu-linux-arm64, "run-id=${{ github.run_id }}-arm64"]
+      [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-arm64"]
     env:
       PLATFORM_PAIR: linux-arm64
     steps:
@@ -145,15 +145,6 @@ jobs:
     if: needs.check_model_server_changes.outputs.changed == 'true'
     runs-on: ubuntu-latest
     steps:
-      - name: Check if stable release version
-        id: check_version
-        run: |
-          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ "${{ github.ref_name }}" != *"cloud"* ]]; then
-            echo "is_stable=true" >> $GITHUB_OUTPUT
-          else
-            echo "is_stable=false" >> $GITHUB_OUTPUT
-          fi
-
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -166,32 +157,20 @@ jobs:
           docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} \
             ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \
             ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
-          if [[ "${{ steps.check_version.outputs.is_stable }}" == "true" ]]; then
+          if [[ "${{ env.LATEST_TAG }}" == "true" ]]; then
             docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:latest \
               ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \
               ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
           fi
-          if [[ "${{ env.EDGE_TAG }}" == "true" ]]; then
-            docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:edge \
-              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \
-              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
-          fi
 
       - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@v3
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
+          TRIVY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          TRIVY_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
         with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: "CRITICAL,HIGH"
+          timeout: "10m"

.github/workflows/docker-build-push-web-container-on-tag.yml

@@ -7,10 +7,7 @@ on:
 
 env:
   REGISTRY_IMAGE: onyxdotapp/onyx-web-server
-
-  # tag nightly builds with "edge"
-  EDGE_TAG: ${{ startsWith(github.ref_name, 'nightly-latest') }}
-
+  LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
   DEPLOYMENT: standalone
 
 jobs:
@@ -48,15 +45,6 @@ jobs:
           platform=${{ matrix.platform }}
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
-      - name: Check if stable release version
-        id: check_version
-        run: |
-          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "is_stable=true" >> $GITHUB_OUTPUT
-          else
-            echo "is_stable=false" >> $GITHUB_OUTPUT
-          fi
-
       - name: Checkout
         uses: actions/checkout@v4
 
@@ -69,8 +57,7 @@ jobs:
             latest=false
           tags: |
             type=raw,value=${{ github.ref_name }}
-            type=raw,value=${{ steps.check_version.outputs.is_stable == 'true' && 'latest' || '' }}
-            type=raw,value=${{ env.EDGE_TAG == 'true' && 'edge' || '' }}
+            type=raw,value=${{ env.LATEST_TAG == 'true' && 'latest' || '' }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -139,8 +126,7 @@ jobs:
             latest=false
           tags: |
             type=raw,value=${{ github.ref_name }}
-            type=raw,value=${{ steps.check_version.outputs.is_stable == 'true' && 'latest' || '' }}
-            type=raw,value=${{ env.EDGE_TAG == 'true' && 'edge' || '' }}
+            type=raw,value=${{ env.LATEST_TAG == 'true' && 'latest' || '' }}
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3
@@ -164,20 +150,12 @@ jobs:
       # https://github.com/aquasecurity/trivy/discussions/7538
       # https://github.com/aquasecurity/trivy-action/issues/389
       - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@v3
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
+          TRIVY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          TRIVY_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
         with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: "CRITICAL,HIGH"

.github/workflows/helm-chart-releases.yml

@@ -1,52 +0,0 @@
-name: Release Onyx Helm Charts
-
-on:
-  push:
-    branches:
-      - main
-
-permissions: write-all
-
-jobs:
-  release:
-    permissions:
-      contents: write
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Install Helm CLI
-        uses: azure/setup-helm@v4
-        with:
-          version: v3.12.1
-
-      - name: Add required Helm repositories
-        run: |
-          helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
-          helm repo add onyx-vespa https://onyx-dot-app.github.io/vespa-helm-charts
-          helm repo add cloudnative-pg https://cloudnative-pg.github.io/charts
-          helm repo add ot-container-kit https://ot-container-kit.github.io/helm-charts
-          helm repo add minio https://charts.min.io/
-          helm repo update
-
-      - name: Build chart dependencies
-        run: |
-          set -euo pipefail
-          for chart_dir in deployment/helm/charts/*; do
-            if [ -f "$chart_dir/Chart.yaml" ]; then
-              echo "Building dependencies for $chart_dir"
-              helm dependency build "$chart_dir"
-            fi
-          done
-
-      - name: Publish Helm charts to gh-pages
-        uses: stefanprodan/helm-gh-pages@v1.7.0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          charts_dir: deployment/helm/charts
-          branch: gh-pages
-          commit_username: ${{ github.actor }}
-          commit_email: ${{ github.actor }}@users.noreply.github.com

.github/workflows/pr-backport-autotrigger.yml

@@ -0,0 +1,124 @@
+name: Backport on Merge
+
+# Note this workflow does not trigger the builds, be sure to manually tag the branches to trigger the builds
+
+on:
+  pull_request:
+    types: [closed] # Later we check for merge so only PRs that go in can get backported
+
+permissions:
+  contents: write
+  actions: write
+
+jobs:
+  backport:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.YUHONG_GH_ACTIONS }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
+          fetch-depth: 0
+
+      - name: Set up Git user
+        run: |
+          git config user.name "Richard Kuo [bot]"
+          git config user.email "rkuo[bot]@onyx.app"
+          git fetch --prune
+
+      - name: Check for Backport Checkbox
+        id: checkbox-check
+        run: |
+          PR_BODY="${{ github.event.pull_request.body }}"
+          if [[ "$PR_BODY" == *"[x] This PR should be backported"* ]]; then
+            echo "backport=true" >> $GITHUB_OUTPUT
+          else
+            echo "backport=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: List and sort release branches
+        id: list-branches
+        run: |
+          git fetch --all --tags
+          BRANCHES=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/release/* | sed 's|origin/release/||' | sort -Vr)
+          BETA=$(echo "$BRANCHES" | head -n 1)
+          STABLE=$(echo "$BRANCHES" | head -n 2 | tail -n 1)
+          echo "beta=release/$BETA" >> $GITHUB_OUTPUT
+          echo "stable=release/$STABLE" >> $GITHUB_OUTPUT
+          # Fetch latest tags for beta and stable
+          LATEST_BETA_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*-beta.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$" | grep -v -- "-cloud" | sort -Vr | head -n 1)
+          LATEST_STABLE_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -Vr | head -n 1)
+
+          # Handle case where no beta tags exist
+          if [[ -z "$LATEST_BETA_TAG" ]]; then
+            NEW_BETA_TAG="v1.0.0-beta.1"
+          else
+            NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '[.-]' '{print $1 "." $2 "." $3 "-beta." ($NF+1)}')
+          fi
+
+          # Increment latest stable tag
+          NEW_STABLE_TAG=$(echo $LATEST_STABLE_TAG | awk -F '.' '{print $1 "." $2 "." ($3+1)}')
+          echo "latest_beta_tag=$LATEST_BETA_TAG" >> $GITHUB_OUTPUT
+          echo "latest_stable_tag=$LATEST_STABLE_TAG" >> $GITHUB_OUTPUT
+          echo "new_beta_tag=$NEW_BETA_TAG" >> $GITHUB_OUTPUT
+          echo "new_stable_tag=$NEW_STABLE_TAG" >> $GITHUB_OUTPUT
+
+      - name: Echo branch and tag information
+        run: |
+          echo "Beta branch: ${{ steps.list-branches.outputs.beta }}"
+          echo "Stable branch: ${{ steps.list-branches.outputs.stable }}"
+          echo "Latest beta tag: ${{ steps.list-branches.outputs.latest_beta_tag }}"
+          echo "Latest stable tag: ${{ steps.list-branches.outputs.latest_stable_tag }}"
+          echo "New beta tag: ${{ steps.list-branches.outputs.new_beta_tag }}"
+          echo "New stable tag: ${{ steps.list-branches.outputs.new_stable_tag }}"
+
+      - name: Trigger Backport
+        if: steps.checkbox-check.outputs.backport == 'true'
+        run: |
+          set -e
+          echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
+
+          # Echo the merge commit SHA
+          echo "Merge commit SHA: ${{ github.event.pull_request.merge_commit_sha }}"
+
+          # Fetch all history for all branches and tags
+          git fetch --prune
+
+          # Reset and prepare the beta branch
+          git checkout ${{ steps.list-branches.outputs.beta }}
+          echo "Last 5 commits on beta branch:"
+          git log -n 5 --pretty=format:"%H"
+          echo ""  # Newline for formatting
+
+          # Cherry-pick the merge commit from the merged PR
+          git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
+            echo "Cherry-pick to beta failed due to conflicts."
+            exit 1
+          }
+
+          # Create new beta branch/tag
+          git tag ${{ steps.list-branches.outputs.new_beta_tag }}
+          # Push the changes and tag to the beta branch using PAT
+          git push origin ${{ steps.list-branches.outputs.beta }}
+          git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
+
+          # Reset and prepare the stable branch
+          git checkout ${{ steps.list-branches.outputs.stable }}
+          echo "Last 5 commits on stable branch:"
+          git log -n 5 --pretty=format:"%H"
+          echo ""  # Newline for formatting
+
+          # Cherry-pick the merge commit from the merged PR
+          git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
+            echo "Cherry-pick to stable failed due to conflicts."
+            exit 1
+          }
+
+          # Create new stable branch/tag
+          git tag ${{ steps.list-branches.outputs.new_stable_tag }}
+          # Push the changes and tag to the stable branch using PAT
+          git push origin ${{ steps.list-branches.outputs.stable }}
+          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}

.github/workflows/pr-external-dependency-unit-tests.yml

@@ -1,111 +0,0 @@
-name: External Dependency Unit Tests
-
-on:
-  merge_group:
-  pull_request:
-    branches: [main]
-
-env:
-  # AWS
-  S3_AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
-  S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
-
-  # MinIO
-  S3_ENDPOINT_URL: "http://localhost:9004"
-
-  # Confluence
-  CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
-  CONFLUENCE_TEST_SPACE: ${{ secrets.CONFLUENCE_TEST_SPACE }}
-  CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }}
-  CONFLUENCE_IS_CLOUD: ${{ secrets.CONFLUENCE_IS_CLOUD }}
-  CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
-  CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
-  CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }}
-
-  # LLMs
-  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-
-jobs:
-  discover-test-dirs:
-    runs-on: ubuntu-latest
-    outputs:
-      test-dirs: ${{ steps.set-matrix.outputs.test-dirs }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      
-      - name: Discover test directories
-        id: set-matrix
-        run: |
-          # Find all subdirectories in backend/tests/external_dependency_unit
-          dirs=$(find backend/tests/external_dependency_unit -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort | jq -R -s -c 'split("\n")[:-1]')
-          echo "test-dirs=$dirs" >> $GITHUB_OUTPUT
-
-  external-dependency-unit-tests:
-    needs: discover-test-dirs
-    # Use larger runner with more resources for Vespa
-    runs-on: [runs-on, runner=16cpu-linux-x64, "run-id=${{ github.run_id }}"]
-    
-    strategy:
-      fail-fast: false
-      matrix:
-        test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }}
-
-    env:
-      PYTHONPATH: ./backend
-      MODEL_SERVER_HOST: "disabled"
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-          cache: "pip"
-          cache-dependency-path: |
-            backend/requirements/default.txt
-            backend/requirements/dev.txt
-
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
-          pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
-          playwright install chromium
-          playwright install-deps chromium
-
-      - name: Set up Standard Dependencies
-        run: |
-          cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d minio relational_db cache index
-
-      - name: Wait for services
-        run: |
-          echo "Waiting for services to be ready..."
-          sleep 30
-          
-          # Wait for Vespa specifically
-          echo "Waiting for Vespa to be ready..."
-          timeout 300 bash -c 'until curl -f -s http://localhost:8081/ApplicationStatus > /dev/null 2>&1; do echo "Vespa not ready, waiting..."; sleep 10; done' || echo "Vespa timeout - continuing anyway"
-          
-          echo "Services should be ready now"
-
-      - name: Run migrations
-        run: |
-          cd backend
-          # Run migrations to head
-          alembic upgrade head
-          alembic heads --verbose
-
-      - name: Run Tests for ${{ matrix.test-dir }}
-        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
-        run: |
-          py.test \
-            --durations=8 \
-            -o junit_family=xunit2 \
-            -xv \
-            --ff \
-            backend/tests/external_dependency_unit/${{ matrix.test-dir }}

.github/workflows/pr-helm-chart-testing.yml

@@ -53,176 +53,9 @@ jobs:
       if: steps.list-changed.outputs.changed == 'true'
       uses: helm/kind-action@v1.12.0
 
-    - name: Pre-install cluster status check
+    - name: Run chart-testing (install)
       if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Pre-install Cluster Status ==="
-        kubectl get nodes -o wide
-        kubectl get pods --all-namespaces
-        kubectl get storageclass
-
-    - name: Add Helm repositories and update
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Adding Helm repositories ==="
-        helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
-        helm repo add vespa https://onyx-dot-app.github.io/vespa-helm-charts
-        helm repo add cloudnative-pg https://cloudnative-pg.github.io/charts
-        helm repo add ot-container-kit https://ot-container-kit.github.io/helm-charts
-        helm repo add minio https://charts.min.io/
-        helm repo update
-
-    - name: Install Redis operator
-      if: steps.list-changed.outputs.changed == 'true'
-      shell: bash
-      run: |
-        echo "=== Installing redis-operator CRDs ==="
-        helm upgrade --install redis-operator ot-container-kit/redis-operator \
-          --namespace redis-operator --create-namespace --wait --timeout 300s
-
-    - name: Pre-pull required images
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Pre-pulling required images to avoid timeout ==="
-        KIND_CLUSTER=$(kubectl config current-context | sed 's/kind-//')
-        echo "Kind cluster: $KIND_CLUSTER"
-
-        IMAGES=(
-          "ghcr.io/cloudnative-pg/cloudnative-pg:1.27.0"
-          "quay.io/opstree/redis:v7.0.15"
-          "docker.io/onyxdotapp/onyx-web-server:latest"
-        )
-
-        for image in "${IMAGES[@]}"; do
-          echo "Pre-pulling $image"
-          if docker pull "$image"; then
-            kind load docker-image "$image" --name "$KIND_CLUSTER" || echo "Failed to load $image into kind"
-          else
-            echo "Failed to pull $image"
-          fi
-        done
-
-        echo "=== Images loaded into Kind cluster ==="
-        docker exec "$KIND_CLUSTER"-control-plane crictl images | grep -E "(cloudnative-pg|redis|onyx)" || echo "Some images may still be loading..."
-
-    - name: Validate chart dependencies
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Validating chart dependencies ==="
-        cd deployment/helm/charts/onyx
-        helm dependency update
-        helm lint .
-
-    - name: Run chart-testing (install) with enhanced monitoring
-      timeout-minutes: 25
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Starting chart installation with monitoring ==="
-        
-        # Function to monitor cluster state
-        monitor_cluster() {
-          while true; do
-            echo "=== Cluster Status Check at $(date) ==="
-            # Only show non-running pods to reduce noise
-            NON_RUNNING_PODS=$(kubectl get pods --all-namespaces --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers 2>/dev/null | wc -l)
-            if [ "$NON_RUNNING_PODS" -gt 0 ]; then
-              echo "Non-running pods:"
-              kubectl get pods --all-namespaces --field-selector=status.phase!=Running,status.phase!=Succeeded
-            else
-              echo "All pods running successfully"
-            fi
-            # Only show recent events if there are issues
-            RECENT_EVENTS=$(kubectl get events --sort-by=.lastTimestamp --all-namespaces --field-selector=type!=Normal 2>/dev/null | tail -5)
-            if [ -n "$RECENT_EVENTS" ]; then
-              echo "Recent warnings/errors:"
-              echo "$RECENT_EVENTS"
-            fi
-            sleep 60
-          done
-        }
-        
-        # Start monitoring in background
-        monitor_cluster &
-        MONITOR_PID=$!
-        
-        # Set up cleanup
-        cleanup() {
-          echo "=== Cleaning up monitoring process ==="
-          kill $MONITOR_PID 2>/dev/null || true
-          echo "=== Final cluster state ==="
-          kubectl get pods --all-namespaces
-          kubectl get events --all-namespaces --sort-by=.lastTimestamp | tail -20
-        }
-        
-        # Trap cleanup on exit
-        trap cleanup EXIT
-        
-        # Run the actual installation with detailed logging
-        echo "=== Starting ct install ==="
-        set +e
-        ct install --all \
-          --helm-extra-set-args="\
-            --set=nginx.enabled=false \
-            --set=minio.enabled=false \
-            --set=vespa.enabled=false \
-            --set=slackbot.enabled=false \
-            --set=postgresql.enabled=true \
-            --set=postgresql.nameOverride=cloudnative-pg \
-            --set=postgresql.cluster.storage.storageClass=standard \
-            --set=redis.enabled=true \
-            --set=redis.storageSpec.volumeClaimTemplate.spec.storageClassName=standard \
-            --set=webserver.replicaCount=1 \
-            --set=api.replicaCount=0 \
-            --set=inferenceCapability.replicaCount=0 \
-            --set=indexCapability.replicaCount=0 \
-            --set=celery_beat.replicaCount=0 \
-            --set=celery_worker_heavy.replicaCount=0 \
-            --set=celery_worker_docfetching.replicaCount=0 \
-            --set=celery_worker_docprocessing.replicaCount=0 \
-            --set=celery_worker_light.replicaCount=0 \
-            --set=celery_worker_monitoring.replicaCount=0 \
-            --set=celery_worker_primary.replicaCount=0 \
-            --set=celery_worker_user_file_processing.replicaCount=0 \
-            --set=celery_worker_user_files_indexing.replicaCount=0" \
-          --helm-extra-args="--timeout 900s --debug" \
-          --debug --config ct.yaml
-        CT_EXIT=$?
-        set -e
-
-        if [[ $CT_EXIT -ne 0 ]]; then
-          echo "ct install failed with exit code $CT_EXIT"
-          exit $CT_EXIT
-        else
-          echo "=== Installation completed successfully ==="
-        fi
-
-        kubectl get pods --all-namespaces
-
-    - name: Post-install verification
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Post-install verification ==="
-        kubectl get pods --all-namespaces
-        kubectl get services --all-namespaces
-        # Only show issues if they exist
-        kubectl describe pods --all-namespaces | grep -A 5 -B 2 "Failed\|Error\|Warning" || echo "No pod issues found"
-
-    - name: Cleanup on failure
-      if: failure() && steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Cleanup on failure ==="
-        echo "=== Final cluster state ==="
-        kubectl get pods --all-namespaces
-        kubectl get events --all-namespaces --sort-by=.lastTimestamp | tail -10
-        
-        echo "=== Pod descriptions for debugging ==="
-        kubectl describe pods --all-namespaces | grep -A 10 -B 3 "Failed\|Error\|Warning\|Pending" || echo "No problematic pods found"
-        
-        echo "=== Recent logs for debugging ==="
-        kubectl logs --all-namespaces --tail=50 | grep -i "error\|timeout\|failed\|pull" || echo "No error logs found"
-
-        echo "=== Helm releases ==="
-        helm list --all-namespaces
+      run: ct install --all --helm-extra-set-args="--set=nginx.enabled=false" --debug --config ct.yaml
       # the following would install only changed charts, but we only have one chart so 
       # don't worry about that for now
       # run: ct install --target-branch ${{ github.event.repository.default_branch }}

.github/workflows/pr-integration-tests.yml

@@ -11,58 +11,23 @@ on:
       - "release/**"
 
 env:
-  # Private Registry Configuration
-  PRIVATE_REGISTRY: experimental-registry.blacksmith.sh:5000
-  PRIVATE_REGISTRY_USERNAME: ${{ secrets.PRIVATE_REGISTRY_USERNAME }}
-  PRIVATE_REGISTRY_PASSWORD: ${{ secrets.PRIVATE_REGISTRY_PASSWORD }}
-
-  # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
   CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
-  CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }}
-  JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
-  JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
-  JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
-  JIRA_API_TOKEN_SCOPED: ${{ secrets.JIRA_API_TOKEN_SCOPED }}
-  PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
-  PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
-  PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
-  PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
+  PLATFORM_PAIR: linux-amd64
 
 jobs:
-  discover-test-dirs:
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    outputs:
-      test-dirs: ${{ steps.set-matrix.outputs.test-dirs }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Discover test directories
-        id: set-matrix
-        run: |
-          # Find all leaf-level directories in both test directories
-          tests_dirs=$(find backend/tests/integration/tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort)
-          connector_dirs=$(find backend/tests/integration/connector_job_tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort)
-
-          # Create JSON array with directory info
-          all_dirs=""
-          for dir in $tests_dirs; do
-            all_dirs="$all_dirs{\"path\":\"tests/$dir\",\"name\":\"tests-$dir\"},"
-          done
-          for dir in $connector_dirs; do
-            all_dirs="$all_dirs{\"path\":\"connector_job_tests/$dir\",\"name\":\"connector-$dir\"},"
-          done
-
-          # Remove trailing comma and wrap in array
-          all_dirs="[${all_dirs%,}]"
-          echo "test-dirs=$all_dirs" >> $GITHUB_OUTPUT
-
-  prepare-build:
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
+  integration-tests:
+    # See https://runs-on.com/runners/linux/
+    runs-on:
+      [
+        runs-on,
+        runner=32cpu-linux-x64,
+        disk=large,
+        "run-id=${{ github.run_id }}",
+      ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -75,12 +40,12 @@ jobs:
           cache-dependency-path: |
             backend/requirements/default.txt
             backend/requirements/dev.txt
-
-      - name: Install Python dependencies
-        run: |
+            backend/requirements/ee.txt
+      - run: |
           python -m pip install --upgrade pip
           pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
+          pip install --retries 5 --timeout 30 -r backend/requirements/ee.txt
 
       - name: Generate OpenAPI schema
         working-directory: ./backend
@@ -98,160 +63,132 @@ jobs:
             -i /local/openapi.json \
             -g python \
             -o /local/onyx_openapi_client \
-            --package-name onyx_openapi_client \
-            --skip-validate-spec \
-            --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=true,SET_OAS3_NULLABLE=true"
-
-      - name: Upload OpenAPI artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: openapi-artifacts
-          path: backend/generated/
-
-  build-backend-image:
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
+            --package-name onyx_openapi_client
 
       - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
+        uses: docker/setup-buildx-action@v3
 
-      - name: Build and push Backend Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}
-          push: true
-          outputs: type=registry
-          no-cache: true
-
-  build-model-server-image:
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Model Server Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile.model_server
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}
-          push: true
-          outputs: type=registry
-          provenance: false
-          no-cache: true
-
-  build-integration-image:
-    needs: prepare-build
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Download OpenAPI artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: openapi-artifacts
-          path: backend/generated/
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push integration test Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/tests/integration/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}
-          push: true
-          outputs: type=registry
-          no-cache: true
-
-  integration-tests:
-    needs:
-      [
-        discover-test-dirs,
-        build-backend-image,
-        build-model-server-image,
-        build-integration-image,
-      ]
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
-      - name: Pull Docker images
+      # tag every docker image with "test" so that we can spin up the correct set
+      # of images during testing
+
+      # We don't need to build the Web Docker image since it's not yet used
+      # in the integration tests. We have a separate action to verify that it builds
+      # successfully.
+      - name: Pull Web Docker image
         run: |
-          # Pull all images from registry in parallel
-          echo "Pulling Docker images in parallel..."
-          # Pull images from private registry
-          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}) &
-          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}) &
-          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}) &
+          docker pull onyxdotapp/onyx-web-server:latest
+          docker tag onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:test
 
-          # Wait for all background jobs to complete
-          wait
-          echo "All Docker images pulled successfully"
+      # we use the runs-on cache for docker builds
+      # in conjunction with runs-on runners, it has better speed and unlimited caching
+      # https://runs-on.com/caching/s3-cache-for-github-actions/
+      # https://runs-on.com/caching/docker/
+      # https://github.com/moby/buildkit#s3-cache-experimental
 
-          # Re-tag to remove registry prefix for docker-compose
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test
+      # images are built and run locally for testing purposes. Not pushed.
+      - name: Build Backend Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-backend:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build Model Server Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.model_server
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-model-server:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build integration test Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/tests/integration/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-integration:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      # Start containers for multi-tenant tests
+      - name: Start Docker containers for multi-tenant tests
+        run: |
+          cd deployment/docker_compose
+          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
+          MULTI_TENANT=true \
+          AUTH_TYPE=cloud \
+          REQUIRE_EMAIL_VERIFICATION=false \
+          DISABLE_TELEMETRY=true \
+          IMAGE_TAG=test \
+          DEV_MODE=true \
+          docker compose -f docker-compose.multitenant-dev.yml -p onyx-stack up -d
+        id: start_docker_multi_tenant
+
+      # In practice, `cloud` Auth type would require OAUTH credentials to be set.
+      - name: Run Multi-Tenant Integration Tests
+        run: |
+          echo "Waiting for 3 minutes to ensure API server is ready..."
+          sleep 180
+          echo "Running integration tests..."
+          docker run --rm --network onyx-stack_default \
+            --name test-runner \
+            -e POSTGRES_HOST=relational_db \
+            -e POSTGRES_USER=postgres \
+            -e POSTGRES_PASSWORD=password \
+            -e DB_READONLY_USER=db_readonly_user \
+            -e DB_READONLY_PASSWORD=password \
+            -e POSTGRES_DB=postgres \
+            -e POSTGRES_USE_NULL_POOL=true \
+            -e VESPA_HOST=index \
+            -e REDIS_HOST=cache \
+            -e API_SERVER_HOST=api_server \
+            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
+            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
+            -e TEST_WEB_HOSTNAME=test-runner \
+            -e AUTH_TYPE=cloud \
+            -e MULTI_TENANT=true \
+            -e REQUIRE_EMAIL_VERIFICATION=false \
+            -e DISABLE_TELEMETRY=true \
+            -e IMAGE_TAG=test \
+            -e DEV_MODE=true \
+            onyxdotapp/onyx-integration:test \
+            /app/tests/integration/multitenant_tests
+        continue-on-error: true
+        id: run_multitenant_tests
+
+      - name: Check multi-tenant test results
+        run: |
+          if [ ${{ steps.run_multitenant_tests.outcome }} == 'failure' ]; then
+            echo "Multi-tenant integration tests failed. Exiting with error."
+            exit 1
+          else
+            echo "All multi-tenant integration tests passed successfully."
+          fi
+
+      - name: Stop multi-tenant Docker containers
+        run: |
+          cd deployment/docker_compose
+          docker compose -f docker-compose.multitenant-dev.yml -p onyx-stack down -v
 
       # NOTE: Use pre-ping/null pool to reduce flakiness due to dropped connections
-      # NOTE: don't need web server for integration tests
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
@@ -264,23 +201,14 @@ jobs:
           IMAGE_TAG=test \
           INTEGRATION_TESTS_MODE=true \
           CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS=0.001 \
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml up \
-            relational_db \
-            index \
-            cache \
-            minio \
-            api_server \
-            inference_model_server \
-            indexing_model_server \
-            background \
-            -d
+          docker compose -f docker-compose.dev.yml -p onyx-stack up -d
         id: start_docker
 
       - name: Wait for service to be ready
         run: |
           echo "Starting wait-for-service script..."
 
-          docker logs -f onyx-api_server-1 &
+          docker logs -f onyx-stack-api_server-1 &
 
           start_time=$(date +%s)
           timeout=300  # 5 minutes in seconds
@@ -316,164 +244,11 @@ jobs:
           docker compose -f docker-compose.mock-it-services.yml \
             -p mock-it-services-stack up -d
 
-      - name: Run Integration Tests for ${{ matrix.test-dir.name }}
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 20
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            echo "Running integration tests for ${{ matrix.test-dir.path }}..."
-            docker run --rm --network onyx_default \
-              --name test-runner \
-              -e POSTGRES_HOST=relational_db \
-              -e POSTGRES_USER=postgres \
-              -e POSTGRES_PASSWORD=password \
-              -e POSTGRES_DB=postgres \
-              -e DB_READONLY_USER=db_readonly_user \
-              -e DB_READONLY_PASSWORD=password \
-              -e POSTGRES_POOL_PRE_PING=true \
-              -e POSTGRES_USE_NULL_POOL=true \
-              -e VESPA_HOST=index \
-              -e REDIS_HOST=cache \
-              -e API_SERVER_HOST=api_server \
-              -e OPENAI_API_KEY=${OPENAI_API_KEY} \
-              -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
-              -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
-              -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
-              -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
-              -e CONFLUENCE_ACCESS_TOKEN_SCOPED=${CONFLUENCE_ACCESS_TOKEN_SCOPED} \
-              -e JIRA_BASE_URL=${JIRA_BASE_URL} \
-              -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
-              -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
-              -e JIRA_API_TOKEN_SCOPED=${JIRA_API_TOKEN_SCOPED} \
-              -e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
-              -e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
-              -e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
-              -e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
-              -e TEST_WEB_HOSTNAME=test-runner \
-              -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
-              -e MOCK_CONNECTOR_SERVER_PORT=8001 \
-              onyxdotapp/onyx-integration:test \
-              /app/tests/integration/${{ matrix.test-dir.path }}
-
-      # ------------------------------------------------------------
-      # Always gather logs BEFORE "down":
-      - name: Dump API server logs
-        if: always()
+      # NOTE: Use pre-ping/null to reduce flakiness due to dropped connections
+      - name: Run Standard Integration Tests
         run: |
-          cd deployment/docker_compose
-          docker compose logs --no-color api_server > $GITHUB_WORKSPACE/api_server.log || true
-
-      - name: Dump all-container logs (optional)
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true
-
-      - name: Upload logs
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: docker-all-logs-${{ matrix.test-dir.name }}
-          path: ${{ github.workspace }}/docker-compose.log
-      # ------------------------------------------------------------
-
-      - name: Stop Docker containers
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose down -v
-
-
-  multitenant-tests:
-    needs:
-      [
-        build-backend-image,
-        build-model-server-image,
-        build-integration-image,
-      ]
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-
-      - name: Pull Docker images
-        run: |
-          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}) &
-          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}) &
-          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}) &
-          wait
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test
-
-      - name: Start Docker containers for multi-tenant tests
-        run: |
-          cd deployment/docker_compose
-          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
-          MULTI_TENANT=true \
-          AUTH_TYPE=cloud \
-          REQUIRE_EMAIL_VERIFICATION=false \
-          DISABLE_TELEMETRY=true \
-          IMAGE_TAG=test \
-          DEV_MODE=true \
-          docker compose -f docker-compose.multitenant-dev.yml up \
-            relational_db \
-            index \
-            cache \
-            minio \
-            api_server \
-            inference_model_server \
-            indexing_model_server \
-            background \
-            -d
-        id: start_docker_multi_tenant
-
-      - name: Wait for service to be ready (multi-tenant)
-        run: |
-          echo "Starting wait-for-service script for multi-tenant..."
-          docker logs -f onyx-api_server-1 &
-          start_time=$(date +%s)
-          timeout=300
-          while true; do
-            current_time=$(date +%s)
-            elapsed_time=$((current_time - start_time))
-            if [ $elapsed_time -ge $timeout ]; then
-              echo "Timeout reached. Service did not become ready in 5 minutes."
-              exit 1
-            fi
-            response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health || echo "curl_error")
-            if [ "$response" = "200" ]; then
-              echo "Service is ready!"
-              break
-            elif [ "$response" = "curl_error" ]; then
-              echo "Curl encountered an error; retrying..."
-            else
-              echo "Service not ready yet (HTTP $response). Retrying in 5 seconds..."
-            fi
-            sleep 5
-          done
-          echo "Finished waiting for service."
-
-      - name: Run Multi-Tenant Integration Tests
-        run: |
-          echo "Running multi-tenant integration tests..."
-          docker run --rm --network onyx_default \
+          echo "Running integration tests..."
+          docker run --rm --network onyx-stack_default \
             --name test-runner \
             -e POSTGRES_HOST=relational_db \
             -e POSTGRES_USER=postgres \
@@ -481,60 +256,58 @@ jobs:
             -e DB_READONLY_USER=db_readonly_user \
             -e DB_READONLY_PASSWORD=password \
             -e POSTGRES_DB=postgres \
+            -e POSTGRES_POOL_PRE_PING=true \
             -e POSTGRES_USE_NULL_POOL=true \
             -e VESPA_HOST=index \
             -e REDIS_HOST=cache \
             -e API_SERVER_HOST=api_server \
             -e OPENAI_API_KEY=${OPENAI_API_KEY} \
             -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
+            -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
+            -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
+            -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
             -e TEST_WEB_HOSTNAME=test-runner \
-            -e AUTH_TYPE=cloud \
-            -e MULTI_TENANT=true \
-            -e SKIP_RESET=true \
-            -e REQUIRE_EMAIL_VERIFICATION=false \
-            -e DISABLE_TELEMETRY=true \
-            -e IMAGE_TAG=test \
-            -e DEV_MODE=true \
+            -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
+            -e MOCK_CONNECTOR_SERVER_PORT=8001 \
             onyxdotapp/onyx-integration:test \
-            /app/tests/integration/multitenant_tests
+            /app/tests/integration/tests \
+            /app/tests/integration/connector_job_tests
+        continue-on-error: true
+        id: run_tests
 
-      - name: Dump API server logs (multi-tenant)
+      - name: Check test results
+        run: |
+          if [ ${{ steps.run_tests.outcome }} == 'failure' ]; then
+            echo "Integration tests failed. Exiting with error."
+            exit 1
+          else
+            echo "All integration tests passed successfully."
+          fi
+
+      # ------------------------------------------------------------
+      # Always gather logs BEFORE "down":
+      - name: Dump API server logs
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.multitenant-dev.yml logs --no-color api_server > $GITHUB_WORKSPACE/api_server_multitenant.log || true
+          docker compose -f docker-compose.dev.yml -p onyx-stack logs --no-color api_server > $GITHUB_WORKSPACE/api_server.log || true
 
-      - name: Dump all-container logs (multi-tenant)
+      - name: Dump all-container logs (optional)
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.multitenant-dev.yml logs --no-color > $GITHUB_WORKSPACE/docker-compose-multitenant.log || true
+          docker compose -f docker-compose.dev.yml -p onyx-stack logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true
 
-      - name: Upload logs (multi-tenant)
+      - name: Upload logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: docker-all-logs-multitenant
-          path: ${{ github.workspace }}/docker-compose-multitenant.log
+          name: docker-all-logs
+          path: ${{ github.workspace }}/docker-compose.log
+      # ------------------------------------------------------------
 
-      - name: Stop multi-tenant Docker containers
+      - name: Stop Docker containers
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.multitenant-dev.yml down -v
-
-  required: 
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    needs: [integration-tests, multitenant-tests]
-    if: ${{ always() }}
-    steps:
-      - uses: actions/github-script@v7
-        with:
-          script: |
-            const needs = ${{ toJSON(needs) }};
-            const failed = Object.values(needs).some(n => n.result !== 'success');
-            if (failed) {
-              core.setFailed('One or more upstream jobs failed or were cancelled.');
-            } else {
-              core.notice('All required jobs succeeded.');
-            }
+          docker compose -f docker-compose.dev.yml -p onyx-stack down -v

.github/workflows/pr-labeler.yml

@@ -1,38 +0,0 @@
-name: PR Labeler
-
-on:
-  pull_request_target:
-    branches:
-      - main
-    types:
-      - opened
-      - reopened
-      - synchronize
-      - edited
-
-permissions:
-  contents: read
-  pull-requests: write
-
-jobs:
-  validate_pr_title:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check PR title for Conventional Commits
-        env:
-          PR_TITLE: ${{ github.event.pull_request.title }}
-        run: |
-          echo "PR Title: $PR_TITLE"
-          if [[ ! "$PR_TITLE" =~ ^(feat|fix|docs|test|ci|refactor|perf|chore|revert|build)(\(.+\))?:\ .+ ]]; then
-            echo "::error::❌ Your PR title does not follow the Conventional Commits format.
-              This check ensures that all pull requests use clear, consistent titles that help automate changelogs and improve project history.
-
-              Please update your PR title to follow the Conventional Commits style.  
-              Here is a link to a blog explaining the reason why we've included the Conventional Commits style into our PR titles: https://xfuture-blog.com/working-with-conventional-commits
-
-              **Here are some examples of valid PR titles:**
-              - feat: add user authentication
-              - fix(login): handle null password error
-              - docs(readme): update installation instructions"
-            exit 1
-          fi

.github/workflows/pr-mit-integration-tests.yml

@@ -5,65 +5,32 @@ concurrency:
 
 on:
   merge_group:
-    types: [checks_requested]
+  pull_request:
+    branches:
+      - main
+      - "release/**"
 
 env:
-  # Private Registry Configuration
-  PRIVATE_REGISTRY: experimental-registry.blacksmith.sh:5000
-  PRIVATE_REGISTRY_USERNAME: ${{ secrets.PRIVATE_REGISTRY_USERNAME }}
-  PRIVATE_REGISTRY_PASSWORD: ${{ secrets.PRIVATE_REGISTRY_PASSWORD }}
-
-  # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
   CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
-  CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }}
-  JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
-  JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
-  JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
-  JIRA_API_TOKEN_SCOPED: ${{ secrets.JIRA_API_TOKEN_SCOPED }}
-  PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
-  PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
-  PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
-  PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
-
+  PLATFORM_PAIR: linux-amd64
 jobs:
-  discover-test-dirs:
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    outputs:
-      test-dirs: ${{ steps.set-matrix.outputs.test-dirs }}
+  integration-tests-mit:
+    # See https://runs-on.com/runners/linux/
+    runs-on:
+      [
+        runs-on,
+        runner=32cpu-linux-x64,
+        disk=large,
+        "run-id=${{ github.run_id }}",
+      ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-
-      - name: Discover test directories
-        id: set-matrix
-        run: |
-          # Find all leaf-level directories in both test directories
-          tests_dirs=$(find backend/tests/integration/tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort)
-          connector_dirs=$(find backend/tests/integration/connector_job_tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort)
-
-          # Create JSON array with directory info
-          all_dirs=""
-          for dir in $tests_dirs; do
-            all_dirs="$all_dirs{\"path\":\"tests/$dir\",\"name\":\"tests-$dir\"},"
-          done
-          for dir in $connector_dirs; do
-            all_dirs="$all_dirs{\"path\":\"connector_job_tests/$dir\",\"name\":\"connector-$dir\"},"
-          done
-
-          # Remove trailing comma and wrap in array
-          all_dirs="[${all_dirs%,}]"
-          echo "test-dirs=$all_dirs" >> $GITHUB_OUTPUT
-
-  prepare-build:
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
+        
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
@@ -72,9 +39,7 @@ jobs:
           cache-dependency-path: |
             backend/requirements/default.txt
             backend/requirements/dev.txt
-
-      - name: Install Python dependencies
-        run: |
+      - run: |
           python -m pip install --upgrade pip
           pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
@@ -95,161 +60,72 @@ jobs:
             -i /local/openapi.json \
             -g python \
             -o /local/onyx_openapi_client \
-            --package-name onyx_openapi_client \
-            --skip-validate-spec \
-            --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=true,SET_OAS3_NULLABLE=true"
-
-      - name: Upload OpenAPI artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: openapi-artifacts
-          path: backend/generated/
-
-  build-backend-image:
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
+            --package-name onyx_openapi_client
+            
       - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
+        uses: docker/setup-buildx-action@v3
 
-      - name: Build and push Backend Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}
-          push: true
-          outputs: type=registry
-          no-cache: true
-
-  build-model-server-image:
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Model Server Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile.model_server
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}
-          push: true
-          outputs: type=registry
-          provenance: false
-          no-cache: true
-
-  build-integration-image:
-    needs: prepare-build
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Download OpenAPI artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: openapi-artifacts
-          path: backend/generated/
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push integration test Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/tests/integration/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}
-          push: true
-          outputs: type=registry
-          no-cache: true
-
-  integration-tests-mit:
-    needs:
-      [
-        discover-test-dirs,
-        build-backend-image,
-        build-model-server-image,
-        build-integration-image,
-      ]
-    # See https://docs.blacksmith.sh/blacksmith-runners/overview
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
-      - name: Pull Docker images
+      # tag every docker image with "test" so that we can spin up the correct set
+      # of images during testing
+
+      # We don't need to build the Web Docker image since it's not yet used
+      # in the integration tests. We have a separate action to verify that it builds
+      # successfully.
+      - name: Pull Web Docker image
         run: |
-          # Pull all images from registry in parallel
-          echo "Pulling Docker images in parallel..."
-          # Pull images from private registry
-          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}) &
-          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}) &
-          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}) &
+          docker pull onyxdotapp/onyx-web-server:latest
+          docker tag onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:test
 
-          # Wait for all background jobs to complete
-          wait
-          echo "All Docker images pulled successfully"
+      # we use the runs-on cache for docker builds
+      # in conjunction with runs-on runners, it has better speed and unlimited caching
+      # https://runs-on.com/caching/s3-cache-for-github-actions/
+      # https://runs-on.com/caching/docker/
+      # https://github.com/moby/buildkit#s3-cache-experimental
 
-          # Re-tag to remove registry prefix for docker-compose
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test
+      # images are built and run locally for testing purposes. Not pushed.
+      - name: Build Backend Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-backend:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build Model Server Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.model_server
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-model-server:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build integration test Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/tests/integration/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-integration:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
       # NOTE: Use pre-ping/null pool to reduce flakiness due to dropped connections
-      # NOTE: don't need web server for integration tests
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
@@ -260,23 +136,14 @@ jobs:
           DISABLE_TELEMETRY=true \
           IMAGE_TAG=test \
           INTEGRATION_TESTS_MODE=true \
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml up \
-            relational_db \
-            index \
-            cache \
-            minio \
-            api_server \
-            inference_model_server \
-            indexing_model_server \
-            background \
-            -d
+          docker compose -f docker-compose.dev.yml -p onyx-stack up -d
         id: start_docker
 
       - name: Wait for service to be ready
         run: |
           echo "Starting wait-for-service script..."
 
-          docker logs -f onyx-api_server-1 &
+          docker logs -f onyx-stack-api_server-1 &
 
           start_time=$(date +%s)
           timeout=300  # 5 minutes in seconds
@@ -313,46 +180,44 @@ jobs:
             -p mock-it-services-stack up -d
 
       # NOTE: Use pre-ping/null to reduce flakiness due to dropped connections
-      - name: Run Integration Tests for ${{ matrix.test-dir.name }}
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 20
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            echo "Running integration tests for ${{ matrix.test-dir.path }}..."
-            docker run --rm --network onyx_default \
-              --name test-runner \
-              -e POSTGRES_HOST=relational_db \
-              -e POSTGRES_USER=postgres \
-              -e POSTGRES_PASSWORD=password \
-              -e POSTGRES_DB=postgres \
-              -e DB_READONLY_USER=db_readonly_user \
-              -e DB_READONLY_PASSWORD=password \
-              -e POSTGRES_POOL_PRE_PING=true \
-              -e POSTGRES_USE_NULL_POOL=true \
-              -e VESPA_HOST=index \
-              -e REDIS_HOST=cache \
-              -e API_SERVER_HOST=api_server \
-              -e OPENAI_API_KEY=${OPENAI_API_KEY} \
-              -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
-              -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
-              -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
-              -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
-              -e CONFLUENCE_ACCESS_TOKEN_SCOPED=${CONFLUENCE_ACCESS_TOKEN_SCOPED} \
-              -e JIRA_BASE_URL=${JIRA_BASE_URL} \
-              -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
-              -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
-              -e JIRA_API_TOKEN_SCOPED=${JIRA_API_TOKEN_SCOPED} \
-              -e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
-              -e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
-              -e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
-              -e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
-              -e TEST_WEB_HOSTNAME=test-runner \
-              -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
-              -e MOCK_CONNECTOR_SERVER_PORT=8001 \
-              onyxdotapp/onyx-integration:test \
-              /app/tests/integration/${{ matrix.test-dir.path }}
+      - name: Run Standard Integration Tests
+        run: |
+          echo "Running integration tests..."
+          docker run --rm --network onyx-stack_default \
+            --name test-runner \
+            -e POSTGRES_HOST=relational_db \
+            -e POSTGRES_USER=postgres \
+            -e POSTGRES_PASSWORD=password \
+            -e POSTGRES_DB=postgres \
+            -e DB_READONLY_USER=db_readonly_user \
+            -e DB_READONLY_PASSWORD=password \
+            -e POSTGRES_POOL_PRE_PING=true \
+            -e POSTGRES_USE_NULL_POOL=true \
+            -e VESPA_HOST=index \
+            -e REDIS_HOST=cache \
+            -e API_SERVER_HOST=api_server \
+            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
+            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
+            -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
+            -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
+            -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
+            -e TEST_WEB_HOSTNAME=test-runner \
+            -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
+            -e MOCK_CONNECTOR_SERVER_PORT=8001 \
+            onyxdotapp/onyx-integration:test \
+            /app/tests/integration/tests \
+            /app/tests/integration/connector_job_tests
+        continue-on-error: true
+        id: run_tests
+
+      - name: Check test results
+        run: |
+          if [ ${{ steps.run_tests.outcome }} == 'failure' ]; then
+            echo "Integration tests failed. Exiting with error."
+            exit 1
+          else
+            echo "All integration tests passed successfully."
+          fi
 
       # ------------------------------------------------------------
       # Always gather logs BEFORE "down":
@@ -360,19 +225,19 @@ jobs:
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose logs --no-color api_server > $GITHUB_WORKSPACE/api_server.log || true
+          docker compose -f docker-compose.dev.yml -p onyx-stack logs --no-color api_server > $GITHUB_WORKSPACE/api_server.log || true
 
       - name: Dump all-container logs (optional)
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true
+          docker compose -f docker-compose.dev.yml -p onyx-stack logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true
 
       - name: Upload logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: docker-all-logs-${{ matrix.test-dir.name }}
+          name: docker-all-logs
           path: ${{ github.workspace }}/docker-compose.log
       # ------------------------------------------------------------
 
@@ -380,21 +245,4 @@ jobs:
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose down -v
-
-  
-  required: 
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    needs: [integration-tests-mit]
-    if: ${{ always() }}
-    steps:
-      - uses: actions/github-script@v7
-        with:
-          script: |
-            const needs = ${{ toJSON(needs) }};
-            const failed = Object.values(needs).some(n => n.result !== 'success');
-            if (failed) {
-              core.setFailed('One or more upstream jobs failed or were cancelled.');
-            } else {
-              core.notice('All required jobs succeeded.');
-            }
+          docker compose -f docker-compose.dev.yml -p onyx-stack down -v

.github/workflows/pr-playwright-tests.yml

@@ -6,171 +6,44 @@ concurrency:
 on: push
 
 env:
-  # AWS ECR Configuration
-  AWS_REGION: ${{ secrets.AWS_REGION || 'us-west-2' }}
-  ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_ECR }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_ECR }}
-  BUILDX_NO_DEFAULT_ATTESTATIONS: 1
-  
-  # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
   GEN_AI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
-
-  # for federated slack tests
-  SLACK_CLIENT_ID: ${{ secrets.SLACK_CLIENT_ID }}
-  SLACK_CLIENT_SECRET: ${{ secrets.SLACK_CLIENT_SECRET }}
-
   MOCK_LLM_RESPONSE: true
+  PYTEST_PLAYWRIGHT_SKIP_INITIAL_RESET: true
 
 jobs:
-  build-web-image:
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Web Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./web
-          file: ./web/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.ECR_REGISTRY }}/integration-test-onyx-web-server:playwright-test-${{ github.run_id }}
-          provenance: false
-          sbom: false
-          push: true
-          outputs: type=registry
-          # no-cache: true
-
-  build-backend-image:
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Backend Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.ECR_REGISTRY }}/integration-test-onyx-backend:playwright-test-${{ github.run_id }}
-          provenance: false
-          sbom: false
-          push: true
-          outputs: type=registry
-          # no-cache: true
-
-  build-model-server-image:
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Model Server Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile.model_server
-          platforms: linux/arm64
-          tags: ${{ env.ECR_REGISTRY }}/integration-test-onyx-model-server:playwright-test-${{ github.run_id }}
-          provenance: false
-          sbom: false
-          push: true
-          outputs: type=registry
-          # no-cache: true
-
   playwright-tests:
-    needs: [build-web-image, build-backend-image, build-model-server-image]
     name: Playwright Tests
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
+
+    # See https://runs-on.com/runners/linux/
+    runs-on:
+      [
+        runs-on,
+        runner=32cpu-linux-x64,
+        disk=large,
+        "run-id=${{ github.run_id }}",
+      ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-
-      - name: Pull Docker images
-        run: |
-          # Pull all images from ECR in parallel
-          echo "Pulling Docker images in parallel..."
-          (docker pull ${{ env.ECR_REGISTRY }}/integration-test-onyx-web-server:playwright-test-${{ github.run_id }}) &
-          (docker pull ${{ env.ECR_REGISTRY }}/integration-test-onyx-backend:playwright-test-${{ github.run_id }}) &
-          (docker pull ${{ env.ECR_REGISTRY }}/integration-test-onyx-model-server:playwright-test-${{ github.run_id }}) &
-
-          # Wait for all background jobs to complete
-          wait
-          echo "All Docker images pulled successfully"
-
-          # Re-tag with expected names for docker-compose
-          docker tag ${{ env.ECR_REGISTRY }}/integration-test-onyx-web-server:playwright-test-${{ github.run_id }} onyxdotapp/onyx-web-server:test
-          docker tag ${{ env.ECR_REGISTRY }}/integration-test-onyx-backend:playwright-test-${{ github.run_id }} onyxdotapp/onyx-backend:test
-          docker tag ${{ env.ECR_REGISTRY }}/integration-test-onyx-model-server:playwright-test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
+          python-version: "3.11"
+          cache: "pip"
+          cache-dependency-path: |
+            backend/requirements/default.txt
+            backend/requirements/dev.txt
+            backend/requirements/model_server.txt
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
+          pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
+          pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt
 
       - name: Setup node
         uses: actions/setup-node@v4
@@ -185,29 +58,79 @@ jobs:
         working-directory: ./web
         run: npx playwright install --with-deps
 
-      - name: Create .env file for Docker Compose
-        run: |
-          cat <<EOF > deployment/docker_compose/.env
-          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
-          AUTH_TYPE=basic
-          GEN_AI_API_KEY=${{ env.OPENAI_API_KEY }}
-          EXA_API_KEY=${{ env.EXA_API_KEY }}
-          REQUIRE_EMAIL_VERIFICATION=false
-          DISABLE_TELEMETRY=true
-          IMAGE_TAG=test
-          EOF
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      # tag every docker image with "test" so that we can spin up the correct set
+      # of images during testing
+
+      # we use the runs-on cache for docker builds
+      # in conjunction with runs-on runners, it has better speed and unlimited caching
+      # https://runs-on.com/caching/s3-cache-for-github-actions/
+      # https://runs-on.com/caching/docker/
+      # https://github.com/moby/buildkit#s3-cache-experimental
+
+      # images are built and run locally for testing purposes. Not pushed.
+
+      - name: Build Web Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./web
+          file: ./web/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-web-server:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/web-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/web-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build Backend Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-backend:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build Model Server Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.model_server
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-model-server:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
+          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
+          AUTH_TYPE=basic \
+          GEN_AI_API_KEY=${{ secrets.OPENAI_API_KEY }} \
+          REQUIRE_EMAIL_VERIFICATION=false \
+          DISABLE_TELEMETRY=true \
+          IMAGE_TAG=test \
+          docker compose -f docker-compose.dev.yml -p danswer-stack up -d
         id: start_docker
 
       - name: Wait for service to be ready
         run: |
           echo "Starting wait-for-service script..."
 
-          docker logs -f onyx-api_server-1 &
+          docker logs -f danswer-stack-api_server-1 &
 
           start_time=$(date +%s)
           timeout=300  # 5 minutes in seconds
@@ -237,18 +160,22 @@ jobs:
           done
           echo "Finished waiting for service."
 
+      - name: Run pytest playwright test init
+        working-directory: ./backend
+        env:
+          PYTEST_IGNORE_SKIP: true
+        run: pytest -s tests/integration/tests/playwright/test_playwright.py
+
       - name: Run Playwright tests
         working-directory: ./web
-        run: |
-          # Create test-results directory to ensure it exists for artifact upload
-          mkdir -p test-results
-          npx playwright test
+        run: npx playwright test
 
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          # Includes test results and debug screenshots
-          name: playwright-test-results-${{ github.run_id }}
+          # Chromatic automatically defaults to the test-results directory.
+          # Replace with the path to your custom directory and adjust the CHROMATIC_ARCHIVE_LOCATION environment variable accordingly.
+          name: test-results
           path: ./web/test-results
           retention-days: 30
 
@@ -257,7 +184,7 @@ jobs:
         if: success() || failure()
         run: |
           cd deployment/docker_compose
-          docker compose logs > docker-compose.log
+          docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log
           mv docker-compose.log ${{ github.workspace }}/docker-compose.log
 
       - name: Upload logs
@@ -270,7 +197,7 @@ jobs:
       - name: Stop Docker containers
         run: |
           cd deployment/docker_compose
-          docker compose down -v
+          docker compose -f docker-compose.dev.yml -p danswer-stack down -v
 
 # NOTE: Chromatic UI diff testing is currently disabled.
 # We are using Playwright for local and CI testing without visual regression checks.

.github/workflows/pr-python-checks.yml

@@ -47,9 +47,7 @@ jobs:
           -i /local/openapi.json \
           -g python \
           -o /local/onyx_openapi_client \
-          --package-name onyx_openapi_client \
-          --skip-validate-spec \
-          --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=true,SET_OAS3_NULLABLE=true"
+          --package-name onyx_openapi_client
             
     - name: Run MyPy
       run: |

.github/workflows/pr-python-connector-tests.yml

@@ -16,17 +16,14 @@ env:
   # Confluence
   CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_TEST_SPACE: ${{ secrets.CONFLUENCE_TEST_SPACE }}
-  CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }}
   CONFLUENCE_IS_CLOUD: ${{ secrets.CONFLUENCE_IS_CLOUD }}
+  CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }}
   CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
-  CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }}
 
   # Jira
-  JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
   JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
   JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
-  JIRA_API_TOKEN_SCOPED: ${{ secrets.JIRA_API_TOKEN_SCOPED }}
 
   # Gong
   GONG_ACCESS_KEY: ${{ secrets.GONG_ACCESS_KEY }}
@@ -52,15 +49,6 @@ env:
   SF_PASSWORD: ${{ secrets.SF_PASSWORD }}
   SF_SECURITY_TOKEN: ${{ secrets.SF_SECURITY_TOKEN }}
 
-  # Hubspot
-  HUBSPOT_ACCESS_TOKEN: ${{ secrets.HUBSPOT_ACCESS_TOKEN }}
-
-  # IMAP
-  IMAP_HOST: ${{ secrets.IMAP_HOST }}
-  IMAP_USERNAME: ${{ secrets.IMAP_USERNAME }}
-  IMAP_PASSWORD: ${{ secrets.IMAP_PASSWORD }}
-  IMAP_MAILBOXES: ${{ secrets.IMAP_MAILBOXES }}
-
   # Airtable
   AIRTABLE_TEST_BASE_ID: ${{ secrets.AIRTABLE_TEST_BASE_ID }}
   AIRTABLE_TEST_TABLE_ID: ${{ secrets.AIRTABLE_TEST_TABLE_ID }}
@@ -98,13 +86,6 @@ env:
   TEAMS_DIRECTORY_ID: ${{ secrets.TEAMS_DIRECTORY_ID }}
   TEAMS_SECRET: ${{ secrets.TEAMS_SECRET }}
 
-  # Bitbucket
-  BITBUCKET_WORKSPACE: ${{ secrets.BITBUCKET_WORKSPACE }}
-  BITBUCKET_REPOSITORIES: ${{ secrets.BITBUCKET_REPOSITORIES }}
-  BITBUCKET_PROJECTS: ${{ secrets.BITBUCKET_PROJECTS }}
-  BITBUCKET_EMAIL: ${{ secrets.BITBUCKET_EMAIL }}
-  BITBUCKET_API_TOKEN: ${{ secrets.BITBUCKET_API_TOKEN }}
-
 jobs:
   connectors-check:
     # See https://runs-on.com/runners/linux/

.github/workflows/pr-python-model-tests.yml

@@ -77,7 +77,7 @@ jobs:
           REQUIRE_EMAIL_VERIFICATION=false \
           DISABLE_TELEMETRY=true \
           IMAGE_TAG=test \
-          docker compose -f docker-compose.model-server-test.yml up -d indexing_model_server
+          docker compose -f docker-compose.model-server-test.yml -p onyx-stack up -d indexing_model_server
         id: start_docker
 
       - name: Wait for service to be ready
@@ -132,7 +132,7 @@ jobs:
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.model-server-test.yml logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true
+          docker compose -f docker-compose.model-server-test.yml -p onyx-stack logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true
 
       - name: Upload logs
         if: always()
@@ -145,5 +145,5 @@ jobs:
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.model-server-test.yml down -v
+          docker compose -f docker-compose.model-server-test.yml -p onyx-stack down -v
           

.github/workflows/pr-python-tests.yml

@@ -31,14 +31,12 @@ jobs:
         cache-dependency-path: |
           backend/requirements/default.txt
           backend/requirements/dev.txt
-          backend/requirements/model_server.txt
 
     - name: Install Dependencies
       run: |
         python -m pip install --upgrade pip
         pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
         pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
-        pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt
 
     - name: Run Tests
       shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"

.gitignore

@@ -17,26 +17,12 @@ backend/tests/regression/answer_quality/test_data.json
 backend/tests/regression/search_quality/eval-*
 backend/tests/regression/search_quality/search_eval_config.yaml
 backend/tests/regression/search_quality/*.json
-backend/onyx/evals/data/
-*.log
 
 # secret files
 .env
 jira_test_env
-settings.json
 
 # others
 /deployment/data/nginx/app.conf
 *.sw?
 /backend/tests/regression/answer_quality/search_test_config.yaml
-*.egg-info
-
-# Local .terraform directories
-**/.terraform/*
-
-# Local .tfstate files
-*.tfstate
-*.tfstate.*
-
-# Local .terraform.lock.hcl file
-.terraform.lock.hcl

.mcp.json.template

@@ -1,8 +0,0 @@
-{
-  "mcpServers": {
-    "onyx-mcp": {
-      "type": "http",
-      "url": "http://localhost:8000/mcp"
-    }
-  }
-}

.pre-commit-config.yaml

@@ -37,15 +37,6 @@ repos:
       additional_dependencies:
       - prettier
 
-  - repo: local
-    hooks:
-      - id: check-lazy-imports
-        name: Check lazy imports are not directly imported
-        entry: python3 backend/scripts/check_lazy_imports.py
-        language: system
-        files: ^backend/.*\.py$
-        pass_filenames: false
-
   # We would like to have a mypy pre-commit hook, but due to the fact that
   # pre-commit runs in it's own isolated environment, we would need to install
   # and keep in sync all dependencies so mypy has access to the appropriate type

.vscode/env_template.txt

@@ -10,7 +10,7 @@ SKIP_WARM_UP=True
 
 # Always keep these on for Dev
 # Logs all model prompts to stdout
-LOG_ONYX_MODEL_INTERACTIONS=True
+LOG_DANSWER_MODEL_INTERACTIONS=True
 # More verbose logging
 LOG_LEVEL=debug
 
@@ -23,9 +23,6 @@ DISABLE_LLM_DOC_RELEVANCE=False
 # Useful if you want to toggle auth on/off (google_oauth/OIDC specifically)
 OAUTH_CLIENT_ID=<REPLACE THIS>
 OAUTH_CLIENT_SECRET=<REPLACE THIS>
-OPENID_CONFIG_URL=<REPLACE THIS>
-SAML_CONF_DIR=/<ABSOLUTE PATH TO ONYX>/onyx/backend/ee/onyx/configs/saml_config
-
 # Generally not useful for dev, we don't generally want to set up an SMTP server for dev
 REQUIRE_EMAIL_VERIFICATION=False
 
@@ -39,8 +36,8 @@ FAST_GEN_AI_MODEL_VERSION=gpt-4o
 
 # For Danswer Slack Bot, overrides the UI values so no need to set this up via UI every time
 # Only needed if using DanswerBot
-#ONYX_BOT_SLACK_APP_TOKEN=<REPLACE THIS>
-#ONYX_BOT_SLACK_BOT_TOKEN=<REPLACE THIS>
+#DANSWER_BOT_SLACK_APP_TOKEN=<REPLACE THIS>
+#DANSWER_BOT_SLACK_BOT_TOKEN=<REPLACE THIS>
 
 
 # Python stuff
@@ -48,8 +45,8 @@ PYTHONPATH=../backend
 PYTHONUNBUFFERED=1
 
 
-# Internet Search
-EXA_API_KEY=<REPLACE THIS>
+# Internet Search 
+BING_API_KEY=<REPLACE THIS>
 
 
 # Enable the full set of Danswer Enterprise Edition features
@@ -61,18 +58,3 @@ AGENT_RETRIEVAL_STATS=False   # Note: This setting will incur substantial re-ran
 AGENT_RERANKING_STATS=True
 AGENT_MAX_QUERY_RETRIEVAL_RESULTS=20
 AGENT_RERANKING_MAX_QUERY_RETRIEVAL_RESULTS=20
-
-# S3 File Store Configuration (MinIO for local development)
-S3_ENDPOINT_URL=http://localhost:9004
-S3_FILE_STORE_BUCKET_NAME=onyx-file-store-bucket
-S3_AWS_ACCESS_KEY_ID=minioadmin
-S3_AWS_SECRET_ACCESS_KEY=minioadmin
-
-# Show extra/uncommon connectors
-SHOW_EXTRA_CONNECTORS=True
-
-# Local langsmith tracing
-LANGSMITH_TRACING="true"
-LANGSMITH_ENDPOINT="https://api.smith.langchain.com"
-LANGSMITH_API_KEY=<REPLACE_THIS>
-LANGSMITH_PROJECT=<REPLACE_THIS>

.vscode/launch.template.jsonc

@@ -1,468 +1,417 @@
 /* Copy this file into '.vscode/launch.json' or merge its contents into your existing configurations. */
 
 {
-  // Use IntelliSense to learn about possible attributes.
-  // Hover to view descriptions of existing attributes.
-  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-  "version": "0.2.0",
-  "compounds": [
-    {
-      // Dummy entry used to label the group
-      "name": "--- Compound ---",
-      "configurations": ["--- Individual ---"],
-      "presentation": {
-        "group": "1"
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "compounds": [
+      {
+        // Dummy entry used to label the group
+        "name": "--- Compound ---",
+        "configurations": ["--- Individual ---"],
+        "presentation": {
+          "group": "1"
+        }
+      },
+      {
+        "name": "Run All Onyx Services",
+        "configurations": [
+          "Web Server",
+          "Model Server",
+          "API Server",
+          "Slack Bot",
+          "Celery primary",
+          "Celery light",
+          "Celery heavy",
+          "Celery indexing",
+          "Celery user files indexing",
+          "Celery beat",
+          "Celery monitoring"
+        ],
+        "presentation": {
+          "group": "1"
+        }
+      },
+      {
+        "name": "Web / Model / API",
+        "configurations": ["Web Server", "Model Server", "API Server"],
+        "presentation": {
+          "group": "1"
+        }
+      },
+      {
+        "name": "Celery (all)",
+        "configurations": [
+          "Celery primary",
+          "Celery light",
+          "Celery heavy",
+          "Celery indexing",
+          "Celery user files indexing",
+          "Celery beat",
+          "Celery monitoring"
+        ],
+        "presentation": {
+          "group": "1"
+        }
       }
-    },
-    {
-      "name": "Run All Onyx Services",
-      "configurations": [
-        "Web Server",
-        "Model Server",
-        "API Server",
-        "Slack Bot",
-        "Celery primary",
-        "Celery light",
-        "Celery heavy",
-        "Celery docfetching",
-        "Celery docprocessing",
-        "Celery beat",
-        "Celery monitoring",
-        "Celery user file processing"
-      ],
-      "presentation": {
-        "group": "1"
-      }
-    },
-    {
-      "name": "Web / Model / API",
-      "configurations": ["Web Server", "Model Server", "API Server"],
-      "presentation": {
-        "group": "1"
-      }
-    },
-    {
-      "name": "Celery (all)",
-      "configurations": [
-        "Celery primary",
-        "Celery light",
-        "Celery heavy",
-        "Celery docfetching",
-        "Celery docprocessing",
-        "Celery beat",
-        "Celery monitoring",
-        "Celery user file processing"
-      ],
-      "presentation": {
-        "group": "1"
+    ],
+    "configurations": [
+      {
+        // Dummy entry used to label the group
+        "name": "--- Individual ---",
+        "type": "node",
+        "request": "launch",
+        "presentation": {
+          "group": "2",
+          "order": 0
+        }
       },
-      "stopAll": true
-    }
-  ],
-  "configurations": [
-    {
-      // Dummy entry used to label the group
-      "name": "--- Individual ---",
-      "type": "node",
-      "request": "launch",
-      "presentation": {
-        "group": "2",
-        "order": 0
-      }
-    },
-    {
-      "name": "Web Server",
-      "type": "node",
-      "request": "launch",
-      "cwd": "${workspaceRoot}/web",
-      "runtimeExecutable": "npm",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "runtimeArgs": ["run", "dev"],
-      "presentation": {
-        "group": "2"
+      {
+        "name": "Web Server",
+        "type": "node",
+        "request": "launch",
+        "cwd": "${workspaceRoot}/web",
+        "runtimeExecutable": "npm",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "runtimeArgs": ["run", "dev"],
+        "presentation": {
+          "group": "2"
+        },
+        "console": "integratedTerminal",
+        "consoleTitle": "Web Server Console"
       },
-      "console": "integratedTerminal",
-      "consoleTitle": "Web Server Console"
-    },
-    {
-      "name": "Model Server",
-      "consoleName": "Model Server",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "uvicorn",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1"
+      {
+        "name": "Model Server",
+        "consoleName": "Model Server",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "uvicorn",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_LEVEL": "DEBUG",
+          "PYTHONUNBUFFERED": "1"
+        },
+        "args": ["model_server.main:app", "--reload", "--port", "9000"],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Model Server Console"
       },
-      "args": ["model_server.main:app", "--reload", "--port", "9000"],
-      "presentation": {
-        "group": "2"
+      {
+        "name": "API Server",
+        "consoleName": "API Server",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "uvicorn",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_DANSWER_MODEL_INTERACTIONS": "True",
+          "LOG_LEVEL": "DEBUG",
+          "PYTHONUNBUFFERED": "1"
+        },
+        "args": ["onyx.main:app", "--reload", "--port", "8080"],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "API Server Console"
       },
-      "consoleTitle": "Model Server Console"
-    },
-    {
-      "name": "API Server",
-      "consoleName": "API Server",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "uvicorn",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_ONYX_MODEL_INTERACTIONS": "True",
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1"
+      // For the listener to access the Slack API,
+      // DANSWER_BOT_SLACK_APP_TOKEN & DANSWER_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
+      {
+        "name": "Slack Bot",
+        "consoleName": "Slack Bot",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "onyx/onyxbot/slack/listener.py",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_LEVEL": "DEBUG",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+        },
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Slack Bot Console"
       },
-      "args": ["onyx.main:app", "--reload", "--port", "8080"],
-      "presentation": {
-        "group": "2"
+      {
+        "name": "Celery primary",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "celery",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_LEVEL": "INFO",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+        },
+        "args": [
+          "-A",
+          "onyx.background.celery.versioned_apps.primary",
+          "worker",
+          "--pool=threads",
+          "--concurrency=4",
+          "--prefetch-multiplier=1",
+          "--loglevel=INFO",
+          "--hostname=primary@%n",
+          "-Q",
+          "celery"
+        ],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Celery primary Console"
       },
-      "consoleTitle": "API Server Console"
-    },
-    // For the listener to access the Slack API,
-    // ONYX_BOT_SLACK_APP_TOKEN & ONYX_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
-    {
-      "name": "Slack Bot",
-      "consoleName": "Slack Bot",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "onyx/onyxbot/slack/listener.py",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
+      {
+        "name": "Celery light",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "celery",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_LEVEL": "INFO",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+        },
+        "args": [
+          "-A",
+          "onyx.background.celery.versioned_apps.light",
+          "worker",
+          "--pool=threads",
+          "--concurrency=64",
+          "--prefetch-multiplier=8",
+          "--loglevel=INFO",
+          "--hostname=light@%n",
+          "-Q",
+          "vespa_metadata_sync,connector_deletion,doc_permissions_upsert"
+        ],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Celery light Console"
       },
-      "presentation": {
-        "group": "2"
+      {
+        "name": "Celery heavy",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "celery",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_LEVEL": "INFO",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+        },
+        "args": [
+          "-A",
+          "onyx.background.celery.versioned_apps.heavy",
+          "worker",
+          "--pool=threads",
+          "--concurrency=4",
+          "--prefetch-multiplier=1",
+          "--loglevel=INFO",
+          "--hostname=heavy@%n",
+          "-Q",
+          "connector_pruning,connector_doc_permissions_sync,connector_external_group_sync"
+        ],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Celery heavy Console"
       },
-      "consoleTitle": "Slack Bot Console"
-    },
-    {
-      "name": "Celery primary",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "celery",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "INFO",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
+      {
+        "name": "Celery indexing",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "celery",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "ENABLE_MULTIPASS_INDEXING": "false",
+          "LOG_LEVEL": "DEBUG",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+        },
+        "args": [
+          "-A",
+          "onyx.background.celery.versioned_apps.indexing",
+          "worker",
+          "--pool=threads",
+          "--concurrency=1",
+          "--prefetch-multiplier=1",
+          "--loglevel=INFO",
+          "--hostname=indexing@%n",
+          "-Q",
+          "connector_indexing"
+        ],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Celery indexing Console"
       },
-      "args": [
-        "-A",
-        "onyx.background.celery.versioned_apps.primary",
-        "worker",
-        "--pool=threads",
-        "--concurrency=4",
-        "--prefetch-multiplier=1",
-        "--loglevel=INFO",
-        "--hostname=primary@%n",
-        "-Q",
-        "celery"
-      ],
-      "presentation": {
-        "group": "2"
+      {
+        "name": "Celery monitoring",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "celery",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {},
+        "args": [
+          "-A",
+          "onyx.background.celery.versioned_apps.monitoring",
+          "worker",
+          "--pool=solo",
+          "--concurrency=1",
+          "--prefetch-multiplier=1",
+          "--loglevel=INFO",
+          "--hostname=monitoring@%n",
+          "-Q",
+          "monitoring"
+        ],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Celery monitoring Console"
       },
-      "consoleTitle": "Celery primary Console"
-    },
-    {
-      "name": "Celery light",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "celery",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "INFO",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
+      {
+        "name": "Celery beat",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "celery",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_LEVEL": "DEBUG",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+        },
+        "args": [
+          "-A",
+          "onyx.background.celery.versioned_apps.beat",
+          "beat",
+          "--loglevel=INFO"
+        ],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Celery beat Console"
       },
-      "args": [
-        "-A",
-        "onyx.background.celery.versioned_apps.light",
-        "worker",
-        "--pool=threads",
-        "--concurrency=64",
-        "--prefetch-multiplier=8",
-        "--loglevel=INFO",
-        "--hostname=light@%n",
-        "-Q",
-        "vespa_metadata_sync,connector_deletion,doc_permissions_upsert,index_attempt_cleanup"
-      ],
-      "presentation": {
-        "group": "2"
+      {
+        "name": "Celery user files indexing",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "celery",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_LEVEL": "DEBUG",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+        },
+        "args": [
+          "-A",
+          "onyx.background.celery.versioned_apps.indexing",
+          "worker",
+          "--pool=threads",
+          "--concurrency=1",
+          "--prefetch-multiplier=1",
+          "--loglevel=INFO",
+          "--hostname=user_files_indexing@%n",
+          "-Q",
+          "user_files_indexing"
+        ],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Celery user files indexing Console"
       },
-      "consoleTitle": "Celery light Console"
-    },
-    {
-      "name": "Celery heavy",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "celery",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "INFO",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
+      {
+        "name": "Pytest",
+        "consoleName": "Pytest",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "pytest",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_LEVEL": "DEBUG",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+        },
+        "args": [
+          "-v"
+          // Specify a sepcific module/test to run or provide nothing to run all tests
+          //"tests/unit/onyx/llm/answering/test_prune_and_merge.py"
+        ],
+        "presentation": {
+          "group": "2"
+        },
+        "consoleTitle": "Pytest Console"
       },
-      "args": [
-        "-A",
-        "onyx.background.celery.versioned_apps.heavy",
-        "worker",
-        "--pool=threads",
-        "--concurrency=4",
-        "--prefetch-multiplier=1",
-        "--loglevel=INFO",
-        "--hostname=heavy@%n",
-        "-Q",
-        "connector_pruning,connector_doc_permissions_sync,connector_external_group_sync"
-      ],
-      "presentation": {
-        "group": "2"
+      {
+        // Dummy entry used to label the group
+        "name": "--- Tasks ---",
+        "type": "node",
+        "request": "launch",
+        "presentation": {
+          "group": "3",
+          "order": 0
+        }
       },
-      "consoleTitle": "Celery heavy Console"
-    },
-    {
-      "name": "Celery docfetching",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "celery",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
+      {
+        "name": "Clear and Restart External Volumes and Containers",
+        "type": "node",
+        "request": "launch",
+        "runtimeExecutable": "bash",
+        "runtimeArgs": [
+          "${workspaceFolder}/backend/scripts/restart_containers.sh"
+        ],
+        "cwd": "${workspaceFolder}",
+        "console": "integratedTerminal",
+        "stopOnEntry": true,
+        "presentation": {
+          "group": "3"
+        }
       },
-      "args": [
-        "-A",
-        "onyx.background.celery.versioned_apps.docfetching",
-        "worker",
-        "--pool=threads",
-        "--concurrency=1",
-        "--prefetch-multiplier=1",
-        "--loglevel=INFO",
-        "--hostname=docfetching@%n",
-        "-Q",
-        "connector_doc_fetching,user_files_indexing"
-      ],
-      "presentation": {
-        "group": "2"
+      {
+        // Celery jobs launched through a single background script (legacy)
+        // Recommend using the "Celery (all)" compound launch instead.
+        "name": "Background Jobs",
+        "consoleName": "Background Jobs",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "scripts/dev_run_background_jobs.py",
+        "cwd": "${workspaceFolder}/backend",
+        "envFile": "${workspaceFolder}/.vscode/.env",
+        "env": {
+          "LOG_DANSWER_MODEL_INTERACTIONS": "True",
+          "LOG_LEVEL": "DEBUG",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+        }
       },
-      "consoleTitle": "Celery docfetching Console",
-      "justMyCode": false
-    },
-    {
-      "name": "Celery docprocessing",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "celery",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "ENABLE_MULTIPASS_INDEXING": "false",
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
+      {
+        "name": "Install Python Requirements",
+        "type": "node",
+        "request": "launch",
+        "runtimeExecutable": "bash",
+        "runtimeArgs": [
+          "-c",
+          "pip install -r backend/requirements/default.txt && pip install -r backend/requirements/dev.txt && pip install -r backend/requirements/ee.txt && pip install -r backend/requirements/model_server.txt"
+        ],
+        "cwd": "${workspaceFolder}",
+        "console": "integratedTerminal",
+        "presentation": {
+          "group": "3"
+        }
       },
-      "args": [
-        "-A",
-        "onyx.background.celery.versioned_apps.docprocessing",
-        "worker",
-        "--pool=threads",
-        "--concurrency=6",
-        "--prefetch-multiplier=1",
-        "--loglevel=INFO",
-        "--hostname=docprocessing@%n",
-        "-Q",
-        "docprocessing"
-      ],
-      "presentation": {
-        "group": "2"
-      },
-      "consoleTitle": "Celery docprocessing Console"
-    },
-    {
-      "name": "Celery beat",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "celery",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
-      },
-      "args": [
-        "-A",
-        "onyx.background.celery.versioned_apps.beat",
-        "beat",
-        "--loglevel=INFO"
-      ],
-      "presentation": {
-        "group": "2"
-      },
-      "consoleTitle": "Celery beat Console"
-    },
-    {
-      "name": "Celery monitoring",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "celery",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {},
-      "args": [
-        "-A",
-        "onyx.background.celery.versioned_apps.monitoring",
-        "worker",
-        "--pool=solo",
-        "--concurrency=1",
-        "--prefetch-multiplier=1",
-        "--loglevel=INFO",
-        "--hostname=monitoring@%n",
-        "-Q",
-        "monitoring"
-      ],
-      "presentation": {
-        "group": "2"
-      },
-      "consoleTitle": "Celery monitoring Console"
-    },
-    {
-      "name": "Celery user file processing",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "celery",
-      "args": [
-        "-A",
-        "onyx.background.celery.versioned_apps.user_file_processing",
-        "worker",
-        "--loglevel=INFO",
-        "--hostname=user_file_processing@%n",
-        "--pool=threads",
-        "-Q",
-        "user_file_processing,user_file_project_sync"
-      ],
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
-      },
-      "presentation": {
-        "group": "2"
-      },
-      "consoleTitle": "Celery user file processing Console"
-    },
-    {
-      "name": "Pytest",
-      "consoleName": "Pytest",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "pytest",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
-      },
-      "args": [
-        "-v"
-        // Specify a specific module/test to run or provide nothing to run all tests
-        // "tests/unit/onyx/llm/answering/test_prune_and_merge.py"
-      ],
-      "presentation": {
-        "group": "2"
-      },
-      "consoleTitle": "Pytest Console"
-    },
-    {
-      // Dummy entry used to label the group
-      "name": "--- Tasks ---",
-      "type": "node",
-      "request": "launch",
-      "presentation": {
-        "group": "3",
-        "order": 0
-      }
-    },
-    {
-      "name": "Clear and Restart External Volumes and Containers",
-      "type": "node",
-      "request": "launch",
-      "runtimeExecutable": "bash",
-      "runtimeArgs": [
-        "${workspaceFolder}/backend/scripts/restart_containers.sh"
-      ],
-      "cwd": "${workspaceFolder}",
-      "console": "integratedTerminal",
-      "stopOnEntry": true,
-      "presentation": {
-        "group": "3"
-      }
-    },
-    {
-      "name": "Eval CLI",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${workspaceFolder}/backend/onyx/evals/eval_cli.py",
-      "cwd": "${workspaceFolder}/backend",
-      "console": "integratedTerminal",
-      "justMyCode": false,
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "presentation": {
-        "group": "3"
-      },
-      "env": {
-        "LOG_LEVEL": "INFO",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
-      },
-      "args": ["--verbose"],
-      "consoleTitle": "Eval CLI Console"
-    },
-    {
-      // Celery jobs launched through a single background script (legacy)
-      // Recommend using the "Celery (all)" compound launch instead.
-      "name": "Background Jobs",
-      "consoleName": "Background Jobs",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "scripts/dev_run_background_jobs.py",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "LOG_ONYX_MODEL_INTERACTIONS": "True",
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "."
-      }
-    },
-    {
-      "name": "Install Python Requirements",
-      "type": "node",
-      "request": "launch",
-      "runtimeExecutable": "bash",
-      "runtimeArgs": [
-        "-c",
-        "pip install -r backend/requirements/default.txt && pip install -r backend/requirements/dev.txt && pip install -r backend/requirements/ee.txt && pip install -r backend/requirements/model_server.txt"
-      ],
-      "cwd": "${workspaceFolder}",
-      "console": "integratedTerminal",
-      "presentation": {
-        "group": "3"
-      }
-    },
     {
       // script to generate the openapi schema
       "name": "Onyx OpenAPI Schema Generator",
@@ -475,7 +424,10 @@
         "PYTHONUNBUFFERED": "1",
         "PYTHONPATH": "."
       },
-      "args": ["--filename", "generated/openapi.json"]
+      "args": [
+        "--filename",
+        "generated/openapi.json",
+      ]
     },
     {
       // script to debug multi tenant db issues
@@ -500,12 +452,13 @@
         "generated/tenants_by_num_docs.csv"
       ]
     },
-    {
-      "name": "Debug React Web App in Chrome",
-      "type": "chrome",
-      "request": "launch",
-      "url": "http://localhost:3000",
-      "webRoot": "${workspaceFolder}/web"
-    }
-  ]
-}
+      {
+        "name": "Debug React Web App in Chrome",
+        "type": "chrome",
+        "request": "launch",
+        "url": "http://localhost:3000",
+        "webRoot": "${workspaceFolder}/web"
+      }
+    ]
+  }
+  

AGENTS.md

@@ -1,295 +0,0 @@
-# AGENTS.md
-
-This file provides guidance to Codex when working with code in this repository.
-
-## KEY NOTES
-
-- If you run into any missing python dependency errors, try running your command with `source backend/.venv/bin/activate` \
-to assume the python venv.
-- To make tests work, check the `.env` file at the root of the project to find an OpenAI key.
-- If using `playwright` to explore the frontend, you can usually log in with username `a@test.com` and password
-`a`. The app can be accessed at `http://localhost:3000`.
-- You should assume that all Onyx services are running. To verify, you can check the `backend/log` directory to
-make sure we see logs coming out from the relevant service.
-- To connect to the Postgres database, use: `docker exec -it onyx-relational_db-1 psql -U postgres -c "<SQL>"`
-- When making calls to the backend, always go through the frontend. E.g. make a call to `http://localhost:3000/api/persona` not `http://localhost:8080/api/persona`
-- Put ALL db operations under the `backend/onyx/db` / `backend/ee/onyx/db` directories. Don't run queries
-outside of those directories.
-
-## Project Overview
-
-**Onyx** (formerly Danswer) is an open-source Gen-AI and Enterprise Search platform that connects to company documents, apps, and people. It features a modular architecture with both Community Edition (MIT licensed) and Enterprise Edition offerings.
-
-
-### Background Workers (Celery)
-
-Onyx uses Celery for asynchronous task processing with multiple specialized workers:
-
-#### Worker Types
-
-1. **Primary Worker** (`celery_app.py`)
-   - Coordinates core background tasks and system-wide operations
-   - Handles connector management, document sync, pruning, and periodic checks
-   - Runs with 4 threads concurrency
-   - Tasks: connector deletion, vespa sync, pruning, LLM model updates, user file sync
-
-2. **Docfetching Worker** (`docfetching`)
-   - Fetches documents from external data sources (connectors)
-   - Spawns docprocessing tasks for each document batch
-   - Implements watchdog monitoring for stuck connectors
-   - Configurable concurrency (default from env)
-
-3. **Docprocessing Worker** (`docprocessing`)
-   - Processes fetched documents through the indexing pipeline:
-     - Upserts documents to PostgreSQL
-     - Chunks documents and adds contextual information
-     - Embeds chunks via model server
-     - Writes chunks to Vespa vector database
-     - Updates document metadata
-   - Configurable concurrency (default from env)
-
-4. **Light Worker** (`light`)
-   - Handles lightweight, fast operations
-   - Tasks: vespa operations, document permissions sync, external group sync
-   - Higher concurrency for quick tasks
-
-5. **Heavy Worker** (`heavy`)
-   - Handles resource-intensive operations
-   - Primary task: document pruning operations
-   - Runs with 4 threads concurrency
-
-6. **KG Processing Worker** (`kg_processing`)
-   - Handles Knowledge Graph processing and clustering
-   - Builds relationships between documents
-   - Runs clustering algorithms
-   - Configurable concurrency
-
-7. **Monitoring Worker** (`monitoring`)
-   - System health monitoring and metrics collection
-   - Monitors Celery queues, process memory, and system status
-   - Single thread (monitoring doesn't need parallelism)
-   - Cloud-specific monitoring tasks
-
-8. **Beat Worker** (`beat`)
-   - Celery's scheduler for periodic tasks
-   - Uses DynamicTenantScheduler for multi-tenant support
-   - Schedules tasks like:
-     - Indexing checks (every 15 seconds)
-     - Connector deletion checks (every 20 seconds)
-     - Vespa sync checks (every 20 seconds)
-     - Pruning checks (every 20 seconds)
-     - KG processing (every 60 seconds)
-     - Monitoring tasks (every 5 minutes)
-     - Cleanup tasks (hourly)
-
-#### Key Features
-
-- **Thread-based Workers**: All workers use thread pools (not processes) for stability
-- **Tenant Awareness**: Multi-tenant support with per-tenant task isolation. There is a 
-middleware layer that automatically finds the appropriate tenant ID when sending tasks 
-via Celery Beat.
-- **Task Prioritization**: High, Medium, Low priority queues
-- **Monitoring**: Built-in heartbeat and liveness checking
-- **Failure Handling**: Automatic retry and failure recovery mechanisms
-- **Redis Coordination**: Inter-process communication via Redis
-- **PostgreSQL State**: Task state and metadata stored in PostgreSQL
-
-
-#### Important Notes
-
-**Defining Tasks**: 
-- Always use `@shared_task` rather than `@celery_app`
-- Put tasks under `background/celery/tasks/` or `ee/background/celery/tasks`
-
-**Defining APIs**:
-When creating new FastAPI APIs, do NOT use the `response_model` field. Instead, just type the
-function.
-
-**Testing Updates**:
-If you make any updates to a celery worker and you want to test these changes, you will need
-to ask me to restart the celery worker. There is no auto-restart on code-change mechanism.
-
-### Code Quality
-```bash
-# Install and run pre-commit hooks
-pre-commit install
-pre-commit run --all-files
-```
-
-NOTE: Always make sure everything is strictly typed (both in Python and Typescript).
-
-## Architecture Overview
-
-### Technology Stack
-- **Backend**: Python 3.11, FastAPI, SQLAlchemy, Alembic, Celery
-- **Frontend**: Next.js 15+, React 18, TypeScript, Tailwind CSS
-- **Database**: PostgreSQL with Redis caching
-- **Search**: Vespa vector database
-- **Auth**: OAuth2, SAML, multi-provider support
-- **AI/ML**: LangChain, LiteLLM, multiple embedding models
-
-### Directory Structure
-
-```
-backend/
-├── onyx/
-│   ├── auth/                    # Authentication & authorization
-│   ├── chat/                    # Chat functionality & LLM interactions
-│   ├── connectors/              # Data source connectors
-│   ├── db/                      # Database models & operations
-│   ├── document_index/          # Vespa integration
-│   ├── federated_connectors/    # External search connectors
-│   ├── llm/                     # LLM provider integrations
-│   └── server/                  # API endpoints & routers
-├── ee/                          # Enterprise Edition features
-├── alembic/                     # Database migrations
-└── tests/                       # Test suites
-
-web/
-├── src/app/                     # Next.js app router pages
-├── src/components/              # Reusable React components
-└── src/lib/                     # Utilities & business logic
-```
-
-## Database & Migrations
-
-### Running Migrations
-```bash
-# Standard migrations
-alembic upgrade head
-
-# Multi-tenant (Enterprise)
-alembic -n schema_private upgrade head
-```
-
-### Creating Migrations
-```bash
-# Auto-generate migration
-alembic revision --autogenerate -m "description"
-
-# Multi-tenant migration
-alembic -n schema_private revision --autogenerate -m "description"
-```
-
-## Testing Strategy
-
-There are 4 main types of tests within Onyx:
-
-### Unit Tests
-These should not assume any Onyx/external services are available to be called.
-Interactions with the outside world should be mocked using `unittest.mock`. Generally, only 
-write these for complex, isolated modules e.g. `citation_processing.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest -xv backend/tests/unit
-```
-
-### External Dependency Unit Tests
-These tests assume that all external dependencies of Onyx are available and callable (e.g. Postgres, Redis, 
-MinIO/S3, Vespa are running + OpenAI can be called + any request to the internet is fine + etc.).
-
-However, the actual Onyx containers are not running and with these tests we call the function to test directly.
-We can also mock components/calls at will. 
-
-The goal with these tests are to minimize mocking while giving some flexibility to mock things that are flakey, 
-need strictly controlled behavior, or need to have their internal behavior validated (e.g. verify a function is called
-with certain args, something that would be impossible with proper integration tests).
-
-A great example of this type of test is `backend/tests/external_dependency_unit/connectors/confluence/test_confluence_group_sync.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest backend/tests/external_dependency_unit
-```
-
-### Integration Tests
-Standard integration tests. Every test in `backend/tests/integration` runs against a real Onyx deployment. We cannot 
-mock anything in these tests. Prefer writing integration tests (or External Dependency Unit Tests if mocking/internal 
-verification is necessary) over any other type of test.
-
-Tests are parallelized at a directory level.
-
-When writing integration tests, make sure to check the root `conftest.py` for useful fixtures + the `backend/tests/integration/common_utils` directory for utilities. Prefer (if one exists), calling the appropriate Manager 
-class in the utils over directly calling the APIs with a library like `requests`. Prefer using fixtures rather than
-calling the utilities directly (e.g. do NOT create admin users with 
-`admin_user = UserManager.create(name="admin_user")`, instead use the `admin_user` fixture).
-
-A great example of this type of test is `backend/tests/integration/dev_apis/test_simple_chat_api.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest backend/tests/integration
-```
-
-### Playwright (E2E) Tests
-These tests are an even more complete version of the Integration Tests mentioned above. Has all services of Onyx 
-running, *including* the Web Server.
-
-Use these tests for anything that requires significant frontend <-> backend coordination.
-
-Tests are located at `web/tests/e2e`. Tests are written in TypeScript.
-
-To run them:
-
-```bash
-npx playwright test <TEST_NAME>
-```
-
-
-## Logs
-
-When (1) writing integration tests or (2) doing live tests (e.g. curl / playwright) you can get access
-to logs via the `backend/log/<service_name>_debug.log` file. All Onyx services (api_server, web_server, celery_X)
-will be tailing their logs to this file. 
-
-
-## Security Considerations
-
-- Never commit API keys or secrets to repository
-- Use encrypted credential storage for connector credentials
-- Follow RBAC patterns for new features
-- Implement proper input validation with Pydantic models
-- Use parameterized queries to prevent SQL injection
-
-## AI/LLM Integration
-
-- Multiple LLM providers supported via LiteLLM
-- Configurable models per feature (chat, search, embeddings)
-- Streaming support for real-time responses
-- Token management and rate limiting
-- Custom prompts and agent actions
-
-## UI/UX Patterns
-
-- Tailwind CSS with design system in `web/src/components/ui/`
-- Radix UI and Headless UI for accessible components
-- SWR for data fetching and caching
-- Form validation with react-hook-form
-- Error handling with popup notifications
-
-## Creating a Plan
-When creating a plan in the `plans` directory, make sure to include at least these elements:
-
-**Issues to Address**
-What the change is meant to do.
-
-**Important Notes**
-Things you come across in your research that are important to the implementation.
-
-**Implementation strategy**
-How you are going to make the changes happen. High level approach.
-
-**Tests**
-What unit (use rarely), external dependency unit, integration, and playwright tests you plan to write to 
-verify the correct behavior. Don't overtest. Usually, a given change only needs one type of test.
-
-Do NOT include these: *Timeline*, *Rollback plan*
-
-This is a minimal list - feel free to include more. Do NOT write code as part of your plan.
-Keep it high level. You can reference certain files or functions though.
-
-Before writing your plan, make sure to do research. Explore the relevant sections in the codebase.

CLAUDE.md

@@ -1,295 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## KEY NOTES
-
-- If you run into any missing python dependency errors, try running your command with `source backend/.venv/bin/activate` \
-to assume the python venv.
-- To make tests work, check the `.env` file at the root of the project to find an OpenAI key.
-- If using `playwright` to explore the frontend, you can usually log in with username `a@test.com` and password
-`a`. The app can be accessed at `http://localhost:3000`.
-- You should assume that all Onyx services are running. To verify, you can check the `backend/log` directory to
-make sure we see logs coming out from the relevant service.
-- To connect to the Postgres database, use: `docker exec -it onyx-relational_db-1 psql -U postgres -c "<SQL>"`
-- When making calls to the backend, always go through the frontend. E.g. make a call to `http://localhost:3000/api/persona` not `http://localhost:8080/api/persona`
-- Put ALL db operations under the `backend/onyx/db` / `backend/ee/onyx/db` directories. Don't run queries
-outside of those directories.
-
-## Project Overview
-
-**Onyx** (formerly Danswer) is an open-source Gen-AI and Enterprise Search platform that connects to company documents, apps, and people. It features a modular architecture with both Community Edition (MIT licensed) and Enterprise Edition offerings.
-
-
-### Background Workers (Celery)
-
-Onyx uses Celery for asynchronous task processing with multiple specialized workers:
-
-#### Worker Types
-
-1. **Primary Worker** (`celery_app.py`)
-   - Coordinates core background tasks and system-wide operations
-   - Handles connector management, document sync, pruning, and periodic checks
-   - Runs with 4 threads concurrency
-   - Tasks: connector deletion, vespa sync, pruning, LLM model updates, user file sync
-
-2. **Docfetching Worker** (`docfetching`)
-   - Fetches documents from external data sources (connectors)
-   - Spawns docprocessing tasks for each document batch
-   - Implements watchdog monitoring for stuck connectors
-   - Configurable concurrency (default from env)
-
-3. **Docprocessing Worker** (`docprocessing`)
-   - Processes fetched documents through the indexing pipeline:
-     - Upserts documents to PostgreSQL
-     - Chunks documents and adds contextual information
-     - Embeds chunks via model server
-     - Writes chunks to Vespa vector database
-     - Updates document metadata
-   - Configurable concurrency (default from env)
-
-4. **Light Worker** (`light`)
-   - Handles lightweight, fast operations
-   - Tasks: vespa operations, document permissions sync, external group sync
-   - Higher concurrency for quick tasks
-
-5. **Heavy Worker** (`heavy`)
-   - Handles resource-intensive operations
-   - Primary task: document pruning operations
-   - Runs with 4 threads concurrency
-
-6. **KG Processing Worker** (`kg_processing`)
-   - Handles Knowledge Graph processing and clustering
-   - Builds relationships between documents
-   - Runs clustering algorithms
-   - Configurable concurrency
-
-7. **Monitoring Worker** (`monitoring`)
-   - System health monitoring and metrics collection
-   - Monitors Celery queues, process memory, and system status
-   - Single thread (monitoring doesn't need parallelism)
-   - Cloud-specific monitoring tasks
-
-8. **Beat Worker** (`beat`)
-   - Celery's scheduler for periodic tasks
-   - Uses DynamicTenantScheduler for multi-tenant support
-   - Schedules tasks like:
-     - Indexing checks (every 15 seconds)
-     - Connector deletion checks (every 20 seconds)
-     - Vespa sync checks (every 20 seconds)
-     - Pruning checks (every 20 seconds)
-     - KG processing (every 60 seconds)
-     - Monitoring tasks (every 5 minutes)
-     - Cleanup tasks (hourly)
-
-#### Key Features
-
-- **Thread-based Workers**: All workers use thread pools (not processes) for stability
-- **Tenant Awareness**: Multi-tenant support with per-tenant task isolation. There is a 
-middleware layer that automatically finds the appropriate tenant ID when sending tasks 
-via Celery Beat.
-- **Task Prioritization**: High, Medium, Low priority queues
-- **Monitoring**: Built-in heartbeat and liveness checking
-- **Failure Handling**: Automatic retry and failure recovery mechanisms
-- **Redis Coordination**: Inter-process communication via Redis
-- **PostgreSQL State**: Task state and metadata stored in PostgreSQL
-
-
-#### Important Notes
-
-**Defining Tasks**: 
-- Always use `@shared_task` rather than `@celery_app`
-- Put tasks under `background/celery/tasks/` or `ee/background/celery/tasks`
-
-**Defining APIs**:
-When creating new FastAPI APIs, do NOT use the `response_model` field. Instead, just type the
-function.
-
-**Testing Updates**:
-If you make any updates to a celery worker and you want to test these changes, you will need
-to ask me to restart the celery worker. There is no auto-restart on code-change mechanism.
-
-### Code Quality
-```bash
-# Install and run pre-commit hooks
-pre-commit install
-pre-commit run --all-files
-```
-
-NOTE: Always make sure everything is strictly typed (both in Python and Typescript).
-
-## Architecture Overview
-
-### Technology Stack
-- **Backend**: Python 3.11, FastAPI, SQLAlchemy, Alembic, Celery
-- **Frontend**: Next.js 15+, React 18, TypeScript, Tailwind CSS
-- **Database**: PostgreSQL with Redis caching
-- **Search**: Vespa vector database
-- **Auth**: OAuth2, SAML, multi-provider support
-- **AI/ML**: LangChain, LiteLLM, multiple embedding models
-
-### Directory Structure
-
-```
-backend/
-├── onyx/
-│   ├── auth/                    # Authentication & authorization
-│   ├── chat/                    # Chat functionality & LLM interactions
-│   ├── connectors/              # Data source connectors
-│   ├── db/                      # Database models & operations
-│   ├── document_index/          # Vespa integration
-│   ├── federated_connectors/    # External search connectors
-│   ├── llm/                     # LLM provider integrations
-│   └── server/                  # API endpoints & routers
-├── ee/                          # Enterprise Edition features
-├── alembic/                     # Database migrations
-└── tests/                       # Test suites
-
-web/
-├── src/app/                     # Next.js app router pages
-├── src/components/              # Reusable React components
-└── src/lib/                     # Utilities & business logic
-```
-
-## Database & Migrations
-
-### Running Migrations
-```bash
-# Standard migrations
-alembic upgrade head
-
-# Multi-tenant (Enterprise)
-alembic -n schema_private upgrade head
-```
-
-### Creating Migrations
-```bash
-# Auto-generate migration
-alembic revision --autogenerate -m "description"
-
-# Multi-tenant migration
-alembic -n schema_private revision --autogenerate -m "description"
-```
-
-## Testing Strategy
-
-There are 4 main types of tests within Onyx:
-
-### Unit Tests
-These should not assume any Onyx/external services are available to be called.
-Interactions with the outside world should be mocked using `unittest.mock`. Generally, only 
-write these for complex, isolated modules e.g. `citation_processing.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest -xv backend/tests/unit
-```
-
-### External Dependency Unit Tests
-These tests assume that all external dependencies of Onyx are available and callable (e.g. Postgres, Redis, 
-MinIO/S3, Vespa are running + OpenAI can be called + any request to the internet is fine + etc.).
-
-However, the actual Onyx containers are not running and with these tests we call the function to test directly.
-We can also mock components/calls at will. 
-
-The goal with these tests are to minimize mocking while giving some flexibility to mock things that are flakey, 
-need strictly controlled behavior, or need to have their internal behavior validated (e.g. verify a function is called
-with certain args, something that would be impossible with proper integration tests).
-
-A great example of this type of test is `backend/tests/external_dependency_unit/connectors/confluence/test_confluence_group_sync.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest backend/tests/external_dependency_unit
-```
-
-### Integration Tests
-Standard integration tests. Every test in `backend/tests/integration` runs against a real Onyx deployment. We cannot 
-mock anything in these tests. Prefer writing integration tests (or External Dependency Unit Tests if mocking/internal 
-verification is necessary) over any other type of test.
-
-Tests are parallelized at a directory level.
-
-When writing integration tests, make sure to check the root `conftest.py` for useful fixtures + the `backend/tests/integration/common_utils` directory for utilities. Prefer (if one exists), calling the appropriate Manager 
-class in the utils over directly calling the APIs with a library like `requests`. Prefer using fixtures rather than
-calling the utilities directly (e.g. do NOT create admin users with 
-`admin_user = UserManager.create(name="admin_user")`, instead use the `admin_user` fixture).
-
-A great example of this type of test is `backend/tests/integration/dev_apis/test_simple_chat_api.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest backend/tests/integration
-```
-
-### Playwright (E2E) Tests
-These tests are an even more complete version of the Integration Tests mentioned above. Has all services of Onyx 
-running, *including* the Web Server.
-
-Use these tests for anything that requires significant frontend <-> backend coordination.
-
-Tests are located at `web/tests/e2e`. Tests are written in TypeScript.
-
-To run them:
-
-```bash
-npx playwright test <TEST_NAME>
-```
-
-
-## Logs
-
-When (1) writing integration tests or (2) doing live tests (e.g. curl / playwright) you can get access
-to logs via the `backend/log/<service_name>_debug.log` file. All Onyx services (api_server, web_server, celery_X)
-will be tailing their logs to this file. 
-
-
-## Security Considerations
-
-- Never commit API keys or secrets to repository
-- Use encrypted credential storage for connector credentials
-- Follow RBAC patterns for new features
-- Implement proper input validation with Pydantic models
-- Use parameterized queries to prevent SQL injection
-
-## AI/LLM Integration
-
-- Multiple LLM providers supported via LiteLLM
-- Configurable models per feature (chat, search, embeddings)
-- Streaming support for real-time responses
-- Token management and rate limiting
-- Custom prompts and agent actions
-
-## UI/UX Patterns
-
-- Tailwind CSS with design system in `web/src/components/ui/`
-- Radix UI and Headless UI for accessible components
-- SWR for data fetching and caching
-- Form validation with react-hook-form
-- Error handling with popup notifications
-
-## Creating a Plan
-When creating a plan in the `plans` directory, make sure to include at least these elements:
-
-**Issues to Address**
-What the change is meant to do.
-
-**Important Notes**
-Things you come across in your research that are important to the implementation.
-
-**Implementation strategy**
-How you are going to make the changes happen. High level approach.
-
-**Tests**
-What unit (use rarely), external dependency unit, integration, and playwright tests you plan to write to 
-verify the correct behavior. Don't overtest. Usually, a given change only needs one type of test.
-
-Do NOT include these: *Timeline*, *Rollback plan*
-
-This is a minimal list - feel free to include more. Do NOT write code as part of your plan.
-Keep it high level. You can reference certain files or functions though.
-
-Before writing your plan, make sure to do research. Explore the relevant sections in the codebase.

CONTRIBUTING.md

@@ -59,7 +59,6 @@ Onyx being a fully functional app, relies on some external software, specificall
 - [Postgres](https://www.postgresql.org/) (Relational DB)
 - [Vespa](https://vespa.ai/) (Vector DB/Search Engine)
 - [Redis](https://redis.io/) (Cache)
-- [MinIO](https://min.io/) (File Store)
 - [Nginx](https://nginx.org/) (Not needed for development flows generally)
 
 > **Note:**
@@ -84,6 +83,10 @@ python -m venv .venv
 source .venv/bin/activate
 ```
 
+> **Note:**
+> This virtual environment MUST NOT be set up WITHIN the onyx directory if you plan on using mypy within certain IDEs.
+> For simplicity, we recommend setting up the virtual environment outside of the onyx directory.
+
 _For Windows, activate the virtual environment using Command Prompt:_
 
 ```bash
@@ -99,15 +102,10 @@ If using PowerShell, the command slightly differs:
 Install the required python dependencies:
 
 ```bash
-pip install -r backend/requirements/default.txt
-pip install -r backend/requirements/dev.txt
-pip install -r backend/requirements/ee.txt
-pip install -r backend/requirements/model_server.txt
-```
-
-Fix vscode/cursor auto-imports:
-```bash
-pip install -e .
+pip install -r onyx/backend/requirements/default.txt
+pip install -r onyx/backend/requirements/dev.txt
+pip install -r onyx/backend/requirements/ee.txt
+pip install -r onyx/backend/requirements/model_server.txt
 ```
 
 Install Playwright for Python (headless browser required by the Web Connector)
@@ -173,10 +171,10 @@ Otherwise, you can follow the instructions below to run the application for deve
 
 You will need Docker installed to run these containers.
 
-First navigate to `onyx/deployment/docker_compose`, then start up Postgres/Vespa/Redis/MinIO with:
+First navigate to `onyx/deployment/docker_compose`, then start up Postgres/Vespa/Redis with:
 
 ```bash
-docker compose up -d index relational_db cache minio
+docker compose -f docker-compose.dev.yml -p onyx-stack up -d index relational_db cache
 ```
 
 (index refers to Vespa, relational_db refers to Postgres, and cache refers to Redis)
@@ -258,7 +256,7 @@ You can run the full Onyx application stack from pre-built images including all
 Navigate to `onyx/deployment/docker_compose` and run:
 
 ```bash
-docker compose up -d
+docker compose -f docker-compose.dev.yml -p onyx-stack up -d
 ```
 
 After Docker pulls and starts these containers, navigate to `http://localhost:3000` to use Onyx.
@@ -266,7 +264,7 @@ After Docker pulls and starts these containers, navigate to `http://localhost:30
 If you want to make changes to Onyx and run those changes in Docker, you can also build a local version of the Onyx container images that incorporates your changes like so:
 
 ```bash
-docker compose up -d --build
+docker compose -f docker-compose.dev.yml -p onyx-stack up -d --build
 ```
 
 

CONTRIBUTING_VSCODE.md

@@ -5,7 +5,7 @@ This guide explains how to set up and use VSCode's debugging capabilities with t
 ## Initial Setup
 
 1. **Environment Setup**:
-   - Copy `.vscode/env_template.txt` to `.vscode/.env`
+   - Copy `.vscode/.env.template` to `.vscode/.env`
    - Fill in the necessary environment variables in `.vscode/.env`
 2. **launch.json**:
    - Copy `.vscode/launch.template.jsonc` to `.vscode/launch.json`
@@ -17,9 +17,10 @@ Before starting, make sure the Docker Daemon is running.
 1. Open the Debug view in VSCode (Cmd+Shift+D on macOS)
 2. From the dropdown at the top, select "Clear and Restart External Volumes and Containers" and press the green play button
 3. From the dropdown at the top, select "Run All Onyx Services" and press the green play button
-4. Now, you can navigate to onyx in your browser (default is http://localhost:3000) and start using the app
-5. You can set breakpoints by clicking to the left of line numbers to help debug while the app is running
-6. Use the debug toolbar to step through code, inspect variables, etc.
+4. CD into web, run "npm i" followed by npm run dev.
+5. Now, you can navigate to onyx in your browser (default is http://localhost:3000) and start using the app
+6. You can set breakpoints by clicking to the left of line numbers to help debug while the app is running
+7. Use the debug toolbar to step through code, inspect variables, etc.
 
 ## Features
 

README.md

@@ -1,103 +1,117 @@
+<!-- ONYX_METADATA={"link": "https://github.com/onyx-dot-app/onyx/blob/main/README.md"} -->
+
 <a name="readme-top"></a>
 
 <h2 align="center">
-    <a href="https://www.onyx.app/"> <img width="50%" src="https://github.com/onyx-dot-app/onyx/blob/logo/OnyxLogoCropped.jpg?raw=true)" /></a>
+<a href="https://www.onyx.app/"> <img width="50%" src="https://github.com/onyx-dot-app/onyx/blob/logo/OnyxLogoCropped.jpg?raw=true)" /></a>
 </h2>
 
-<p align="center">Open Source AI Platform</p>
+<p align="center">
+<p align="center">Open Source Gen-AI + Enterprise Search.</p>
 
 <p align="center">
-    <a href="https://discord.gg/TDJ59cGV2X" target="_blank">
-        <img src="https://img.shields.io/badge/discord-join-blue.svg?logo=discord&logoColor=white" alt="Discord">
-    </a>
-    <a href="https://docs.onyx.app/" target="_blank">
-        <img src="https://img.shields.io/badge/docs-view-blue" alt="Documentation">
-    </a>
-    <a href="https://docs.onyx.app/" target="_blank">
-        <img src="https://img.shields.io/website?url=https://www.onyx.app&up_message=visit&up_color=blue" alt="Documentation">
-    </a>
-    <a href="https://github.com/onyx-dot-app/onyx/blob/main/LICENSE" target="_blank">
-        <img src="https://img.shields.io/static/v1?label=license&message=MIT&color=blue" alt="License">
-    </a>
+<a href="https://docs.onyx.app/" target="_blank">
+    <img src="https://img.shields.io/badge/docs-view-blue" alt="Documentation">
+</a>
+<a href="https://join.slack.com/t/onyx-dot-app/shared_invite/zt-34lu4m7xg-TsKGO6h8PDvR5W27zTdyhA" target="_blank">
+    <img src="https://img.shields.io/badge/slack-join-blue.svg?logo=slack" alt="Slack">
+</a>
+<a href="https://discord.gg/TDJ59cGV2X" target="_blank">
+    <img src="https://img.shields.io/badge/discord-join-blue.svg?logo=discord&logoColor=white" alt="Discord">
+</a>
+<a href="https://github.com/onyx-dot-app/onyx/blob/main/README.md" target="_blank">
+    <img src="https://img.shields.io/static/v1?label=license&message=MIT&color=blue" alt="License">
+</a>
 </p>
 
+<strong>[Onyx](https://www.onyx.app/)</strong> (formerly Danswer) is the AI platform connected to your company's docs, apps, and people.
+Onyx provides a feature rich Chat interface and plugs into any LLM of your choice.
+Keep knowledge and access controls sync-ed across over 40 connectors like Google Drive, Slack, Confluence, Salesforce, etc.
+Create custom AI agents with unique prompts, knowledge, and actions that the agents can take.
+Onyx can be deployed securely anywhere and for any scale - on a laptop, on-premise, or to cloud.
 
 
-**[Onyx](https://www.onyx.app/)** is a feature-rich, self-hostable Chat UI that works with any LLM. It is easy to deploy and can run in a completely airgapped environment.
+<h3>Feature Highlights</h3>
 
-Onyx comes loaded with advanced features like Agents, Web Search, RAG, MCP, Deep Research, Connectors to 40+ knowledge sources, and more.
+**Deep research over your team's knowledge:**
 
-> [!TIP]
-> Run Onyx with one command (or see deployment section below):
-> ```
-> curl -fsSL https://raw.githubusercontent.com/onyx-dot-app/onyx/main/deployment/docker_compose/install.sh > install.sh && chmod +x install.sh && ./install.sh
-> ```
+https://private-user-images.githubusercontent.com/32520769/414509312-48392e83-95d0-4fb5-8650-a396e05e0a32.mp4?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3Mzk5Mjg2MzYsIm5iZiI6MTczOTkyODMzNiwicGF0aCI6Ii8zMjUyMDc2OS80MTQ1MDkzMTItNDgzOTJlODMtOTVkMC00ZmI1LTg2NTAtYTM5NmUwNWUwYTMyLm1wND9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNTAyMTklMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjUwMjE5VDAxMjUzNlomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPWFhMzk5Njg2Y2Y5YjFmNDNiYTQ2YzM5ZTg5YWJiYTU2NWMyY2YwNmUyODE2NWUxMDRiMWQxZWJmODI4YTA0MTUmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.a9D8A0sgKE9AoaoE-mfFbJ6_OKYeqaf7TZ4Han2JfW8
 
-****
+
+**Use Onyx as a secure AI Chat with any LLM:**
 
 ![Onyx Chat Silent Demo](https://github.com/onyx-dot-app/onyx/releases/download/v0.21.1/OnyxChatSilentDemo.gif)
 
 
+**Easily set up connectors to your apps:**
 
-## ⭐ Features
-- **🤖 Custom Agents:** Build AI Agents with unique instructions, knowledge and actions.
-- **🌍 Web Search:** Browse the web with Google PSE, Exa, and Serper as well as an in-house scraper or Firecrawl.
-- **🔍 RAG:** Best in class hybrid-search + knowledge graph for uploaded files and ingested documents from connectors. 
-- **🔄 Connectors:** Pull knowledge, metadata, and access information from over 40 applications.
-- **🔬 Deep Research:** Get in depth answers with an agentic multi-step search.
-- **▶️ Actions & MCP:** Give AI Agents the ability to interact with external systems.
-- **💻 Code Interpreter:** Execute code to analyze data, render graphs and create files.
-- **🎨 Image Generation:** Generate images based on user prompts.
-- **👥 Collaboration:** Chat sharing, feedback gathering, user management, usage analytics, and more.
-
-Onyx works with all LLMs (like OpenAI, Anthropic, Gemini, etc.) and self-hosted LLMs (like Ollama, vLLM, etc.)
-
-To learn more about the features, check out our [documentation](https://docs.onyx.app/welcome)!
+![Onyx Connector Silent Demo](https://github.com/onyx-dot-app/onyx/releases/download/v0.21.1/OnyxConnectorSilentDemo.gif)
 
 
+**Access Onyx where your team already works:**
 
-## 🚀 Deployment
-Onyx supports deployments in Docker, Kubernetes, Terraform, along with guides for major cloud providers.
-
-See guides below:
-- [Docker](https://docs.onyx.app/deployment/local/docker) or [Quickstart](https://docs.onyx.app/deployment/getting_started/quickstart) (best for most users)
-- [Kubernetes](https://docs.onyx.app/deployment/local/kubernetes) (best for large teams)
-- [Terraform](https://docs.onyx.app/deployment/local/terraform) (best for teams already using Terraform)
-- Cloud specific guides (best if specifically using [AWS EKS](https://docs.onyx.app/deployment/cloud/aws/eks), [Azure VMs](https://docs.onyx.app/deployment/cloud/azure), etc.)
-
-> [!TIP]  
-> **To try Onyx for free without deploying, check out [Onyx Cloud](https://cloud.onyx.app/signup)**.
+![Onyx Bot Demo](https://github.com/onyx-dot-app/onyx/releases/download/v0.21.1/OnyxBot.png)
 
 
+## Deployment
+**To try it out for free and get started in seconds, check out [Onyx Cloud](https://cloud.onyx.app/signup)**.
 
-## 🔍 Other Notable Benefits
-Onyx is built for teams of all sizes, from individual users to the largest global enterprises.
+Onyx can also be run locally (even on a laptop) or deployed on a virtual machine with a single
+`docker compose` command. Checkout our [docs](https://docs.onyx.app/quickstart) to learn more.
 
-- **Enterprise Search**: far more than simple RAG, Onyx has custom indexing and retrieval that remains performant and accurate for scales of up to tens of millions of documents.
-- **Security**: SSO (OIDC/SAML/OAuth2), RBAC, encryption of credentials, etc.
-- **Management UI**: different user roles such as basic, curator, and admin.
-- **Document Permissioning**: mirrors user access from external apps for RAG use cases.
+We also have built-in support for high-availability/scalable deployment on Kubernetes.
+References [here](https://github.com/onyx-dot-app/onyx/tree/main/deployment).
 
 
+## 🔍 Other Notable Benefits of Onyx
+- Custom deep learning models for indexing and inference time, only through Onyx + learning from user feedback.
+- Flexible security features like SSO (OIDC/SAML/OAuth2), RBAC, encryption of credentials, etc.
+- Knowledge curation features like document-sets, query history, usage analytics, etc.
+- Scalable deployment options tested up to many tens of thousands users and hundreds of millions of documents.
+
 
 ## 🚧 Roadmap
-To see ongoing and upcoming projects, check out our [roadmap](https://github.com/orgs/onyx-dot-app/projects/2)!
+- New methods in information retrieval (StructRAG, LightGraphRAG, etc.)
+- Personalized Search
+- Organizational understanding and ability to locate and suggest experts from your team.
+- Code Search
+- SQL and Structured Query Language
 
 
+## 🔌 Connectors
+Keep knowledge and access up to sync across 40+ connectors:
+
+- Google Drive
+- Confluence
+- Slack
+- Gmail
+- Salesforce
+- Microsoft Sharepoint
+- Github
+- Jira
+- Zendesk
+- Gong
+- Microsoft Teams
+- Dropbox
+- Local Files
+- Websites
+- And more ...
+
+See the full list [here](https://docs.onyx.app/connectors).
+
 
 ## 📚 Licensing
 There are two editions of Onyx:
 
-- Onyx Community Edition (CE) is available freely under the MIT license.
+- Onyx Community Edition (CE) is available freely under the MIT Expat license. Simply follow the Deployment guide above.
 - Onyx Enterprise Edition (EE) includes extra features that are primarily useful for larger organizations.
 For feature details, check out [our website](https://www.onyx.app/pricing).
 
-
-
-## 👪 Community
-Join our open source community on **[Discord](https://discord.gg/TDJ59cGV2X)**!
-
+To try the Onyx Enterprise Edition:
+1. Checkout [Onyx Cloud](https://cloud.onyx.app/signup).
+2. For self-hosting the Enterprise Edition, contact us at [founders@onyx.app](mailto:founders@onyx.app) or book a call with us on our [Cal](https://cal.com/team/onyx/founders).
 
 
 ## 💡 Contributing
 Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.
+

backend/Dockerfile

@@ -12,8 +12,7 @@ ARG ONYX_VERSION=0.0.0-dev
 # DO_NOT_TRACK is used to disable telemetry for Unstructured
 ENV ONYX_VERSION=${ONYX_VERSION} \
     DANSWER_RUNNING_IN_DOCKER="true" \
-    DO_NOT_TRACK="true" \
-    PLAYWRIGHT_BROWSERS_PATH="/app/.cache/ms-playwright"
+    DO_NOT_TRACK="true"
 
 
 RUN echo "ONYX_VERSION: ${ONYX_VERSION}"
@@ -38,7 +37,8 @@ RUN apt-get update && \
         pkg-config \
         gcc \
         nano \
-        vim && \
+        vim \
+        postgresql-client && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get clean
 
@@ -78,9 +78,6 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/* && \
     rm -f /usr/local/lib/python3.11/site-packages/tornado/test/test.key
 
-# Install postgresql-client for easy manual tests
-# Install it here to avoid it being cleaned up above
-RUN apt-get update && apt-get install -y postgresql-client
 
 # Pre-downloading models for setups with limited egress
 RUN python -c "from tokenizers import Tokenizer; \
@@ -117,14 +114,6 @@ COPY ./assets /app/assets
 
 ENV PYTHONPATH=/app
 
-# Create non-root user for security best practices
-RUN groupadd -g 1001 onyx && \
-    useradd -u 1001 -g onyx -m -s /bin/bash onyx && \
-    chown -R onyx:onyx /app && \
-    mkdir -p /var/log/onyx && \
-    chmod 755 /var/log/onyx && \
-    chown onyx:onyx /var/log/onyx
-
 # Default command which does nothing
 # This container is used by api server and background which specify their own CMD
 CMD ["tail", "-f", "/dev/null"]

backend/Dockerfile.model_server

@@ -9,36 +9,11 @@ visit https://github.com/onyx-dot-app/onyx."
 # Default ONYX_VERSION, typically overriden during builds by GitHub Actions.
 ARG ONYX_VERSION=0.0.0-dev
 ENV ONYX_VERSION=${ONYX_VERSION} \
-    DANSWER_RUNNING_IN_DOCKER="true" \
-    HF_HOME=/app/.cache/huggingface
+    DANSWER_RUNNING_IN_DOCKER="true"
+
 
 RUN echo "ONYX_VERSION: ${ONYX_VERSION}"
 
-# Create non-root user for security best practices
-RUN mkdir -p /app && \
-    groupadd -g 1001 onyx && \
-    useradd -u 1001 -g onyx -m -s /bin/bash onyx  && \
-    chown -R onyx:onyx /app && \
-    mkdir -p /var/log/onyx && \
-    chmod 755 /var/log/onyx && \
-    chown onyx:onyx /var/log/onyx
-
-# --- add toolchain needed for Rust/Python builds (fastuuid) ---
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH
-
-RUN set -eux; \
-    apt-get update && apt-get install -y --no-install-recommends \
-        build-essential \
-        pkg-config \
-        curl \
-        ca-certificates \
-    && rm -rf /var/lib/apt/lists/* \
-    # Install latest stable Rust (supports Cargo.lock v4)
-    && curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable \
-    && rustc --version && cargo --version
-
 COPY ./requirements/model_server.txt /tmp/requirements.txt
 RUN pip install --no-cache-dir --upgrade \
         --retries 5 \
@@ -63,11 +38,9 @@ snapshot_download('mixedbread-ai/mxbai-rerank-xsmall-v1'); \
 from sentence_transformers import SentenceTransformer; \
 SentenceTransformer(model_name_or_path='nomic-ai/nomic-embed-text-v1', trust_remote_code=True);"
 
-# In case the user has volumes mounted to /app/.cache/huggingface that they've downloaded while
-# running Onyx, move the current contents of the cache folder to a temporary location to ensure 
-# it's preserved in order to combine with the user's cache contents
-RUN mv /app/.cache/huggingface /app/.cache/temp_huggingface && \
-    chown -R onyx:onyx /app
+# In case the user has volumes mounted to /root/.cache/huggingface that they've downloaded while
+# running Onyx, don't overwrite it with the built in cache folder
+RUN mv /root/.cache/huggingface /root/.cache/temp_huggingface
 
 WORKDIR /app
 

backend/alembic/README.md

@@ -20,44 +20,3 @@ To run all un-applied migrations:
 To undo migrations:
 `alembic downgrade -X`
 where X is the number of migrations you want to undo from the current state
-
-### Multi-tenant migrations
-
-For multi-tenant deployments, you can use additional options:
-
-**Upgrade all tenants:**
-```bash
-alembic -x upgrade_all_tenants=true upgrade head
-```
-
-**Upgrade specific schemas:**
-```bash
-# Single schema
-alembic -x schemas=tenant_12345678-1234-1234-1234-123456789012 upgrade head
-
-# Multiple schemas (comma-separated)
-alembic -x schemas=tenant_12345678-1234-1234-1234-123456789012,public,another_tenant upgrade head
-```
-
-**Upgrade tenants within an alphabetical range:**
-```bash
-# Upgrade tenants 100-200 when sorted alphabetically (positions 100 to 200)
-alembic -x upgrade_all_tenants=true -x tenant_range_start=100 -x tenant_range_end=200 upgrade head
-
-# Upgrade tenants starting from position 1000 alphabetically
-alembic -x upgrade_all_tenants=true -x tenant_range_start=1000 upgrade head
-
-# Upgrade first 500 tenants alphabetically
-alembic -x upgrade_all_tenants=true -x tenant_range_end=500 upgrade head
-```
-
-**Continue on error (for batch operations):**
-```bash
-alembic -x upgrade_all_tenants=true -x continue=true upgrade head
-```
-
-The tenant range filtering works by:
-1. Sorting tenant IDs alphabetically
-2. Using 1-based position numbers (1st, 2nd, 3rd tenant, etc.)
-3. Filtering to the specified range of positions
-4. Non-tenant schemas (like 'public') are always included

backend/alembic/env.py

@@ -1,12 +1,12 @@
 from typing import Any, Literal
-from onyx.db.engine.iam_auth import get_iam_auth_token
+from onyx.db.engine import get_iam_auth_token
 from onyx.configs.app_configs import USE_IAM_AUTH
 from onyx.configs.app_configs import POSTGRES_HOST
 from onyx.configs.app_configs import POSTGRES_PORT
 from onyx.configs.app_configs import POSTGRES_USER
 from onyx.configs.app_configs import AWS_REGION_NAME
-from onyx.db.engine.sql_engine import build_connection_string
-from onyx.db.engine.tenant_utils import get_all_tenant_ids
+from onyx.db.engine import build_connection_string
+from onyx.db.engine import get_all_tenant_ids
 from sqlalchemy import event
 from sqlalchemy import pool
 from sqlalchemy import text
@@ -21,14 +21,10 @@ from alembic import context
 from sqlalchemy.ext.asyncio import create_async_engine
 from sqlalchemy.sql.schema import SchemaItem
 from onyx.configs.constants import SSL_CERT_FILE
-from shared_configs.configs import (
-    MULTI_TENANT,
-    POSTGRES_DEFAULT_SCHEMA,
-    TENANT_ID_PREFIX,
-)
+from shared_configs.configs import MULTI_TENANT, POSTGRES_DEFAULT_SCHEMA
 from onyx.db.models import Base
 from celery.backends.database.session import ResultModelBase  # type: ignore
-from onyx.db.engine.sql_engine import SqlEngine
+from onyx.db.engine import SqlEngine
 
 # Make sure in alembic.ini [logger_root] level=INFO is set or most logging will be
 # hidden! (defaults to level=WARN)
@@ -73,67 +69,15 @@ def include_object(
     return True
 
 
-def filter_tenants_by_range(
-    tenant_ids: list[str], start_range: int | None = None, end_range: int | None = None
-) -> list[str]:
-    """
-    Filter tenant IDs by alphabetical position range.
-
-    Args:
-        tenant_ids: List of tenant IDs to filter
-        start_range: Starting position in alphabetically sorted list (1-based, inclusive)
-        end_range: Ending position in alphabetically sorted list (1-based, inclusive)
-
-    Returns:
-        Filtered list of tenant IDs in their original order
-    """
-    if start_range is None and end_range is None:
-        return tenant_ids
-
-    # Separate tenant IDs from non-tenant schemas
-    tenant_schemas = [tid for tid in tenant_ids if tid.startswith(TENANT_ID_PREFIX)]
-    non_tenant_schemas = [
-        tid for tid in tenant_ids if not tid.startswith(TENANT_ID_PREFIX)
-    ]
-
-    # Sort tenant schemas alphabetically.
-    # NOTE: can cause missed schemas if a schema is created in between workers
-    # fetching of all tenant IDs. We accept this risk for now. Just re-running
-    # the migration will fix the issue.
-    sorted_tenant_schemas = sorted(tenant_schemas)
-
-    # Apply range filtering (0-based indexing)
-    start_idx = start_range if start_range is not None else 0
-    end_idx = end_range if end_range is not None else len(sorted_tenant_schemas)
-
-    # Ensure indices are within bounds
-    start_idx = max(0, start_idx)
-    end_idx = min(len(sorted_tenant_schemas), end_idx)
-
-    # Get the filtered tenant schemas
-    filtered_tenant_schemas = sorted_tenant_schemas[start_idx:end_idx]
-
-    # Combine with non-tenant schemas and preserve original order
-    filtered_tenants = []
-    for tenant_id in tenant_ids:
-        if tenant_id in filtered_tenant_schemas or tenant_id in non_tenant_schemas:
-            filtered_tenants.append(tenant_id)
-
-    return filtered_tenants
-
-
-def get_schema_options() -> (
-    tuple[bool, bool, bool, int | None, int | None, list[str] | None]
-):
+def get_schema_options() -> tuple[str, bool, bool, bool]:
     x_args_raw = context.get_x_argument()
     x_args = {}
     for arg in x_args_raw:
-        if "=" in arg:
-            key, value = arg.split("=", 1)
-            x_args[key.strip()] = value.strip()
-        else:
-            raise ValueError(f"Invalid argument: {arg}")
-
+        for pair in arg.split(","):
+            if "=" in pair:
+                key, value = pair.split("=", 1)
+                x_args[key.strip()] = value.strip()
+    schema_name = x_args.get("schema", POSTGRES_DEFAULT_SCHEMA)
     create_schema = x_args.get("create_schema", "true").lower() == "true"
     upgrade_all_tenants = x_args.get("upgrade_all_tenants", "false").lower() == "true"
 
@@ -141,81 +85,17 @@ def get_schema_options() -> (
     # only applies to online migrations
     continue_on_error = x_args.get("continue", "false").lower() == "true"
 
-    # Tenant range filtering
-    tenant_range_start = None
-    tenant_range_end = None
-
-    if "tenant_range_start" in x_args:
-        try:
-            tenant_range_start = int(x_args["tenant_range_start"])
-        except ValueError:
-            raise ValueError(
-                f"Invalid tenant_range_start value: {x_args['tenant_range_start']}. Must be an integer."
-            )
-
-    if "tenant_range_end" in x_args:
-        try:
-            tenant_range_end = int(x_args["tenant_range_end"])
-        except ValueError:
-            raise ValueError(
-                f"Invalid tenant_range_end value: {x_args['tenant_range_end']}. Must be an integer."
-            )
-
-    # Validate range
-    if tenant_range_start is not None and tenant_range_end is not None:
-        if tenant_range_start > tenant_range_end:
-            raise ValueError(
-                f"tenant_range_start ({tenant_range_start}) cannot be greater than tenant_range_end ({tenant_range_end})"
-            )
-
-    # Specific schema names filtering (replaces both schema_name and the old tenant_ids approach)
-    schemas = None
-    if "schemas" in x_args:
-        schema_names_str = x_args["schemas"].strip()
-        if schema_names_str:
-            # Split by comma and strip whitespace
-            schemas = [
-                name.strip() for name in schema_names_str.split(",") if name.strip()
-            ]
-            if schemas:
-                logger.info(f"Specific schema names specified: {schemas}")
-
-    # Validate that only one method is used at a time
-    range_filtering = tenant_range_start is not None or tenant_range_end is not None
-    specific_filtering = schemas is not None and len(schemas) > 0
-
-    if range_filtering and specific_filtering:
+    if (
+        MULTI_TENANT
+        and schema_name == POSTGRES_DEFAULT_SCHEMA
+        and not upgrade_all_tenants
+    ):
         raise ValueError(
-            "Cannot use both tenant range filtering (tenant_range_start/tenant_range_end) "
-            "and specific schema filtering (schemas) at the same time. "
-            "Please use only one filtering method."
+            "Cannot run default migrations in public schema when multi-tenancy is enabled. "
+            "Please specify a tenant-specific schema."
         )
 
-    if upgrade_all_tenants and specific_filtering:
-        raise ValueError(
-            "Cannot use both upgrade_all_tenants=true and schemas at the same time. "
-            "Use either upgrade_all_tenants=true for all tenants, or schemas for specific schemas."
-        )
-
-    # If any filtering parameters are specified, we're not doing the default single schema migration
-    if range_filtering:
-        upgrade_all_tenants = True
-
-    # Validate multi-tenant requirements
-    if MULTI_TENANT and not upgrade_all_tenants and not specific_filtering:
-        raise ValueError(
-            "In multi-tenant mode, you must specify either upgrade_all_tenants=true "
-            "or provide schemas. Cannot run default migration."
-        )
-
-    return (
-        create_schema,
-        upgrade_all_tenants,
-        continue_on_error,
-        tenant_range_start,
-        tenant_range_end,
-        schemas,
-    )
+    return schema_name, create_schema, upgrade_all_tenants, continue_on_error
 
 
 def do_run_migrations(
@@ -262,17 +142,12 @@ def provide_iam_token_for_alembic(
 
 async def run_async_migrations() -> None:
     (
+        schema_name,
         create_schema,
         upgrade_all_tenants,
         continue_on_error,
-        tenant_range_start,
-        tenant_range_end,
-        schemas,
     ) = get_schema_options()
 
-    if not schemas and not MULTI_TENANT:
-        schemas = [POSTGRES_DEFAULT_SCHEMA]
-
     # without init_engine, subsequent engine calls fail hard intentionally
     SqlEngine.init_engine(pool_size=20, max_overflow=5)
 
@@ -289,50 +164,12 @@ async def run_async_migrations() -> None:
         ) -> None:
             provide_iam_token_for_alembic(dialect, conn_rec, cargs, cparams)
 
-    if schemas:
-        # Use specific schema names directly without fetching all tenants
-        logger.info(f"Migrating specific schema names: {schemas}")
-
-        i_schema = 0
-        num_schemas = len(schemas)
-        for schema in schemas:
-            i_schema += 1
-            logger.info(
-                f"Migrating schema: index={i_schema} num_schemas={num_schemas} schema={schema}"
-            )
-            try:
-                async with engine.connect() as connection:
-                    await connection.run_sync(
-                        do_run_migrations,
-                        schema_name=schema,
-                        create_schema=create_schema,
-                    )
-            except Exception as e:
-                logger.error(f"Error migrating schema {schema}: {e}")
-                if not continue_on_error:
-                    logger.error("--continue=true is not set, raising exception!")
-                    raise
-
-                logger.warning("--continue=true is set, continuing to next schema.")
-
-    elif upgrade_all_tenants:
+    if upgrade_all_tenants:
         tenant_schemas = get_all_tenant_ids()
 
-        filtered_tenant_schemas = filter_tenants_by_range(
-            tenant_schemas, tenant_range_start, tenant_range_end
-        )
-
-        if tenant_range_start is not None or tenant_range_end is not None:
-            logger.info(
-                f"Filtering tenants by range: start={tenant_range_start}, end={tenant_range_end}"
-            )
-            logger.info(
-                f"Total tenants: {len(tenant_schemas)}, Filtered tenants: {len(filtered_tenant_schemas)}"
-            )
-
         i_tenant = 0
-        num_tenants = len(filtered_tenant_schemas)
-        for schema in filtered_tenant_schemas:
+        num_tenants = len(tenant_schemas)
+        for schema in tenant_schemas:
             i_tenant += 1
             logger.info(
                 f"Migrating schema: index={i_tenant} num_tenants={num_tenants} schema={schema}"
@@ -353,13 +190,17 @@ async def run_async_migrations() -> None:
                 logger.warning("--continue=true is set, continuing to next schema.")
 
     else:
-        # This should not happen in the new design since we require either
-        # upgrade_all_tenants=true or schemas in multi-tenant mode
-        # and for non-multi-tenant mode, we should use schemas with the default schema
-        raise ValueError(
-            "No migration target specified. Use either upgrade_all_tenants=true for all tenants "
-            "or schemas for specific schemas."
-        )
+        try:
+            logger.info(f"Migrating schema: {schema_name}")
+            async with engine.connect() as connection:
+                await connection.run_sync(
+                    do_run_migrations,
+                    schema_name=schema_name,
+                    create_schema=create_schema,
+                )
+        except Exception as e:
+            logger.error(f"Error migrating schema {schema_name}: {e}")
+            raise
 
     await engine.dispose()
 
@@ -380,37 +221,10 @@ def run_migrations_offline() -> None:
     # without init_engine, subsequent engine calls fail hard intentionally
     SqlEngine.init_engine(pool_size=20, max_overflow=5)
 
-    (
-        create_schema,
-        upgrade_all_tenants,
-        continue_on_error,
-        tenant_range_start,
-        tenant_range_end,
-        schemas,
-    ) = get_schema_options()
+    schema_name, _, upgrade_all_tenants, continue_on_error = get_schema_options()
     url = build_connection_string()
 
-    if schemas:
-        # Use specific schema names directly without fetching all tenants
-        logger.info(f"Migrating specific schema names: {schemas}")
-
-        for schema in schemas:
-            logger.info(f"Migrating schema: {schema}")
-            context.configure(
-                url=url,
-                target_metadata=target_metadata,  # type: ignore
-                literal_binds=True,
-                include_object=include_object,
-                version_table_schema=schema,
-                include_schemas=True,
-                script_location=config.get_main_option("script_location"),
-                dialect_opts={"paramstyle": "named"},
-            )
-
-            with context.begin_transaction():
-                context.run_migrations()
-
-    elif upgrade_all_tenants:
+    if upgrade_all_tenants:
         engine = create_async_engine(url)
 
         if USE_IAM_AUTH:
@@ -424,19 +238,7 @@ def run_migrations_offline() -> None:
         tenant_schemas = get_all_tenant_ids()
         engine.sync_engine.dispose()
 
-        filtered_tenant_schemas = filter_tenants_by_range(
-            tenant_schemas, tenant_range_start, tenant_range_end
-        )
-
-        if tenant_range_start is not None or tenant_range_end is not None:
-            logger.info(
-                f"Filtering tenants by range: start={tenant_range_start}, end={tenant_range_end}"
-            )
-            logger.info(
-                f"Total tenants: {len(tenant_schemas)}, Filtered tenants: {len(filtered_tenant_schemas)}"
-            )
-
-        for schema in filtered_tenant_schemas:
+        for schema in tenant_schemas:
             logger.info(f"Migrating schema: {schema}")
             context.configure(
                 url=url,
@@ -452,12 +254,21 @@ def run_migrations_offline() -> None:
             with context.begin_transaction():
                 context.run_migrations()
     else:
-        # This should not happen in the new design
-        raise ValueError(
-            "No migration target specified. Use either upgrade_all_tenants=true for all tenants "
-            "or schemas for specific schemas."
+        logger.info(f"Migrating schema: {schema_name}")
+        context.configure(
+            url=url,
+            target_metadata=target_metadata,  # type: ignore
+            literal_binds=True,
+            include_object=include_object,
+            version_table_schema=schema_name,
+            include_schemas=True,
+            script_location=config.get_main_option("script_location"),
+            dialect_opts={"paramstyle": "named"},
         )
 
+        with context.begin_transaction():
+            context.run_migrations()
+
 
 def run_migrations_online() -> None:
     logger.info("run_migrations_online starting.")

backend/alembic/versions/0816326d83aa_add_federated_connector_tables.py

@@ -1,72 +0,0 @@
-"""add federated connector tables
-
-Revision ID: 0816326d83aa
-Revises: 12635f6655b7
-Create Date: 2025-06-29 14:09:45.109518
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-
-# revision identifiers, used by Alembic.
-revision = "0816326d83aa"
-down_revision = "12635f6655b7"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Create federated_connector table
-    op.create_table(
-        "federated_connector",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("source", sa.String(), nullable=False),
-        sa.Column("credentials", sa.LargeBinary(), nullable=False),
-        sa.PrimaryKeyConstraint("id"),
-    )
-
-    # Create federated_connector_oauth_token table
-    op.create_table(
-        "federated_connector_oauth_token",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("federated_connector_id", sa.Integer(), nullable=False),
-        sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column("token", sa.LargeBinary(), nullable=False),
-        sa.Column("expires_at", sa.DateTime(), nullable=True),
-        sa.ForeignKeyConstraint(
-            ["federated_connector_id"], ["federated_connector.id"], ondelete="CASCADE"
-        ),
-        sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("id"),
-    )
-
-    # Create federated_connector__document_set table
-    op.create_table(
-        "federated_connector__document_set",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("federated_connector_id", sa.Integer(), nullable=False),
-        sa.Column("document_set_id", sa.Integer(), nullable=False),
-        sa.Column("entities", postgresql.JSONB(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["federated_connector_id"], ["federated_connector.id"], ondelete="CASCADE"
-        ),
-        sa.ForeignKeyConstraint(
-            ["document_set_id"], ["document_set.id"], ondelete="CASCADE"
-        ),
-        sa.PrimaryKeyConstraint("id"),
-        sa.UniqueConstraint(
-            "federated_connector_id",
-            "document_set_id",
-            name="uq_federated_connector_document_set",
-        ),
-    )
-
-
-def downgrade() -> None:
-    # Drop tables in reverse order due to foreign key dependencies
-    op.drop_table("federated_connector__document_set")
-    op.drop_table("federated_connector_oauth_token")
-    op.drop_table("federated_connector")

backend/alembic/versions/0cd424f32b1d_user_file_data_preparation_and_backfill.py

@@ -1,389 +0,0 @@
-"""Migration 2: User file data preparation and backfill
-
-Revision ID: 0cd424f32b1d
-Revises: 9b66d3156fc6
-Create Date: 2025-09-22 09:44:42.727034
-
-This migration populates the new columns added in migration 1.
-It prepares data for the UUID transition and relationship migration.
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-import logging
-
-logger = logging.getLogger("alembic.runtime.migration")
-
-# revision identifiers, used by Alembic.
-revision = "0cd424f32b1d"
-down_revision = "9b66d3156fc6"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """Populate new columns with data."""
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    # === Step 1: Populate user_file.new_id ===
-    user_file_columns = [col["name"] for col in inspector.get_columns("user_file")]
-    has_new_id = "new_id" in user_file_columns
-
-    if has_new_id:
-        logger.info("Populating user_file.new_id with UUIDs...")
-
-        # Count rows needing UUIDs
-        null_count = bind.execute(
-            text("SELECT COUNT(*) FROM user_file WHERE new_id IS NULL")
-        ).scalar_one()
-
-        if null_count > 0:
-            logger.info(f"Generating UUIDs for {null_count} user_file records...")
-
-            # Populate in batches to avoid long locks
-            batch_size = 10000
-            total_updated = 0
-
-            while True:
-                result = bind.execute(
-                    text(
-                        """
-                    UPDATE user_file
-                    SET new_id = gen_random_uuid()
-                    WHERE new_id IS NULL
-                    AND id IN (
-                        SELECT id FROM user_file
-                        WHERE new_id IS NULL
-                        LIMIT :batch_size
-                    )
-                """
-                    ),
-                    {"batch_size": batch_size},
-                )
-
-                updated = result.rowcount
-                total_updated += updated
-
-                if updated < batch_size:
-                    break
-
-                logger.info(f"  Updated {total_updated}/{null_count} records...")
-
-            logger.info(f"Generated UUIDs for {total_updated} user_file records")
-
-        # Verify all records have UUIDs
-        remaining_null = bind.execute(
-            text("SELECT COUNT(*) FROM user_file WHERE new_id IS NULL")
-        ).scalar_one()
-
-        if remaining_null > 0:
-            raise Exception(
-                f"Failed to populate all user_file.new_id values ({remaining_null} NULL)"
-            )
-
-        # Lock down the column
-        op.alter_column("user_file", "new_id", nullable=False)
-        op.alter_column("user_file", "new_id", server_default=None)
-        logger.info("Locked down user_file.new_id column")
-
-    # === Step 2: Populate persona__user_file.user_file_id_uuid ===
-    persona_user_file_columns = [
-        col["name"] for col in inspector.get_columns("persona__user_file")
-    ]
-
-    if has_new_id and "user_file_id_uuid" in persona_user_file_columns:
-        logger.info("Populating persona__user_file.user_file_id_uuid...")
-
-        # Count rows needing update
-        null_count = bind.execute(
-            text(
-                """
-            SELECT COUNT(*) FROM persona__user_file
-            WHERE user_file_id IS NOT NULL AND user_file_id_uuid IS NULL
-        """
-            )
-        ).scalar_one()
-
-        if null_count > 0:
-            logger.info(f"Updating {null_count} persona__user_file records...")
-
-            # Update in batches
-            batch_size = 10000
-            total_updated = 0
-
-            while True:
-                result = bind.execute(
-                    text(
-                        """
-                    UPDATE persona__user_file p
-                    SET user_file_id_uuid = uf.new_id
-                    FROM user_file uf
-                    WHERE p.user_file_id = uf.id
-                    AND p.user_file_id_uuid IS NULL
-                    AND p.persona_id IN (
-                        SELECT persona_id
-                        FROM persona__user_file
-                        WHERE user_file_id_uuid IS NULL
-                        LIMIT :batch_size
-                    )
-                """
-                    ),
-                    {"batch_size": batch_size},
-                )
-
-                updated = result.rowcount
-                total_updated += updated
-
-                if updated < batch_size:
-                    break
-
-                logger.info(f"  Updated {total_updated}/{null_count} records...")
-
-            logger.info(f"Updated {total_updated} persona__user_file records")
-
-        # Verify all records are populated
-        remaining_null = bind.execute(
-            text(
-                """
-            SELECT COUNT(*) FROM persona__user_file
-            WHERE user_file_id IS NOT NULL AND user_file_id_uuid IS NULL
-        """
-            )
-        ).scalar_one()
-
-        if remaining_null > 0:
-            raise Exception(
-                f"Failed to populate all persona__user_file.user_file_id_uuid values ({remaining_null} NULL)"
-            )
-
-        op.alter_column("persona__user_file", "user_file_id_uuid", nullable=False)
-        logger.info("Locked down persona__user_file.user_file_id_uuid column")
-
-    # === Step 3: Create user_project records from chat_folder ===
-    if "chat_folder" in inspector.get_table_names():
-        logger.info("Creating user_project records from chat_folder...")
-
-        result = bind.execute(
-            text(
-                """
-            INSERT INTO user_project (user_id, name)
-            SELECT cf.user_id, cf.name
-            FROM chat_folder cf
-            WHERE NOT EXISTS (
-                SELECT 1
-                FROM user_project up
-                WHERE up.user_id = cf.user_id AND up.name = cf.name
-            )
-        """
-            )
-        )
-
-        logger.info(f"Created {result.rowcount} user_project records from chat_folder")
-
-    # === Step 4: Populate chat_session.project_id ===
-    chat_session_columns = [
-        col["name"] for col in inspector.get_columns("chat_session")
-    ]
-
-    if "folder_id" in chat_session_columns and "project_id" in chat_session_columns:
-        logger.info("Populating chat_session.project_id...")
-
-        # Count sessions needing update
-        null_count = bind.execute(
-            text(
-                """
-            SELECT COUNT(*) FROM chat_session
-            WHERE project_id IS NULL AND folder_id IS NOT NULL
-        """
-            )
-        ).scalar_one()
-
-        if null_count > 0:
-            logger.info(f"Updating {null_count} chat_session records...")
-
-            result = bind.execute(
-                text(
-                    """
-                UPDATE chat_session cs
-                SET project_id = up.id
-                FROM chat_folder cf
-                JOIN user_project up ON up.user_id = cf.user_id AND up.name = cf.name
-                WHERE cs.folder_id = cf.id AND cs.project_id IS NULL
-            """
-                )
-            )
-
-            logger.info(f"Updated {result.rowcount} chat_session records")
-
-        # Verify all records are populated
-        remaining_null = bind.execute(
-            text(
-                """
-            SELECT COUNT(*) FROM chat_session
-            WHERE project_id IS NULL AND folder_id IS NOT NULL
-        """
-            )
-        ).scalar_one()
-
-        if remaining_null > 0:
-            logger.warning(
-                f"Warning: {remaining_null} chat_session records could not be mapped to projects"
-            )
-
-    # === Step 5: Update plaintext FileRecord IDs/display names to UUID scheme ===
-    # Prior to UUID migration, plaintext cache files were stored with file_id like 'plain_text_<int_id>'.
-    # After migration, we use 'plaintext_<uuid>' (note the name change to 'plaintext_').
-    # This step remaps existing FileRecord rows to the new naming while preserving object_key/bucket.
-    logger.info("Updating plaintext FileRecord ids and display names to UUID scheme...")
-
-    # Count legacy plaintext records that can be mapped to UUID user_file ids
-    count_query = text(
-        """
-        SELECT COUNT(*)
-        FROM file_record fr
-        JOIN user_file uf ON fr.file_id = CONCAT('plaintext_', uf.id::text)
-        WHERE LOWER(fr.file_origin::text) = 'plaintext_cache'
-        """
-    )
-    legacy_count = bind.execute(count_query).scalar_one()
-
-    if legacy_count and legacy_count > 0:
-        logger.info(f"Found {legacy_count} legacy plaintext file records to update")
-
-        # Update display_name first for readability (safe regardless of rename)
-        bind.execute(
-            text(
-                """
-                UPDATE file_record fr
-                SET display_name = CONCAT('Plaintext for user file ', uf.new_id::text)
-                FROM user_file uf
-                WHERE LOWER(fr.file_origin::text) = 'plaintext_cache'
-                    AND fr.file_id = CONCAT('plaintext_', uf.id::text)
-                """
-            )
-        )
-
-        # Remap file_id from 'plaintext_<int>' -> 'plaintext_<uuid>' using transitional new_id
-        # Use a single UPDATE ... WHERE file_id LIKE 'plain_text_%'
-        # and ensure it aligns to existing user_file ids to avoid renaming unrelated rows
-        result = bind.execute(
-            text(
-                """
-                UPDATE file_record fr
-                SET file_id = CONCAT('plaintext_', uf.new_id::text)
-                FROM user_file uf
-                WHERE LOWER(fr.file_origin::text) = 'plaintext_cache'
-                    AND fr.file_id = CONCAT('plaintext_', uf.id::text)
-                """
-            )
-        )
-        logger.info(
-            f"Updated {result.rowcount} plaintext file_record ids to UUID scheme"
-        )
-
-    # === Step 6: Ensure document_id_migrated default TRUE and backfill existing FALSE ===
-    # New records should default to migrated=True so the migration task won't run for them.
-    # Existing rows that had a legacy document_id should be marked as not migrated to be processed.
-
-    # Backfill existing records: if document_id is not null, set to FALSE
-    bind.execute(
-        text(
-            """
-            UPDATE user_file
-            SET document_id_migrated = FALSE
-            WHERE document_id IS NOT NULL
-            """
-        )
-    )
-
-    # === Step 7: Backfill user_file.status from index_attempt ===
-    logger.info("Backfilling user_file.status from index_attempt...")
-
-    # Update user_file status based on latest index attempt
-    # Using CTEs instead of temp tables for asyncpg compatibility
-    result = bind.execute(
-        text(
-            """
-        WITH latest_attempt AS (
-            SELECT DISTINCT ON (ia.connector_credential_pair_id)
-                ia.connector_credential_pair_id,
-                ia.status
-            FROM index_attempt ia
-            ORDER BY ia.connector_credential_pair_id, ia.time_updated DESC
-        ),
-        uf_to_ccp AS (
-            SELECT DISTINCT uf.id AS uf_id, ccp.id AS cc_pair_id
-            FROM user_file uf
-            JOIN document_by_connector_credential_pair dcc
-                ON dcc.id = REPLACE(uf.document_id, 'USER_FILE_CONNECTOR__', 'FILE_CONNECTOR__')
-            JOIN connector_credential_pair ccp
-                ON ccp.connector_id = dcc.connector_id
-                AND ccp.credential_id = dcc.credential_id
-        )
-        UPDATE user_file uf
-        SET status = CASE
-            WHEN la.status IN ('NOT_STARTED', 'IN_PROGRESS') THEN 'PROCESSING'
-            WHEN la.status = 'SUCCESS' THEN 'COMPLETED'
-            ELSE 'FAILED'
-        END
-        FROM uf_to_ccp ufc
-        LEFT JOIN latest_attempt la
-            ON la.connector_credential_pair_id = ufc.cc_pair_id
-        WHERE uf.id = ufc.uf_id
-        AND uf.status = 'PROCESSING'
-    """
-        )
-    )
-
-    logger.info(f"Updated status for {result.rowcount} user_file records")
-
-    logger.info("Migration 2 (data preparation) completed successfully")
-
-
-def downgrade() -> None:
-    """Reset populated data to allow clean downgrade of schema."""
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    logger.info("Starting downgrade of data preparation...")
-
-    # Reset user_file columns to allow nulls before data removal
-    if "user_file" in inspector.get_table_names():
-        columns = [col["name"] for col in inspector.get_columns("user_file")]
-
-        if "new_id" in columns:
-            op.alter_column(
-                "user_file",
-                "new_id",
-                nullable=True,
-                server_default=sa.text("gen_random_uuid()"),
-            )
-            # Optionally clear the data
-            # bind.execute(text("UPDATE user_file SET new_id = NULL"))
-            logger.info("Reset user_file.new_id to nullable")
-
-    # Reset persona__user_file.user_file_id_uuid
-    if "persona__user_file" in inspector.get_table_names():
-        columns = [col["name"] for col in inspector.get_columns("persona__user_file")]
-
-        if "user_file_id_uuid" in columns:
-            op.alter_column("persona__user_file", "user_file_id_uuid", nullable=True)
-            # Optionally clear the data
-            # bind.execute(text("UPDATE persona__user_file SET user_file_id_uuid = NULL"))
-            logger.info("Reset persona__user_file.user_file_id_uuid to nullable")
-
-    # Note: We don't delete user_project records or reset chat_session.project_id
-    # as these might be in use and can be handled by the schema downgrade
-
-    # Reset user_file.status to default
-    if "user_file" in inspector.get_table_names():
-        columns = [col["name"] for col in inspector.get_columns("user_file")]
-        if "status" in columns:
-            bind.execute(text("UPDATE user_file SET status = 'PROCESSING'"))
-            logger.info("Reset user_file.status to default")
-
-    logger.info("Downgrade completed successfully")

backend/alembic/versions/12635f6655b7_drive_canonical_ids.py

@@ -1,596 +0,0 @@
-"""drive-canonical-ids
-
-Revision ID: 12635f6655b7
-Revises: 58c50ef19f08
-Create Date: 2025-06-20 14:44:54.241159
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from urllib.parse import urlparse, urlunparse
-from httpx import HTTPStatusError
-import httpx
-from onyx.document_index.factory import get_default_document_index
-from onyx.db.search_settings import SearchSettings
-from onyx.document_index.vespa.shared_utils.utils import get_vespa_http_client
-from onyx.document_index.vespa.shared_utils.utils import (
-    replace_invalid_doc_id_characters,
-)
-from onyx.document_index.vespa_constants import DOCUMENT_ID_ENDPOINT
-from onyx.utils.logger import setup_logger
-import os
-
-logger = setup_logger()
-
-# revision identifiers, used by Alembic.
-revision = "12635f6655b7"
-down_revision = "58c50ef19f08"
-branch_labels = None
-depends_on = None
-
-SKIP_CANON_DRIVE_IDS = os.environ.get("SKIP_CANON_DRIVE_IDS", "true").lower() == "true"
-
-
-def active_search_settings() -> tuple[SearchSettings, SearchSettings | None]:
-    result = op.get_bind().execute(
-        sa.text(
-            """
-        SELECT * FROM search_settings WHERE status = 'PRESENT' ORDER BY id DESC LIMIT 1
-        """
-        )
-    )
-    search_settings_fetch = result.fetchall()
-    search_settings = (
-        SearchSettings(**search_settings_fetch[0]._asdict())
-        if search_settings_fetch
-        else None
-    )
-
-    result2 = op.get_bind().execute(
-        sa.text(
-            """
-        SELECT * FROM search_settings WHERE status = 'FUTURE' ORDER BY id DESC LIMIT 1
-        """
-        )
-    )
-    search_settings_future_fetch = result2.fetchall()
-    search_settings_future = (
-        SearchSettings(**search_settings_future_fetch[0]._asdict())
-        if search_settings_future_fetch
-        else None
-    )
-
-    if not isinstance(search_settings, SearchSettings):
-        raise RuntimeError(
-            "current search settings is of type " + str(type(search_settings))
-        )
-    if (
-        not isinstance(search_settings_future, SearchSettings)
-        and search_settings_future is not None
-    ):
-        raise RuntimeError(
-            "future search settings is of type " + str(type(search_settings_future))
-        )
-
-    return search_settings, search_settings_future
-
-
-def normalize_google_drive_url(url: str) -> str:
-    """Remove query parameters from Google Drive URLs to create canonical document IDs.
-    NOTE: copied from drive doc_conversion.py
-    """
-    parsed_url = urlparse(url)
-    parsed_url = parsed_url._replace(query="")
-    spl_path = parsed_url.path.split("/")
-    if spl_path and (spl_path[-1] in ["edit", "view", "preview"]):
-        spl_path.pop()
-        parsed_url = parsed_url._replace(path="/".join(spl_path))
-    # Remove query parameters and reconstruct URL
-    return urlunparse(parsed_url)
-
-
-def get_google_drive_documents_from_database() -> list[dict]:
-    """Get all Google Drive documents from the database."""
-    bind = op.get_bind()
-    result = bind.execute(
-        sa.text(
-            """
-            SELECT d.id
-            FROM document d
-            JOIN document_by_connector_credential_pair dcc ON d.id = dcc.id
-            JOIN connector_credential_pair cc ON dcc.connector_id = cc.connector_id
-                AND dcc.credential_id = cc.credential_id
-            JOIN connector c ON cc.connector_id = c.id
-            WHERE c.source = 'GOOGLE_DRIVE'
-        """
-        )
-    )
-
-    documents = []
-    for row in result:
-        documents.append({"document_id": row.id})
-
-    return documents
-
-
-def update_document_id_in_database(
-    old_doc_id: str, new_doc_id: str, index_name: str
-) -> None:
-    """Update document IDs in all relevant database tables using copy-and-swap approach."""
-    bind = op.get_bind()
-
-    # print(f"Updating database tables for document {old_doc_id} -> {new_doc_id}")
-
-    # Check if new document ID already exists
-    result = bind.execute(
-        sa.text("SELECT COUNT(*) FROM document WHERE id = :new_id"),
-        {"new_id": new_doc_id},
-    )
-    row = result.fetchone()
-    if row and row[0] > 0:
-        # print(f"Document with ID {new_doc_id} already exists, deleting old one")
-        delete_document_from_db(old_doc_id, index_name)
-        return
-
-    # Step 1: Create a new document row with the new ID (copy all fields from old row)
-    # Use a conservative approach to handle columns that might not exist in all installations
-    try:
-        bind.execute(
-            sa.text(
-                """
-                INSERT INTO document (id, from_ingestion_api, boost, hidden, semantic_id,
-                                    link, doc_updated_at, primary_owners, secondary_owners,
-                                    external_user_emails, external_user_group_ids, is_public,
-                                    chunk_count, last_modified, last_synced, kg_stage, kg_processing_time)
-                SELECT :new_id, from_ingestion_api, boost, hidden, semantic_id,
-                       link, doc_updated_at, primary_owners, secondary_owners,
-                       external_user_emails, external_user_group_ids, is_public,
-                       chunk_count, last_modified, last_synced, kg_stage, kg_processing_time
-                FROM document
-                WHERE id = :old_id
-            """
-            ),
-            {"new_id": new_doc_id, "old_id": old_doc_id},
-        )
-        # print(f"Successfully updated database tables for document {old_doc_id} -> {new_doc_id}")
-    except Exception as e:
-        # If the full INSERT fails, try a more basic version with only core columns
-        logger.warning(f"Full INSERT failed, trying basic version: {e}")
-        bind.execute(
-            sa.text(
-                """
-                INSERT INTO document (id, from_ingestion_api, boost, hidden, semantic_id,
-                                    link, doc_updated_at, primary_owners, secondary_owners)
-                SELECT :new_id, from_ingestion_api, boost, hidden, semantic_id,
-                       link, doc_updated_at, primary_owners, secondary_owners
-                FROM document
-                WHERE id = :old_id
-            """
-            ),
-            {"new_id": new_doc_id, "old_id": old_doc_id},
-        )
-
-    # Step 2: Update all foreign key references to point to the new ID
-
-    # Update document_by_connector_credential_pair table
-    bind.execute(
-        sa.text(
-            "UPDATE document_by_connector_credential_pair SET id = :new_id WHERE id = :old_id"
-        ),
-        {"new_id": new_doc_id, "old_id": old_doc_id},
-    )
-    # print(f"Successfully updated document_by_connector_credential_pair table for document {old_doc_id} -> {new_doc_id}")
-
-    # Update search_doc table (stores search results for chat replay)
-    # This is critical for agent functionality
-    bind.execute(
-        sa.text(
-            "UPDATE search_doc SET document_id = :new_id WHERE document_id = :old_id"
-        ),
-        {"new_id": new_doc_id, "old_id": old_doc_id},
-    )
-    # print(f"Successfully updated search_doc table for document {old_doc_id} -> {new_doc_id}")
-    # Update document_retrieval_feedback table (user feedback on documents)
-    bind.execute(
-        sa.text(
-            "UPDATE document_retrieval_feedback SET document_id = :new_id WHERE document_id = :old_id"
-        ),
-        {"new_id": new_doc_id, "old_id": old_doc_id},
-    )
-    # print(f"Successfully updated document_retrieval_feedback table for document {old_doc_id} -> {new_doc_id}")
-    # Update document__tag table (document-tag relationships)
-    bind.execute(
-        sa.text(
-            "UPDATE document__tag SET document_id = :new_id WHERE document_id = :old_id"
-        ),
-        {"new_id": new_doc_id, "old_id": old_doc_id},
-    )
-    # print(f"Successfully updated document__tag table for document {old_doc_id} -> {new_doc_id}")
-    # Update user_file table (user uploaded files linked to documents)
-    bind.execute(
-        sa.text(
-            "UPDATE user_file SET document_id = :new_id WHERE document_id = :old_id"
-        ),
-        {"new_id": new_doc_id, "old_id": old_doc_id},
-    )
-    # print(f"Successfully updated user_file table for document {old_doc_id} -> {new_doc_id}")
-    # Update KG and chunk_stats tables (these may not exist in all installations)
-    try:
-        # Update kg_entity table
-        bind.execute(
-            sa.text(
-                "UPDATE kg_entity SET document_id = :new_id WHERE document_id = :old_id"
-            ),
-            {"new_id": new_doc_id, "old_id": old_doc_id},
-        )
-        # print(f"Successfully updated kg_entity table for document {old_doc_id} -> {new_doc_id}")
-        # Update kg_entity_extraction_staging table
-        bind.execute(
-            sa.text(
-                "UPDATE kg_entity_extraction_staging SET document_id = :new_id WHERE document_id = :old_id"
-            ),
-            {"new_id": new_doc_id, "old_id": old_doc_id},
-        )
-        # print(f"Successfully updated kg_entity_extraction_staging table for document {old_doc_id} -> {new_doc_id}")
-        # Update kg_relationship table
-        bind.execute(
-            sa.text(
-                "UPDATE kg_relationship SET source_document = :new_id WHERE source_document = :old_id"
-            ),
-            {"new_id": new_doc_id, "old_id": old_doc_id},
-        )
-        # print(f"Successfully updated kg_relationship table for document {old_doc_id} -> {new_doc_id}")
-        # Update kg_relationship_extraction_staging table
-        bind.execute(
-            sa.text(
-                "UPDATE kg_relationship_extraction_staging SET source_document = :new_id WHERE source_document = :old_id"
-            ),
-            {"new_id": new_doc_id, "old_id": old_doc_id},
-        )
-        # print(f"Successfully updated kg_relationship_extraction_staging table for document {old_doc_id} -> {new_doc_id}")
-        # Update chunk_stats table
-        bind.execute(
-            sa.text(
-                "UPDATE chunk_stats SET document_id = :new_id WHERE document_id = :old_id"
-            ),
-            {"new_id": new_doc_id, "old_id": old_doc_id},
-        )
-        # print(f"Successfully updated chunk_stats table for document {old_doc_id} -> {new_doc_id}")
-        # Update chunk_stats ID field which includes document_id
-        bind.execute(
-            sa.text(
-                """
-                UPDATE chunk_stats
-                SET id = REPLACE(id, :old_id, :new_id)
-                WHERE id LIKE :old_id_pattern
-            """
-            ),
-            {
-                "new_id": new_doc_id,
-                "old_id": old_doc_id,
-                "old_id_pattern": f"{old_doc_id}__%",
-            },
-        )
-        # print(f"Successfully updated chunk_stats ID field for document {old_doc_id} -> {new_doc_id}")
-    except Exception as e:
-        logger.warning(f"Some KG/chunk tables may not exist or failed to update: {e}")
-
-    # Step 3: Delete the old document row (this should now be safe since all FKs point to new row)
-    bind.execute(
-        sa.text("DELETE FROM document WHERE id = :old_id"), {"old_id": old_doc_id}
-    )
-    # print(f"Successfully deleted document {old_doc_id} from database")
-
-
-def _visit_chunks(
-    *,
-    http_client: httpx.Client,
-    index_name: str,
-    selection: str,
-    continuation: str | None = None,
-) -> tuple[list[dict], str | None]:
-    """Helper that calls the /document/v1 visit API once and returns (docs, next_token)."""
-
-    # Use the same URL as the document API, but with visit-specific params
-    base_url = DOCUMENT_ID_ENDPOINT.format(index_name=index_name)
-
-    params: dict[str, str] = {
-        "selection": selection,
-        "wantedDocumentCount": "1000",
-    }
-    if continuation:
-        params["continuation"] = continuation
-
-    # print(f"Visiting chunks for selection '{selection}' with params {params}")
-    resp = http_client.get(base_url, params=params, timeout=None)
-    # print(f"Visited chunks for document {selection}")
-    resp.raise_for_status()
-
-    payload = resp.json()
-    return payload.get("documents", []), payload.get("continuation")
-
-
-def delete_document_chunks_from_vespa(index_name: str, doc_id: str) -> None:
-    """Delete all chunks for *doc_id* from Vespa using continuation-token paging (no offset)."""
-
-    total_deleted = 0
-    # Use exact match instead of contains - Document Selector Language doesn't support contains
-    selection = f'{index_name}.document_id=="{doc_id}"'
-
-    with get_vespa_http_client() as http_client:
-        continuation: str | None = None
-        while True:
-            docs, continuation = _visit_chunks(
-                http_client=http_client,
-                index_name=index_name,
-                selection=selection,
-                continuation=continuation,
-            )
-
-            if not docs:
-                break
-
-            for doc in docs:
-                vespa_full_id = doc.get("id")
-                if not vespa_full_id:
-                    continue
-
-                vespa_doc_uuid = vespa_full_id.split("::")[-1]
-                delete_url = f"{DOCUMENT_ID_ENDPOINT.format(index_name=index_name)}/{vespa_doc_uuid}"
-
-                try:
-                    resp = http_client.delete(delete_url)
-                    resp.raise_for_status()
-                    total_deleted += 1
-                except Exception as e:
-                    print(f"Failed to delete chunk {vespa_doc_uuid}: {e}")
-
-            if not continuation:
-                break
-
-
-def update_document_id_in_vespa(
-    index_name: str, old_doc_id: str, new_doc_id: str
-) -> None:
-    """Update all chunks' document_id field from *old_doc_id* to *new_doc_id* using continuation paging."""
-
-    clean_new_doc_id = replace_invalid_doc_id_characters(new_doc_id)
-
-    # Use exact match instead of contains - Document Selector Language doesn't support contains
-    selection = f'{index_name}.document_id=="{old_doc_id}"'
-
-    with get_vespa_http_client() as http_client:
-        continuation: str | None = None
-        while True:
-            # print(f"Visiting chunks for document {old_doc_id} -> {new_doc_id}")
-            docs, continuation = _visit_chunks(
-                http_client=http_client,
-                index_name=index_name,
-                selection=selection,
-                continuation=continuation,
-            )
-
-            if not docs:
-                break
-
-            for doc in docs:
-                vespa_full_id = doc.get("id")
-                if not vespa_full_id:
-                    continue
-
-                vespa_doc_uuid = vespa_full_id.split("::")[-1]
-                vespa_url = f"{DOCUMENT_ID_ENDPOINT.format(index_name=index_name)}/{vespa_doc_uuid}"
-
-                update_request = {
-                    "fields": {"document_id": {"assign": clean_new_doc_id}}
-                }
-
-                try:
-                    resp = http_client.put(vespa_url, json=update_request)
-                    resp.raise_for_status()
-                except Exception as e:
-                    print(f"Failed to update chunk {vespa_doc_uuid}: {e}")
-                    raise
-
-            if not continuation:
-                break
-
-
-def delete_document_from_db(current_doc_id: str, index_name: str) -> None:
-    # Delete all foreign key references first, then delete the document
-    try:
-        bind = op.get_bind()
-
-        # Delete from agent-related tables first (order matters due to foreign keys)
-        # Delete from agent__sub_query__search_doc first since it references search_doc
-        bind.execute(
-            sa.text(
-                """
-                DELETE FROM agent__sub_query__search_doc
-                WHERE search_doc_id IN (
-                    SELECT id FROM search_doc WHERE document_id = :doc_id
-                )
-                """
-            ),
-            {"doc_id": current_doc_id},
-        )
-
-        # Delete from chat_message__search_doc
-        bind.execute(
-            sa.text(
-                """
-                DELETE FROM chat_message__search_doc
-                WHERE search_doc_id IN (
-                    SELECT id FROM search_doc WHERE document_id = :doc_id
-                )
-                """
-            ),
-            {"doc_id": current_doc_id},
-        )
-
-        # Now we can safely delete from search_doc
-        bind.execute(
-            sa.text("DELETE FROM search_doc WHERE document_id = :doc_id"),
-            {"doc_id": current_doc_id},
-        )
-
-        # Delete from document_by_connector_credential_pair
-        bind.execute(
-            sa.text(
-                "DELETE FROM document_by_connector_credential_pair WHERE id = :doc_id"
-            ),
-            {"doc_id": current_doc_id},
-        )
-
-        # Delete from other tables that reference this document
-        bind.execute(
-            sa.text(
-                "DELETE FROM document_retrieval_feedback WHERE document_id = :doc_id"
-            ),
-            {"doc_id": current_doc_id},
-        )
-
-        bind.execute(
-            sa.text("DELETE FROM document__tag WHERE document_id = :doc_id"),
-            {"doc_id": current_doc_id},
-        )
-
-        bind.execute(
-            sa.text("DELETE FROM user_file WHERE document_id = :doc_id"),
-            {"doc_id": current_doc_id},
-        )
-
-        # Delete from KG tables if they exist
-        try:
-            bind.execute(
-                sa.text("DELETE FROM kg_entity WHERE document_id = :doc_id"),
-                {"doc_id": current_doc_id},
-            )
-
-            bind.execute(
-                sa.text(
-                    "DELETE FROM kg_entity_extraction_staging WHERE document_id = :doc_id"
-                ),
-                {"doc_id": current_doc_id},
-            )
-
-            bind.execute(
-                sa.text("DELETE FROM kg_relationship WHERE source_document = :doc_id"),
-                {"doc_id": current_doc_id},
-            )
-
-            bind.execute(
-                sa.text(
-                    "DELETE FROM kg_relationship_extraction_staging WHERE source_document = :doc_id"
-                ),
-                {"doc_id": current_doc_id},
-            )
-
-            bind.execute(
-                sa.text("DELETE FROM chunk_stats WHERE document_id = :doc_id"),
-                {"doc_id": current_doc_id},
-            )
-
-            bind.execute(
-                sa.text("DELETE FROM chunk_stats WHERE id LIKE :doc_id_pattern"),
-                {"doc_id_pattern": f"{current_doc_id}__%"},
-            )
-
-        except Exception as e:
-            logger.warning(
-                f"Some KG/chunk tables may not exist or failed to delete from: {e}"
-            )
-
-        # Finally delete the document itself
-        bind.execute(
-            sa.text("DELETE FROM document WHERE id = :doc_id"),
-            {"doc_id": current_doc_id},
-        )
-
-        # Delete chunks from vespa
-        delete_document_chunks_from_vespa(index_name, current_doc_id)
-
-    except Exception as e:
-        print(f"Failed to delete duplicate document {current_doc_id}: {e}")
-        # Continue with other documents instead of failing the entire migration
-
-
-def upgrade() -> None:
-    if SKIP_CANON_DRIVE_IDS:
-        return
-    current_search_settings, future_search_settings = active_search_settings()
-    document_index = get_default_document_index(
-        current_search_settings,
-        future_search_settings,
-    )
-
-    # Get the index name
-    if hasattr(document_index, "index_name"):
-        index_name = document_index.index_name
-    else:
-        # Default index name if we can't get it from the document_index
-        index_name = "danswer_index"
-
-    # Get all Google Drive documents from the database (this is faster and more reliable)
-    gdrive_documents = get_google_drive_documents_from_database()
-
-    if not gdrive_documents:
-        return
-
-    # Track normalized document IDs to detect duplicates
-    all_normalized_doc_ids = set()
-    updated_count = 0
-
-    for doc_info in gdrive_documents:
-        current_doc_id = doc_info["document_id"]
-        normalized_doc_id = normalize_google_drive_url(current_doc_id)
-
-        print(f"Processing document {current_doc_id} -> {normalized_doc_id}")
-        # Check for duplicates
-        if normalized_doc_id in all_normalized_doc_ids:
-            # print(f"Deleting duplicate document {current_doc_id}")
-            delete_document_from_db(current_doc_id, index_name)
-            continue
-
-        all_normalized_doc_ids.add(normalized_doc_id)
-
-        # If the document ID already doesn't have query parameters, skip it
-        if current_doc_id == normalized_doc_id:
-            # print(f"Skipping document {current_doc_id} -> {normalized_doc_id} because it already has no query parameters")
-            continue
-
-        try:
-            # Update both database and Vespa in order
-            # Database first to ensure consistency
-            update_document_id_in_database(
-                current_doc_id, normalized_doc_id, index_name
-            )
-
-            # For Vespa, we can now use the original document IDs since we're using contains matching
-            update_document_id_in_vespa(index_name, current_doc_id, normalized_doc_id)
-            updated_count += 1
-            # print(f"Finished updating document {current_doc_id} -> {normalized_doc_id}")
-        except Exception as e:
-            print(f"Failed to update document {current_doc_id}: {e}")
-
-            if isinstance(e, HTTPStatusError):
-                print(f"HTTPStatusError: {e}")
-                print(f"Response: {e.response.text}")
-                print(f"Status: {e.response.status_code}")
-                print(f"Headers: {e.response.headers}")
-                print(f"Request: {e.request.url}")
-                print(f"Request headers: {e.request.headers}")
-            # Note: Rollback is complex with copy-and-swap approach since the old document is already deleted
-            # In case of failure, manual intervention may be required
-            # Continue with other documents instead of failing the entire migration
-            continue
-
-    logger.info(f"Migration complete. Updated {updated_count} Google Drive documents")
-
-
-def downgrade() -> None:
-    # this is a one way migration, so no downgrade.
-    # It wouldn't make sense to store the extra query parameters
-    # and duplicate documents to allow a reversal.
-    pass

backend/alembic/versions/16c37a30adf2_user_file_relationship_migration.py

@@ -1,261 +0,0 @@
-"""Migration 3: User file relationship migration
-
-Revision ID: 16c37a30adf2
-Revises: 0cd424f32b1d
-Create Date: 2025-09-22 09:47:34.175596
-
-This migration converts folder-based relationships to project-based relationships.
-It migrates persona__user_folder to persona__user_file and populates project__user_file.
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-import logging
-
-logger = logging.getLogger("alembic.runtime.migration")
-
-# revision identifiers, used by Alembic.
-revision = "16c37a30adf2"
-down_revision = "0cd424f32b1d"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """Migrate folder-based relationships to project-based relationships."""
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    # === Step 1: Migrate persona__user_folder to persona__user_file ===
-    table_names = inspector.get_table_names()
-
-    if "persona__user_folder" in table_names and "user_file" in table_names:
-        user_file_columns = [col["name"] for col in inspector.get_columns("user_file")]
-        has_new_id = "new_id" in user_file_columns
-
-        if has_new_id and "folder_id" in user_file_columns:
-            logger.info(
-                "Migrating persona__user_folder relationships to persona__user_file..."
-            )
-
-            # Count relationships to migrate (asyncpg-compatible)
-            count_query = text(
-                """
-                SELECT COUNT(*)
-                FROM (
-                    SELECT DISTINCT puf.persona_id, uf.id
-                    FROM persona__user_folder puf
-                    JOIN user_file uf ON uf.folder_id = puf.user_folder_id
-                    WHERE NOT EXISTS (
-                        SELECT 1
-                        FROM persona__user_file p2
-                        WHERE p2.persona_id = puf.persona_id
-                        AND p2.user_file_id = uf.id
-                    )
-                ) AS distinct_pairs
-            """
-            )
-            to_migrate = bind.execute(count_query).scalar_one()
-
-            if to_migrate > 0:
-                logger.info(f"Creating {to_migrate} persona-file relationships...")
-
-                # Migrate in batches to avoid memory issues
-                batch_size = 10000
-                total_inserted = 0
-
-                while True:
-                    # Insert batch directly using subquery (asyncpg compatible)
-                    result = bind.execute(
-                        text(
-                            """
-                        INSERT INTO persona__user_file (persona_id, user_file_id, user_file_id_uuid)
-                        SELECT DISTINCT puf.persona_id, uf.id as file_id, uf.new_id
-                        FROM persona__user_folder puf
-                        JOIN user_file uf ON uf.folder_id = puf.user_folder_id
-                        WHERE NOT EXISTS (
-                            SELECT 1
-                            FROM persona__user_file p2
-                            WHERE p2.persona_id = puf.persona_id
-                            AND p2.user_file_id = uf.id
-                        )
-                        LIMIT :batch_size
-                    """
-                        ),
-                        {"batch_size": batch_size},
-                    )
-
-                    inserted = result.rowcount
-                    total_inserted += inserted
-
-                    if inserted < batch_size:
-                        break
-
-                    logger.info(
-                        f"  Migrated {total_inserted}/{to_migrate} relationships..."
-                    )
-
-                logger.info(
-                    f"Created {total_inserted} persona__user_file relationships"
-                )
-
-    # === Step 2: Add foreign key for chat_session.project_id ===
-    chat_session_fks = inspector.get_foreign_keys("chat_session")
-    fk_exists = any(
-        fk["name"] == "fk_chat_session_project_id" for fk in chat_session_fks
-    )
-
-    if not fk_exists:
-        logger.info("Adding foreign key constraint for chat_session.project_id...")
-        op.create_foreign_key(
-            "fk_chat_session_project_id",
-            "chat_session",
-            "user_project",
-            ["project_id"],
-            ["id"],
-        )
-        logger.info("Added foreign key constraint")
-
-    # === Step 3: Populate project__user_file from user_file.folder_id ===
-    user_file_columns = [col["name"] for col in inspector.get_columns("user_file")]
-    has_new_id = "new_id" in user_file_columns
-
-    if has_new_id and "folder_id" in user_file_columns:
-        logger.info("Populating project__user_file from folder relationships...")
-
-        # Count relationships to create
-        count_query = text(
-            """
-            SELECT COUNT(*)
-            FROM user_file uf
-            WHERE uf.folder_id IS NOT NULL
-            AND NOT EXISTS (
-                SELECT 1
-                FROM project__user_file puf
-                WHERE puf.project_id = uf.folder_id
-                AND puf.user_file_id = uf.new_id
-            )
-        """
-        )
-        to_create = bind.execute(count_query).scalar_one()
-
-        if to_create > 0:
-            logger.info(f"Creating {to_create} project-file relationships...")
-
-            # Insert in batches
-            batch_size = 10000
-            total_inserted = 0
-
-            while True:
-                result = bind.execute(
-                    text(
-                        """
-                    INSERT INTO project__user_file (project_id, user_file_id)
-                    SELECT uf.folder_id, uf.new_id
-                    FROM user_file uf
-                    WHERE uf.folder_id IS NOT NULL
-                    AND NOT EXISTS (
-                        SELECT 1
-                        FROM project__user_file puf
-                        WHERE puf.project_id = uf.folder_id
-                        AND puf.user_file_id = uf.new_id
-                    )
-                    LIMIT :batch_size
-                    ON CONFLICT (project_id, user_file_id) DO NOTHING
-                """
-                    ),
-                    {"batch_size": batch_size},
-                )
-
-                inserted = result.rowcount
-                total_inserted += inserted
-
-                if inserted < batch_size:
-                    break
-
-                logger.info(f"  Created {total_inserted}/{to_create} relationships...")
-
-            logger.info(f"Created {total_inserted} project__user_file relationships")
-
-    # === Step 4: Create index on chat_session.project_id ===
-    try:
-        indexes = [ix.get("name") for ix in inspector.get_indexes("chat_session")]
-    except Exception:
-        indexes = []
-
-    if "ix_chat_session_project_id" not in indexes:
-        logger.info("Creating index on chat_session.project_id...")
-        op.create_index(
-            "ix_chat_session_project_id", "chat_session", ["project_id"], unique=False
-        )
-        logger.info("Created index")
-
-    logger.info("Migration 3 (relationship migration) completed successfully")
-
-
-def downgrade() -> None:
-    """Remove migrated relationships and constraints."""
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    logger.info("Starting downgrade of relationship migration...")
-
-    # Drop index on chat_session.project_id
-    try:
-        indexes = [ix.get("name") for ix in inspector.get_indexes("chat_session")]
-        if "ix_chat_session_project_id" in indexes:
-            op.drop_index("ix_chat_session_project_id", "chat_session")
-            logger.info("Dropped index on chat_session.project_id")
-    except Exception:
-        pass
-
-    # Drop foreign key constraint
-    try:
-        chat_session_fks = inspector.get_foreign_keys("chat_session")
-        fk_exists = any(
-            fk["name"] == "fk_chat_session_project_id" for fk in chat_session_fks
-        )
-        if fk_exists:
-            op.drop_constraint(
-                "fk_chat_session_project_id", "chat_session", type_="foreignkey"
-            )
-            logger.info("Dropped foreign key constraint on chat_session.project_id")
-    except Exception:
-        pass
-
-    # Clear project__user_file relationships (but keep the table for migration 1 to handle)
-    if "project__user_file" in inspector.get_table_names():
-        result = bind.execute(text("DELETE FROM project__user_file"))
-        logger.info(f"Cleared {result.rowcount} records from project__user_file")
-
-    # Remove migrated persona__user_file relationships
-    # Only remove those that came from folder relationships
-    if all(
-        table in inspector.get_table_names()
-        for table in ["persona__user_file", "persona__user_folder", "user_file"]
-    ):
-        user_file_columns = [col["name"] for col in inspector.get_columns("user_file")]
-        if "folder_id" in user_file_columns:
-            result = bind.execute(
-                text(
-                    """
-                DELETE FROM persona__user_file puf
-                WHERE EXISTS (
-                    SELECT 1
-                    FROM user_file uf
-                    JOIN persona__user_folder puf2
-                        ON puf2.user_folder_id = uf.folder_id
-                    WHERE puf.persona_id = puf2.persona_id
-                    AND puf.user_file_id = uf.id
-                )
-            """
-                )
-            )
-            logger.info(
-                f"Removed {result.rowcount} migrated persona__user_file relationships"
-            )
-
-    logger.info("Downgrade completed successfully")

backend/alembic/versions/27c6ecc08586_permission_framework.py

@@ -144,34 +144,27 @@ def upgrade() -> None:
 
 def downgrade() -> None:
     op.execute("TRUNCATE TABLE index_attempt")
-    conn = op.get_bind()
-    inspector = sa.inspect(conn)
-    existing_columns = {col["name"] for col in inspector.get_columns("index_attempt")}
-
-    if "input_type" not in existing_columns:
-        op.add_column(
-            "index_attempt",
-            sa.Column("input_type", sa.VARCHAR(), autoincrement=False, nullable=False),
-        )
-
-    if "source" not in existing_columns:
-        op.add_column(
-            "index_attempt",
-            sa.Column("source", sa.VARCHAR(), autoincrement=False, nullable=False),
-        )
-
-    if "connector_specific_config" not in existing_columns:
-        op.add_column(
-            "index_attempt",
-            sa.Column(
-                "connector_specific_config",
-                postgresql.JSONB(astext_type=sa.Text()),
-                autoincrement=False,
-                nullable=False,
-            ),
-        )
+    op.add_column(
+        "index_attempt",
+        sa.Column("input_type", sa.VARCHAR(), autoincrement=False, nullable=False),
+    )
+    op.add_column(
+        "index_attempt",
+        sa.Column("source", sa.VARCHAR(), autoincrement=False, nullable=False),
+    )
+    op.add_column(
+        "index_attempt",
+        sa.Column(
+            "connector_specific_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            autoincrement=False,
+            nullable=False,
+        ),
+    )
 
     # Check if the constraint exists before dropping
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
     constraints = inspector.get_foreign_keys("index_attempt")
 
     if any(
@@ -190,12 +183,8 @@ def downgrade() -> None:
             "fk_index_attempt_connector_id", "index_attempt", type_="foreignkey"
         )
 
-    if "credential_id" in existing_columns:
-        op.drop_column("index_attempt", "credential_id")
-
-    if "connector_id" in existing_columns:
-        op.drop_column("index_attempt", "connector_id")
-
-    op.execute("DROP TABLE IF EXISTS connector_credential_pair CASCADE")
-    op.execute("DROP TABLE IF EXISTS credential CASCADE")
-    op.execute("DROP TABLE IF EXISTS connector CASCADE")
+    op.drop_column("index_attempt", "credential_id")
+    op.drop_column("index_attempt", "connector_id")
+    op.drop_table("connector_credential_pair")
+    op.drop_table("credential")
+    op.drop_table("connector")

backend/alembic/versions/2b75d0a8ffcb_user_file_schema_cleanup.py

@@ -1,218 +0,0 @@
-"""Migration 6: User file schema cleanup
-
-Revision ID: 2b75d0a8ffcb
-Revises: 3a78dba1080a
-Create Date: 2025-09-22 10:09:26.375377
-
-This migration removes legacy columns and tables after data migration is complete.
-It should only be run after verifying all data has been successfully migrated.
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-import logging
-
-logger = logging.getLogger("alembic.runtime.migration")
-
-# revision identifiers, used by Alembic.
-revision = "2b75d0a8ffcb"
-down_revision = "3a78dba1080a"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """Remove legacy columns and tables."""
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    logger.info("Starting schema cleanup...")
-
-    # === Step 1: Verify data migration is complete ===
-    logger.info("Verifying data migration completion...")
-
-    # Check if any chat sessions still have folder_id references
-    chat_session_columns = [
-        col["name"] for col in inspector.get_columns("chat_session")
-    ]
-    if "folder_id" in chat_session_columns:
-        orphaned_count = bind.execute(
-            text(
-                """
-            SELECT COUNT(*) FROM chat_session
-            WHERE folder_id IS NOT NULL AND project_id IS NULL
-        """
-            )
-        ).scalar_one()
-
-        if orphaned_count > 0:
-            logger.warning(
-                f"WARNING: {orphaned_count} chat_session records still have "
-                f"folder_id without project_id. Proceeding anyway."
-            )
-
-    # === Step 2: Drop chat_session.folder_id ===
-    if "folder_id" in chat_session_columns:
-        logger.info("Dropping chat_session.folder_id...")
-
-        # Drop foreign key constraint first
-        op.execute(
-            "ALTER TABLE chat_session DROP CONSTRAINT IF EXISTS chat_session_folder_fk"
-        )
-
-        # Drop the column
-        op.drop_column("chat_session", "folder_id")
-        logger.info("Dropped chat_session.folder_id")
-
-    # === Step 3: Drop persona__user_folder table ===
-    if "persona__user_folder" in inspector.get_table_names():
-        logger.info("Dropping persona__user_folder table...")
-
-        # Check for any remaining data
-        remaining = bind.execute(
-            text("SELECT COUNT(*) FROM persona__user_folder")
-        ).scalar_one()
-
-        if remaining > 0:
-            logger.warning(
-                f"WARNING: Dropping persona__user_folder with {remaining} records"
-            )
-
-        op.drop_table("persona__user_folder")
-        logger.info("Dropped persona__user_folder table")
-
-    # === Step 4: Drop chat_folder table ===
-    if "chat_folder" in inspector.get_table_names():
-        logger.info("Dropping chat_folder table...")
-
-        # Check for any remaining data
-        remaining = bind.execute(text("SELECT COUNT(*) FROM chat_folder")).scalar_one()
-
-        if remaining > 0:
-            logger.warning(f"WARNING: Dropping chat_folder with {remaining} records")
-
-        op.drop_table("chat_folder")
-        logger.info("Dropped chat_folder table")
-
-    # === Step 5: Drop user_file legacy columns ===
-    user_file_columns = [col["name"] for col in inspector.get_columns("user_file")]
-
-    # Drop folder_id
-    if "folder_id" in user_file_columns:
-        logger.info("Dropping user_file.folder_id...")
-        op.drop_column("user_file", "folder_id")
-        logger.info("Dropped user_file.folder_id")
-
-    # Drop cc_pair_id (already handled in migration 5, but be sure)
-    if "cc_pair_id" in user_file_columns:
-        logger.info("Dropping user_file.cc_pair_id...")
-
-        # Drop any remaining foreign key constraints
-        bind.execute(
-            text(
-                """
-            DO $$
-            DECLARE r RECORD;
-            BEGIN
-              FOR r IN (
-                SELECT conname
-                FROM pg_constraint c
-                JOIN pg_class t ON c.conrelid = t.oid
-                WHERE c.contype = 'f'
-                  AND t.relname = 'user_file'
-                  AND EXISTS (
-                    SELECT 1 FROM pg_attribute a
-                    WHERE a.attrelid = t.oid
-                    AND a.attname = 'cc_pair_id'
-                  )
-              ) LOOP
-                EXECUTE format('ALTER TABLE user_file DROP CONSTRAINT IF EXISTS %I', r.conname);
-              END LOOP;
-            END$$;
-        """
-            )
-        )
-
-        op.drop_column("user_file", "cc_pair_id")
-        logger.info("Dropped user_file.cc_pair_id")
-
-    # === Step 6: Clean up any remaining constraints ===
-    logger.info("Cleaning up remaining constraints...")
-
-    # Drop any unique constraints on removed columns
-    op.execute(
-        "ALTER TABLE user_file DROP CONSTRAINT IF EXISTS user_file_cc_pair_id_key"
-    )
-
-    logger.info("Migration 6 (schema cleanup) completed successfully")
-    logger.info("Legacy schema has been fully removed")
-
-
-def downgrade() -> None:
-    """Recreate dropped columns and tables (structure only, no data)."""
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    logger.warning("Downgrading schema cleanup - recreating structure only, no data!")
-
-    # Recreate user_file columns
-    if "user_file" in inspector.get_table_names():
-        columns = [col["name"] for col in inspector.get_columns("user_file")]
-
-        if "cc_pair_id" not in columns:
-            op.add_column(
-                "user_file", sa.Column("cc_pair_id", sa.Integer(), nullable=True)
-            )
-
-        if "folder_id" not in columns:
-            op.add_column(
-                "user_file", sa.Column("folder_id", sa.Integer(), nullable=True)
-            )
-
-    # Recreate chat_folder table
-    if "chat_folder" not in inspector.get_table_names():
-        op.create_table(
-            "chat_folder",
-            sa.Column("id", sa.Integer(), nullable=False),
-            sa.Column("user_id", sa.UUID(), nullable=False),
-            sa.Column("name", sa.String(), nullable=False),
-            sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
-            sa.PrimaryKeyConstraint("id"),
-            sa.ForeignKeyConstraint(
-                ["user_id"], ["user.id"], name="chat_folder_user_fk"
-            ),
-        )
-
-    # Recreate persona__user_folder table
-    if "persona__user_folder" not in inspector.get_table_names():
-        op.create_table(
-            "persona__user_folder",
-            sa.Column("persona_id", sa.Integer(), nullable=False),
-            sa.Column("user_folder_id", sa.Integer(), nullable=False),
-            sa.PrimaryKeyConstraint("persona_id", "user_folder_id"),
-            sa.ForeignKeyConstraint(["persona_id"], ["persona.id"]),
-            sa.ForeignKeyConstraint(["user_folder_id"], ["user_project.id"]),
-        )
-
-    # Add folder_id back to chat_session
-    if "chat_session" in inspector.get_table_names():
-        columns = [col["name"] for col in inspector.get_columns("chat_session")]
-        if "folder_id" not in columns:
-            op.add_column(
-                "chat_session", sa.Column("folder_id", sa.Integer(), nullable=True)
-            )
-
-            # Add foreign key if chat_folder exists
-            if "chat_folder" in inspector.get_table_names():
-                op.create_foreign_key(
-                    "chat_session_folder_fk",
-                    "chat_session",
-                    "chat_folder",
-                    ["folder_id"],
-                    ["id"],
-                )
-
-    logger.info("Downgrade completed - structure recreated but data is lost")

backend/alembic/versions/2f95e36923e6_add_indexing_coordination.py

@@ -1,115 +0,0 @@
-"""add_indexing_coordination
-
-Revision ID: 2f95e36923e6
-Revises: 0816326d83aa
-Create Date: 2025-07-10 16:17:57.762182
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "2f95e36923e6"
-down_revision = "0816326d83aa"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add database-based coordination fields (replacing Redis fencing)
-    op.add_column(
-        "index_attempt", sa.Column("celery_task_id", sa.String(), nullable=True)
-    )
-    op.add_column(
-        "index_attempt",
-        sa.Column(
-            "cancellation_requested",
-            sa.Boolean(),
-            nullable=False,
-            server_default="false",
-        ),
-    )
-
-    # Add batch coordination fields (replacing FileStore state)
-    op.add_column(
-        "index_attempt", sa.Column("total_batches", sa.Integer(), nullable=True)
-    )
-    op.add_column(
-        "index_attempt",
-        sa.Column(
-            "completed_batches", sa.Integer(), nullable=False, server_default="0"
-        ),
-    )
-    op.add_column(
-        "index_attempt",
-        sa.Column(
-            "total_failures_batch_level",
-            sa.Integer(),
-            nullable=False,
-            server_default="0",
-        ),
-    )
-    op.add_column(
-        "index_attempt",
-        sa.Column("total_chunks", sa.Integer(), nullable=False, server_default="0"),
-    )
-
-    # Progress tracking for stall detection
-    op.add_column(
-        "index_attempt",
-        sa.Column("last_progress_time", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.add_column(
-        "index_attempt",
-        sa.Column(
-            "last_batches_completed_count",
-            sa.Integer(),
-            nullable=False,
-            server_default="0",
-        ),
-    )
-
-    # Heartbeat tracking for worker liveness detection
-    op.add_column(
-        "index_attempt",
-        sa.Column(
-            "heartbeat_counter", sa.Integer(), nullable=False, server_default="0"
-        ),
-    )
-    op.add_column(
-        "index_attempt",
-        sa.Column(
-            "last_heartbeat_value", sa.Integer(), nullable=False, server_default="0"
-        ),
-    )
-    op.add_column(
-        "index_attempt",
-        sa.Column("last_heartbeat_time", sa.DateTime(timezone=True), nullable=True),
-    )
-
-    # Add index for coordination queries
-    op.create_index(
-        "ix_index_attempt_active_coordination",
-        "index_attempt",
-        ["connector_credential_pair_id", "search_settings_id", "status"],
-    )
-
-
-def downgrade() -> None:
-    # Remove the new index
-    op.drop_index("ix_index_attempt_active_coordination", table_name="index_attempt")
-
-    # Remove the new columns
-    op.drop_column("index_attempt", "last_batches_completed_count")
-    op.drop_column("index_attempt", "last_progress_time")
-    op.drop_column("index_attempt", "last_heartbeat_time")
-    op.drop_column("index_attempt", "last_heartbeat_value")
-    op.drop_column("index_attempt", "heartbeat_counter")
-    op.drop_column("index_attempt", "total_chunks")
-    op.drop_column("index_attempt", "total_failures_batch_level")
-    op.drop_column("index_attempt", "completed_batches")
-    op.drop_column("index_attempt", "total_batches")
-    op.drop_column("index_attempt", "cancellation_requested")
-    op.drop_column("index_attempt", "celery_task_id")

backend/alembic/versions/36e9220ab794_update_kg_trigger_functions.py

@@ -1,136 +0,0 @@
-"""update_kg_trigger_functions
-
-Revision ID: 36e9220ab794
-Revises: c9e2cd766c29
-Create Date: 2025-06-22 17:33:25.833733
-
-"""
-
-from alembic import op
-from sqlalchemy.orm import Session
-from sqlalchemy import text
-from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
-
-# revision identifiers, used by Alembic.
-revision = "36e9220ab794"
-down_revision = "c9e2cd766c29"
-branch_labels = None
-depends_on = None
-
-
-def _get_tenant_contextvar(session: Session) -> str:
-    """Get the current schema for the migration"""
-    current_tenant = session.execute(text("SELECT current_schema()")).scalar()
-    if isinstance(current_tenant, str):
-        return current_tenant
-    else:
-        raise ValueError("Current tenant is not a string")
-
-
-def upgrade() -> None:
-
-    bind = op.get_bind()
-    session = Session(bind=bind)
-
-    # Create kg_entity trigger to update kg_entity.name and its trigrams
-    tenant_id = _get_tenant_contextvar(session)
-    alphanum_pattern = r"[^a-z0-9]+"
-    truncate_length = 1000
-    function = "update_kg_entity_name"
-    op.execute(
-        text(
-            f"""
-            CREATE OR REPLACE FUNCTION "{tenant_id}".{function}()
-            RETURNS TRIGGER AS $$
-            DECLARE
-                name text;
-                cleaned_name text;
-            BEGIN
-                -- Set name to semantic_id if document_id is not NULL
-                IF NEW.document_id IS NOT NULL THEN
-                    SELECT lower(semantic_id) INTO name
-                    FROM "{tenant_id}".document
-                    WHERE id = NEW.document_id;
-                ELSE
-                    name = lower(NEW.name);
-                END IF;
-
-                -- Clean name and truncate if too long
-                cleaned_name = regexp_replace(
-                    name,
-                    '{alphanum_pattern}', '', 'g'
-                );
-                IF length(cleaned_name) > {truncate_length} THEN
-                    cleaned_name = left(cleaned_name, {truncate_length});
-                END IF;
-
-                -- Set name and name trigrams
-                NEW.name = name;
-                NEW.name_trigrams = {POSTGRES_DEFAULT_SCHEMA}.show_trgm(cleaned_name);
-                RETURN NEW;
-            END;
-            $$ LANGUAGE plpgsql;
-            """
-        )
-    )
-    trigger = f"{function}_trigger"
-    op.execute(f'DROP TRIGGER IF EXISTS {trigger} ON "{tenant_id}".kg_entity')
-    op.execute(
-        f"""
-        CREATE TRIGGER {trigger}
-            BEFORE INSERT OR UPDATE OF name
-            ON "{tenant_id}".kg_entity
-            FOR EACH ROW
-            EXECUTE FUNCTION "{tenant_id}".{function}();
-        """
-    )
-
-    # Create kg_entity trigger to update kg_entity.name and its trigrams
-    function = "update_kg_entity_name_from_doc"
-    op.execute(
-        text(
-            f"""
-            CREATE OR REPLACE FUNCTION "{tenant_id}".{function}()
-            RETURNS TRIGGER AS $$
-            DECLARE
-                doc_name text;
-                cleaned_name text;
-            BEGIN
-                doc_name = lower(NEW.semantic_id);
-
-                -- Clean name and truncate if too long
-                cleaned_name = regexp_replace(
-                    doc_name,
-                    '{alphanum_pattern}', '', 'g'
-                );
-                IF length(cleaned_name) > {truncate_length} THEN
-                    cleaned_name = left(cleaned_name, {truncate_length});
-                END IF;
-
-                -- Set name and name trigrams for all entities referencing this document
-                UPDATE "{tenant_id}".kg_entity
-                SET
-                    name = doc_name,
-                    name_trigrams = {POSTGRES_DEFAULT_SCHEMA}.show_trgm(cleaned_name)
-                WHERE document_id = NEW.id;
-                RETURN NEW;
-            END;
-            $$ LANGUAGE plpgsql;
-            """
-        )
-    )
-    trigger = f"{function}_trigger"
-    op.execute(f'DROP TRIGGER IF EXISTS {trigger} ON "{tenant_id}".document')
-    op.execute(
-        f"""
-        CREATE TRIGGER {trigger}
-            AFTER UPDATE OF semantic_id
-            ON "{tenant_id}".document
-            FOR EACH ROW
-            EXECUTE FUNCTION "{tenant_id}".{function}();
-        """
-    )
-
-
-def downgrade() -> None:
-    pass

backend/alembic/versions/3a78dba1080a_user_file_legacy_data_cleanup.py

@@ -1,298 +0,0 @@
-"""Migration 5: User file legacy data cleanup
-
-Revision ID: 3a78dba1080a
-Revises: 7cc3fcc116c1
-Create Date: 2025-09-22 10:04:27.986294
-
-This migration removes legacy user-file documents and connector_credential_pairs.
-It performs bulk deletions of obsolete data after the UUID migration.
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql as psql
-from sqlalchemy import text
-import logging
-from typing import List
-import uuid
-
-logger = logging.getLogger("alembic.runtime.migration")
-
-# revision identifiers, used by Alembic.
-revision = "3a78dba1080a"
-down_revision = "7cc3fcc116c1"
-branch_labels = None
-depends_on = None
-
-
-def batch_delete(
-    bind: sa.engine.Connection,
-    table_name: str,
-    id_column: str,
-    ids: List[str | int | uuid.UUID],
-    batch_size: int = 1000,
-    id_type: str = "int",
-) -> int:
-    """Delete records in batches to avoid memory issues and timeouts."""
-    total_count = len(ids)
-    if total_count == 0:
-        return 0
-
-    logger.info(
-        f"Starting batch deletion of {total_count} records from {table_name}..."
-    )
-
-    # Determine appropriate ARRAY type
-    if id_type == "uuid":
-        array_type = psql.ARRAY(psql.UUID(as_uuid=True))
-    elif id_type == "int":
-        array_type = psql.ARRAY(sa.Integer())
-    else:
-        array_type = psql.ARRAY(sa.String())
-
-    total_deleted = 0
-    failed_batches = []
-
-    for i in range(0, total_count, batch_size):
-        batch_ids = ids[i : i + batch_size]
-        try:
-            stmt = text(
-                f"DELETE FROM {table_name} WHERE {id_column} = ANY(:ids)"
-            ).bindparams(sa.bindparam("ids", value=batch_ids, type_=array_type))
-            result = bind.execute(stmt)
-            total_deleted += result.rowcount
-
-            # Log progress every 10 batches or at completion
-            batch_num = (i // batch_size) + 1
-            if batch_num % 10 == 0 or i + batch_size >= total_count:
-                logger.info(
-                    f"  Deleted {min(i + batch_size, total_count)}/{total_count} records "
-                    f"({total_deleted} actual) from {table_name}"
-                )
-        except Exception as e:
-            logger.error(f"Failed to delete batch {(i // batch_size) + 1}: {e}")
-            failed_batches.append((i, min(i + batch_size, total_count)))
-
-    if failed_batches:
-        logger.warning(
-            f"Failed to delete {len(failed_batches)} batches from {table_name}. "
-            f"Total deleted: {total_deleted}/{total_count}"
-        )
-        # Fail the migration to avoid silently succeeding on partial cleanup
-        raise RuntimeError(
-            f"Batch deletion failed for {table_name}: "
-            f"{len(failed_batches)} failed batches out of "
-            f"{(total_count + batch_size - 1) // batch_size}."
-        )
-
-    return total_deleted
-
-
-def upgrade() -> None:
-    """Remove legacy user-file documents and connector_credential_pairs."""
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    logger.info("Starting legacy data cleanup...")
-
-    # === Step 1: Identify and delete user-file documents ===
-    logger.info("Identifying user-file documents to delete...")
-
-    # Get document IDs to delete
-    doc_rows = bind.execute(
-        text(
-            """
-        SELECT DISTINCT dcc.id AS document_id
-        FROM document_by_connector_credential_pair dcc
-        JOIN connector_credential_pair u
-          ON u.connector_id = dcc.connector_id
-         AND u.credential_id = dcc.credential_id
-        WHERE u.is_user_file IS TRUE
-    """
-        )
-    ).fetchall()
-
-    doc_ids = [r[0] for r in doc_rows]
-
-    if doc_ids:
-        logger.info(f"Found {len(doc_ids)} user-file documents to delete")
-
-        # Delete dependent rows first
-        tables_to_clean = [
-            ("document_retrieval_feedback", "document_id"),
-            ("document__tag", "document_id"),
-            ("chunk_stats", "document_id"),
-        ]
-
-        for table_name, column_name in tables_to_clean:
-            if table_name in inspector.get_table_names():
-                # document_id is a string in these tables
-                deleted = batch_delete(
-                    bind, table_name, column_name, doc_ids, id_type="str"
-                )
-                logger.info(f"Deleted {deleted} records from {table_name}")
-
-        # Delete document_by_connector_credential_pair entries
-        deleted = batch_delete(
-            bind, "document_by_connector_credential_pair", "id", doc_ids, id_type="str"
-        )
-        logger.info(f"Deleted {deleted} document_by_connector_credential_pair records")
-
-        # Delete documents themselves
-        deleted = batch_delete(bind, "document", "id", doc_ids, id_type="str")
-        logger.info(f"Deleted {deleted} document records")
-    else:
-        logger.info("No user-file documents found to delete")
-
-    # === Step 2: Clean up user-file connector_credential_pairs ===
-    logger.info("Cleaning up user-file connector_credential_pairs...")
-
-    # Get cc_pair IDs
-    cc_pair_rows = bind.execute(
-        text(
-            """
-        SELECT id AS cc_pair_id
-        FROM connector_credential_pair
-        WHERE is_user_file IS TRUE
-    """
-        )
-    ).fetchall()
-
-    cc_pair_ids = [r[0] for r in cc_pair_rows]
-
-    if cc_pair_ids:
-        logger.info(
-            f"Found {len(cc_pair_ids)} user-file connector_credential_pairs to clean up"
-        )
-
-        # Delete related records
-        # Clean child tables first to satisfy foreign key constraints,
-        # then the parent tables
-        tables_to_clean = [
-            ("index_attempt_errors", "connector_credential_pair_id"),
-            ("index_attempt", "connector_credential_pair_id"),
-            ("background_error", "cc_pair_id"),
-            ("document_set__connector_credential_pair", "connector_credential_pair_id"),
-            ("user_group__connector_credential_pair", "cc_pair_id"),
-        ]
-
-        for table_name, column_name in tables_to_clean:
-            if table_name in inspector.get_table_names():
-                deleted = batch_delete(
-                    bind, table_name, column_name, cc_pair_ids, id_type="int"
-                )
-                logger.info(f"Deleted {deleted} records from {table_name}")
-
-    # === Step 3: Identify connectors and credentials to delete ===
-    logger.info("Identifying orphaned connectors and credentials...")
-
-    # Get connectors used only by user-file cc_pairs
-    connector_rows = bind.execute(
-        text(
-            """
-        SELECT DISTINCT ccp.connector_id
-        FROM connector_credential_pair ccp
-        WHERE ccp.is_user_file IS TRUE
-          AND ccp.connector_id != 0  -- Exclude system default
-          AND NOT EXISTS (
-            SELECT 1
-            FROM connector_credential_pair c2
-            WHERE c2.connector_id = ccp.connector_id
-              AND c2.is_user_file IS NOT TRUE
-          )
-    """
-        )
-    ).fetchall()
-
-    userfile_only_connector_ids = [r[0] for r in connector_rows]
-
-    # Get credentials used only by user-file cc_pairs
-    credential_rows = bind.execute(
-        text(
-            """
-        SELECT DISTINCT ccp.credential_id
-        FROM connector_credential_pair ccp
-        WHERE ccp.is_user_file IS TRUE
-          AND ccp.credential_id != 0  -- Exclude public/default
-          AND NOT EXISTS (
-            SELECT 1
-            FROM connector_credential_pair c2
-            WHERE c2.credential_id = ccp.credential_id
-              AND c2.is_user_file IS NOT TRUE
-          )
-    """
-        )
-    ).fetchall()
-
-    userfile_only_credential_ids = [r[0] for r in credential_rows]
-
-    # === Step 4: Delete the cc_pairs themselves ===
-    if cc_pair_ids:
-        # Remove FK dependency from user_file first
-        bind.execute(
-            text(
-                """
-            DO $$
-            DECLARE r RECORD;
-            BEGIN
-              FOR r IN (
-                SELECT conname
-                FROM pg_constraint c
-                JOIN pg_class t ON c.conrelid = t.oid
-                JOIN pg_class ft ON c.confrelid = ft.oid
-                WHERE c.contype = 'f'
-                  AND t.relname = 'user_file'
-                  AND ft.relname = 'connector_credential_pair'
-              ) LOOP
-                EXECUTE format('ALTER TABLE user_file DROP CONSTRAINT IF EXISTS %I', r.conname);
-              END LOOP;
-            END$$;
-        """
-            )
-        )
-
-        # Delete cc_pairs
-        deleted = batch_delete(
-            bind, "connector_credential_pair", "id", cc_pair_ids, id_type="int"
-        )
-        logger.info(f"Deleted {deleted} connector_credential_pair records")
-
-    # === Step 5: Delete orphaned connectors ===
-    if userfile_only_connector_ids:
-        deleted = batch_delete(
-            bind, "connector", "id", userfile_only_connector_ids, id_type="int"
-        )
-        logger.info(f"Deleted {deleted} orphaned connector records")
-
-    # === Step 6: Delete orphaned credentials ===
-    if userfile_only_credential_ids:
-        # Clean up credential__user_group mappings first
-        deleted = batch_delete(
-            bind,
-            "credential__user_group",
-            "credential_id",
-            userfile_only_credential_ids,
-            id_type="int",
-        )
-        logger.info(f"Deleted {deleted} credential__user_group records")
-
-        # Delete credentials
-        deleted = batch_delete(
-            bind, "credential", "id", userfile_only_credential_ids, id_type="int"
-        )
-        logger.info(f"Deleted {deleted} orphaned credential records")
-
-    logger.info("Migration 5 (legacy data cleanup) completed successfully")
-
-
-def downgrade() -> None:
-    """Cannot restore deleted data - requires backup restoration."""
-
-    logger.error("CRITICAL: Downgrading data cleanup cannot restore deleted data!")
-    logger.error("Data restoration requires backup files or database backup.")
-
-    raise NotImplementedError(
-        "Downgrade of legacy data cleanup is not supported. "
-        "Deleted data must be restored from backups."
-    )

backend/alembic/versions/3bd4c84fe72f_improved_index.py

@@ -21,14 +21,22 @@ depends_on = None
 # an outage by creating an index without using CONCURRENTLY. This migration:
 #
 # 1. Creates more efficient full-text search capabilities using tsvector columns and GIN indexes
-# 2. Adds indexes to both chat_message and chat_session tables for comprehensive search
-# 3. Note: CONCURRENTLY was removed due to operational issues
+# 2. Uses CONCURRENTLY for all index creation to prevent table locking
+# 3. Explicitly manages transactions with COMMIT statements to allow CONCURRENTLY to work
+# (see: https://www.postgresql.org/docs/9.4/sql-createindex.html#SQL-CREATEINDEX-CONCURRENTLY)
+# (see: https://github.com/sqlalchemy/alembic/issues/277)
+# 4. Adds indexes to both chat_message and chat_session tables for comprehensive search
 
 
 def upgrade() -> None:
     # First, drop any existing indexes to avoid conflicts
-    op.execute("DROP INDEX IF EXISTS idx_chat_message_tsv;")
-    op.execute("DROP INDEX IF EXISTS idx_chat_session_desc_tsv;")
+    op.execute("COMMIT")
+    op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_chat_message_tsv;")
+
+    op.execute("COMMIT")
+    op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_chat_session_desc_tsv;")
+
+    op.execute("COMMIT")
     op.execute("DROP INDEX IF EXISTS idx_chat_message_message_lower;")
 
     # Drop existing columns if they exist
@@ -44,9 +52,12 @@ def upgrade() -> None:
         """
     )
 
+    # Commit the current transaction before creating concurrent indexes
+    op.execute("COMMIT")
+
     op.execute(
         """
-        CREATE INDEX IF NOT EXISTS idx_chat_message_tsv
+        CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_chat_message_tsv
         ON chat_message
         USING GIN (message_tsv)
         """
@@ -61,9 +72,12 @@ def upgrade() -> None:
         """
     )
 
+    # Commit again before creating the second concurrent index
+    op.execute("COMMIT")
+
     op.execute(
         """
-        CREATE INDEX IF NOT EXISTS idx_chat_session_desc_tsv
+        CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_chat_session_desc_tsv
         ON chat_session
         USING GIN (description_tsv)
         """
@@ -71,9 +85,12 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    # Drop the indexes first
-    op.execute("DROP INDEX IF EXISTS idx_chat_message_tsv;")
-    op.execute("DROP INDEX IF EXISTS idx_chat_session_desc_tsv;")
+    # Drop the indexes first (use CONCURRENTLY for dropping too)
+    op.execute("COMMIT")
+    op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_chat_message_tsv;")
+
+    op.execute("COMMIT")
+    op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_chat_session_desc_tsv;")
 
     # Then drop the columns
     op.execute("ALTER TABLE chat_message DROP COLUMN IF EXISTS message_tsv;")

backend/alembic/versions/3fc5d75723b3_add_doc_metadata_field_in_document_model.py

@@ -1,30 +0,0 @@
-"""add_doc_metadata_field_in_document_model
-
-Revision ID: 3fc5d75723b3
-Revises: 2f95e36923e6
-Create Date: 2025-07-28 18:45:37.985406
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = "3fc5d75723b3"
-down_revision = "2f95e36923e6"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "document",
-        sa.Column(
-            "doc_metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("document", "doc_metadata")

backend/alembic/versions/495cb26ce93e_create_knowlege_graph_tables.py

@@ -15,7 +15,7 @@ from datetime import datetime, timedelta
 from onyx.configs.app_configs import DB_READONLY_USER
 from onyx.configs.app_configs import DB_READONLY_PASSWORD
 from shared_configs.configs import MULTI_TENANT
-from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA_STANDARD_VALUE
 
 
 # revision identifiers, used by Alembic.
@@ -80,7 +80,6 @@ def upgrade() -> None:
         )
     )
 
-    op.execute("DROP TABLE IF EXISTS kg_config CASCADE")
     op.create_table(
         "kg_config",
         sa.Column("id", sa.Integer(), primary_key=True, nullable=False, index=True),
@@ -124,7 +123,6 @@ def upgrade() -> None:
         ],
     )
 
-    op.execute("DROP TABLE IF EXISTS kg_entity_type CASCADE")
     op.create_table(
         "kg_entity_type",
         sa.Column("id_name", sa.String(), primary_key=True, nullable=False, index=True),
@@ -158,7 +156,6 @@ def upgrade() -> None:
         ),
     )
 
-    op.execute("DROP TABLE IF EXISTS kg_relationship_type CASCADE")
     # Create KGRelationshipType table
     op.create_table(
         "kg_relationship_type",
@@ -197,7 +194,6 @@ def upgrade() -> None:
         ),
     )
 
-    op.execute("DROP TABLE IF EXISTS kg_relationship_type_extraction_staging CASCADE")
     # Create KGRelationshipTypeExtractionStaging table
     op.create_table(
         "kg_relationship_type_extraction_staging",
@@ -231,8 +227,6 @@ def upgrade() -> None:
         ),
     )
 
-    op.execute("DROP TABLE IF EXISTS kg_entity CASCADE")
-
     # Create KGEntity table
     op.create_table(
         "kg_entity",
@@ -287,7 +281,6 @@ def upgrade() -> None:
         "ix_entity_name_search", "kg_entity", ["name", "entity_type_id_name"]
     )
 
-    op.execute("DROP TABLE IF EXISTS kg_entity_extraction_staging CASCADE")
     # Create KGEntityExtractionStaging table
     op.create_table(
         "kg_entity_extraction_staging",
@@ -337,7 +330,6 @@ def upgrade() -> None:
         ["name", "entity_type_id_name"],
     )
 
-    op.execute("DROP TABLE IF EXISTS kg_relationship CASCADE")
     # Create KGRelationship table
     op.create_table(
         "kg_relationship",
@@ -379,7 +371,6 @@ def upgrade() -> None:
         "ix_kg_relationship_nodes", "kg_relationship", ["source_node", "target_node"]
     )
 
-    op.execute("DROP TABLE IF EXISTS kg_relationship_extraction_staging CASCADE")
     # Create KGRelationshipExtractionStaging table
     op.create_table(
         "kg_relationship_extraction_staging",
@@ -423,7 +414,6 @@ def upgrade() -> None:
         ["source_node", "target_node"],
     )
 
-    op.execute("DROP TABLE IF EXISTS kg_term CASCADE")
     # Create KGTerm table
     op.create_table(
         "kg_term",
@@ -477,11 +467,11 @@ def upgrade() -> None:
 
     # Create GIN index for clustering and normalization
     op.execute(
-        "CREATE INDEX IF NOT EXISTS idx_kg_entity_clustering_trigrams "
-        f"ON kg_entity USING GIN (name {POSTGRES_DEFAULT_SCHEMA}.gin_trgm_ops)"
+        "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_kg_entity_clustering_trigrams "
+        f"ON kg_entity USING GIN (name {POSTGRES_DEFAULT_SCHEMA_STANDARD_VALUE}.gin_trgm_ops)"
     )
     op.execute(
-        "CREATE INDEX IF NOT EXISTS idx_kg_entity_normalization_trigrams "
+        "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_kg_entity_normalization_trigrams "
         "ON kg_entity USING GIN (name_trigrams)"
     )
 
@@ -518,7 +508,7 @@ def upgrade() -> None:
 
                 -- Set name and name trigrams
                 NEW.name = name;
-                NEW.name_trigrams = {POSTGRES_DEFAULT_SCHEMA}.show_trgm(cleaned_name);
+                NEW.name_trigrams = {POSTGRES_DEFAULT_SCHEMA_STANDARD_VALUE}.show_trgm(cleaned_name);
                 RETURN NEW;
             END;
             $$ LANGUAGE plpgsql;
@@ -563,7 +553,7 @@ def upgrade() -> None:
                 UPDATE kg_entity
                 SET
                     name = doc_name,
-                    name_trigrams = {POSTGRES_DEFAULT_SCHEMA}.show_trgm(cleaned_name)
+                    name_trigrams = {POSTGRES_DEFAULT_SCHEMA_STANDARD_VALUE}.show_trgm(cleaned_name)
                 WHERE document_id = NEW.id;
                 RETURN NEW;
             END;
@@ -635,8 +625,9 @@ def downgrade() -> None:
         op.execute(f"DROP FUNCTION IF EXISTS {function}()")
 
     # Drop index
-    op.execute("DROP INDEX IF EXISTS idx_kg_entity_clustering_trigrams")
-    op.execute("DROP INDEX IF EXISTS idx_kg_entity_normalization_trigrams")
+    op.execute("COMMIT")  # Commit to allow CONCURRENTLY
+    op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_kg_entity_clustering_trigrams")
+    op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_kg_entity_normalization_trigrams")
 
     # Drop tables in reverse order of creation to handle dependencies
     op.drop_table("kg_term")

backend/alembic/versions/505c488f6662_merge_default_assistants_into_unified.py

@@ -1,380 +0,0 @@
-"""merge_default_assistants_into_unified
-
-Revision ID: 505c488f6662
-Revises: d09fc20a3c66
-Create Date: 2025-09-09 19:00:56.816626
-
-"""
-
-import json
-from typing import Any
-from typing import NamedTuple
-from uuid import UUID
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "505c488f6662"
-down_revision = "d09fc20a3c66"
-branch_labels = None
-depends_on = None
-
-# Constants for the unified assistant
-UNIFIED_ASSISTANT_NAME = "Assistant"
-UNIFIED_ASSISTANT_DESCRIPTION = (
-    "Your AI assistant with search, web browsing, and image generation capabilities."
-)
-UNIFIED_ASSISTANT_NUM_CHUNKS = 25
-UNIFIED_ASSISTANT_DISPLAY_PRIORITY = 0
-UNIFIED_ASSISTANT_LLM_FILTER_EXTRACTION = True
-UNIFIED_ASSISTANT_LLM_RELEVANCE_FILTER = False
-UNIFIED_ASSISTANT_RECENCY_BIAS = "AUTO"  # NOTE: needs to be capitalized
-UNIFIED_ASSISTANT_CHUNKS_ABOVE = 0
-UNIFIED_ASSISTANT_CHUNKS_BELOW = 0
-UNIFIED_ASSISTANT_DATETIME_AWARE = True
-
-# NOTE: tool specific prompts are handled on the fly and automatically injected
-# into the prompt before passing to the LLM.
-DEFAULT_SYSTEM_PROMPT = """
-You are a highly capable, thoughtful, and precise assistant. Your goal is to deeply understand the \
-user's intent, ask clarifying questions when needed, think step-by-step through complex problems, \
-provide clear and accurate answers, and proactively anticipate helpful follow-up information. Always \
-prioritize being truthful, nuanced, insightful, and efficient.
-The current date is [[CURRENT_DATETIME]]
-
-You use different text styles, bolding, emojis (sparingly), block quotes, and other formatting to make \
-your responses more readable and engaging.
-You use proper Markdown and LaTeX to format your responses for math, scientific, and chemical formulas, \
-symbols, etc.: '$$\\n[expression]\\n$$' for standalone cases and '\\( [expression] \\)' when inline.
-For code you prefer to use Markdown and specify the language.
-You can use Markdown horizontal rules (---) to separate sections of your responses.
-You can use Markdown tables to format your responses for data, lists, and other structured information.
-""".strip()
-
-
-INSERT_DICT: dict[str, Any] = {
-    "name": UNIFIED_ASSISTANT_NAME,
-    "description": UNIFIED_ASSISTANT_DESCRIPTION,
-    "system_prompt": DEFAULT_SYSTEM_PROMPT,
-    "num_chunks": UNIFIED_ASSISTANT_NUM_CHUNKS,
-    "display_priority": UNIFIED_ASSISTANT_DISPLAY_PRIORITY,
-    "llm_filter_extraction": UNIFIED_ASSISTANT_LLM_FILTER_EXTRACTION,
-    "llm_relevance_filter": UNIFIED_ASSISTANT_LLM_RELEVANCE_FILTER,
-    "recency_bias": UNIFIED_ASSISTANT_RECENCY_BIAS,
-    "chunks_above": UNIFIED_ASSISTANT_CHUNKS_ABOVE,
-    "chunks_below": UNIFIED_ASSISTANT_CHUNKS_BELOW,
-    "datetime_aware": UNIFIED_ASSISTANT_DATETIME_AWARE,
-}
-
-GENERAL_ASSISTANT_ID = -1
-ART_ASSISTANT_ID = -3
-
-
-class UserRow(NamedTuple):
-    """Typed representation of user row from database query."""
-
-    id: UUID
-    chosen_assistants: list[int] | None
-    visible_assistants: list[int] | None
-    hidden_assistants: list[int] | None
-    pinned_assistants: list[int] | None
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # Start transaction
-    conn.execute(sa.text("BEGIN"))
-
-    try:
-        # Step 1: Create or update the unified assistant (ID 0)
-        search_assistant = conn.execute(
-            sa.text("SELECT * FROM persona WHERE id = 0")
-        ).fetchone()
-
-        if search_assistant:
-            # Update existing Search assistant to be the unified assistant
-            conn.execute(
-                sa.text(
-                    """
-                    UPDATE persona
-                    SET name = :name,
-                        description = :description,
-                        system_prompt = :system_prompt,
-                        num_chunks = :num_chunks,
-                        is_default_persona = true,
-                        is_visible = true,
-                        deleted = false,
-                        display_priority = :display_priority,
-                        llm_filter_extraction = :llm_filter_extraction,
-                        llm_relevance_filter = :llm_relevance_filter,
-                        recency_bias = :recency_bias,
-                        chunks_above = :chunks_above,
-                        chunks_below = :chunks_below,
-                        datetime_aware = :datetime_aware,
-                        starter_messages = null
-                    WHERE id = 0
-                """
-                ),
-                INSERT_DICT,
-            )
-        else:
-            # Create new unified assistant with ID 0
-            conn.execute(
-                sa.text(
-                    """
-                    INSERT INTO persona (
-                        id, name, description, system_prompt, num_chunks,
-                        is_default_persona, is_visible, deleted, display_priority,
-                        llm_filter_extraction, llm_relevance_filter, recency_bias,
-                        chunks_above, chunks_below, datetime_aware, starter_messages,
-                        builtin_persona
-                    ) VALUES (
-                        0, :name, :description, :system_prompt, :num_chunks,
-                        true, true, false, :display_priority, :llm_filter_extraction,
-                        :llm_relevance_filter, :recency_bias, :chunks_above, :chunks_below,
-                        :datetime_aware, null, true
-                    )
-                """
-                ),
-                INSERT_DICT,
-            )
-
-        # Step 2: Mark ALL builtin assistants as deleted (except the unified assistant ID 0)
-        conn.execute(
-            sa.text(
-                """
-                UPDATE persona
-                SET deleted = true, is_visible = false, is_default_persona = false
-                WHERE builtin_persona = true AND id != 0
-            """
-            )
-        )
-
-        # Step 3: Add all built-in tools to the unified assistant
-        # First, get the tool IDs for SearchTool, ImageGenerationTool, and WebSearchTool
-        search_tool = conn.execute(
-            sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'SearchTool'")
-        ).fetchone()
-
-        if not search_tool:
-            raise ValueError(
-                "SearchTool not found in database. Ensure tools migration has run first."
-            )
-
-        image_gen_tool = conn.execute(
-            sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'ImageGenerationTool'")
-        ).fetchone()
-
-        if not image_gen_tool:
-            raise ValueError(
-                "ImageGenerationTool not found in database. Ensure tools migration has run first."
-            )
-
-        # WebSearchTool is optional - may not be configured
-        web_search_tool = conn.execute(
-            sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'WebSearchTool'")
-        ).fetchone()
-
-        # Clear existing tool associations for persona 0
-        conn.execute(sa.text("DELETE FROM persona__tool WHERE persona_id = 0"))
-
-        # Add tools to the unified assistant
-        conn.execute(
-            sa.text(
-                """
-                INSERT INTO persona__tool (persona_id, tool_id)
-                VALUES (0, :tool_id)
-                ON CONFLICT DO NOTHING
-            """
-            ),
-            {"tool_id": search_tool[0]},
-        )
-
-        conn.execute(
-            sa.text(
-                """
-                INSERT INTO persona__tool (persona_id, tool_id)
-                VALUES (0, :tool_id)
-                ON CONFLICT DO NOTHING
-            """
-            ),
-            {"tool_id": image_gen_tool[0]},
-        )
-
-        if web_search_tool:
-            conn.execute(
-                sa.text(
-                    """
-                    INSERT INTO persona__tool (persona_id, tool_id)
-                    VALUES (0, :tool_id)
-                    ON CONFLICT DO NOTHING
-                """
-                ),
-                {"tool_id": web_search_tool[0]},
-            )
-
-        # Step 4: Migrate existing chat sessions from all builtin assistants to unified assistant
-        conn.execute(
-            sa.text(
-                """
-                UPDATE chat_session
-                SET persona_id = 0
-                WHERE persona_id IN (
-                    SELECT id FROM persona WHERE builtin_persona = true AND id != 0
-                )
-            """
-            )
-        )
-
-        # Step 5: Migrate user preferences - remove references to all builtin assistants
-        # First, get all builtin assistant IDs (except 0)
-        builtin_assistants_result = conn.execute(
-            sa.text(
-                """
-                SELECT id FROM persona
-                WHERE builtin_persona = true AND id != 0
-            """
-            )
-        ).fetchall()
-        builtin_assistant_ids = [row[0] for row in builtin_assistants_result]
-
-        # Get all users with preferences
-        users_result = conn.execute(
-            sa.text(
-                """
-                SELECT id, chosen_assistants, visible_assistants,
-                       hidden_assistants, pinned_assistants
-                FROM "user"
-            """
-            )
-        ).fetchall()
-
-        for user_row in users_result:
-            user = UserRow(*user_row)
-            user_id: UUID = user.id
-            updates: dict[str, Any] = {}
-
-            # Remove all builtin assistants from chosen_assistants
-            if user.chosen_assistants:
-                new_chosen: list[int] = [
-                    assistant_id
-                    for assistant_id in user.chosen_assistants
-                    if assistant_id not in builtin_assistant_ids
-                ]
-                if new_chosen != user.chosen_assistants:
-                    updates["chosen_assistants"] = json.dumps(new_chosen)
-
-            # Remove all builtin assistants from visible_assistants
-            if user.visible_assistants:
-                new_visible: list[int] = [
-                    assistant_id
-                    for assistant_id in user.visible_assistants
-                    if assistant_id not in builtin_assistant_ids
-                ]
-                if new_visible != user.visible_assistants:
-                    updates["visible_assistants"] = json.dumps(new_visible)
-
-            # Add all builtin assistants to hidden_assistants
-            if user.hidden_assistants:
-                new_hidden: list[int] = list(user.hidden_assistants)
-                for old_id in builtin_assistant_ids:
-                    if old_id not in new_hidden:
-                        new_hidden.append(old_id)
-                if new_hidden != user.hidden_assistants:
-                    updates["hidden_assistants"] = json.dumps(new_hidden)
-            else:
-                updates["hidden_assistants"] = json.dumps(builtin_assistant_ids)
-
-            # Remove all builtin assistants from pinned_assistants
-            if user.pinned_assistants:
-                new_pinned: list[int] = [
-                    assistant_id
-                    for assistant_id in user.pinned_assistants
-                    if assistant_id not in builtin_assistant_ids
-                ]
-                if new_pinned != user.pinned_assistants:
-                    updates["pinned_assistants"] = json.dumps(new_pinned)
-
-            # Apply updates if any
-            if updates:
-                set_clause = ", ".join([f"{k} = :{k}" for k in updates.keys()])
-                updates["user_id"] = str(user_id)  # Convert UUID to string for SQL
-                conn.execute(
-                    sa.text(f'UPDATE "user" SET {set_clause} WHERE id = :user_id'),
-                    updates,
-                )
-
-        # Commit transaction
-        conn.execute(sa.text("COMMIT"))
-
-    except Exception as e:
-        # Rollback on error
-        conn.execute(sa.text("ROLLBACK"))
-        raise e
-
-
-def downgrade() -> None:
-    conn = op.get_bind()
-
-    # Start transaction
-    conn.execute(sa.text("BEGIN"))
-
-    try:
-        # Only restore General (ID -1) and Art (ID -3) assistants
-        # Step 1: Keep Search assistant (ID 0) as default but restore original state
-        conn.execute(
-            sa.text(
-                """
-                UPDATE persona
-                SET is_default_persona = true,
-                    is_visible = true,
-                    deleted = false
-                WHERE id = 0
-            """
-            )
-        )
-
-        # Step 2: Restore General assistant (ID -1)
-        conn.execute(
-            sa.text(
-                """
-                UPDATE persona
-                SET deleted = false,
-                    is_visible = true,
-                    is_default_persona = true
-                WHERE id = :general_assistant_id
-            """
-            ),
-            {"general_assistant_id": GENERAL_ASSISTANT_ID},
-        )
-
-        # Step 3: Restore Art assistant (ID -3)
-        conn.execute(
-            sa.text(
-                """
-                UPDATE persona
-                SET deleted = false,
-                    is_visible = true,
-                    is_default_persona = true
-                WHERE id = :art_assistant_id
-            """
-            ),
-            {"art_assistant_id": ART_ASSISTANT_ID},
-        )
-
-        # Note: We don't restore the original tool associations, names, or descriptions
-        # as those would require more complex logic to determine original state.
-        # We also cannot restore original chat session persona_ids as we don't
-        # have the original mappings.
-        # Other builtin assistants remain deleted as per the requirement.
-
-        # Commit transaction
-        conn.execute(sa.text("COMMIT"))
-
-    except Exception as e:
-        # Rollback on error
-        conn.execute(sa.text("ROLLBACK"))
-        raise e

backend/alembic/versions/58c50ef19f08_add_stale_column_to_user__external_user_.py

@@ -1,90 +0,0 @@
-"""add stale column to external user group tables
-
-Revision ID: 58c50ef19f08
-Revises: 7b9b952abdf6
-Create Date: 2025-06-25 14:08:14.162380
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "58c50ef19f08"
-down_revision = "7b9b952abdf6"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add the stale column with default value False to user__external_user_group_id
-    op.add_column(
-        "user__external_user_group_id",
-        sa.Column("stale", sa.Boolean(), nullable=False, server_default="false"),
-    )
-
-    # Create index for efficient querying of stale rows by cc_pair_id
-    op.create_index(
-        "ix_user__external_user_group_id_cc_pair_id_stale",
-        "user__external_user_group_id",
-        ["cc_pair_id", "stale"],
-        unique=False,
-    )
-
-    # Create index for efficient querying of all stale rows
-    op.create_index(
-        "ix_user__external_user_group_id_stale",
-        "user__external_user_group_id",
-        ["stale"],
-        unique=False,
-    )
-
-    # Add the stale column with default value False to public_external_user_group
-    op.add_column(
-        "public_external_user_group",
-        sa.Column("stale", sa.Boolean(), nullable=False, server_default="false"),
-    )
-
-    # Create index for efficient querying of stale rows by cc_pair_id
-    op.create_index(
-        "ix_public_external_user_group_cc_pair_id_stale",
-        "public_external_user_group",
-        ["cc_pair_id", "stale"],
-        unique=False,
-    )
-
-    # Create index for efficient querying of all stale rows
-    op.create_index(
-        "ix_public_external_user_group_stale",
-        "public_external_user_group",
-        ["stale"],
-        unique=False,
-    )
-
-
-def downgrade() -> None:
-    # Drop the indices for public_external_user_group first
-    op.drop_index(
-        "ix_public_external_user_group_stale", table_name="public_external_user_group"
-    )
-    op.drop_index(
-        "ix_public_external_user_group_cc_pair_id_stale",
-        table_name="public_external_user_group",
-    )
-
-    # Drop the stale column from public_external_user_group
-    op.drop_column("public_external_user_group", "stale")
-
-    # Drop the indices for user__external_user_group_id
-    op.drop_index(
-        "ix_user__external_user_group_id_stale",
-        table_name="user__external_user_group_id",
-    )
-    op.drop_index(
-        "ix_user__external_user_group_id_cc_pair_id_stale",
-        table_name="user__external_user_group_id",
-    )
-
-    # Drop the stale column from user__external_user_group_id
-    op.drop_column("user__external_user_group_id", "stale")

backend/alembic/versions/5ae8240accb3_add_research_agent_database_tables_and_.py

@@ -1,115 +0,0 @@
-"""add research agent database tables and chat message research fields
-
-Revision ID: 5ae8240accb3
-Revises: b558f51620b4
-Create Date: 2025-08-06 14:29:24.691388
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-
-# revision identifiers, used by Alembic.
-revision = "5ae8240accb3"
-down_revision = "b558f51620b4"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add research_type and research_plan columns to chat_message table
-    op.add_column(
-        "chat_message",
-        sa.Column("research_type", sa.String(), nullable=True),
-    )
-    op.add_column(
-        "chat_message",
-        sa.Column("research_plan", postgresql.JSONB(), nullable=True),
-    )
-
-    # Create research_agent_iteration table
-    op.create_table(
-        "research_agent_iteration",
-        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column(
-            "primary_question_id",
-            sa.Integer(),
-            sa.ForeignKey("chat_message.id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        sa.Column("iteration_nr", sa.Integer(), nullable=False),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.func.now(),
-            nullable=False,
-        ),
-        sa.Column("purpose", sa.String(), nullable=True),
-        sa.Column("reasoning", sa.String(), nullable=True),
-        sa.PrimaryKeyConstraint("id"),
-        sa.UniqueConstraint(
-            "primary_question_id",
-            "iteration_nr",
-            name="_research_agent_iteration_unique_constraint",
-        ),
-    )
-
-    # Create research_agent_iteration_sub_step table
-    op.create_table(
-        "research_agent_iteration_sub_step",
-        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column(
-            "primary_question_id",
-            sa.Integer(),
-            sa.ForeignKey("chat_message.id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        sa.Column(
-            "parent_question_id",
-            sa.Integer(),
-            sa.ForeignKey("research_agent_iteration_sub_step.id", ondelete="CASCADE"),
-            nullable=True,
-        ),
-        sa.Column("iteration_nr", sa.Integer(), nullable=False),
-        sa.Column("iteration_sub_step_nr", sa.Integer(), nullable=False),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.func.now(),
-            nullable=False,
-        ),
-        sa.Column("sub_step_instructions", sa.String(), nullable=True),
-        sa.Column(
-            "sub_step_tool_id",
-            sa.Integer(),
-            sa.ForeignKey("tool.id"),
-            nullable=True,
-        ),
-        sa.Column("reasoning", sa.String(), nullable=True),
-        sa.Column("sub_answer", sa.String(), nullable=True),
-        sa.Column("cited_doc_results", postgresql.JSONB(), nullable=True),
-        sa.Column("claims", postgresql.JSONB(), nullable=True),
-        sa.Column("generated_images", postgresql.JSONB(), nullable=True),
-        sa.Column("additional_data", postgresql.JSONB(), nullable=True),
-        sa.PrimaryKeyConstraint("id"),
-        sa.ForeignKeyConstraint(
-            ["primary_question_id", "iteration_nr"],
-            [
-                "research_agent_iteration.primary_question_id",
-                "research_agent_iteration.iteration_nr",
-            ],
-            ondelete="CASCADE",
-        ),
-    )
-
-
-def downgrade() -> None:
-    # Drop tables in reverse order
-    op.drop_table("research_agent_iteration_sub_step")
-    op.drop_table("research_agent_iteration")
-
-    # Remove columns from chat_message table
-    op.drop_column("chat_message", "research_plan")
-    op.drop_column("chat_message", "research_type")

backend/alembic/versions/62c3a055a141_add_file_names_to_file_connector_config.py

@@ -1,132 +0,0 @@
-"""add file names to file connector config
-
-Revision ID: 62c3a055a141
-Revises: 3fc5d75723b3
-Create Date: 2025-07-30 17:01:24.417551
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-import json
-import os
-import logging
-
-
-# revision identifiers, used by Alembic.
-revision = "62c3a055a141"
-down_revision = "3fc5d75723b3"
-branch_labels = None
-depends_on = None
-
-SKIP_FILE_NAME_MIGRATION = (
-    os.environ.get("SKIP_FILE_NAME_MIGRATION", "true").lower() == "true"
-)
-
-logger = logging.getLogger("alembic.runtime.migration")
-
-
-def upgrade() -> None:
-    if SKIP_FILE_NAME_MIGRATION:
-        logger.info(
-            "Skipping file name migration. Hint: set SKIP_FILE_NAME_MIGRATION=false to run this migration"
-        )
-        return
-    logger.info("Running file name migration")
-    # Get connection
-    conn = op.get_bind()
-
-    # Get all FILE connectors with their configs
-    file_connectors = conn.execute(
-        sa.text(
-            """
-            SELECT id, connector_specific_config
-            FROM connector
-            WHERE source = 'FILE'
-        """
-        )
-    ).fetchall()
-
-    for connector_id, config in file_connectors:
-        # Parse config if it's a string
-        if isinstance(config, str):
-            config = json.loads(config)
-
-        # Get file_locations list
-        file_locations = config.get("file_locations", [])
-
-        # Get display names for each file_id
-        file_names = []
-        for file_id in file_locations:
-            result = conn.execute(
-                sa.text(
-                    """
-                    SELECT display_name
-                    FROM file_record
-                    WHERE file_id = :file_id
-                """
-                ),
-                {"file_id": file_id},
-            ).fetchone()
-
-            if result:
-                file_names.append(result[0])
-            else:
-                file_names.append(file_id)  # Should not happen
-
-        # Add file_names to config
-        new_config = dict(config)
-        new_config["file_names"] = file_names
-
-        # Update the connector
-        conn.execute(
-            sa.text(
-                """
-                UPDATE connector
-                SET connector_specific_config = :new_config
-                WHERE id = :connector_id
-            """
-            ),
-            {"connector_id": connector_id, "new_config": json.dumps(new_config)},
-        )
-
-
-def downgrade() -> None:
-    # Get connection
-    conn = op.get_bind()
-
-    # Remove file_names from all FILE connectors
-    file_connectors = conn.execute(
-        sa.text(
-            """
-            SELECT id, connector_specific_config
-            FROM connector
-            WHERE source = 'FILE'
-        """
-        )
-    ).fetchall()
-
-    for connector_id, config in file_connectors:
-        # Parse config if it's a string
-        if isinstance(config, str):
-            config = json.loads(config)
-
-        # Remove file_names if it exists
-        if "file_names" in config:
-            new_config = dict(config)
-            del new_config["file_names"]
-
-            # Update the connector
-            conn.execute(
-                sa.text(
-                    """
-                    UPDATE connector
-                    SET connector_specific_config = :new_config
-                    WHERE id = :connector_id
-                """
-                ),
-                {
-                    "connector_id": connector_id,
-                    "new_config": json.dumps(new_config),
-                },
-            )

backend/alembic/versions/64bd5677aeb6_add_image_input_support_to_model_config.py

@@ -1,37 +0,0 @@
-"""Add image input support to model config
-
-Revision ID: 64bd5677aeb6
-Revises: b30353be4eec
-Create Date: 2025-09-28 15:48:12.003612
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "64bd5677aeb6"
-down_revision = "b30353be4eec"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "model_configuration",
-        sa.Column("supports_image_input", sa.Boolean(), nullable=True),
-    )
-
-    # Seems to be left over from when model visibility was introduced and a nullable field.
-    # Set any null is_visible values to False
-    connection = op.get_bind()
-    connection.execute(
-        sa.text(
-            "UPDATE model_configuration SET is_visible = false WHERE is_visible IS NULL"
-        )
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("model_configuration", "supports_image_input")

backend/alembic/versions/7b9b952abdf6_update_entities.py

@@ -1,318 +0,0 @@
-"""update-entities
-
-Revision ID: 7b9b952abdf6
-Revises: 36e9220ab794
-Create Date: 2025-06-23 20:24:08.139201
-
-"""
-
-import json
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "7b9b952abdf6"
-down_revision = "36e9220ab794"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # new entity type metadata_attribute_conversion
-    new_entity_type_conversion = {
-        "LINEAR": {
-            "team": {"name": "team", "keep": True, "implication_property": None},
-            "state": {"name": "state", "keep": True, "implication_property": None},
-            "priority": {
-                "name": "priority",
-                "keep": True,
-                "implication_property": None,
-            },
-            "estimate": {
-                "name": "estimate",
-                "keep": True,
-                "implication_property": None,
-            },
-            "created_at": {
-                "name": "created_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "started_at": {
-                "name": "started_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "completed_at": {
-                "name": "completed_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "due_date": {
-                "name": "due_date",
-                "keep": True,
-                "implication_property": None,
-            },
-            "creator": {
-                "name": "creator",
-                "keep": False,
-                "implication_property": {
-                    "implied_entity_type": "from_email",
-                    "implied_relationship_name": "is_creator_of",
-                },
-            },
-            "assignee": {
-                "name": "assignee",
-                "keep": False,
-                "implication_property": {
-                    "implied_entity_type": "from_email",
-                    "implied_relationship_name": "is_assignee_of",
-                },
-            },
-        },
-        "JIRA": {
-            "issuetype": {
-                "name": "subtype",
-                "keep": True,
-                "implication_property": None,
-            },
-            "status": {"name": "status", "keep": True, "implication_property": None},
-            "priority": {
-                "name": "priority",
-                "keep": True,
-                "implication_property": None,
-            },
-            "project_name": {
-                "name": "project",
-                "keep": True,
-                "implication_property": None,
-            },
-            "created": {
-                "name": "created_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "updated": {
-                "name": "updated_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "resolution_date": {
-                "name": "completed_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "duedate": {"name": "due_date", "keep": True, "implication_property": None},
-            "reporter_email": {
-                "name": "creator",
-                "keep": False,
-                "implication_property": {
-                    "implied_entity_type": "from_email",
-                    "implied_relationship_name": "is_creator_of",
-                },
-            },
-            "assignee_email": {
-                "name": "assignee",
-                "keep": False,
-                "implication_property": {
-                    "implied_entity_type": "from_email",
-                    "implied_relationship_name": "is_assignee_of",
-                },
-            },
-            "key": {"name": "key", "keep": True, "implication_property": None},
-            "parent": {"name": "parent", "keep": True, "implication_property": None},
-        },
-        "GITHUB_PR": {
-            "repo": {"name": "repository", "keep": True, "implication_property": None},
-            "state": {"name": "state", "keep": True, "implication_property": None},
-            "num_commits": {
-                "name": "num_commits",
-                "keep": True,
-                "implication_property": None,
-            },
-            "num_files_changed": {
-                "name": "num_files_changed",
-                "keep": True,
-                "implication_property": None,
-            },
-            "labels": {"name": "labels", "keep": True, "implication_property": None},
-            "merged": {"name": "merged", "keep": True, "implication_property": None},
-            "merged_at": {
-                "name": "merged_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "closed_at": {
-                "name": "closed_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "created_at": {
-                "name": "created_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "updated_at": {
-                "name": "updated_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "user": {
-                "name": "creator",
-                "keep": False,
-                "implication_property": {
-                    "implied_entity_type": "from_email",
-                    "implied_relationship_name": "is_creator_of",
-                },
-            },
-            "assignees": {
-                "name": "assignees",
-                "keep": False,
-                "implication_property": {
-                    "implied_entity_type": "from_email",
-                    "implied_relationship_name": "is_assignee_of",
-                },
-            },
-        },
-        "GITHUB_ISSUE": {
-            "repo": {"name": "repository", "keep": True, "implication_property": None},
-            "state": {"name": "state", "keep": True, "implication_property": None},
-            "labels": {"name": "labels", "keep": True, "implication_property": None},
-            "closed_at": {
-                "name": "closed_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "created_at": {
-                "name": "created_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "updated_at": {
-                "name": "updated_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "user": {
-                "name": "creator",
-                "keep": False,
-                "implication_property": {
-                    "implied_entity_type": "from_email",
-                    "implied_relationship_name": "is_creator_of",
-                },
-            },
-            "assignees": {
-                "name": "assignees",
-                "keep": False,
-                "implication_property": {
-                    "implied_entity_type": "from_email",
-                    "implied_relationship_name": "is_assignee_of",
-                },
-            },
-        },
-        "FIREFLIES": {},
-        "ACCOUNT": {},
-        "OPPORTUNITY": {
-            "name": {"name": "name", "keep": True, "implication_property": None},
-            "stage_name": {"name": "stage", "keep": True, "implication_property": None},
-            "type": {"name": "type", "keep": True, "implication_property": None},
-            "amount": {"name": "amount", "keep": True, "implication_property": None},
-            "fiscal_year": {
-                "name": "fiscal_year",
-                "keep": True,
-                "implication_property": None,
-            },
-            "fiscal_quarter": {
-                "name": "fiscal_quarter",
-                "keep": True,
-                "implication_property": None,
-            },
-            "is_closed": {
-                "name": "is_closed",
-                "keep": True,
-                "implication_property": None,
-            },
-            "close_date": {
-                "name": "close_date",
-                "keep": True,
-                "implication_property": None,
-            },
-            "probability": {
-                "name": "close_probability",
-                "keep": True,
-                "implication_property": None,
-            },
-            "created_date": {
-                "name": "created_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "last_modified_date": {
-                "name": "updated_at",
-                "keep": True,
-                "implication_property": None,
-            },
-            "account": {
-                "name": "account",
-                "keep": False,
-                "implication_property": {
-                    "implied_entity_type": "ACCOUNT",
-                    "implied_relationship_name": "is_account_of",
-                },
-            },
-        },
-        "VENDOR": {},
-        "EMPLOYEE": {},
-    }
-
-    current_entity_types = conn.execute(
-        sa.text("SELECT id_name, attributes from kg_entity_type")
-    ).all()
-    for entity_type, attributes in current_entity_types:
-        # delete removed entity types
-        if entity_type not in new_entity_type_conversion:
-            op.execute(
-                sa.text(f"DELETE FROM kg_entity_type WHERE id_name = '{entity_type}'")
-            )
-            continue
-
-        # update entity type attributes
-        if "metadata_attributes" in attributes:
-            del attributes["metadata_attributes"]
-        attributes["metadata_attribute_conversion"] = new_entity_type_conversion[
-            entity_type
-        ]
-        attributes_str = json.dumps(attributes).replace("'", "''")
-        op.execute(
-            sa.text(
-                f"UPDATE kg_entity_type SET attributes = '{attributes_str}'"
-                f"WHERE id_name = '{entity_type}'"
-            ),
-        )
-
-
-def downgrade() -> None:
-    conn = op.get_bind()
-
-    current_entity_types = conn.execute(
-        sa.text("SELECT id_name, attributes from kg_entity_type")
-    ).all()
-    for entity_type, attributes in current_entity_types:
-        conversion = {}
-        if "metadata_attribute_conversion" in attributes:
-            conversion = attributes.pop("metadata_attribute_conversion")
-        attributes["metadata_attributes"] = {
-            attr: prop["name"] for attr, prop in conversion.items() if prop["keep"]
-        }
-
-        attributes_str = json.dumps(attributes).replace("'", "''")
-        op.execute(
-            sa.text(
-                f"UPDATE kg_entity_type SET attributes = '{attributes_str}'"
-                f"WHERE id_name = '{entity_type}'"
-            ),
-        )

backend/alembic/versions/7cc3fcc116c1_user_file_uuid_primary_key_swap.py

@@ -1,193 +0,0 @@
-"""Migration 4: User file UUID primary key swap
-
-Revision ID: 7cc3fcc116c1
-Revises: 16c37a30adf2
-Create Date: 2025-09-22 09:54:38.292952
-
-This migration performs the critical UUID primary key swap on user_file table.
-It updates all foreign key references to use UUIDs instead of integers.
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql as psql
-import logging
-
-logger = logging.getLogger("alembic.runtime.migration")
-
-# revision identifiers, used by Alembic.
-revision = "7cc3fcc116c1"
-down_revision = "16c37a30adf2"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """Swap user_file primary key from integer to UUID."""
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    # Verify we're in the expected state
-    user_file_columns = [col["name"] for col in inspector.get_columns("user_file")]
-    if "new_id" not in user_file_columns:
-        logger.warning(
-            "user_file.new_id not found - migration may have already been applied"
-        )
-        return
-
-    logger.info("Starting UUID primary key swap...")
-
-    # === Step 1: Update persona__user_file foreign key to UUID ===
-    logger.info("Updating persona__user_file foreign key...")
-
-    # Drop existing foreign key constraints
-    op.execute(
-        "ALTER TABLE persona__user_file DROP CONSTRAINT IF EXISTS persona__user_file_user_file_id_uuid_fkey"
-    )
-    op.execute(
-        "ALTER TABLE persona__user_file DROP CONSTRAINT IF EXISTS persona__user_file_user_file_id_fkey"
-    )
-
-    # Create new foreign key to user_file.new_id
-    op.create_foreign_key(
-        "persona__user_file_user_file_id_fkey",
-        "persona__user_file",
-        "user_file",
-        local_cols=["user_file_id_uuid"],
-        remote_cols=["new_id"],
-    )
-
-    # Drop the old integer column and rename UUID column
-    op.execute("ALTER TABLE persona__user_file DROP COLUMN IF EXISTS user_file_id")
-    op.alter_column(
-        "persona__user_file",
-        "user_file_id_uuid",
-        new_column_name="user_file_id",
-        existing_type=psql.UUID(as_uuid=True),
-        nullable=False,
-    )
-
-    # Recreate composite primary key
-    op.execute(
-        "ALTER TABLE persona__user_file DROP CONSTRAINT IF EXISTS persona__user_file_pkey"
-    )
-    op.execute(
-        "ALTER TABLE persona__user_file ADD PRIMARY KEY (persona_id, user_file_id)"
-    )
-
-    logger.info("Updated persona__user_file to use UUID foreign key")
-
-    # === Step 2: Perform the primary key swap on user_file ===
-    logger.info("Swapping user_file primary key to UUID...")
-
-    # Drop the primary key constraint
-    op.execute("ALTER TABLE user_file DROP CONSTRAINT IF EXISTS user_file_pkey")
-
-    # Drop the old id column and rename new_id to id
-    op.execute("ALTER TABLE user_file DROP COLUMN IF EXISTS id")
-    op.alter_column(
-        "user_file",
-        "new_id",
-        new_column_name="id",
-        existing_type=psql.UUID(as_uuid=True),
-        nullable=False,
-    )
-
-    # Set default for new inserts
-    op.alter_column(
-        "user_file",
-        "id",
-        existing_type=psql.UUID(as_uuid=True),
-        server_default=sa.text("gen_random_uuid()"),
-    )
-
-    # Create new primary key
-    op.execute("ALTER TABLE user_file ADD PRIMARY KEY (id)")
-
-    logger.info("Swapped user_file primary key to UUID")
-
-    # === Step 3: Update foreign key constraints ===
-    logger.info("Updating foreign key constraints...")
-
-    # Recreate persona__user_file foreign key to point to user_file.id
-    # Drop existing FK first to break dependency on the unique constraint
-    op.execute(
-        "ALTER TABLE persona__user_file DROP CONSTRAINT IF EXISTS persona__user_file_user_file_id_fkey"
-    )
-    # Drop the unique constraint on (formerly) new_id BEFORE recreating the FK,
-    # so the FK will bind to the primary key instead of the unique index.
-    op.execute("ALTER TABLE user_file DROP CONSTRAINT IF EXISTS uq_user_file_new_id")
-    # Now recreate FK to the primary key column
-    op.create_foreign_key(
-        "persona__user_file_user_file_id_fkey",
-        "persona__user_file",
-        "user_file",
-        local_cols=["user_file_id"],
-        remote_cols=["id"],
-    )
-
-    # Add foreign keys for project__user_file
-    existing_fks = inspector.get_foreign_keys("project__user_file")
-
-    has_user_file_fk = any(
-        fk.get("referred_table") == "user_file"
-        and fk.get("constrained_columns") == ["user_file_id"]
-        for fk in existing_fks
-    )
-
-    if not has_user_file_fk:
-        op.create_foreign_key(
-            "fk_project__user_file_user_file_id",
-            "project__user_file",
-            "user_file",
-            ["user_file_id"],
-            ["id"],
-        )
-        logger.info("Added project__user_file -> user_file foreign key")
-
-    has_project_fk = any(
-        fk.get("referred_table") == "user_project"
-        and fk.get("constrained_columns") == ["project_id"]
-        for fk in existing_fks
-    )
-
-    if not has_project_fk:
-        op.create_foreign_key(
-            "fk_project__user_file_project_id",
-            "project__user_file",
-            "user_project",
-            ["project_id"],
-            ["id"],
-        )
-        logger.info("Added project__user_file -> user_project foreign key")
-
-    # === Step 4: Mark files for document_id migration ===
-    logger.info("Marking files for background document_id migration...")
-
-    logger.info("Migration 4 (UUID primary key swap) completed successfully")
-    logger.info(
-        "NOTE: Background task will update document IDs in Vespa and search_doc"
-    )
-
-
-def downgrade() -> None:
-    """Revert UUID primary key back to integer (data destructive!)."""
-
-    logger.error("CRITICAL: Downgrading UUID primary key swap is data destructive!")
-    logger.error(
-        "This will break all UUID-based references created after the migration."
-    )
-    logger.error("Only proceed if absolutely necessary and have backups.")
-
-    # The downgrade would need to:
-    # 1. Add back integer columns
-    # 2. Generate new sequential IDs
-    # 3. Update all foreign key references
-    # 4. Swap primary keys back
-    # This is complex and risky, so we raise an error instead
-
-    raise NotImplementedError(
-        "Downgrade of UUID primary key swap is not supported due to data loss risk. "
-        "Manual intervention with data backup/restore is required."
-    )

backend/alembic/versions/7ed603b64d5a_add_mcp_server_and_connection_config_.py

@@ -1,249 +0,0 @@
-"""add_mcp_server_and_connection_config_models
-
-Revision ID: 7ed603b64d5a
-Revises: b329d00a9ea6
-Create Date: 2025-07-28 17:35:59.900680
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-from onyx.db.enums import MCPAuthenticationType
-
-# revision identifiers, used by Alembic.
-revision = "7ed603b64d5a"
-down_revision = "b329d00a9ea6"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """Create tables and columns for MCP Server support"""
-
-    # 1. MCP Server main table (no FK constraints yet to avoid circular refs)
-    op.create_table(
-        "mcp_server",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("owner", sa.String(), nullable=False),
-        sa.Column("name", sa.String(), nullable=False),
-        sa.Column("description", sa.String(), nullable=True),
-        sa.Column("server_url", sa.String(), nullable=False),
-        sa.Column(
-            "auth_type",
-            sa.Enum(
-                MCPAuthenticationType,
-                name="mcp_authentication_type",
-                native_enum=False,
-            ),
-            nullable=False,
-        ),
-        sa.Column("admin_connection_config_id", sa.Integer(), nullable=True),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),  # type: ignore
-            nullable=False,
-        ),
-        sa.Column(
-            "updated_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),  # type: ignore
-            nullable=False,
-        ),
-    )
-
-    # 2. MCP Connection Config table (can reference mcp_server now that it exists)
-    op.create_table(
-        "mcp_connection_config",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("mcp_server_id", sa.Integer(), nullable=True),
-        sa.Column("user_email", sa.String(), nullable=False, default=""),
-        sa.Column("config", sa.LargeBinary(), nullable=False),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),  # type: ignore
-            nullable=False,
-        ),
-        sa.Column(
-            "updated_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),  # type: ignore
-            nullable=False,
-        ),
-        sa.ForeignKeyConstraint(
-            ["mcp_server_id"], ["mcp_server.id"], ondelete="CASCADE"
-        ),
-    )
-
-    # Helpful indexes
-    op.create_index(
-        "ix_mcp_connection_config_server_user",
-        "mcp_connection_config",
-        ["mcp_server_id", "user_email"],
-    )
-    op.create_index(
-        "ix_mcp_connection_config_user_email",
-        "mcp_connection_config",
-        ["user_email"],
-    )
-
-    # 3. Add the back-references from mcp_server to connection configs
-    op.create_foreign_key(
-        "mcp_server_admin_config_fk",
-        "mcp_server",
-        "mcp_connection_config",
-        ["admin_connection_config_id"],
-        ["id"],
-        ondelete="SET NULL",
-    )
-
-    # 4. Association / access-control tables
-    op.create_table(
-        "mcp_server__user",
-        sa.Column("mcp_server_id", sa.Integer(), primary_key=True),
-        sa.Column("user_id", sa.UUID(), primary_key=True),
-        sa.ForeignKeyConstraint(
-            ["mcp_server_id"], ["mcp_server.id"], ondelete="CASCADE"
-        ),
-        sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
-    )
-
-    op.create_table(
-        "mcp_server__user_group",
-        sa.Column("mcp_server_id", sa.Integer(), primary_key=True),
-        sa.Column("user_group_id", sa.Integer(), primary_key=True),
-        sa.ForeignKeyConstraint(
-            ["mcp_server_id"], ["mcp_server.id"], ondelete="CASCADE"
-        ),
-        sa.ForeignKeyConstraint(["user_group_id"], ["user_group.id"]),
-    )
-
-    # 5. Update existing `tool` table – allow tools to belong to an MCP server
-    op.add_column(
-        "tool",
-        sa.Column("mcp_server_id", sa.Integer(), nullable=True),
-    )
-    # Add column for MCP tool input schema
-    op.add_column(
-        "tool",
-        sa.Column("mcp_input_schema", postgresql.JSONB(), nullable=True),
-    )
-    op.create_foreign_key(
-        "tool_mcp_server_fk",
-        "tool",
-        "mcp_server",
-        ["mcp_server_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )
-
-    # 6. Update persona__tool foreign keys to cascade delete
-    # This ensures that when a tool is deleted (including via MCP server deletion),
-    # the corresponding persona__tool rows are also deleted
-    op.drop_constraint(
-        "persona__tool_tool_id_fkey", "persona__tool", type_="foreignkey"
-    )
-    op.drop_constraint(
-        "persona__tool_persona_id_fkey", "persona__tool", type_="foreignkey"
-    )
-
-    op.create_foreign_key(
-        "persona__tool_persona_id_fkey",
-        "persona__tool",
-        "persona",
-        ["persona_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )
-    op.create_foreign_key(
-        "persona__tool_tool_id_fkey",
-        "persona__tool",
-        "tool",
-        ["tool_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )
-
-    # 7. Update research_agent_iteration_sub_step foreign key to SET NULL on delete
-    # This ensures that when a tool is deleted, the sub_step_tool_id is set to NULL
-    # instead of causing a foreign key constraint violation
-    op.drop_constraint(
-        "research_agent_iteration_sub_step_sub_step_tool_id_fkey",
-        "research_agent_iteration_sub_step",
-        type_="foreignkey",
-    )
-    op.create_foreign_key(
-        "research_agent_iteration_sub_step_sub_step_tool_id_fkey",
-        "research_agent_iteration_sub_step",
-        "tool",
-        ["sub_step_tool_id"],
-        ["id"],
-        ondelete="SET NULL",
-    )
-
-
-def downgrade() -> None:
-    """Drop all MCP-related tables / columns"""
-
-    # # # 1. Drop FK & columns from tool
-    # op.drop_constraint("tool_mcp_server_fk", "tool", type_="foreignkey")
-    op.execute("DELETE FROM tool WHERE mcp_server_id IS NOT NULL")
-
-    op.drop_constraint(
-        "research_agent_iteration_sub_step_sub_step_tool_id_fkey",
-        "research_agent_iteration_sub_step",
-        type_="foreignkey",
-    )
-    op.create_foreign_key(
-        "research_agent_iteration_sub_step_sub_step_tool_id_fkey",
-        "research_agent_iteration_sub_step",
-        "tool",
-        ["sub_step_tool_id"],
-        ["id"],
-    )
-
-    # Restore original persona__tool foreign keys (without CASCADE)
-    op.drop_constraint(
-        "persona__tool_persona_id_fkey", "persona__tool", type_="foreignkey"
-    )
-    op.drop_constraint(
-        "persona__tool_tool_id_fkey", "persona__tool", type_="foreignkey"
-    )
-
-    op.create_foreign_key(
-        "persona__tool_persona_id_fkey",
-        "persona__tool",
-        "persona",
-        ["persona_id"],
-        ["id"],
-    )
-    op.create_foreign_key(
-        "persona__tool_tool_id_fkey",
-        "persona__tool",
-        "tool",
-        ["tool_id"],
-        ["id"],
-    )
-    op.drop_column("tool", "mcp_input_schema")
-    op.drop_column("tool", "mcp_server_id")
-
-    # 2. Drop association tables
-    op.drop_table("mcp_server__user_group")
-    op.drop_table("mcp_server__user")
-
-    # 3. Drop FK from mcp_server to connection configs
-    op.drop_constraint("mcp_server_admin_config_fk", "mcp_server", type_="foreignkey")
-
-    # 4. Drop connection config indexes & table
-    op.drop_index(
-        "ix_mcp_connection_config_user_email", table_name="mcp_connection_config"
-    )
-    op.drop_index(
-        "ix_mcp_connection_config_server_user", table_name="mcp_connection_config"
-    )
-    op.drop_table("mcp_connection_config")
-
-    # 5. Finally drop mcp_server table
-    op.drop_table("mcp_server")

backend/alembic/versions/8818cf73fa1a_drop_include_citations.py

@@ -1,38 +0,0 @@
-"""drop include citations
-
-Revision ID: 8818cf73fa1a
-Revises: 7ed603b64d5a
-Create Date: 2025-09-02 19:43:50.060680
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = "8818cf73fa1a"
-down_revision = "7ed603b64d5a"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.drop_column("prompt", "include_citations")
-
-
-def downgrade() -> None:
-    op.add_column(
-        "prompt",
-        sa.Column(
-            "include_citations",
-            sa.BOOLEAN(),
-            autoincrement=False,
-            nullable=True,
-        ),
-    )
-    # Set include_citations based on prompt name: FALSE for ImageGeneration, TRUE for others
-    op.execute(
-        sa.text(
-            "UPDATE prompt SET include_citations = CASE WHEN name = 'ImageGeneration' THEN FALSE ELSE TRUE END"
-        )
-    )

backend/alembic/versions/90e3b9af7da4_tag_fix.py

@@ -1,341 +0,0 @@
-"""tag-fix
-
-Revision ID: 90e3b9af7da4
-Revises: 62c3a055a141
-Create Date: 2025-08-01 20:58:14.607624
-
-"""
-
-import json
-import logging
-import os
-
-from typing import cast
-from typing import Generator
-
-from alembic import op
-import sqlalchemy as sa
-
-from onyx.document_index.factory import get_default_document_index
-from onyx.document_index.vespa_constants import DOCUMENT_ID_ENDPOINT
-from onyx.db.search_settings import SearchSettings
-from onyx.configs.app_configs import AUTH_TYPE
-from onyx.configs.constants import AuthType
-from onyx.document_index.vespa.shared_utils.utils import get_vespa_http_client
-
-logger = logging.getLogger("alembic.runtime.migration")
-
-
-# revision identifiers, used by Alembic.
-revision = "90e3b9af7da4"
-down_revision = "62c3a055a141"
-branch_labels = None
-depends_on = None
-
-SKIP_TAG_FIX = os.environ.get("SKIP_TAG_FIX", "true").lower() == "true"
-
-# override for cloud
-if AUTH_TYPE == AuthType.CLOUD:
-    SKIP_TAG_FIX = True
-
-
-def set_is_list_for_known_tags() -> None:
-    """
-    Sets is_list to true for all tags that are known to be lists.
-    """
-    LIST_METADATA: list[tuple[str, str]] = [
-        ("CLICKUP", "tags"),
-        ("CONFLUENCE", "labels"),
-        ("DISCOURSE", "tags"),
-        ("FRESHDESK", "emails"),
-        ("GITHUB", "assignees"),
-        ("GITHUB", "labels"),
-        ("GURU", "tags"),
-        ("GURU", "folders"),
-        ("HUBSPOT", "associated_contact_ids"),
-        ("HUBSPOT", "associated_company_ids"),
-        ("HUBSPOT", "associated_deal_ids"),
-        ("HUBSPOT", "associated_ticket_ids"),
-        ("JIRA", "labels"),
-        ("MEDIAWIKI", "categories"),
-        ("ZENDESK", "labels"),
-        ("ZENDESK", "content_tags"),
-    ]
-
-    bind = op.get_bind()
-    for source, key in LIST_METADATA:
-        bind.execute(
-            sa.text(
-                f"""
-                UPDATE tag
-                SET is_list = true
-                WHERE tag_key = '{key}'
-                AND source = '{source}'
-                """
-            )
-        )
-
-
-def set_is_list_for_list_tags() -> None:
-    """
-    Sets is_list to true for all tags which have multiple values for a given
-    document, key, and source triplet. This only works if we remove old tags
-    from the database.
-    """
-    bind = op.get_bind()
-    bind.execute(
-        sa.text(
-            """
-            UPDATE tag
-            SET is_list = true
-            FROM (
-                SELECT DISTINCT tag.tag_key, tag.source
-                FROM tag
-                JOIN document__tag ON tag.id = document__tag.tag_id
-                GROUP BY tag.tag_key, tag.source, document__tag.document_id
-                HAVING count(*) > 1
-            ) AS list_tags
-            WHERE tag.tag_key = list_tags.tag_key
-            AND tag.source = list_tags.source
-            """
-        )
-    )
-
-
-def log_list_tags() -> None:
-    bind = op.get_bind()
-    result = bind.execute(
-        sa.text(
-            """
-            SELECT DISTINCT source, tag_key
-            FROM tag
-            WHERE is_list
-            ORDER BY source, tag_key
-            """
-        )
-    ).fetchall()
-    logger.info(
-        "List tags:\n" + "\n".join(f"  {source}: {key}" for source, key in result)
-    )
-
-
-def remove_old_tags() -> None:
-    """
-    Removes old tags from the database.
-    Previously, there was a bug where if a document got indexed with a tag and then
-    the document got reindexed, the old tag would not be removed.
-    This function removes those old tags by comparing it against the tags in vespa.
-    """
-    current_search_settings, future_search_settings = active_search_settings()
-    document_index = get_default_document_index(
-        current_search_settings, future_search_settings
-    )
-
-    # Get the index name
-    if hasattr(document_index, "index_name"):
-        index_name = document_index.index_name
-    else:
-        # Default index name if we can't get it from the document_index
-        index_name = "danswer_index"
-
-    for batch in _get_batch_documents_with_multiple_tags():
-        n_deleted = 0
-
-        for document_id in batch:
-            true_metadata = _get_vespa_metadata(document_id, index_name)
-            tags = _get_document_tags(document_id)
-
-            # identify document__tags to delete
-            to_delete: list[str] = []
-            for tag_id, tag_key, tag_value in tags:
-                true_val = true_metadata.get(tag_key, "")
-                if (isinstance(true_val, list) and tag_value not in true_val) or (
-                    isinstance(true_val, str) and tag_value != true_val
-                ):
-                    to_delete.append(str(tag_id))
-
-            if not to_delete:
-                continue
-
-            # delete old document__tags
-            bind = op.get_bind()
-            result = bind.execute(
-                sa.text(
-                    f"""
-                    DELETE FROM document__tag
-                    WHERE document_id = '{document_id}'
-                    AND tag_id IN ({','.join(to_delete)})
-                    """
-                )
-            )
-            n_deleted += result.rowcount
-        logger.info(f"Processed {len(batch)} documents and deleted {n_deleted} tags")
-
-
-def active_search_settings() -> tuple[SearchSettings, SearchSettings | None]:
-    result = op.get_bind().execute(
-        sa.text(
-            """
-        SELECT * FROM search_settings WHERE status = 'PRESENT' ORDER BY id DESC LIMIT 1
-        """
-        )
-    )
-    search_settings_fetch = result.fetchall()
-    search_settings = (
-        SearchSettings(**search_settings_fetch[0]._asdict())
-        if search_settings_fetch
-        else None
-    )
-
-    result2 = op.get_bind().execute(
-        sa.text(
-            """
-        SELECT * FROM search_settings WHERE status = 'FUTURE' ORDER BY id DESC LIMIT 1
-        """
-        )
-    )
-    search_settings_future_fetch = result2.fetchall()
-    search_settings_future = (
-        SearchSettings(**search_settings_future_fetch[0]._asdict())
-        if search_settings_future_fetch
-        else None
-    )
-
-    if not isinstance(search_settings, SearchSettings):
-        raise RuntimeError(
-            "current search settings is of type " + str(type(search_settings))
-        )
-    if (
-        not isinstance(search_settings_future, SearchSettings)
-        and search_settings_future is not None
-    ):
-        raise RuntimeError(
-            "future search settings is of type " + str(type(search_settings_future))
-        )
-
-    return search_settings, search_settings_future
-
-
-def _get_batch_documents_with_multiple_tags(
-    batch_size: int = 128,
-) -> Generator[list[str], None, None]:
-    """
-    Returns a list of document ids which contain a one to many tag.
-    The document may either contain a list metadata value, or may contain leftover
-    old tags from reindexing.
-    """
-    offset_clause = ""
-    bind = op.get_bind()
-
-    while True:
-        batch = bind.execute(
-            sa.text(
-                f"""
-                SELECT DISTINCT document__tag.document_id
-                FROM tag
-                JOIN document__tag ON tag.id = document__tag.tag_id
-                GROUP BY tag.tag_key, tag.source, document__tag.document_id
-                HAVING count(*) > 1 {offset_clause}
-                ORDER BY document__tag.document_id
-                LIMIT {batch_size}
-                """
-            )
-        ).fetchall()
-        if not batch:
-            break
-        doc_ids = [document_id for document_id, in batch]
-        yield doc_ids
-        offset_clause = f"AND document__tag.document_id > '{doc_ids[-1]}'"
-
-
-def _get_vespa_metadata(
-    document_id: str, index_name: str
-) -> dict[str, str | list[str]]:
-    url = DOCUMENT_ID_ENDPOINT.format(index_name=index_name)
-
-    # Document-Selector language
-    selection = (
-        f"{index_name}.document_id=='{document_id}' and {index_name}.chunk_id==0"
-    )
-
-    params: dict[str, str | int] = {
-        "selection": selection,
-        "wantedDocumentCount": 1,
-        "fieldSet": f"{index_name}:metadata",
-    }
-
-    with get_vespa_http_client() as client:
-        resp = client.get(url, params=params)
-        resp.raise_for_status()
-
-    docs = resp.json().get("documents", [])
-    if not docs:
-        raise RuntimeError(f"No chunk-0 found for document {document_id}")
-
-    # for some reason, metadata is a string
-    metadata = docs[0]["fields"]["metadata"]
-    return json.loads(metadata)
-
-
-def _get_document_tags(document_id: str) -> list[tuple[int, str, str]]:
-    bind = op.get_bind()
-    result = bind.execute(
-        sa.text(
-            f"""
-            SELECT tag.id, tag.tag_key, tag.tag_value
-            FROM tag
-            JOIN document__tag ON tag.id = document__tag.tag_id
-            WHERE document__tag.document_id = '{document_id}'
-            """
-        )
-    ).fetchall()
-    return cast(list[tuple[int, str, str]], result)
-
-
-def upgrade() -> None:
-    op.add_column(
-        "tag",
-        sa.Column("is_list", sa.Boolean(), nullable=False, server_default="false"),
-    )
-    op.drop_constraint(
-        constraint_name="_tag_key_value_source_uc",
-        table_name="tag",
-        type_="unique",
-    )
-    op.create_unique_constraint(
-        constraint_name="_tag_key_value_source_list_uc",
-        table_name="tag",
-        columns=["tag_key", "tag_value", "source", "is_list"],
-    )
-    set_is_list_for_known_tags()
-
-    if SKIP_TAG_FIX:
-        logger.warning(
-            "Skipping removal of old tags. "
-            "This can cause issues when using the knowledge graph, or "
-            "when filtering for documents by tags."
-        )
-        log_list_tags()
-        return
-
-    remove_old_tags()
-    set_is_list_for_list_tags()
-
-    # debug
-    log_list_tags()
-
-
-def downgrade() -> None:
-    # the migration adds and populates the is_list column, and removes old bugged tags
-    # there isn't a point in adding back the bugged tags, so we just drop the column
-    op.drop_constraint(
-        constraint_name="_tag_key_value_source_list_uc",
-        table_name="tag",
-        type_="unique",
-    )
-    op.create_unique_constraint(
-        constraint_name="_tag_key_value_source_uc",
-        table_name="tag",
-        columns=["tag_key", "tag_value", "source"],
-    )
-    op.drop_column("tag", "is_list")

backend/alembic/versions/9b66d3156fc6_user_file_schema_additions.py

@@ -1,257 +0,0 @@
-"""Migration 1: User file schema additions
-
-Revision ID: 9b66d3156fc6
-Revises: b4ef3ae0bf6e
-Create Date: 2025-09-22 09:42:06.086732
-
-This migration adds new columns and tables without modifying existing data.
-It is safe to run and can be easily rolled back.
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql as psql
-import logging
-
-logger = logging.getLogger("alembic.runtime.migration")
-# revision identifiers, used by Alembic.
-revision = "9b66d3156fc6"
-down_revision = "b4ef3ae0bf6e"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """Add new columns and tables without modifying existing data."""
-
-    # Enable pgcrypto for UUID generation
-    op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    # === USER_FILE: Add new columns ===
-    logger.info("Adding new columns to user_file table...")
-
-    user_file_columns = [col["name"] for col in inspector.get_columns("user_file")]
-
-    # Check if ID is already UUID (in case of re-run after partial migration)
-    id_is_uuid = any(
-        col["name"] == "id" and "uuid" in str(col["type"]).lower()
-        for col in inspector.get_columns("user_file")
-    )
-
-    # Add transitional UUID column only if ID is not already UUID
-    if "new_id" not in user_file_columns and not id_is_uuid:
-        op.add_column(
-            "user_file",
-            sa.Column(
-                "new_id",
-                psql.UUID(as_uuid=True),
-                nullable=True,
-                server_default=sa.text("gen_random_uuid()"),
-            ),
-        )
-        op.create_unique_constraint("uq_user_file_new_id", "user_file", ["new_id"])
-        logger.info("Added new_id column to user_file")
-
-    # Add status column
-    if "status" not in user_file_columns:
-        op.add_column(
-            "user_file",
-            sa.Column(
-                "status",
-                sa.Enum(
-                    "PROCESSING",
-                    "COMPLETED",
-                    "FAILED",
-                    "CANCELED",
-                    name="userfilestatus",
-                    native_enum=False,
-                ),
-                nullable=False,
-                server_default="PROCESSING",
-            ),
-        )
-        logger.info("Added status column to user_file")
-
-    # Add other tracking columns
-    if "chunk_count" not in user_file_columns:
-        op.add_column(
-            "user_file", sa.Column("chunk_count", sa.Integer(), nullable=True)
-        )
-        logger.info("Added chunk_count column to user_file")
-
-    if "last_accessed_at" not in user_file_columns:
-        op.add_column(
-            "user_file",
-            sa.Column("last_accessed_at", sa.DateTime(timezone=True), nullable=True),
-        )
-        logger.info("Added last_accessed_at column to user_file")
-
-    if "needs_project_sync" not in user_file_columns:
-        op.add_column(
-            "user_file",
-            sa.Column(
-                "needs_project_sync",
-                sa.Boolean(),
-                nullable=False,
-                server_default=sa.text("false"),
-            ),
-        )
-        logger.info("Added needs_project_sync column to user_file")
-
-    if "last_project_sync_at" not in user_file_columns:
-        op.add_column(
-            "user_file",
-            sa.Column(
-                "last_project_sync_at", sa.DateTime(timezone=True), nullable=True
-            ),
-        )
-        logger.info("Added last_project_sync_at column to user_file")
-
-    if "document_id_migrated" not in user_file_columns:
-        op.add_column(
-            "user_file",
-            sa.Column(
-                "document_id_migrated",
-                sa.Boolean(),
-                nullable=False,
-                server_default=sa.text("true"),
-            ),
-        )
-        logger.info("Added document_id_migrated column to user_file")
-
-    # === USER_FOLDER -> USER_PROJECT rename ===
-    table_names = set(inspector.get_table_names())
-
-    if "user_folder" in table_names:
-        logger.info("Updating user_folder table...")
-        # Make description nullable first
-        op.alter_column("user_folder", "description", nullable=True)
-
-        # Rename table if user_project doesn't exist
-        if "user_project" not in table_names:
-            op.execute("ALTER TABLE user_folder RENAME TO user_project")
-            logger.info("Renamed user_folder to user_project")
-    elif "user_project" in table_names:
-        # If already renamed, ensure column nullability
-        project_cols = [col["name"] for col in inspector.get_columns("user_project")]
-        if "description" in project_cols:
-            op.alter_column("user_project", "description", nullable=True)
-
-    # Add instructions column to user_project
-    inspector = sa.inspect(bind)  # Refresh after rename
-    if "user_project" in inspector.get_table_names():
-        project_columns = [col["name"] for col in inspector.get_columns("user_project")]
-        if "instructions" not in project_columns:
-            op.add_column(
-                "user_project",
-                sa.Column("instructions", sa.String(), nullable=True),
-            )
-            logger.info("Added instructions column to user_project")
-
-    # === CHAT_SESSION: Add project_id ===
-    chat_session_columns = [
-        col["name"] for col in inspector.get_columns("chat_session")
-    ]
-    if "project_id" not in chat_session_columns:
-        op.add_column(
-            "chat_session",
-            sa.Column("project_id", sa.Integer(), nullable=True),
-        )
-        logger.info("Added project_id column to chat_session")
-
-    # === PERSONA__USER_FILE: Add UUID column ===
-    persona_user_file_columns = [
-        col["name"] for col in inspector.get_columns("persona__user_file")
-    ]
-    if "user_file_id_uuid" not in persona_user_file_columns:
-        op.add_column(
-            "persona__user_file",
-            sa.Column("user_file_id_uuid", psql.UUID(as_uuid=True), nullable=True),
-        )
-        logger.info("Added user_file_id_uuid column to persona__user_file")
-
-    # === PROJECT__USER_FILE: Create new table ===
-    if "project__user_file" not in inspector.get_table_names():
-        op.create_table(
-            "project__user_file",
-            sa.Column("project_id", sa.Integer(), nullable=False),
-            sa.Column("user_file_id", psql.UUID(as_uuid=True), nullable=False),
-            sa.PrimaryKeyConstraint("project_id", "user_file_id"),
-        )
-        op.create_index(
-            "idx_project__user_file_user_file_id",
-            "project__user_file",
-            ["user_file_id"],
-        )
-        logger.info("Created project__user_file table")
-
-    logger.info("Migration 1 (schema additions) completed successfully")
-
-
-def downgrade() -> None:
-    """Remove added columns and tables."""
-
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-
-    logger.info("Starting downgrade of schema additions...")
-
-    # Drop project__user_file table
-    if "project__user_file" in inspector.get_table_names():
-        op.drop_index("idx_project__user_file_user_file_id", "project__user_file")
-        op.drop_table("project__user_file")
-        logger.info("Dropped project__user_file table")
-
-    # Remove columns from persona__user_file
-    if "persona__user_file" in inspector.get_table_names():
-        columns = [col["name"] for col in inspector.get_columns("persona__user_file")]
-        if "user_file_id_uuid" in columns:
-            op.drop_column("persona__user_file", "user_file_id_uuid")
-            logger.info("Dropped user_file_id_uuid from persona__user_file")
-
-    # Remove columns from chat_session
-    if "chat_session" in inspector.get_table_names():
-        columns = [col["name"] for col in inspector.get_columns("chat_session")]
-        if "project_id" in columns:
-            op.drop_column("chat_session", "project_id")
-            logger.info("Dropped project_id from chat_session")
-
-    # Rename user_project back to user_folder and remove instructions
-    if "user_project" in inspector.get_table_names():
-        columns = [col["name"] for col in inspector.get_columns("user_project")]
-        if "instructions" in columns:
-            op.drop_column("user_project", "instructions")
-        op.execute("ALTER TABLE user_project RENAME TO user_folder")
-        op.alter_column("user_folder", "description", nullable=False)
-        logger.info("Renamed user_project back to user_folder")
-
-    # Remove columns from user_file
-    if "user_file" in inspector.get_table_names():
-        columns = [col["name"] for col in inspector.get_columns("user_file")]
-
-        columns_to_drop = [
-            "document_id_migrated",
-            "last_project_sync_at",
-            "needs_project_sync",
-            "last_accessed_at",
-            "chunk_count",
-            "status",
-        ]
-
-        for col in columns_to_drop:
-            if col in columns:
-                op.drop_column("user_file", col)
-                logger.info(f"Dropped {col} from user_file")
-
-        if "new_id" in columns:
-            op.drop_constraint("uq_user_file_new_id", "user_file", type_="unique")
-            op.drop_column("user_file", "new_id")
-            logger.info("Dropped new_id from user_file")
-
-    # Drop enum type if no columns use it
-    bind.execute(sa.text("DROP TYPE IF EXISTS userfilestatus"))
-
-    logger.info("Downgrade completed successfully")

backend/alembic/versions/abbfec3a5ac5_merge_prompt_into_persona.py

@@ -1,225 +0,0 @@
-"""merge prompt into persona
-
-Revision ID: abbfec3a5ac5
-Revises: 8818cf73fa1a
-Create Date: 2024-12-19 12:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = "abbfec3a5ac5"
-down_revision = "8818cf73fa1a"
-branch_labels = None
-depends_on = None
-
-
-MAX_PROMPT_LENGTH = 5_000_000
-
-
-def upgrade() -> None:
-    """NOTE: Prompts without any Personas will just be lost."""
-    # Step 1: Add new columns to persona table (only if they don't exist)
-
-    # Check if columns exist before adding them
-    connection = op.get_bind()
-    inspector = sa.inspect(connection)
-    existing_columns = [col["name"] for col in inspector.get_columns("persona")]
-
-    if "system_prompt" not in existing_columns:
-        op.add_column(
-            "persona",
-            sa.Column(
-                "system_prompt", sa.String(length=MAX_PROMPT_LENGTH), nullable=True
-            ),
-        )
-
-    if "task_prompt" not in existing_columns:
-        op.add_column(
-            "persona",
-            sa.Column(
-                "task_prompt", sa.String(length=MAX_PROMPT_LENGTH), nullable=True
-            ),
-        )
-
-    if "datetime_aware" not in existing_columns:
-        op.add_column(
-            "persona",
-            sa.Column(
-                "datetime_aware", sa.Boolean(), nullable=False, server_default="true"
-            ),
-        )
-
-    # Step 2: Migrate data from prompt table to persona table (only if tables exist)
-    existing_tables = inspector.get_table_names()
-
-    if "prompt" in existing_tables and "persona__prompt" in existing_tables:
-        # For personas that have associated prompts, copy the prompt data
-        op.execute(
-            """
-            UPDATE persona
-            SET
-                system_prompt = p.system_prompt,
-                task_prompt = p.task_prompt,
-                datetime_aware = p.datetime_aware
-            FROM (
-                -- Get the first prompt for each persona (in case there are multiple)
-                SELECT DISTINCT ON (pp.persona_id)
-                    pp.persona_id,
-                    pr.system_prompt,
-                    pr.task_prompt,
-                    pr.datetime_aware
-                FROM persona__prompt pp
-                JOIN prompt pr ON pp.prompt_id = pr.id
-            ) p
-            WHERE persona.id = p.persona_id
-        """
-        )
-
-        # Step 3: Update chat_message references
-        # Since chat messages referenced prompt_id, we need to update them to use persona_id
-        # This is complex as we need to map from prompt_id to persona_id
-
-        # Check if chat_message has prompt_id column
-        chat_message_columns = [
-            col["name"] for col in inspector.get_columns("chat_message")
-        ]
-        if "prompt_id" in chat_message_columns:
-            op.execute(
-                """
-                ALTER TABLE chat_message
-                DROP CONSTRAINT IF EXISTS chat_message__prompt_fk
-            """
-            )
-            op.drop_column("chat_message", "prompt_id")
-
-    # Step 4: Handle personas without prompts - set default values if needed (always run this)
-    op.execute(
-        """
-        UPDATE persona
-        SET
-            system_prompt = COALESCE(system_prompt, ''),
-            task_prompt = COALESCE(task_prompt, '')
-        WHERE system_prompt IS NULL OR task_prompt IS NULL
-    """
-    )
-
-    # Step 5: Drop the persona__prompt association table (if it exists)
-    if "persona__prompt" in existing_tables:
-        op.drop_table("persona__prompt")
-
-    # Step 6: Drop the prompt table (if it exists)
-    if "prompt" in existing_tables:
-        op.drop_table("prompt")
-
-    # Step 7: Make system_prompt and task_prompt non-nullable after migration (only if they exist)
-    op.alter_column(
-        "persona",
-        "system_prompt",
-        existing_type=sa.String(length=MAX_PROMPT_LENGTH),
-        nullable=False,
-        server_default=None,
-    )
-
-    op.alter_column(
-        "persona",
-        "task_prompt",
-        existing_type=sa.String(length=MAX_PROMPT_LENGTH),
-        nullable=False,
-        server_default=None,
-    )
-
-
-def downgrade() -> None:
-    # Step 1: Recreate the prompt table
-    op.create_table(
-        "prompt",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=True),
-        sa.Column("name", sa.String(), nullable=False),
-        sa.Column("description", sa.String(), nullable=False),
-        sa.Column("system_prompt", sa.String(length=MAX_PROMPT_LENGTH), nullable=False),
-        sa.Column("task_prompt", sa.String(length=MAX_PROMPT_LENGTH), nullable=False),
-        sa.Column(
-            "datetime_aware", sa.Boolean(), nullable=False, server_default="true"
-        ),
-        sa.Column(
-            "default_prompt", sa.Boolean(), nullable=False, server_default="false"
-        ),
-        sa.Column("deleted", sa.Boolean(), nullable=False, server_default="false"),
-        sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("id"),
-    )
-
-    # Step 2: Recreate the persona__prompt association table
-    op.create_table(
-        "persona__prompt",
-        sa.Column("persona_id", sa.Integer(), nullable=False),
-        sa.Column("prompt_id", sa.Integer(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["persona_id"],
-            ["persona.id"],
-        ),
-        sa.ForeignKeyConstraint(
-            ["prompt_id"],
-            ["prompt.id"],
-        ),
-        sa.PrimaryKeyConstraint("persona_id", "prompt_id"),
-    )
-
-    # Step 3: Migrate data back from persona to prompt table
-    op.execute(
-        """
-        INSERT INTO prompt (
-            name,
-            description,
-            system_prompt,
-            task_prompt,
-            datetime_aware,
-            default_prompt,
-            deleted,
-            user_id
-        )
-        SELECT
-            CONCAT('Prompt for ', name),
-            description,
-            system_prompt,
-            task_prompt,
-            datetime_aware,
-            is_default_persona,
-            deleted,
-            user_id
-        FROM persona
-        WHERE system_prompt IS NOT NULL AND system_prompt != ''
-        RETURNING id, name
-        """
-    )
-
-    # Step 4: Re-establish persona__prompt relationships
-    op.execute(
-        """
-        INSERT INTO persona__prompt (persona_id, prompt_id)
-        SELECT
-            p.id as persona_id,
-            pr.id as prompt_id
-        FROM persona p
-        JOIN prompt pr ON pr.name = CONCAT('Prompt for ', p.name)
-        WHERE p.system_prompt IS NOT NULL AND p.system_prompt != ''
-    """
-    )
-
-    # Step 5: Add prompt_id column back to chat_message
-    op.add_column("chat_message", sa.Column("prompt_id", sa.Integer(), nullable=True))
-
-    # Step 6: Re-establish foreign key constraint
-    op.create_foreign_key(
-        "chat_message__prompt_fk", "chat_message", "prompt", ["prompt_id"], ["id"]
-    )
-
-    # Step 7: Remove columns from persona table
-    op.drop_column("persona", "datetime_aware")
-    op.drop_column("persona", "task_prompt")
-    op.drop_column("persona", "system_prompt")

backend/alembic/versions/b30353be4eec_add_mcp_auth_performer.py

@@ -1,123 +0,0 @@
-"""add_mcp_auth_performer
-
-Revision ID: b30353be4eec
-Revises: 2b75d0a8ffcb
-Create Date: 2025-09-13 14:58:08.413534
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from onyx.db.enums import MCPAuthenticationPerformer, MCPTransport
-
-
-# revision identifiers, used by Alembic.
-revision = "b30353be4eec"
-down_revision = "2b75d0a8ffcb"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """moving to a better way of handling auth performer and transport"""
-    # Add nullable column first for backward compatibility
-    op.add_column(
-        "mcp_server",
-        sa.Column(
-            "auth_performer",
-            sa.Enum(MCPAuthenticationPerformer, native_enum=False),
-            nullable=True,
-        ),
-    )
-
-    op.add_column(
-        "mcp_server",
-        sa.Column(
-            "transport",
-            sa.Enum(MCPTransport, native_enum=False),
-            nullable=True,
-        ),
-    )
-
-    # # Backfill values using existing data and inference rules
-    bind = op.get_bind()
-
-    # 1) OAUTH servers are always PER_USER
-    bind.execute(
-        sa.text(
-            """
-        UPDATE mcp_server
-        SET auth_performer = 'PER_USER'
-        WHERE auth_type = 'OAUTH'
-        """
-        )
-    )
-
-    # 2) If there is no admin connection config, mark as ADMIN (and not set yet)
-    bind.execute(
-        sa.text(
-            """
-        UPDATE mcp_server
-        SET auth_performer = 'ADMIN'
-        WHERE admin_connection_config_id IS NULL
-          AND auth_performer IS NULL
-        """
-        )
-    )
-
-    # 3) If there exists any user-specific connection config (user_email != ''), mark as PER_USER
-    bind.execute(
-        sa.text(
-            """
-        UPDATE mcp_server AS ms
-        SET auth_performer = 'PER_USER'
-        FROM mcp_connection_config AS mcc
-        WHERE mcc.mcp_server_id = ms.id
-          AND COALESCE(mcc.user_email, '') <> ''
-          AND ms.auth_performer IS NULL
-        """
-        )
-    )
-
-    # 4) Default any remaining nulls to ADMIN (covers API_TOKEN admin-managed and NONE)
-    bind.execute(
-        sa.text(
-            """
-        UPDATE mcp_server
-        SET auth_performer = 'ADMIN'
-        WHERE auth_performer IS NULL
-        """
-        )
-    )
-
-    # Finally, make the column non-nullable
-    op.alter_column(
-        "mcp_server",
-        "auth_performer",
-        existing_type=sa.Enum(MCPAuthenticationPerformer, native_enum=False),
-        nullable=False,
-    )
-
-    # Backfill transport for existing rows to STREAMABLE_HTTP, then make non-nullable
-    bind.execute(
-        sa.text(
-            """
-        UPDATE mcp_server
-        SET transport = 'STREAMABLE_HTTP'
-        WHERE transport IS NULL
-        """
-        )
-    )
-
-    op.alter_column(
-        "mcp_server",
-        "transport",
-        existing_type=sa.Enum(MCPTransport, native_enum=False),
-        nullable=False,
-    )
-
-
-def downgrade() -> None:
-    """remove cols"""
-    op.drop_column("mcp_server", "transport")
-    op.drop_column("mcp_server", "auth_performer")

backend/alembic/versions/b329d00a9ea6_adding_assistant_specific_user_.py

@@ -1,38 +0,0 @@
-"""Adding assistant-specific user preferences
-
-Revision ID: b329d00a9ea6
-Revises: f9b8c7d6e5a4
-Create Date: 2025-08-26 23:14:44.592985
-
-"""
-
-from alembic import op
-import fastapi_users_db_sqlalchemy
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = "b329d00a9ea6"
-down_revision = "f9b8c7d6e5a4"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "assistant__user_specific_config",
-        sa.Column("assistant_id", sa.Integer(), nullable=False),
-        sa.Column(
-            "user_id",
-            fastapi_users_db_sqlalchemy.generics.GUID(),
-            nullable=False,
-        ),
-        sa.Column("disabled_tool_ids", postgresql.ARRAY(sa.Integer()), nullable=False),
-        sa.ForeignKeyConstraint(["assistant_id"], ["persona.id"], ondelete="CASCADE"),
-        sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("assistant_id", "user_id"),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("assistant__user_specific_config")

backend/alembic/versions/b4ef3ae0bf6e_add_user_oauth_token_to_slack_bot.py

@@ -1,27 +0,0 @@
-"""add_user_oauth_token_to_slack_bot
-
-Revision ID: b4ef3ae0bf6e
-Revises: 505c488f6662
-Create Date: 2025-08-26 17:47:41.788462
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "b4ef3ae0bf6e"
-down_revision = "505c488f6662"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add user_token column to slack_bot table
-    op.add_column("slack_bot", sa.Column("user_token", sa.LargeBinary(), nullable=True))
-
-
-def downgrade() -> None:
-    # Remove user_token column from slack_bot table
-    op.drop_column("slack_bot", "user_token")

backend/alembic/versions/b558f51620b4_pause_finished_user_file_connectors.py

@@ -1,33 +0,0 @@
-"""Pause finished user file connectors
-
-Revision ID: b558f51620b4
-Revises: 90e3b9af7da4
-Create Date: 2025-08-15 17:17:02.456704
-
-"""
-
-from alembic import op
-
-
-# revision identifiers, used by Alembic.
-revision = "b558f51620b4"
-down_revision = "90e3b9af7da4"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Set all user file connector credential pairs with ACTIVE status to PAUSED
-    # This ensures user files don't continue to run indexing tasks after processing
-    op.execute(
-        """
-        UPDATE connector_credential_pair
-        SET status = 'PAUSED'
-        WHERE is_user_file = true
-        AND status = 'ACTIVE'
-        """
-    )
-
-
-def downgrade() -> None:
-    pass

backend/alembic/versions/b7ec9b5b505f_adjust_prompt_length.py

@@ -1,43 +0,0 @@
-"""adjust prompt length
-
-Revision ID: b7ec9b5b505f
-Revises: abbfec3a5ac5
-Create Date: 2025-09-10 18:51:15.629197
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "b7ec9b5b505f"
-down_revision = "abbfec3a5ac5"
-branch_labels = None
-depends_on = None
-
-
-MAX_PROMPT_LENGTH = 5_000_000
-
-
-def upgrade() -> None:
-    # NOTE: need to run this since the previous migration PREVIOUSLY set the length to 8000
-    op.alter_column(
-        "persona",
-        "system_prompt",
-        existing_type=sa.String(length=8000),
-        type_=sa.String(length=MAX_PROMPT_LENGTH),
-        existing_nullable=False,
-    )
-    op.alter_column(
-        "persona",
-        "task_prompt",
-        existing_type=sa.String(length=8000),
-        type_=sa.String(length=MAX_PROMPT_LENGTH),
-        existing_nullable=False,
-    )
-
-
-def downgrade() -> None:
-    # Downgrade not necessary
-    pass

backend/alembic/versions/bd7c3bf8beba_migrate_agent_responses_to_research_.py

@@ -1,147 +0,0 @@
-"""migrate_agent_sub_questions_to_research_iterations
-
-Revision ID: bd7c3bf8beba
-Revises: f8a9b2c3d4e5
-Create Date: 2025-08-18 11:33:27.098287
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "bd7c3bf8beba"
-down_revision = "f8a9b2c3d4e5"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Get connection to execute raw SQL
-    connection = op.get_bind()
-
-    # First, insert data into research_agent_iteration table
-    # This creates one iteration record per primary_question_id using the earliest time_created
-    connection.execute(
-        sa.text(
-            """
-            INSERT INTO research_agent_iteration (primary_question_id, created_at, iteration_nr, purpose, reasoning)
-            SELECT
-                primary_question_id,
-                MIN(time_created) as created_at,
-                1 as iteration_nr,
-                'Generating and researching subquestions' as purpose,
-                '(No previous reasoning)' as reasoning
-            FROM agent__sub_question
-            JOIN chat_message on agent__sub_question.primary_question_id = chat_message.id
-            WHERE primary_question_id IS NOT NULL
-                AND chat_message.is_agentic = true
-            GROUP BY primary_question_id
-            ON CONFLICT DO NOTHING;
-        """
-        )
-    )
-
-    # Then, insert data into research_agent_iteration_sub_step table
-    # This migrates each sub-question as a sub-step
-    connection.execute(
-        sa.text(
-            """
-            INSERT INTO research_agent_iteration_sub_step (
-                primary_question_id,
-                iteration_nr,
-                iteration_sub_step_nr,
-                created_at,
-                sub_step_instructions,
-                sub_step_tool_id,
-                sub_answer,
-                cited_doc_results
-            )
-            SELECT
-                primary_question_id,
-                1 as iteration_nr,
-                level_question_num as iteration_sub_step_nr,
-                time_created as created_at,
-                sub_question as sub_step_instructions,
-                1 as sub_step_tool_id,
-                sub_answer,
-                sub_question_doc_results as cited_doc_results
-            FROM agent__sub_question
-            JOIN chat_message on agent__sub_question.primary_question_id = chat_message.id
-            WHERE chat_message.is_agentic = true
-            AND primary_question_id IS NOT NULL
-            ON CONFLICT DO NOTHING;
-        """
-        )
-    )
-
-    # Update chat_message records: set legacy agentic type and answer purpose for existing agentic messages
-    connection.execute(
-        sa.text(
-            """
-            UPDATE chat_message
-            SET research_answer_purpose = 'ANSWER'
-            WHERE is_agentic = true
-            AND research_type IS NULL and
-                message_type = 'ASSISTANT';
-        """
-        )
-    )
-    connection.execute(
-        sa.text(
-            """
-            UPDATE chat_message
-            SET research_type = 'LEGACY_AGENTIC'
-            WHERE is_agentic = true
-            AND research_type IS NULL;
-        """
-        )
-    )
-
-
-def downgrade() -> None:
-    # Get connection to execute raw SQL
-    connection = op.get_bind()
-
-    # Note: This downgrade removes all research agent iteration data
-    # There's no way to perfectly restore the original agent__sub_question data
-    # if it was deleted after this migration
-
-    # Delete all research_agent_iteration_sub_step records that were migrated
-    connection.execute(
-        sa.text(
-            """
-            DELETE FROM research_agent_iteration_sub_step
-            USING chat_message
-            WHERE research_agent_iteration_sub_step.primary_question_id = chat_message.id
-            AND chat_message.research_type = 'LEGACY_AGENTIC';
-        """
-        )
-    )
-
-    # Delete all research_agent_iteration records that were migrated
-    connection.execute(
-        sa.text(
-            """
-            DELETE FROM research_agent_iteration
-            USING chat_message
-            WHERE research_agent_iteration.primary_question_id = chat_message.id
-            AND chat_message.research_type = 'LEGACY_AGENTIC';
-        """
-        )
-    )
-
-    # Revert chat_message updates: clear research fields for legacy agentic messages
-    connection.execute(
-        sa.text(
-            """
-            UPDATE chat_message
-            SET research_type = NULL,
-                research_answer_purpose = NULL
-            WHERE is_agentic = true
-            AND research_type = 'LEGACY_AGENTIC'
-            AND message_type = 'ASSISTANT';
-        """
-        )
-    )

backend/alembic/versions/c9e2cd766c29_add_s3_file_store_table.py

@@ -1,315 +0,0 @@
-"""modify_file_store_for_external_storage
-
-Revision ID: c9e2cd766c29
-Revises: 03bf8be6b53a
-Create Date: 2025-06-13 14:02:09.867679
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.orm import Session
-from sqlalchemy import text
-from typing import cast, Any
-
-from botocore.exceptions import ClientError
-
-from onyx.db._deprecated.pg_file_store import delete_lobj_by_id, read_lobj
-from onyx.file_store.file_store import get_s3_file_store
-from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
-
-# revision identifiers, used by Alembic.
-revision = "c9e2cd766c29"
-down_revision = "03bf8be6b53a"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    try:
-        # Modify existing file_store table to support external storage
-        op.rename_table("file_store", "file_record")
-
-        # Make lobj_oid nullable (for external storage files)
-        op.alter_column("file_record", "lobj_oid", nullable=True)
-
-        # Add external storage columns with generic names
-        op.add_column(
-            "file_record", sa.Column("bucket_name", sa.String(), nullable=True)
-        )
-        op.add_column(
-            "file_record", sa.Column("object_key", sa.String(), nullable=True)
-        )
-
-        # Add timestamps for tracking
-        op.add_column(
-            "file_record",
-            sa.Column(
-                "created_at",
-                sa.DateTime(timezone=True),
-                server_default=sa.func.now(),
-                nullable=False,
-            ),
-        )
-        op.add_column(
-            "file_record",
-            sa.Column(
-                "updated_at",
-                sa.DateTime(timezone=True),
-                server_default=sa.func.now(),
-                nullable=False,
-            ),
-        )
-
-        op.alter_column("file_record", "file_name", new_column_name="file_id")
-    except Exception as e:
-        if "does not exist" in str(e) or 'relation "file_store" does not exist' in str(
-            e
-        ):
-            print(
-                f"Ran into error - {e}. Likely means we had a partial success in the past, continuing..."
-            )
-        else:
-            raise
-
-    print(
-        "External storage configured - migrating files from PostgreSQL to external storage..."
-    )
-    # if we fail midway through this, we'll have a partial success. Running the migration
-    # again should allow us to continue.
-    _migrate_files_to_external_storage()
-    print("File migration completed successfully!")
-
-    # Remove lobj_oid column
-    op.drop_column("file_record", "lobj_oid")
-
-
-def downgrade() -> None:
-    """Revert schema changes and migrate files from external storage back to PostgreSQL large objects."""
-
-    print(
-        "Reverting to PostgreSQL-backed file store – migrating files from external storage …"
-    )
-
-    # 1. Ensure `lobj_oid` exists on the current `file_record` table (nullable for now).
-    op.add_column("file_record", sa.Column("lobj_oid", sa.Integer(), nullable=True))
-
-    # 2. Move content from external storage back into PostgreSQL large objects (table is still
-    #    called `file_record` so application code continues to work during the copy).
-    try:
-        _migrate_files_to_postgres()
-    except Exception:
-        print("Error during downgrade migration, rolling back …")
-        op.drop_column("file_record", "lobj_oid")
-        raise
-
-    # 3. After migration every row should now have `lobj_oid` populated – mark NOT NULL.
-    op.alter_column("file_record", "lobj_oid", nullable=False)
-
-    # 4. Remove columns that are only relevant to external storage.
-    op.drop_column("file_record", "updated_at")
-    op.drop_column("file_record", "created_at")
-    op.drop_column("file_record", "object_key")
-    op.drop_column("file_record", "bucket_name")
-
-    # 5. Rename `file_id` back to `file_name` (still on `file_record`).
-    op.alter_column("file_record", "file_id", new_column_name="file_name")
-
-    # 6. Finally, rename the table back to its original name expected by the legacy codebase.
-    op.rename_table("file_record", "file_store")
-
-    print(
-        "Downgrade migration completed – files are now stored inside PostgreSQL again."
-    )
-
-
-# -----------------------------------------------------------------------------
-# Helper: migrate from external storage (S3/MinIO) back into PostgreSQL large objects
-
-
-def _migrate_files_to_postgres() -> None:
-    """Move any files whose content lives in external S3-compatible storage back into PostgreSQL.
-
-    The logic mirrors *inverse* of `_migrate_files_to_external_storage` used on upgrade.
-    """
-
-    # Obtain DB session from Alembic context
-    bind = op.get_bind()
-    session = Session(bind=bind)
-
-    # Fetch rows that have external storage pointers (bucket/object_key not NULL)
-    result = session.execute(
-        text(
-            "SELECT file_id, bucket_name, object_key FROM file_record "
-            "WHERE bucket_name IS NOT NULL AND object_key IS NOT NULL"
-        )
-    )
-
-    files_to_migrate = [row[0] for row in result.fetchall()]
-    total_files = len(files_to_migrate)
-
-    if total_files == 0:
-        print("No files found in external storage to migrate back to PostgreSQL.")
-        return
-
-    print(f"Found {total_files} files to migrate back to PostgreSQL large objects.")
-
-    _set_tenant_contextvar(session)
-    migrated_count = 0
-
-    # only create external store if we have files to migrate. This line
-    # makes it so we need to have S3/MinIO configured to run this migration.
-    external_store = get_s3_file_store()
-
-    for i, file_id in enumerate(files_to_migrate, 1):
-        print(f"Migrating file {i}/{total_files}: {file_id}")
-
-        # Read file content from external storage (always binary)
-        try:
-            file_io = external_store.read_file(
-                file_id=file_id, mode="b", use_tempfile=True
-            )
-            file_io.seek(0)
-
-            # Import lazily to avoid circular deps at Alembic runtime
-            from onyx.db._deprecated.pg_file_store import (
-                create_populate_lobj,
-            )  # noqa: E402
-
-            # Create new Postgres large object and populate it
-            lobj_oid = create_populate_lobj(content=file_io, db_session=session)
-
-            # Update DB row: set lobj_oid, clear bucket/object_key
-            session.execute(
-                text(
-                    "UPDATE file_record SET lobj_oid = :lobj_oid, bucket_name = NULL, "
-                    "object_key = NULL WHERE file_id = :file_id"
-                ),
-                {"lobj_oid": lobj_oid, "file_id": file_id},
-            )
-        except ClientError as e:
-            if "NoSuchKey" in str(e):
-                print(
-                    f"File {file_id} not found in external storage. Deleting from database."
-                )
-                session.execute(
-                    text("DELETE FROM file_record WHERE file_id = :file_id"),
-                    {"file_id": file_id},
-                )
-            else:
-                raise
-
-        migrated_count += 1
-        print(f"✓ Successfully migrated file {i}/{total_files}: {file_id}")
-
-    # Flush the SQLAlchemy session so statements are sent to the DB, but **do not**
-    # commit the transaction.  The surrounding Alembic migration will commit once
-    # the *entire* downgrade succeeds.  This keeps the whole downgrade atomic and
-    # avoids leaving the database in a partially-migrated state if a later schema
-    # operation fails.
-    session.flush()
-
-    print(
-        f"Migration back to PostgreSQL completed: {migrated_count} files staged for commit."
-    )
-
-
-def _migrate_files_to_external_storage() -> None:
-    """Migrate files from PostgreSQL large objects to external storage"""
-    # Get database session
-    bind = op.get_bind()
-    session = Session(bind=bind)
-    external_store = get_s3_file_store()
-
-    # Find all files currently stored in PostgreSQL (lobj_oid is not null)
-    result = session.execute(
-        text(
-            "SELECT file_id FROM file_record WHERE lobj_oid IS NOT NULL "
-            "AND bucket_name IS NULL AND object_key IS NULL"
-        )
-    )
-
-    files_to_migrate = [row[0] for row in result.fetchall()]
-    total_files = len(files_to_migrate)
-
-    if total_files == 0:
-        print("No files found in PostgreSQL storage to migrate.")
-        return
-
-    # might need to move this above the if statement when creating a new multi-tenant
-    # system. VERY extreme edge case.
-    external_store.initialize()
-    print(f"Found {total_files} files to migrate from PostgreSQL to external storage.")
-
-    _set_tenant_contextvar(session)
-    migrated_count = 0
-
-    for i, file_id in enumerate(files_to_migrate, 1):
-        print(f"Migrating file {i}/{total_files}: {file_id}")
-
-        # Read file record to get metadata
-        file_record = session.execute(
-            text("SELECT * FROM file_record WHERE file_id = :file_id"),
-            {"file_id": file_id},
-        ).fetchone()
-
-        if file_record is None:
-            print(f"File {file_id} not found in PostgreSQL storage.")
-            continue
-
-        lobj_id = cast(int, file_record.lobj_oid)  # type: ignore
-        file_metadata = cast(Any, file_record.file_metadata)  # type: ignore
-
-        # Read file content from PostgreSQL
-        try:
-            file_content = read_lobj(
-                lobj_id, db_session=session, mode="b", use_tempfile=True
-            )
-        except Exception as e:
-            if "large object" in str(e) and "does not exist" in str(e):
-                print(f"File {file_id} not found in PostgreSQL storage.")
-                continue
-            else:
-                raise
-
-        # Handle file_metadata type conversion
-        file_metadata = None
-        if file_metadata is not None:
-            if isinstance(file_metadata, dict):
-                file_metadata = file_metadata
-            else:
-                # Convert other types to dict if possible, otherwise None
-                try:
-                    file_metadata = dict(file_record.file_metadata)  # type: ignore
-                except (TypeError, ValueError):
-                    file_metadata = None
-
-        # Save to external storage (this will handle the database record update and cleanup)
-        # NOTE: this WILL .commit() the transaction.
-        external_store.save_file(
-            file_id=file_id,
-            content=file_content,
-            display_name=file_record.display_name,
-            file_origin=file_record.file_origin,
-            file_type=file_record.file_type,
-            file_metadata=file_metadata,
-        )
-        delete_lobj_by_id(lobj_id, db_session=session)
-
-        migrated_count += 1
-        print(f"✓ Successfully migrated file {i}/{total_files}: {file_id}")
-
-    # See note above – flush but do **not** commit so the outer Alembic transaction
-    # controls atomicity.
-    session.flush()
-
-    print(
-        f"Migration completed: {migrated_count} files staged for commit to external storage."
-    )
-
-
-def _set_tenant_contextvar(session: Session) -> None:
-    """Set the tenant contextvar to the default schema"""
-    current_tenant = session.execute(text("SELECT current_schema()")).scalar()
-    print(f"Migrating files for tenant: {current_tenant}")
-    CURRENT_TENANT_ID_CONTEXTVAR.set(current_tenant)

backend/alembic/versions/d09fc20a3c66_seed_builtin_tools.py

@@ -1,152 +0,0 @@
-"""seed_builtin_tools
-
-Revision ID: d09fc20a3c66
-Revises: b7ec9b5b505f
-Create Date: 2025-09-09 19:32:16.824373
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "d09fc20a3c66"
-down_revision = "b7ec9b5b505f"
-branch_labels = None
-depends_on = None
-
-
-# Tool definitions - core tools that should always be seeded
-# Names/in_code_tool_id are the same as the class names in the tool_implementations package
-BUILT_IN_TOOLS = [
-    {
-        "name": "SearchTool",
-        "display_name": "Internal Search",
-        "description": "The Search Action allows the Assistant to search through connected knowledge to help build an answer.",
-        "in_code_tool_id": "SearchTool",
-    },
-    {
-        "name": "ImageGenerationTool",
-        "display_name": "Image Generation",
-        "description": (
-            "The Image Generation Action allows the assistant to use DALL-E 3 or GPT-IMAGE-1 to generate images. "
-            "The action will be used when the user asks the assistant to generate an image."
-        ),
-        "in_code_tool_id": "ImageGenerationTool",
-    },
-    {
-        "name": "WebSearchTool",
-        "display_name": "Web Search",
-        "description": (
-            "The Web Search Action allows the assistant "
-            "to perform internet searches for up-to-date information."
-        ),
-        "in_code_tool_id": "WebSearchTool",
-    },
-    {
-        "name": "KnowledgeGraphTool",
-        "display_name": "Knowledge Graph Search",
-        "description": (
-            "The Knowledge Graph Search Action allows the assistant to search the "
-            "Knowledge Graph for information. This tool can (for now) only be active in the KG Beta Assistant, "
-            "and it requires the Knowledge Graph to be enabled."
-        ),
-        "in_code_tool_id": "KnowledgeGraphTool",
-    },
-    {
-        "name": "OktaProfileTool",
-        "display_name": "Okta Profile",
-        "description": (
-            "The Okta Profile Action allows the assistant to fetch the current user's information from Okta. "
-            "This may include the user's name, email, phone number, address, and other details such as their "
-            "manager and direct reports."
-        ),
-        "in_code_tool_id": "OktaProfileTool",
-    },
-]
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # Start transaction
-    conn.execute(sa.text("BEGIN"))
-
-    try:
-        # Get existing tools to check what already exists
-        existing_tools = conn.execute(
-            sa.text(
-                "SELECT in_code_tool_id FROM tool WHERE in_code_tool_id IS NOT NULL"
-            )
-        ).fetchall()
-        existing_tool_ids = {row[0] for row in existing_tools}
-
-        # Insert or update built-in tools
-        for tool in BUILT_IN_TOOLS:
-            in_code_id = tool["in_code_tool_id"]
-
-            # Handle historical rename: InternetSearchTool -> WebSearchTool
-            if (
-                in_code_id == "WebSearchTool"
-                and "WebSearchTool" not in existing_tool_ids
-                and "InternetSearchTool" in existing_tool_ids
-            ):
-                # Rename the existing InternetSearchTool row in place and update fields
-                conn.execute(
-                    sa.text(
-                        """
-                        UPDATE tool
-                        SET name = :name,
-                            display_name = :display_name,
-                            description = :description,
-                            in_code_tool_id = :in_code_tool_id
-                        WHERE in_code_tool_id = 'InternetSearchTool'
-                        """
-                    ),
-                    tool,
-                )
-                # Keep the local view of existing ids in sync to avoid duplicate insert
-                existing_tool_ids.discard("InternetSearchTool")
-                existing_tool_ids.add("WebSearchTool")
-                continue
-
-            if in_code_id in existing_tool_ids:
-                # Update existing tool
-                conn.execute(
-                    sa.text(
-                        """
-                        UPDATE tool
-                        SET name = :name,
-                            display_name = :display_name,
-                            description = :description
-                        WHERE in_code_tool_id = :in_code_tool_id
-                        """
-                    ),
-                    tool,
-                )
-            else:
-                # Insert new tool
-                conn.execute(
-                    sa.text(
-                        """
-                        INSERT INTO tool (name, display_name, description, in_code_tool_id)
-                        VALUES (:name, :display_name, :description, :in_code_tool_id)
-                        """
-                    ),
-                    tool,
-                )
-
-        # Commit transaction
-        conn.execute(sa.text("COMMIT"))
-
-    except Exception as e:
-        # Rollback on error
-        conn.execute(sa.text("ROLLBACK"))
-        raise e
-
-
-def downgrade() -> None:
-    # We don't remove the tools on downgrade since it's totally fine to just
-    # have them around. If we upgrade again, it will be a no-op.
-    pass

backend/alembic/versions/da42808081e3_migrate_jira_connectors_to_new_format.py

@@ -11,7 +11,7 @@ import sqlalchemy as sa
 import json
 
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.jira.utils import extract_jira_project
+from onyx.connectors.onyx_jira.utils import extract_jira_project
 
 
 # revision identifiers, used by Alembic.

backend/alembic/versions/df0c7ad8a076_added_deletion_attempt_table.py

@@ -18,13 +18,11 @@ depends_on: None = None
 
 
 def upgrade() -> None:
-    op.execute("DROP TABLE IF EXISTS document CASCADE")
     op.create_table(
         "document",
         sa.Column("id", sa.String(), nullable=False),
         sa.PrimaryKeyConstraint("id"),
     )
-    op.execute("DROP TABLE IF EXISTS chunk CASCADE")
     op.create_table(
         "chunk",
         sa.Column("id", sa.String(), nullable=False),
@@ -45,7 +43,6 @@ def upgrade() -> None:
         ),
         sa.PrimaryKeyConstraint("id", "document_store_type"),
     )
-    op.execute("DROP TABLE IF EXISTS deletion_attempt CASCADE")
     op.create_table(
         "deletion_attempt",
         sa.Column("id", sa.Integer(), nullable=False),
@@ -87,7 +84,6 @@ def upgrade() -> None:
         ),
         sa.PrimaryKeyConstraint("id"),
     )
-    op.execute("DROP TABLE IF EXISTS document_by_connector_credential_pair CASCADE")
     op.create_table(
         "document_by_connector_credential_pair",
         sa.Column("id", sa.String(), nullable=False),
@@ -110,10 +106,7 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    # upstream tables first
     op.drop_table("document_by_connector_credential_pair")
     op.drop_table("deletion_attempt")
     op.drop_table("chunk")
-
-    # Alembic op.drop_table() has no "cascade" flag – issue raw SQL
-    op.execute("DROP TABLE IF EXISTS document CASCADE")
+    op.drop_table("document")

backend/alembic/versions/f8a9b2c3d4e5_add_research_answer_purpose_to_chat_message.py

@@ -1,30 +0,0 @@
-"""add research_answer_purpose to chat_message
-
-Revision ID: f8a9b2c3d4e5
-Revises: 5ae8240accb3
-Create Date: 2025-01-27 12:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "f8a9b2c3d4e5"
-down_revision = "5ae8240accb3"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add research_answer_purpose column to chat_message table
-    op.add_column(
-        "chat_message",
-        sa.Column("research_answer_purpose", sa.String(), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    # Remove research_answer_purpose column from chat_message table
-    op.drop_column("chat_message", "research_answer_purpose")

backend/alembic/versions/f9b8c7d6e5a4_update_parent_question_id_foreign_key_to_research_agent_iteration.py

@@ -1,69 +0,0 @@
-"""remove foreign key constraints from research_agent_iteration_sub_step
-
-Revision ID: f9b8c7d6e5a4
-Revises: bd7c3bf8beba
-Create Date: 2025-01-27 12:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "f9b8c7d6e5a4"
-down_revision = "bd7c3bf8beba"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Drop the existing foreign key constraint for parent_question_id
-    op.drop_constraint(
-        "research_agent_iteration_sub_step_parent_question_id_fkey",
-        "research_agent_iteration_sub_step",
-        type_="foreignkey",
-    )
-
-    # Drop the parent_question_id column entirely
-    op.drop_column("research_agent_iteration_sub_step", "parent_question_id")
-
-    # Drop the foreign key constraint for primary_question_id to chat_message.id
-    # (keep the column as it's needed for the composite foreign key)
-    op.drop_constraint(
-        "research_agent_iteration_sub_step_primary_question_id_fkey",
-        "research_agent_iteration_sub_step",
-        type_="foreignkey",
-    )
-
-
-def downgrade() -> None:
-    # Restore the foreign key constraint for primary_question_id to chat_message.id
-    op.create_foreign_key(
-        "research_agent_iteration_sub_step_primary_question_id_fkey",
-        "research_agent_iteration_sub_step",
-        "chat_message",
-        ["primary_question_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )
-
-    # Add back the parent_question_id column
-    op.add_column(
-        "research_agent_iteration_sub_step",
-        sa.Column(
-            "parent_question_id",
-            sa.Integer(),
-            nullable=True,
-        ),
-    )
-
-    # Restore the foreign key constraint pointing to research_agent_iteration_sub_step.id
-    op.create_foreign_key(
-        "research_agent_iteration_sub_step_parent_question_id_fkey",
-        "research_agent_iteration_sub_step",
-        "research_agent_iteration_sub_step",
-        ["parent_question_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )

backend/alembic_tenants/env.py

@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import create_async_engine
 from sqlalchemy.schema import SchemaItem
 
 from alembic import context
-from onyx.db.engine.sql_engine import build_connection_string
+from onyx.db.engine import build_connection_string
 from onyx.db.models import PublicBase
 
 # this is the Alembic Config object, which provides

backend/ee/onyx/background/celery/apps/heavy.py

@@ -16,7 +16,7 @@ from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import FileType
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import QueryHistoryType
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
+from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.tasks import delete_task_with_id
 from onyx.db.tasks import mark_task_as_finished_with_id
 from onyx.db.tasks import mark_task_as_started_with_id
@@ -35,13 +35,7 @@ logger = setup_logger()
     trail=False,
 )
 def export_query_history_task(
-    self: Task,
-    *,
-    start: datetime,
-    end: datetime,
-    start_time: datetime,
-    # Need to include the tenant_id since the TenantAwareTask needs this
-    tenant_id: str,
+    self: Task, *, start: datetime, end: datetime, start_time: datetime
 ) -> None:
     if not self.request.id:
         raise RuntimeError("No task id defined for this task; cannot identify it")
@@ -91,7 +85,8 @@ def export_query_history_task(
     with get_session_with_current_tenant() as db_session:
         try:
             stream.seek(0)
-            get_default_file_store().save_file(
+            get_default_file_store(db_session).save_file(
+                file_name=report_name,
                 content=stream,
                 display_name=report_name,
                 file_origin=FileOrigin.QUERY_HISTORY_CSV,
@@ -101,7 +96,6 @@ def export_query_history_task(
                     "end": end.isoformat(),
                     "start_time": start_time.isoformat(),
                 },
-                file_id=report_name,
             )
 
             delete_task_with_id(

backend/ee/onyx/background/celery/apps/primary.py

@@ -1,4 +1,133 @@
+from datetime import datetime
+from datetime import timezone
+from uuid import UUID
+
+from celery import shared_task
+from celery import Task
+
+from ee.onyx.background.celery_utils import should_perform_chat_ttl_check
+from ee.onyx.background.task_name_builders import name_chat_ttl_task
+from ee.onyx.server.reporting.usage_export_generation import create_new_usage_report
 from onyx.background.celery.apps.primary import celery_app
+from onyx.configs.app_configs import JOB_TIMEOUT
+from onyx.configs.constants import OnyxCeleryTask
+from onyx.db.chat import delete_chat_session
+from onyx.db.chat import get_chat_sessions_older_than
+from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.enums import TaskStatus
+from onyx.db.tasks import mark_task_as_finished_with_id
+from onyx.db.tasks import register_task
+from onyx.server.settings.store import load_settings
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+# mark as EE for all tasks in this file
+
+
+@shared_task(
+    name=OnyxCeleryTask.PERFORM_TTL_MANAGEMENT_TASK,
+    ignore_result=True,
+    soft_time_limit=JOB_TIMEOUT,
+    bind=True,
+    trail=False,
+)
+def perform_ttl_management_task(
+    self: Task, retention_limit_days: int, *, tenant_id: str
+) -> None:
+    task_id = self.request.id
+    if not task_id:
+        raise RuntimeError("No task id defined for this task; cannot identify it")
+
+    start_time = datetime.now(tz=timezone.utc)
+
+    user_id: UUID | None = None
+    session_id: UUID | None = None
+    try:
+        with get_session_with_current_tenant() as db_session:
+            # we generally want to move off this, but keeping for now
+            register_task(
+                db_session=db_session,
+                task_name=name_chat_ttl_task(retention_limit_days, tenant_id),
+                task_id=task_id,
+                status=TaskStatus.STARTED,
+                start_time=start_time,
+            )
+
+            old_chat_sessions = get_chat_sessions_older_than(
+                retention_limit_days, db_session
+            )
+
+        for user_id, session_id in old_chat_sessions:
+            # one session per delete so that we don't blow up if a deletion fails.
+            with get_session_with_current_tenant() as db_session:
+                delete_chat_session(
+                    user_id,
+                    session_id,
+                    db_session,
+                    include_deleted=True,
+                    hard_delete=True,
+                )
+
+        with get_session_with_current_tenant() as db_session:
+            mark_task_as_finished_with_id(
+                db_session=db_session,
+                task_id=task_id,
+                success=True,
+            )
+
+    except Exception:
+        logger.exception(
+            "delete_chat_session exceptioned. "
+            f"user_id={user_id} session_id={session_id}"
+        )
+        with get_session_with_current_tenant() as db_session:
+            mark_task_as_finished_with_id(
+                db_session=db_session,
+                task_id=task_id,
+                success=False,
+            )
+        raise
+
+
+#####
+# Periodic Tasks
+#####
+
+
+@celery_app.task(
+    name=OnyxCeleryTask.CHECK_TTL_MANAGEMENT_TASK,
+    ignore_result=True,
+    soft_time_limit=JOB_TIMEOUT,
+)
+def check_ttl_management_task(*, tenant_id: str) -> None:
+    """Runs periodically to check if any ttl tasks should be run and adds them
+    to the queue"""
+
+    settings = load_settings()
+    retention_limit_days = settings.maximum_chat_retention_days
+    with get_session_with_current_tenant() as db_session:
+        if should_perform_chat_ttl_check(retention_limit_days, db_session):
+            perform_ttl_management_task.apply_async(
+                kwargs=dict(
+                    retention_limit_days=retention_limit_days, tenant_id=tenant_id
+                ),
+            )
+
+
+@celery_app.task(
+    name=OnyxCeleryTask.AUTOGENERATE_USAGE_REPORT_TASK,
+    ignore_result=True,
+    soft_time_limit=JOB_TIMEOUT,
+)
+def autogenerate_usage_report_task(*, tenant_id: str) -> None:
+    """This generates usage report under the /admin/generate-usage/report endpoint"""
+    with get_session_with_current_tenant() as db_session:
+        create_new_usage_report(
+            db_session=db_session,
+            user_id=None,
+            period=None,
+        )
 
 
 celery_app.autodiscover_tasks(
@@ -6,7 +135,5 @@ celery_app.autodiscover_tasks(
         "ee.onyx.background.celery.tasks.doc_permission_syncing",
         "ee.onyx.background.celery.tasks.external_group_syncing",
         "ee.onyx.background.celery.tasks.cloud",
-        "ee.onyx.background.celery.tasks.ttl_management",
-        "ee.onyx.background.celery.tasks.usage_reporting",
     ]
 )

backend/ee/onyx/background/celery/tasks/beat_schedule.py

@@ -20,36 +20,39 @@ from shared_configs.configs import MULTI_TENANT
 
 ee_beat_system_tasks: list[dict] = []
 
-ee_beat_task_templates: list[dict] = [
-    {
-        "name": "autogenerate-usage-report",
-        "task": OnyxCeleryTask.GENERATE_USAGE_REPORT_TASK,
-        "schedule": timedelta(days=30),
-        "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
-            "expires": BEAT_EXPIRES_DEFAULT,
+ee_beat_task_templates: list[dict] = []
+ee_beat_task_templates.extend(
+    [
+        {
+            "name": "autogenerate-usage-report",
+            "task": OnyxCeleryTask.AUTOGENERATE_USAGE_REPORT_TASK,
+            "schedule": timedelta(days=30),
+            "options": {
+                "priority": OnyxCeleryPriority.MEDIUM,
+                "expires": BEAT_EXPIRES_DEFAULT,
+            },
         },
-    },
-    {
-        "name": "check-ttl-management",
-        "task": OnyxCeleryTask.CHECK_TTL_MANAGEMENT_TASK,
-        "schedule": timedelta(hours=CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS),
-        "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
-            "expires": BEAT_EXPIRES_DEFAULT,
+        {
+            "name": "check-ttl-management",
+            "task": OnyxCeleryTask.CHECK_TTL_MANAGEMENT_TASK,
+            "schedule": timedelta(hours=CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS),
+            "options": {
+                "priority": OnyxCeleryPriority.MEDIUM,
+                "expires": BEAT_EXPIRES_DEFAULT,
+            },
         },
-    },
-    {
-        "name": "export-query-history-cleanup-task",
-        "task": OnyxCeleryTask.EXPORT_QUERY_HISTORY_CLEANUP_TASK,
-        "schedule": timedelta(hours=1),
-        "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
-            "expires": BEAT_EXPIRES_DEFAULT,
-            "queue": OnyxCeleryQueues.CSV_GENERATION,
+        {
+            "name": "export-query-history-cleanup-task",
+            "task": OnyxCeleryTask.EXPORT_QUERY_HISTORY_CLEANUP_TASK,
+            "schedule": timedelta(hours=1),
+            "options": {
+                "priority": OnyxCeleryPriority.MEDIUM,
+                "expires": BEAT_EXPIRES_DEFAULT,
+                "queue": OnyxCeleryQueues.CSV_GENERATION,
+            },
         },
-    },
-]
+    ]
+)
 
 ee_tasks_to_schedule: list[dict] = []
 
@@ -57,7 +60,7 @@ if not MULTI_TENANT:
     ee_tasks_to_schedule = [
         {
             "name": "autogenerate-usage-report",
-            "task": OnyxCeleryTask.GENERATE_USAGE_REPORT_TASK,
+            "task": OnyxCeleryTask.AUTOGENERATE_USAGE_REPORT_TASK,
             "schedule": timedelta(days=30),  # TODO: change this to config flag
             "options": {
                 "priority": OnyxCeleryPriority.MEDIUM,

backend/ee/onyx/background/celery/tasks/cleanup/tasks.py

@@ -6,7 +6,7 @@ from celery import shared_task
 from ee.onyx.db.query_history import get_all_query_history_export_tasks
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.db.engine.sql_engine import get_session_with_tenant
+from onyx.db.engine import get_session_with_tenant
 from onyx.db.enums import TaskStatus
 from onyx.db.tasks import delete_task_with_id
 from onyx.utils.logger import setup_logger

backend/ee/onyx/background/celery/tasks/cloud/tasks.py

@@ -13,7 +13,7 @@ from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.db.engine.tenant_utils import get_all_tenant_ids
+from onyx.db.engine import get_all_tenant_ids
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import redis_lock_dump
 from shared_configs.configs import IGNORED_SYNCING_TENANT_LIST

backend/ee/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -30,7 +30,6 @@ from onyx.background.celery.celery_redis import celery_find_task
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
-from onyx.background.celery.tasks.beat_schedule import CLOUD_BEAT_MULTIPLIER_DEFAULT
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import CELERY_PERMISSIONS_SYNC_LOCK_TIMEOUT
@@ -47,10 +46,9 @@ from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.db.connector import mark_cc_pair_as_permissions_synced
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.document import get_document_ids_for_connector_credential_pair
-from onyx.db.document import get_documents_for_connector_credential_pair_limited_columns
 from onyx.db.document import upsert_document_by_connector_credential_pair
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
-from onyx.db.engine.sql_engine import get_session_with_tenant
+from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.engine import get_session_with_tenant
 from onyx.db.enums import AccessType
 from onyx.db.enums import ConnectorCredentialPairStatus
 from onyx.db.enums import SyncStatus
@@ -59,9 +57,7 @@ from onyx.db.models import ConnectorCredentialPair
 from onyx.db.sync_record import insert_sync_record
 from onyx.db.sync_record import update_sync_record_status
 from onyx.db.users import batch_add_ext_perm_user_if_not_exists
-from onyx.db.utils import DocumentRow
 from onyx.db.utils import is_retryable_sqlalchemy_error
-from onyx.db.utils import SortOrder
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.redis.redis_connector import RedisConnector
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
@@ -77,7 +73,6 @@ from onyx.utils.logger import LoggerContextVars
 from onyx.utils.logger import setup_logger
 from onyx.utils.telemetry import optional_telemetry
 from onyx.utils.telemetry import RecordType
-from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 
@@ -92,24 +87,6 @@ LIGHT_SOFT_TIME_LIMIT = 105
 LIGHT_TIME_LIMIT = LIGHT_SOFT_TIME_LIMIT + 15
 
 
-def _get_fence_validation_block_expiration() -> int:
-    """
-    Compute the expiration time for the fence validation block signal.
-    Base expiration is 300 seconds, multiplied by the beat multiplier only in MULTI_TENANT mode.
-    """
-    base_expiration = 300  # seconds
-
-    if not MULTI_TENANT:
-        return base_expiration
-
-    try:
-        beat_multiplier = OnyxRuntime.get_beat_multiplier()
-    except Exception:
-        beat_multiplier = CLOUD_BEAT_MULTIPLIER_DEFAULT
-
-    return int(base_expiration * beat_multiplier)
-
-
 """Jobs / utils for kicking off doc permissions sync tasks."""
 
 
@@ -217,11 +194,7 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str) -> bool | None
                     "Exception while validating permission sync fences"
                 )
 
-            r.set(
-                OnyxRedisSignals.BLOCK_VALIDATE_PERMISSION_SYNC_FENCES,
-                1,
-                ex=_get_fence_validation_block_expiration(),
-            )
+            r.set(OnyxRedisSignals.BLOCK_VALIDATE_PERMISSION_SYNC_FENCES, 1, ex=300)
 
         # use a lookup table to find active fences. We still have to verify the fence
         # exists since it is an optimization and not the source of truth.
@@ -425,7 +398,7 @@ def connector_permission_sync_generator_task(
 
     lock: RedisLock = r.lock(
         OnyxRedisLocks.CONNECTOR_DOC_PERMISSIONS_SYNC_LOCK_PREFIX
-        + f"_{redis_connector.cc_pair_id}",
+        + f"_{redis_connector.id}",
         timeout=CELERY_PERMISSIONS_SYNC_LOCK_TIMEOUT,
         thread_local=False,
     )
@@ -452,7 +425,6 @@ def connector_permission_sync_generator_task(
                 created = validate_ccpair_for_user(
                     cc_pair.connector.id,
                     cc_pair.credential.id,
-                    cc_pair.access_type,
                     db_session,
                     enforce_creation=False,
                 )
@@ -501,31 +473,16 @@ def connector_permission_sync_generator_task(
             # this is can be used to determine documents that are "missing" and thus
             # should no longer be accessible. The decision as to whether we should find
             # every document during the doc sync process is connector-specific.
-            def fetch_all_existing_docs_fn(
-                sort_order: SortOrder | None = None,
-            ) -> list[DocumentRow]:
-                result = get_documents_for_connector_credential_pair_limited_columns(
-                    db_session=db_session,
-                    connector_id=cc_pair.connector.id,
-                    credential_id=cc_pair.credential.id,
-                    sort_order=sort_order,
-                )
-                return list(result)
-
-            def fetch_all_existing_docs_ids_fn() -> list[str]:
-                result = get_document_ids_for_connector_credential_pair(
+            def fetch_all_existing_docs_fn() -> list[str]:
+                return get_document_ids_for_connector_credential_pair(
                     db_session=db_session,
                     connector_id=cc_pair.connector.id,
                     credential_id=cc_pair.credential.id,
                 )
-                return result
 
             doc_sync_func = sync_config.doc_sync_config.doc_sync_func
             document_external_accesses = doc_sync_func(
-                cc_pair,
-                fetch_all_existing_docs_fn,
-                fetch_all_existing_docs_ids_fn,
-                callback,
+                cc_pair, fetch_all_existing_docs_fn, callback
             )
 
             task_logger.info(
@@ -640,6 +597,91 @@ def document_update_permissions(
     return True
 
 
+# NOTE(rkuo): Deprecating this due to degenerate behavior in Redis from sending
+# large permissions through celery (over 1MB in size)
+# @shared_task(
+#     name=OnyxCeleryTask.UPDATE_EXTERNAL_DOCUMENT_PERMISSIONS_TASK,
+#     soft_time_limit=LIGHT_SOFT_TIME_LIMIT,
+#     time_limit=LIGHT_TIME_LIMIT,
+#     max_retries=DOCUMENT_PERMISSIONS_UPDATE_MAX_RETRIES,
+#     bind=True,
+# )
+# def update_external_document_permissions_task(
+#     self: Task,
+#     tenant_id: str,
+#     serialized_doc_external_access: dict,
+#     source_string: str,
+#     connector_id: int,
+#     credential_id: int,
+# ) -> bool:
+#     start = time.monotonic()
+
+#     completion_status = OnyxCeleryTaskCompletionStatus.UNDEFINED
+
+#     document_external_access = DocExternalAccess.from_dict(
+#         serialized_doc_external_access
+#     )
+#     doc_id = document_external_access.doc_id
+#     external_access = document_external_access.external_access
+
+#     try:
+#         with get_session_with_current_tenant() as db_session:
+#             # Add the users to the DB if they don't exist
+#             batch_add_ext_perm_user_if_not_exists(
+#                 db_session=db_session,
+#                 emails=list(external_access.external_user_emails),
+#                 continue_on_error=True,
+#             )
+#             # Then upsert the document's external permissions
+#             created_new_doc = upsert_document_external_perms(
+#                 db_session=db_session,
+#                 doc_id=doc_id,
+#                 external_access=external_access,
+#                 source_type=DocumentSource(source_string),
+#             )
+
+#             if created_new_doc:
+#                 # If a new document was created, we associate it with the cc_pair
+#                 upsert_document_by_connector_credential_pair(
+#                     db_session=db_session,
+#                     connector_id=connector_id,
+#                     credential_id=credential_id,
+#                     document_ids=[doc_id],
+#                 )
+
+#             elapsed = time.monotonic() - start
+#             task_logger.info(
+#                 f"connector_id={connector_id} "
+#                 f"doc={doc_id} "
+#                 f"action=update_permissions "
+#                 f"elapsed={elapsed:.2f}"
+#             )
+
+#         completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
+#     except Exception as e:
+#         error_msg = format_error_for_logging(e)
+#         task_logger.warning(
+#             f"Exception in update_external_document_permissions_task: connector_id={connector_id} doc_id={doc_id} {error_msg}"
+#         )
+#         task_logger.exception(
+#             f"update_external_document_permissions_task exceptioned: "
+#             f"connector_id={connector_id} doc_id={doc_id}"
+#         )
+#         completion_status = OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
+#     finally:
+#         task_logger.info(
+#             f"update_external_document_permissions_task completed: status={completion_status.value} doc={doc_id}"
+#         )
+
+#     if completion_status != OnyxCeleryTaskCompletionStatus.SUCCEEDED:
+#         return False
+
+#     task_logger.info(
+#         f"update_external_document_permissions_task finished: connector_id={connector_id} doc_id={doc_id}"
+#     )
+#     return True
+
+
 def validate_permission_sync_fences(
     tenant_id: str,
     r: Redis,

backend/ee/onyx/background/celery/tasks/external_group_syncing/tasks.py

@@ -20,9 +20,7 @@ from ee.onyx.background.celery.tasks.external_group_syncing.group_sync_utils imp
 from ee.onyx.db.connector_credential_pair import get_all_auto_sync_cc_pairs
 from ee.onyx.db.connector_credential_pair import get_cc_pairs_by_source
 from ee.onyx.db.external_perm import ExternalUserGroup
-from ee.onyx.db.external_perm import mark_old_external_groups_as_stale
-from ee.onyx.db.external_perm import remove_stale_external_groups
-from ee.onyx.db.external_perm import upsert_external_groups
+from ee.onyx.db.external_perm import replace_user__ext_group_for_cc_pair
 from ee.onyx.external_permissions.sync_params import (
     get_all_cc_pair_agnostic_group_sync_sources,
 )
@@ -30,7 +28,6 @@ from ee.onyx.external_permissions.sync_params import get_source_perm_sync_config
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_find_task
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
-from onyx.background.celery.tasks.beat_schedule import CLOUD_BEAT_MULTIPLIER_DEFAULT
 from onyx.background.error_logging import emit_background_error
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.constants import CELERY_EXTERNAL_GROUP_SYNC_LOCK_TIMEOUT
@@ -42,8 +39,9 @@ from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
+from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
+from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.enums import AccessType
 from onyx.db.enums import ConnectorCredentialPairStatus
 from onyx.db.enums import SyncStatus
@@ -58,34 +56,19 @@ from onyx.redis.redis_connector_ext_group_sync import (
 )
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import get_redis_replica_client
-from onyx.server.runtime.onyx_runtime import OnyxRuntime
 from onyx.server.utils import make_short_id
 from onyx.utils.logger import format_error_for_logging
 from onyx.utils.logger import setup_logger
-from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 
 
-_EXTERNAL_GROUP_BATCH_SIZE = 100
+EXTERNAL_GROUPS_UPDATE_MAX_RETRIES = 3
 
 
-def _get_fence_validation_block_expiration() -> int:
-    """
-    Compute the expiration time for the fence validation block signal.
-    Base expiration is 300 seconds, multiplied by the beat multiplier only in MULTI_TENANT mode.
-    """
-    base_expiration = 300  # seconds
-
-    if not MULTI_TENANT:
-        return base_expiration
-
-    try:
-        beat_multiplier = OnyxRuntime.get_beat_multiplier()
-    except Exception:
-        beat_multiplier = CLOUD_BEAT_MULTIPLIER_DEFAULT
-
-    return int(base_expiration * beat_multiplier)
+# 5 seconds more than RetryDocumentIndex STOP_AFTER+MAX_WAIT
+LIGHT_SOFT_TIME_LIMIT = 105
+LIGHT_TIME_LIMIT = LIGHT_SOFT_TIME_LIMIT + 15
 
 
 def _is_external_group_sync_due(cc_pair: ConnectorCredentialPair) -> bool:
@@ -93,7 +76,7 @@ def _is_external_group_sync_due(cc_pair: ConnectorCredentialPair) -> bool:
 
     if cc_pair.access_type != AccessType.SYNC:
         task_logger.error(
-            f"Received non-sync CC Pair {cc_pair.id} for external "
+            f"Recieved non-sync CC Pair {cc_pair.id} for external "
             f"group sync. Actual access type: {cc_pair.access_type}"
         )
         return False
@@ -215,11 +198,7 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str) -> bool | None:
                     "Exception while validating external group sync fences"
                 )
 
-            r.set(
-                OnyxRedisSignals.BLOCK_VALIDATE_EXTERNAL_GROUP_SYNC_FENCES,
-                1,
-                ex=_get_fence_validation_block_expiration(),
-            )
+            r.set(OnyxRedisSignals.BLOCK_VALIDATE_EXTERNAL_GROUP_SYNC_FENCES, 1, ex=300)
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -383,7 +362,7 @@ def connector_external_group_sync_generator_task(
 
     lock: RedisLock = r.lock(
         OnyxRedisLocks.CONNECTOR_EXTERNAL_GROUP_SYNC_LOCK_PREFIX
-        + f"_{redis_connector.cc_pair_id}",
+        + f"_{redis_connector.id}",
         timeout=CELERY_EXTERNAL_GROUP_SYNC_LOCK_TIMEOUT,
     )
 
@@ -398,12 +377,63 @@ def connector_external_group_sync_generator_task(
         payload.started = datetime.now(timezone.utc)
         redis_connector.external_group_sync.set_fence(payload)
 
-        _perform_external_group_sync(
-            cc_pair_id=cc_pair_id,
-            tenant_id=tenant_id,
-        )
-
         with get_session_with_current_tenant() as db_session:
+            cc_pair = get_connector_credential_pair_from_id(
+                db_session=db_session,
+                cc_pair_id=cc_pair_id,
+                eager_load_credential=True,
+            )
+            if cc_pair is None:
+                raise ValueError(
+                    f"No connector credential pair found for id: {cc_pair_id}"
+                )
+
+            source_type = cc_pair.connector.source
+            sync_config = get_source_perm_sync_config(source_type)
+            if sync_config is None:
+                msg = (
+                    f"No sync config found for {source_type} for cc_pair: {cc_pair_id}"
+                )
+                emit_background_error(msg, cc_pair_id=cc_pair_id)
+                raise ValueError(msg)
+
+            if sync_config.group_sync_config is None:
+                msg = f"No group sync config found for {source_type} for cc_pair: {cc_pair_id}"
+                emit_background_error(msg, cc_pair_id=cc_pair_id)
+                raise ValueError(msg)
+
+            ext_group_sync_func = sync_config.group_sync_config.group_sync_func
+
+            logger.info(
+                f"Syncing external groups for {source_type} for cc_pair: {cc_pair_id}"
+            )
+            external_user_groups: list[ExternalUserGroup] = []
+            try:
+                external_user_groups = ext_group_sync_func(tenant_id, cc_pair)
+            except ConnectorValidationError as e:
+                # TODO: add some notification to the admins here
+                logger.exception(
+                    f"Error syncing external groups for {source_type} for cc_pair: {cc_pair_id} {e}"
+                )
+                raise e
+
+            logger.info(
+                f"Syncing {len(external_user_groups)} external user groups for {source_type}"
+            )
+            logger.debug(f"New external user groups: {external_user_groups}")
+
+            replace_user__ext_group_for_cc_pair(
+                db_session=db_session,
+                cc_pair_id=cc_pair.id,
+                group_defs=external_user_groups,
+                source=cc_pair.connector.source,
+            )
+            logger.info(
+                f"Synced {len(external_user_groups)} external user groups for {source_type}"
+            )
+
+            mark_all_relevant_cc_pairs_as_external_group_synced(db_session, cc_pair)
+
             update_sync_record_status(
                 db_session=db_session,
                 entity_id=cc_pair_id,
@@ -445,81 +475,6 @@ def connector_external_group_sync_generator_task(
     )
 
 
-def _perform_external_group_sync(
-    cc_pair_id: int,
-    tenant_id: str,
-) -> None:
-    with get_session_with_current_tenant() as db_session:
-        cc_pair = get_connector_credential_pair_from_id(
-            db_session=db_session,
-            cc_pair_id=cc_pair_id,
-            eager_load_credential=True,
-        )
-        if cc_pair is None:
-            raise ValueError(f"No connector credential pair found for id: {cc_pair_id}")
-
-        source_type = cc_pair.connector.source
-        sync_config = get_source_perm_sync_config(source_type)
-        if sync_config is None:
-            msg = f"No sync config found for {source_type} for cc_pair: {cc_pair_id}"
-            emit_background_error(msg, cc_pair_id=cc_pair_id)
-            raise ValueError(msg)
-
-        if sync_config.group_sync_config is None:
-            msg = f"No group sync config found for {source_type} for cc_pair: {cc_pair_id}"
-            emit_background_error(msg, cc_pair_id=cc_pair_id)
-            raise ValueError(msg)
-
-        ext_group_sync_func = sync_config.group_sync_config.group_sync_func
-
-        logger.info(
-            f"Marking old external groups as stale for {source_type} for cc_pair: {cc_pair_id}"
-        )
-        mark_old_external_groups_as_stale(db_session, cc_pair_id)
-
-        logger.info(
-            f"Syncing external groups for {source_type} for cc_pair: {cc_pair_id}"
-        )
-        external_user_group_batch: list[ExternalUserGroup] = []
-        try:
-            external_user_group_generator = ext_group_sync_func(tenant_id, cc_pair)
-            for external_user_group in external_user_group_generator:
-                external_user_group_batch.append(external_user_group)
-                if len(external_user_group_batch) >= _EXTERNAL_GROUP_BATCH_SIZE:
-                    logger.debug(
-                        f"New external user groups: {external_user_group_batch}"
-                    )
-                    upsert_external_groups(
-                        db_session=db_session,
-                        cc_pair_id=cc_pair_id,
-                        external_groups=external_user_group_batch,
-                        source=cc_pair.connector.source,
-                    )
-                    external_user_group_batch = []
-
-            if external_user_group_batch:
-                logger.debug(f"New external user groups: {external_user_group_batch}")
-                upsert_external_groups(
-                    db_session=db_session,
-                    cc_pair_id=cc_pair_id,
-                    external_groups=external_user_group_batch,
-                    source=cc_pair.connector.source,
-                )
-        except Exception as e:
-            # TODO: add some notification to the admins here
-            logger.exception(
-                f"Error syncing external groups for {source_type} for cc_pair: {cc_pair_id} {e}"
-            )
-            raise e
-
-        logger.info(
-            f"Removing stale external groups for {source_type} for cc_pair: {cc_pair_id}"
-        )
-        remove_stale_external_groups(db_session, cc_pair_id)
-
-        mark_all_relevant_cc_pairs_as_external_group_synced(db_session, cc_pair)
-
-
 def validate_external_group_sync_fences(
     tenant_id: str,
     celery_app: Celery,

backend/ee/onyx/background/celery/tasks/tenant_provisioning/tasks.py

@@ -19,7 +19,7 @@ from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.db.engine.sql_engine import get_session_with_shared_schema
+from onyx.db.engine import get_session_with_shared_schema
 from onyx.db.models import AvailableTenant
 from onyx.redis.redis_pool import get_redis_client
 from shared_configs.configs import MULTI_TENANT

backend/ee/onyx/background/celery/tasks/ttl_management/tasks.py

@@ -1,106 +0,0 @@
-from datetime import datetime
-from datetime import timezone
-from uuid import UUID
-
-from celery import shared_task
-from celery import Task
-
-from ee.onyx.background.celery_utils import should_perform_chat_ttl_check
-from ee.onyx.background.task_name_builders import name_chat_ttl_task
-from onyx.configs.app_configs import JOB_TIMEOUT
-from onyx.configs.constants import OnyxCeleryTask
-from onyx.db.chat import delete_chat_session
-from onyx.db.chat import get_chat_sessions_older_than
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
-from onyx.db.enums import TaskStatus
-from onyx.db.tasks import mark_task_as_finished_with_id
-from onyx.db.tasks import register_task
-from onyx.server.settings.store import load_settings
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-@shared_task(
-    name=OnyxCeleryTask.PERFORM_TTL_MANAGEMENT_TASK,
-    ignore_result=True,
-    soft_time_limit=JOB_TIMEOUT,
-    bind=True,
-    trail=False,
-)
-def perform_ttl_management_task(
-    self: Task, retention_limit_days: int, *, tenant_id: str
-) -> None:
-    task_id = self.request.id
-    if not task_id:
-        raise RuntimeError("No task id defined for this task; cannot identify it")
-
-    start_time = datetime.now(tz=timezone.utc)
-
-    user_id: UUID | None = None
-    session_id: UUID | None = None
-    try:
-        with get_session_with_current_tenant() as db_session:
-            # we generally want to move off this, but keeping for now
-            register_task(
-                db_session=db_session,
-                task_name=name_chat_ttl_task(retention_limit_days, tenant_id),
-                task_id=task_id,
-                status=TaskStatus.STARTED,
-                start_time=start_time,
-            )
-
-            old_chat_sessions = get_chat_sessions_older_than(
-                retention_limit_days, db_session
-            )
-
-        for user_id, session_id in old_chat_sessions:
-            # one session per delete so that we don't blow up if a deletion fails.
-            with get_session_with_current_tenant() as db_session:
-                delete_chat_session(
-                    user_id,
-                    session_id,
-                    db_session,
-                    include_deleted=True,
-                    hard_delete=True,
-                )
-
-        with get_session_with_current_tenant() as db_session:
-            mark_task_as_finished_with_id(
-                db_session=db_session,
-                task_id=task_id,
-                success=True,
-            )
-
-    except Exception:
-        logger.exception(
-            "delete_chat_session exceptioned. "
-            f"user_id={user_id} session_id={session_id}"
-        )
-        with get_session_with_current_tenant() as db_session:
-            mark_task_as_finished_with_id(
-                db_session=db_session,
-                task_id=task_id,
-                success=False,
-            )
-        raise
-
-
-@shared_task(
-    name=OnyxCeleryTask.CHECK_TTL_MANAGEMENT_TASK,
-    ignore_result=True,
-    soft_time_limit=JOB_TIMEOUT,
-)
-def check_ttl_management_task(*, tenant_id: str) -> None:
-    """Runs periodically to check if any ttl tasks should be run and adds them
-    to the queue"""
-
-    settings = load_settings()
-    retention_limit_days = settings.maximum_chat_retention_days
-    with get_session_with_current_tenant() as db_session:
-        if should_perform_chat_ttl_check(retention_limit_days, db_session):
-            perform_ttl_management_task.apply_async(
-                kwargs=dict(
-                    retention_limit_days=retention_limit_days, tenant_id=tenant_id
-                ),
-            )

backend/ee/onyx/background/celery/tasks/usage_reporting/tasks.py

@@ -1,46 +0,0 @@
-from datetime import datetime
-from uuid import UUID
-
-from celery import shared_task
-from celery import Task
-
-from ee.onyx.server.reporting.usage_export_generation import create_new_usage_report
-from onyx.configs.app_configs import JOB_TIMEOUT
-from onyx.configs.constants import OnyxCeleryTask
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-@shared_task(
-    name=OnyxCeleryTask.GENERATE_USAGE_REPORT_TASK,
-    ignore_result=True,
-    soft_time_limit=JOB_TIMEOUT,
-    bind=True,
-    trail=False,
-)
-def generate_usage_report_task(
-    self: Task,
-    *,
-    tenant_id: str,
-    user_id: str | None = None,
-    period_from: str | None = None,
-    period_to: str | None = None,
-) -> None:
-    """User-initiated usage report generation task"""
-    # Parse period if provided
-    period = None
-    if period_from and period_to:
-        period = (
-            datetime.fromisoformat(period_from),
-            datetime.fromisoformat(period_to),
-        )
-
-    # Generate the report
-    with get_session_with_current_tenant() as db_session:
-        create_new_usage_report(
-            db_session=db_session,
-            user_id=UUID(user_id) if user_id else None,
-            period=period,
-        )

backend/ee/onyx/chat/process_message.py

@@ -0,0 +1,38 @@
+from ee.onyx.server.query_and_chat.models import OneShotQAResponse
+from onyx.chat.models import AllCitations
+from onyx.chat.models import LLMRelevanceFilterResponse
+from onyx.chat.models import OnyxAnswerPiece
+from onyx.chat.models import QADocsResponse
+from onyx.chat.models import StreamingError
+from onyx.chat.process_message import ChatPacketStream
+from onyx.server.query_and_chat.models import ChatMessageDetail
+from onyx.utils.timing import log_function_time
+
+
+@log_function_time()
+def gather_stream_for_answer_api(
+    packets: ChatPacketStream,
+) -> OneShotQAResponse:
+    response = OneShotQAResponse()
+
+    answer = ""
+    for packet in packets:
+        if isinstance(packet, OnyxAnswerPiece) and packet.answer_piece:
+            answer += packet.answer_piece
+        elif isinstance(packet, QADocsResponse):
+            response.docs = packet
+            # Extraneous, provided for backwards compatibility
+            response.rephrase = packet.rephrased_query
+        elif isinstance(packet, StreamingError):
+            response.error_msg = packet.error
+        elif isinstance(packet, ChatMessageDetail):
+            response.chat_message_id = packet.message_id
+        elif isinstance(packet, LLMRelevanceFilterResponse):
+            response.llm_selected_doc_indices = packet.llm_selected_doc_indices
+        elif isinstance(packet, AllCitations):
+            response.citations = packet.citations
+
+    if answer:
+        response.answer = answer
+
+    return response

backend/ee/onyx/configs/app_configs.py

@@ -53,16 +53,6 @@ CONFLUENCE_ANONYMOUS_ACCESS_IS_PUBLIC = (
 )
 
 
-#####
-# JIRA
-#####
-
-# In seconds, default is 30 minutes
-JIRA_PERMISSION_DOC_SYNC_FREQUENCY = int(
-    os.environ.get("JIRA_PERMISSION_DOC_SYNC_FREQUENCY") or 30 * 60
-)
-
-
 #####
 # Google Drive
 #####
@@ -71,19 +61,6 @@ GOOGLE_DRIVE_PERMISSION_GROUP_SYNC_FREQUENCY = int(
 )
 
 
-#####
-# GitHub
-#####
-# In seconds, default is 5 minutes
-GITHUB_PERMISSION_DOC_SYNC_FREQUENCY = int(
-    os.environ.get("GITHUB_PERMISSION_DOC_SYNC_FREQUENCY") or 5 * 60
-)
-# In seconds, default is 5 minutes
-GITHUB_PERMISSION_GROUP_SYNC_FREQUENCY = int(
-    os.environ.get("GITHUB_PERMISSION_GROUP_SYNC_FREQUENCY") or 5 * 60
-)
-
-
 #####
 # Slack
 #####
@@ -94,28 +71,6 @@ SLACK_PERMISSION_DOC_SYNC_FREQUENCY = int(
 NUM_PERMISSION_WORKERS = int(os.environ.get("NUM_PERMISSION_WORKERS") or 2)
 
 
-#####
-# Teams
-#####
-# In seconds, default is 5 minutes
-TEAMS_PERMISSION_DOC_SYNC_FREQUENCY = int(
-    os.environ.get("TEAMS_PERMISSION_DOC_SYNC_FREQUENCY") or 5 * 60
-)
-
-#####
-# SharePoint
-#####
-# In seconds, default is 30 minutes
-SHAREPOINT_PERMISSION_DOC_SYNC_FREQUENCY = int(
-    os.environ.get("SHAREPOINT_PERMISSION_DOC_SYNC_FREQUENCY") or 30 * 60
-)
-
-# In seconds, default is 5 minutes
-SHAREPOINT_PERMISSION_GROUP_SYNC_FREQUENCY = int(
-    os.environ.get("SHAREPOINT_PERMISSION_GROUP_SYNC_FREQUENCY") or 5 * 60
-)
-
-
 ####
 # Celery Job Frequency
 ####

backend/ee/onyx/connectors/perm_sync_valid.py

@@ -1,28 +0,0 @@
-from onyx.connectors.confluence.connector import ConfluenceConnector
-from onyx.connectors.google_drive.connector import GoogleDriveConnector
-from onyx.connectors.interfaces import BaseConnector
-
-
-def validate_confluence_perm_sync(connector: ConfluenceConnector) -> None:
-    """
-    Validate that the connector is configured correctly for permissions syncing.
-    """
-
-
-def validate_drive_perm_sync(connector: GoogleDriveConnector) -> None:
-    """
-    Validate that the connector is configured correctly for permissions syncing.
-    """
-
-
-def validate_perm_sync(connector: BaseConnector) -> None:
-    """
-    Override this if your connector needs to validate permissions syncing.
-    Raise an exception if invalid, otherwise do nothing.
-
-    Default is a no-op (always successful).
-    """
-    if isinstance(connector, ConfluenceConnector):
-        validate_confluence_perm_sync(connector)
-    elif isinstance(connector, GoogleDriveConnector):
-        validate_drive_perm_sync(connector)

backend/ee/onyx/db/external_perm.py

@@ -4,7 +4,6 @@ from uuid import UUID
 from pydantic import BaseModel
 from sqlalchemy import delete
 from sqlalchemy import select
-from sqlalchemy import update
 from sqlalchemy.orm import Session
 
 from onyx.access.utils import build_ext_group_name_for_onyx
@@ -63,41 +62,20 @@ def delete_public_external_group_for_cc_pair__no_commit(
     )
 
 
-def mark_old_external_groups_as_stale(
+def replace_user__ext_group_for_cc_pair(
     db_session: Session,
     cc_pair_id: int,
-) -> None:
-    db_session.execute(
-        update(User__ExternalUserGroupId)
-        .where(User__ExternalUserGroupId.cc_pair_id == cc_pair_id)
-        .values(stale=True)
-    )
-    db_session.execute(
-        update(PublicExternalUserGroup)
-        .where(PublicExternalUserGroup.cc_pair_id == cc_pair_id)
-        .values(stale=True)
-    )
-
-
-def upsert_external_groups(
-    db_session: Session,
-    cc_pair_id: int,
-    external_groups: list[ExternalUserGroup],
+    group_defs: list[ExternalUserGroup],
     source: DocumentSource,
 ) -> None:
     """
-    Performs a true upsert operation for external user groups:
-    - For existing groups (same user_id, external_user_group_id, cc_pair_id), updates the stale flag to False
-    - For new groups, inserts them with stale=False
-    - For public groups, uses upsert logic as well
+    This function clears all existing external user group relations for a given cc_pair_id
+    and replaces them with the new group definitions and commits the changes.
     """
-    # If there are no groups to add, return early
-    if not external_groups:
-        return
 
     # collect all emails from all groups to batch add all users at once for efficiency
     all_group_member_emails = set()
-    for external_group in external_groups:
+    for external_group in group_defs:
         for user_email in external_group.user_emails:
             all_group_member_emails.add(user_email)
 
@@ -108,17 +86,26 @@ def upsert_external_groups(
         emails=list(all_group_member_emails),
     )
 
-    # map emails to ids
-    email_id_map = {user.email.lower(): user.id for user in all_group_members}
+    delete_user__ext_group_for_cc_pair__no_commit(
+        db_session=db_session,
+        cc_pair_id=cc_pair_id,
+    )
+    delete_public_external_group_for_cc_pair__no_commit(
+        db_session=db_session,
+        cc_pair_id=cc_pair_id,
+    )
 
-    # Process each external group
-    for external_group in external_groups:
+    # map emails to ids
+    email_id_map = {user.email: user.id for user in all_group_members}
+
+    # use these ids to create new external user group relations relating group_id to user_ids
+    new_external_permissions: list[User__ExternalUserGroupId] = []
+    new_public_external_groups: list[PublicExternalUserGroup] = []
+    for external_group in group_defs:
         external_group_id = build_ext_group_name_for_onyx(
             ext_group_name=external_group.id,
             source=source,
         )
-
-        # Handle user-group mappings
         for user_email in external_group.user_emails:
             user_id = email_id_map.get(user_email.lower())
             if user_id is None:
@@ -127,71 +114,24 @@ def upsert_external_groups(
                     f" with email {user_email} not found"
                 )
                 continue
-
-            # Check if the user-group mapping already exists
-            existing_user_group = db_session.scalar(
-                select(User__ExternalUserGroupId).where(
-                    User__ExternalUserGroupId.user_id == user_id,
-                    User__ExternalUserGroupId.external_user_group_id
-                    == external_group_id,
-                    User__ExternalUserGroupId.cc_pair_id == cc_pair_id,
-                )
-            )
-
-            if existing_user_group:
-                # Update existing record
-                existing_user_group.stale = False
-            else:
-                # Insert new record
-                new_user_group = User__ExternalUserGroupId(
+            new_external_permissions.append(
+                User__ExternalUserGroupId(
                     user_id=user_id,
                     external_user_group_id=external_group_id,
                     cc_pair_id=cc_pair_id,
-                    stale=False,
-                )
-                db_session.add(new_user_group)
-
-        # Handle public group if needed
-        if external_group.gives_anyone_access:
-            # Check if the public group already exists
-            existing_public_group = db_session.scalar(
-                select(PublicExternalUserGroup).where(
-                    PublicExternalUserGroup.external_user_group_id == external_group_id,
-                    PublicExternalUserGroup.cc_pair_id == cc_pair_id,
                 )
             )
 
-            if existing_public_group:
-                # Update existing record
-                existing_public_group.stale = False
-            else:
-                # Insert new record
-                new_public_group = PublicExternalUserGroup(
+        if external_group.gives_anyone_access:
+            new_public_external_groups.append(
+                PublicExternalUserGroup(
                     external_user_group_id=external_group_id,
                     cc_pair_id=cc_pair_id,
-                    stale=False,
                 )
-                db_session.add(new_public_group)
+            )
 
-    db_session.commit()
-
-
-def remove_stale_external_groups(
-    db_session: Session,
-    cc_pair_id: int,
-) -> None:
-    db_session.execute(
-        delete(User__ExternalUserGroupId).where(
-            User__ExternalUserGroupId.cc_pair_id == cc_pair_id,
-            User__ExternalUserGroupId.stale.is_(True),
-        )
-    )
-    db_session.execute(
-        delete(PublicExternalUserGroup).where(
-            PublicExternalUserGroup.cc_pair_id == cc_pair_id,
-            PublicExternalUserGroup.stale.is_(True),
-        )
-    )
+    db_session.add_all(new_external_permissions)
+    db_session.add_all(new_public_external_groups)
     db_session.commit()
 
 

backend/ee/onyx/db/usage_export.py

@@ -114,24 +114,12 @@ def get_all_usage_reports(db_session: Session) -> list[UsageReportMetadata]:
 
 
 def get_usage_report_data(
-    report_display_name: str,
+    db_session: Session,
+    report_name: str,
 ) -> IO:
-    """
-    Get the usage report data from the file store.
-
-    Args:
-        db_session: The database session.
-        report_display_name: The display name of the usage report. Also assumes
-                             that the file is stored with this as the ID in the file store.
-
-    Returns:
-        The usage report data.
-    """
-    file_store = get_default_file_store()
+    file_store = get_default_file_store(db_session)
     # usage report may be very large, so don't load it all into memory
-    return file_store.read_file(
-        file_id=report_display_name, mode="b", use_tempfile=True
-    )
+    return file_store.read_file(file_name=report_name, mode="b", use_tempfile=True)
 
 
 def write_usage_report(

backend/ee/onyx/db/user_group.py

@@ -128,14 +128,11 @@ def validate_object_creation_for_user(
     target_group_ids: list[int] | None = None,
     object_is_public: bool | None = None,
     object_is_perm_sync: bool | None = None,
-    object_is_owned_by_user: bool = False,
-    object_is_new: bool = False,
 ) -> None:
     """
     All users can create/edit permission synced objects if they don't specify a group
     All admin actions are allowed.
-    Curators and global curators can create public objects.
-    Prevents other non-admins from creating/editing:
+    Prevents non-admins from creating/editing:
     - public objects
     - objects with no groups
     - objects that belong to a group they don't curate
@@ -146,23 +143,13 @@ def validate_object_creation_for_user(
     if not user or user.role == UserRole.ADMIN:
         return
 
-    # Allow curators and global curators to create public objects
-    # w/o associated groups IF the object is new/owned by them
-    if (
-        object_is_public
-        and user.role in [UserRole.CURATOR, UserRole.GLOBAL_CURATOR]
-        and (object_is_new or object_is_owned_by_user)
-    ):
-        return
-
-    if object_is_public and user.role == UserRole.BASIC:
-        detail = "User does not have permission to create public objects"
+    if object_is_public:
+        detail = "User does not have permission to create public credentials"
         logger.error(detail)
         raise HTTPException(
             status_code=400,
             detail=detail,
         )
-
     if not target_group_ids:
         detail = "Curators must specify 1+ groups"
         logger.error(detail)

backend/ee/onyx/document_index/vespa/app_config/cloud-services.xml.jinja

@@ -18,9 +18,9 @@
             <!-- <document type="danswer_chunk" mode="index" /> -->
 {{ document_elements }}
         </documents>
-        <nodes count="60">
-            <resources vcpu="8.0" memory="128.0Gb" architecture="arm64" storage-type="local"
-                disk="475.0Gb" />
+        <nodes count="75">
+            <resources vcpu="8.0" memory="64.0Gb" architecture="arm64" storage-type="local"
+                disk="474.0Gb" />
         </nodes>
         <engine>
             <proton>

backend/ee/onyx/external_permissions/confluence/doc_sync.py

@@ -6,12 +6,11 @@ https://confluence.atlassian.com/conf85/check-who-can-view-a-page-1283360557.htm
 from collections.abc import Generator
 
 from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsFunction
-from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsIdsFunction
-from ee.onyx.external_permissions.utils import generic_doc_sync
 from onyx.access.models import DocExternalAccess
-from onyx.configs.constants import DocumentSource
+from onyx.access.models import ExternalAccess
 from onyx.connectors.confluence.connector import ConfluenceConnector
 from onyx.connectors.credentials_provider import OnyxDBCredentialsProvider
+from onyx.connectors.models import SlimDocument
 from onyx.db.models import ConnectorCredentialPair
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
@@ -20,13 +19,9 @@ from shared_configs.contextvars import get_current_tenant_id
 logger = setup_logger()
 
 
-CONFLUENCE_DOC_SYNC_LABEL = "confluence_doc_sync"
-
-
 def confluence_doc_sync(
     cc_pair: ConnectorCredentialPair,
     fetch_all_existing_docs_fn: FetchAllDocumentsFunction,
-    fetch_all_existing_docs_ids_fn: FetchAllDocumentsIdsFunction,
     callback: IndexingHeartbeatInterface | None,
 ) -> Generator[DocExternalAccess, None, None]:
     """
@@ -34,6 +29,7 @@ def confluence_doc_sync(
     Compares fetched documents against existing documents in the DB for the connector.
     If a document exists in the DB but not in the Confluence fetch, it's marked as restricted.
     """
+    logger.info(f"Starting confluence doc sync for CC Pair ID: {cc_pair.id}")
     confluence_connector = ConfluenceConnector(
         **cc_pair.connector.connector_specific_config
     )
@@ -43,11 +39,52 @@ def confluence_doc_sync(
     )
     confluence_connector.set_credentials_provider(provider)
 
-    yield from generic_doc_sync(
-        cc_pair=cc_pair,
-        fetch_all_existing_docs_ids_fn=fetch_all_existing_docs_ids_fn,
-        callback=callback,
-        doc_source=DocumentSource.CONFLUENCE,
-        slim_connector=confluence_connector,
-        label=CONFLUENCE_DOC_SYNC_LABEL,
-    )
+    slim_docs: list[SlimDocument] = []
+    logger.info("Fetching all slim documents from confluence")
+    for doc_batch in confluence_connector.retrieve_all_slim_documents(
+        callback=callback
+    ):
+        logger.info(f"Got {len(doc_batch)} slim documents from confluence")
+        if callback:
+            if callback.should_stop():
+                raise RuntimeError("confluence_doc_sync: Stop signal detected")
+
+            callback.progress("confluence_doc_sync", 1)
+
+        slim_docs.extend(doc_batch)
+
+    # Find documents that are no longer accessible in Confluence
+    logger.info(f"Querying existing document IDs for CC Pair ID: {cc_pair.id}")
+    existing_doc_ids = fetch_all_existing_docs_fn()
+
+    # Find missing doc IDs
+    fetched_doc_ids = {doc.id for doc in slim_docs}
+    missing_doc_ids = set(existing_doc_ids) - fetched_doc_ids
+
+    # Yield access removal for missing docs. Better to be safe.
+    if missing_doc_ids:
+        logger.warning(
+            f"Found {len(missing_doc_ids)} documents that are in the DB but "
+            "not present in Confluence fetch. Making them inaccessible."
+        )
+        for missing_id in missing_doc_ids:
+            logger.warning(f"Removing access for document ID: {missing_id}")
+            yield DocExternalAccess(
+                doc_id=missing_id,
+                external_access=ExternalAccess(
+                    external_user_emails=set(),
+                    external_user_group_ids=set(),
+                    is_public=False,
+                ),
+            )
+
+    for doc in slim_docs:
+        if not doc.external_access:
+            raise RuntimeError(f"No external access found for document ID: {doc.id}")
+
+        yield DocExternalAccess(
+            doc_id=doc.id,
+            external_access=doc.external_access,
+        )
+
+    logger.info("Finished confluence doc sync")

backend/ee/onyx/external_permissions/confluence/group_sync.py

@@ -1,5 +1,3 @@
-from collections.abc import Generator
-
 from ee.onyx.db.external_perm import ExternalUserGroup
 from ee.onyx.external_permissions.confluence.constants import ALL_CONF_EMAILS_GROUP_NAME
 from onyx.background.error_logging import emit_background_error
@@ -67,7 +65,7 @@ def _build_group_member_email_map(
 def confluence_group_sync(
     tenant_id: str,
     cc_pair: ConnectorCredentialPair,
-) -> Generator[ExternalUserGroup, None, None]:
+) -> list[ExternalUserGroup]:
     provider = OnyxDBCredentialsProvider(tenant_id, "confluence", cc_pair.credential_id)
     is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
     wiki_base: str = cc_pair.connector.connector_specific_config["wiki_base"]
@@ -91,10 +89,10 @@ def confluence_group_sync(
         confluence_client=confluence_client,
         cc_pair_id=cc_pair.id,
     )
-
+    onyx_groups: list[ExternalUserGroup] = []
     all_found_emails = set()
     for group_id, group_member_emails in group_member_email_map.items():
-        yield (
+        onyx_groups.append(
             ExternalUserGroup(
                 id=group_id,
                 user_emails=list(group_member_emails),
@@ -109,4 +107,6 @@ def confluence_group_sync(
             id=ALL_CONF_EMAILS_GROUP_NAME,
             user_emails=list(all_found_emails),
         )
-        yield all_found_group
+        onyx_groups.append(all_found_group)
+
+    return onyx_groups

backend/ee/onyx/external_permissions/confluence/space_access.py

@@ -124,9 +124,9 @@ def get_space_permission(
         and not space_permissions.external_user_group_ids
     ):
         logger.warning(
-            f"No permissions found for space '{space_key}'. This is very unlikely "
-            "to be correct and is more likely caused by an access token with "
-            "insufficient permissions. Make sure that the access token has Admin "
+            f"No permissions found for space '{space_key}'. This is very unlikely"
+            "to be correct and is more likely caused by an access token with"
+            "insufficient permissions. Make sure that the access token has Admin"
             f"permissions for space '{space_key}'"
         )
 

backend/ee/onyx/external_permissions/github/doc_sync.py

@@ -1,294 +0,0 @@
-import json
-from collections.abc import Generator
-
-from github import Github
-from github.Repository import Repository
-
-from ee.onyx.external_permissions.github.utils import fetch_repository_team_slugs
-from ee.onyx.external_permissions.github.utils import form_collaborators_group_id
-from ee.onyx.external_permissions.github.utils import form_organization_group_id
-from ee.onyx.external_permissions.github.utils import (
-    form_outside_collaborators_group_id,
-)
-from ee.onyx.external_permissions.github.utils import get_external_access_permission
-from ee.onyx.external_permissions.github.utils import get_repository_visibility
-from ee.onyx.external_permissions.github.utils import GitHubVisibility
-from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsFunction
-from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsIdsFunction
-from onyx.access.models import DocExternalAccess
-from onyx.access.utils import build_ext_group_name_for_onyx
-from onyx.configs.constants import DocumentSource
-from onyx.connectors.github.connector import DocMetadata
-from onyx.connectors.github.connector import GithubConnector
-from onyx.db.models import ConnectorCredentialPair
-from onyx.db.utils import DocumentRow
-from onyx.db.utils import SortOrder
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-GITHUB_DOC_SYNC_LABEL = "github_doc_sync"
-
-
-def github_doc_sync(
-    cc_pair: ConnectorCredentialPair,
-    fetch_all_existing_docs_fn: FetchAllDocumentsFunction,
-    fetch_all_existing_docs_ids_fn: FetchAllDocumentsIdsFunction,
-    callback: IndexingHeartbeatInterface | None = None,
-) -> Generator[DocExternalAccess, None, None]:
-    """
-    Sync GitHub documents with external access permissions.
-
-    This function checks each repository for visibility/team changes and updates
-    document permissions accordingly without using checkpoints.
-    """
-    logger.info(f"Starting GitHub document sync for CC pair ID: {cc_pair.id}")
-
-    # Initialize GitHub connector with credentials
-    github_connector: GithubConnector = GithubConnector(
-        **cc_pair.connector.connector_specific_config
-    )
-
-    github_connector.load_credentials(cc_pair.credential.credential_json)
-    logger.info("GitHub connector credentials loaded successfully")
-
-    if not github_connector.github_client:
-        logger.error("GitHub client initialization failed")
-        raise ValueError("github_client is required")
-
-    # Get all repositories from GitHub API
-    logger.info("Fetching all repositories from GitHub API")
-    try:
-        repos = []
-        if github_connector.repositories:
-            if "," in github_connector.repositories:
-                # Multiple repositories specified
-                repos = github_connector.get_github_repos(
-                    github_connector.github_client
-                )
-            else:
-                # Single repository
-                repos = [
-                    github_connector.get_github_repo(github_connector.github_client)
-                ]
-        else:
-            # All repositories
-            repos = github_connector.get_all_repos(github_connector.github_client)
-
-        logger.info(f"Found {len(repos)} repositories to check")
-    except Exception as e:
-        logger.error(f"Failed to fetch repositories: {e}")
-        raise
-
-    repo_to_doc_list_map: dict[str, list[DocumentRow]] = {}
-    # sort order is ascending because we want to get the oldest documents first
-    existing_docs: list[DocumentRow] = fetch_all_existing_docs_fn(
-        sort_order=SortOrder.ASC
-    )
-    logger.info(f"Found {len(existing_docs)} documents to check")
-    for doc in existing_docs:
-        try:
-            doc_metadata = DocMetadata.model_validate_json(json.dumps(doc.doc_metadata))
-            if doc_metadata.repo not in repo_to_doc_list_map:
-                repo_to_doc_list_map[doc_metadata.repo] = []
-            repo_to_doc_list_map[doc_metadata.repo].append(doc)
-        except Exception as e:
-            logger.error(f"Failed to parse doc metadata: {e} for doc {doc.id}")
-            continue
-    logger.info(f"Found {len(repo_to_doc_list_map)} documents to check")
-    # Process each repository individually
-    for repo in repos:
-        try:
-            logger.info(f"Processing repository: {repo.id} (name: {repo.name})")
-            repo_doc_list: list[DocumentRow] = repo_to_doc_list_map.get(
-                repo.full_name, []
-            )
-            if not repo_doc_list:
-                logger.warning(
-                    f"No documents found for repository {repo.id} ({repo.name})"
-                )
-                continue
-
-            current_external_group_ids = repo_doc_list[0].external_user_group_ids or []
-            # Check if repository has any permission changes
-            has_changes = _check_repository_for_changes(
-                repo=repo,
-                github_client=github_connector.github_client,
-                current_external_group_ids=current_external_group_ids,
-            )
-
-            if has_changes:
-                logger.info(
-                    f"Repository {repo.id} ({repo.name}) has changes, updating documents"
-                )
-
-                # Get new external access permissions for this repository
-                new_external_access = get_external_access_permission(
-                    repo, github_connector.github_client
-                )
-
-                logger.info(
-                    f"Found {len(repo_doc_list)} documents for repository {repo.full_name}"
-                )
-
-                # Yield updated external access for each document
-                for doc in repo_doc_list:
-                    if callback:
-                        callback.progress(GITHUB_DOC_SYNC_LABEL, 1)
-
-                    yield DocExternalAccess(
-                        doc_id=doc.id,
-                        external_access=new_external_access,
-                    )
-            else:
-                logger.info(
-                    f"Repository {repo.id} ({repo.name}) has no changes, skipping"
-                )
-        except Exception as e:
-            logger.error(f"Error processing repository {repo.id} ({repo.name}): {e}")
-
-    logger.info(f"GitHub document sync completed for CC pair ID: {cc_pair.id}")
-
-
-def _check_repository_for_changes(
-    repo: Repository,
-    github_client: Github,
-    current_external_group_ids: list[str],
-) -> bool:
-    """
-    Check if repository has any permission changes (visibility or team updates).
-    """
-    logger.info(f"Checking repository {repo.id} ({repo.name}) for changes")
-
-    # Check for repository visibility changes using the sample document data
-    if _is_repo_visibility_changed_from_groups(
-        repo=repo,
-        current_external_group_ids=current_external_group_ids,
-    ):
-        logger.info(f"Repository {repo.id} ({repo.name}) has visibility changes")
-        return True
-
-    # Check for team membership changes if repository is private
-    if get_repository_visibility(
-        repo
-    ) == GitHubVisibility.PRIVATE and _teams_updated_from_groups(
-        repo=repo,
-        github_client=github_client,
-        current_external_group_ids=current_external_group_ids,
-    ):
-        logger.info(f"Repository {repo.id} ({repo.name}) has team changes")
-        return True
-
-    logger.info(f"Repository {repo.id} ({repo.name}) has no changes")
-    return False
-
-
-def _is_repo_visibility_changed_from_groups(
-    repo: Repository,
-    current_external_group_ids: list[str],
-) -> bool:
-    """
-    Check if repository visibility has changed by analyzing existing external group IDs.
-
-    Args:
-        repo: GitHub repository object
-        current_external_group_ids: List of external group IDs from existing document
-
-    Returns:
-        True if visibility has changed
-    """
-    current_repo_visibility = get_repository_visibility(repo)
-    logger.info(f"Current repository visibility: {current_repo_visibility.value}")
-
-    # Build expected group IDs for current visibility
-    collaborators_group_id = build_ext_group_name_for_onyx(
-        source=DocumentSource.GITHUB,
-        ext_group_name=form_collaborators_group_id(repo.id),
-    )
-
-    org_group_id = None
-    if repo.organization:
-        org_group_id = build_ext_group_name_for_onyx(
-            source=DocumentSource.GITHUB,
-            ext_group_name=form_organization_group_id(repo.organization.id),
-        )
-
-    # Determine existing visibility from group IDs
-    has_collaborators_group = collaborators_group_id in current_external_group_ids
-    has_org_group = org_group_id and org_group_id in current_external_group_ids
-
-    if has_collaborators_group:
-        existing_repo_visibility = GitHubVisibility.PRIVATE
-    elif has_org_group:
-        existing_repo_visibility = GitHubVisibility.INTERNAL
-    else:
-        existing_repo_visibility = GitHubVisibility.PUBLIC
-
-    logger.info(f"Inferred existing visibility: {existing_repo_visibility.value}")
-
-    visibility_changed = existing_repo_visibility != current_repo_visibility
-    if visibility_changed:
-        logger.info(
-            f"Visibility changed for repo {repo.id} ({repo.name}): "
-            f"{existing_repo_visibility.value} -> {current_repo_visibility.value}"
-        )
-
-    return visibility_changed
-
-
-def _teams_updated_from_groups(
-    repo: Repository,
-    github_client: Github,
-    current_external_group_ids: list[str],
-) -> bool:
-    """
-    Check if repository team memberships have changed using existing group IDs.
-    """
-    # Fetch current team slugs for the repository
-    current_teams = fetch_repository_team_slugs(repo=repo, github_client=github_client)
-    logger.info(
-        f"Current teams for repository {repo.id} (name: {repo.name}): {current_teams}"
-    )
-
-    # Build group IDs to exclude from team comparison (non-team groups)
-    collaborators_group_id = build_ext_group_name_for_onyx(
-        source=DocumentSource.GITHUB,
-        ext_group_name=form_collaborators_group_id(repo.id),
-    )
-    outside_collaborators_group_id = build_ext_group_name_for_onyx(
-        source=DocumentSource.GITHUB,
-        ext_group_name=form_outside_collaborators_group_id(repo.id),
-    )
-    non_team_group_ids = {collaborators_group_id, outside_collaborators_group_id}
-
-    # Extract existing team IDs from current external group IDs
-    existing_team_ids = set()
-    for group_id in current_external_group_ids:
-        # Skip all non-team groups, keep only team groups
-        if group_id not in non_team_group_ids:
-            existing_team_ids.add(group_id)
-
-    # Note: existing_team_ids from DB are already prefixed (e.g., "github__team-slug")
-    # but current_teams from API are raw team slugs, so we need to add the prefix
-    current_team_ids = set()
-    for team_slug in current_teams:
-        team_group_id = build_ext_group_name_for_onyx(
-            source=DocumentSource.GITHUB,
-            ext_group_name=team_slug,
-        )
-        current_team_ids.add(team_group_id)
-
-    logger.info(
-        f"Existing team IDs: {existing_team_ids}, Current team IDs: {current_team_ids}"
-    )
-
-    # Compare actual team IDs to detect changes
-    teams_changed = current_team_ids != existing_team_ids
-    if teams_changed:
-        logger.info(
-            f"Team changes detected for repo {repo.id} (name: {repo.name}): "
-            f"existing={existing_team_ids}, current={current_team_ids}"
-        )
-
-    return teams_changed

backend/ee/onyx/external_permissions/github/group_sync.py

@@ -1,46 +0,0 @@
-from collections.abc import Generator
-
-from github import Repository
-
-from ee.onyx.db.external_perm import ExternalUserGroup
-from ee.onyx.external_permissions.github.utils import get_external_user_group
-from onyx.connectors.github.connector import GithubConnector
-from onyx.db.models import ConnectorCredentialPair
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-def github_group_sync(
-    tenant_id: str,
-    cc_pair: ConnectorCredentialPair,
-) -> Generator[ExternalUserGroup, None, None]:
-    github_connector: GithubConnector = GithubConnector(
-        **cc_pair.connector.connector_specific_config
-    )
-    github_connector.load_credentials(cc_pair.credential.credential_json)
-    if not github_connector.github_client:
-        raise ValueError("github_client is required")
-
-    logger.info("Starting GitHub group sync...")
-    repos: list[Repository.Repository] = []
-    if github_connector.repositories:
-        if "," in github_connector.repositories:
-            # Multiple repositories specified
-            repos = github_connector.get_github_repos(github_connector.github_client)
-        else:
-            # Single repository (backward compatibility)
-            repos = [github_connector.get_github_repo(github_connector.github_client)]
-    else:
-        # All repositories
-        repos = github_connector.get_all_repos(github_connector.github_client)
-
-    for repo in repos:
-        try:
-            for external_group in get_external_user_group(
-                repo, github_connector.github_client
-            ):
-                logger.info(f"External group: {external_group}")
-                yield external_group
-        except Exception as e:
-            logger.error(f"Error processing repository {repo.id} ({repo.name}): {e}")

backend/ee/onyx/external_permissions/github/utils.py

@@ -1,488 +0,0 @@
-from collections.abc import Callable
-from enum import Enum
-from typing import List
-from typing import Optional
-from typing import Tuple
-from typing import TypeVar
-
-from github import Github
-from github import RateLimitExceededException
-from github.GithubException import GithubException
-from github.NamedUser import NamedUser
-from github.Organization import Organization
-from github.PaginatedList import PaginatedList
-from github.Repository import Repository
-from github.Team import Team
-from pydantic import BaseModel
-
-from ee.onyx.db.external_perm import ExternalUserGroup
-from onyx.access.models import ExternalAccess
-from onyx.access.utils import build_ext_group_name_for_onyx
-from onyx.configs.constants import DocumentSource
-from onyx.connectors.github.rate_limit_utils import sleep_after_rate_limit_exception
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-class GitHubVisibility(Enum):
-    """GitHub repository visibility options."""
-
-    PUBLIC = "public"
-    PRIVATE = "private"
-    INTERNAL = "internal"
-
-
-MAX_RETRY_COUNT = 3
-
-T = TypeVar("T")
-
-# Higher-order function to wrap GitHub operations with retry and exception handling
-
-
-def _run_with_retry(
-    operation: Callable[[], T],
-    description: str,
-    github_client: Github,
-    retry_count: int = 0,
-) -> Optional[T]:
-    """Execute a GitHub operation with retry on rate limit and exception handling."""
-    logger.debug(f"Starting operation '{description}', attempt {retry_count + 1}")
-    try:
-        result = operation()
-        logger.debug(f"Operation '{description}' completed successfully")
-        return result
-    except RateLimitExceededException:
-        if retry_count < MAX_RETRY_COUNT:
-            sleep_after_rate_limit_exception(github_client)
-            logger.warning(
-                f"Rate limit exceeded while {description}. Retrying... "
-                f"(attempt {retry_count + 1}/{MAX_RETRY_COUNT})"
-            )
-            return _run_with_retry(
-                operation, description, github_client, retry_count + 1
-            )
-        else:
-            error_msg = f"Max retries exceeded for {description}"
-            logger.exception(error_msg)
-            raise RuntimeError(error_msg)
-    except GithubException as e:
-        logger.warning(f"GitHub API error during {description}: {e}")
-        return None
-    except Exception as e:
-        logger.exception(f"Unexpected error during {description}: {e}")
-        return None
-
-
-class UserInfo(BaseModel):
-    """Represents a GitHub user with their basic information."""
-
-    login: str
-    name: Optional[str] = None
-    email: Optional[str] = None
-
-
-class TeamInfo(BaseModel):
-    """Represents a GitHub team with its members."""
-
-    name: str
-    slug: str
-    members: List[UserInfo]
-
-
-def _fetch_organization_members(
-    github_client: Github, org_name: str, retry_count: int = 0
-) -> List[UserInfo]:
-    """Fetch all organization members including owners and regular members."""
-    org_members: List[UserInfo] = []
-    logger.info(f"Fetching organization members for {org_name}")
-
-    org = _run_with_retry(
-        lambda: github_client.get_organization(org_name),
-        f"get organization {org_name}",
-        github_client,
-    )
-    if not org:
-        logger.error(f"Failed to fetch organization {org_name}")
-        raise RuntimeError(f"Failed to fetch organization {org_name}")
-
-    member_objs: PaginatedList[NamedUser] | list[NamedUser] = (
-        _run_with_retry(
-            lambda: org.get_members(filter_="all"),
-            f"get members for organization {org_name}",
-            github_client,
-        )
-        or []
-    )
-
-    for member in member_objs:
-        user_info = UserInfo(login=member.login, name=member.name, email=member.email)
-        org_members.append(user_info)
-
-    logger.info(f"Fetched {len(org_members)} members for organization {org_name}")
-    return org_members
-
-
-def _fetch_repository_teams_detailed(
-    repo: Repository, github_client: Github, retry_count: int = 0
-) -> List[TeamInfo]:
-    """Fetch teams with access to the repository and their members."""
-    teams_data: List[TeamInfo] = []
-    logger.info(f"Fetching teams for repository {repo.full_name}")
-
-    team_objs: PaginatedList[Team] | list[Team] = (
-        _run_with_retry(
-            lambda: repo.get_teams(),
-            f"get teams for repository {repo.full_name}",
-            github_client,
-        )
-        or []
-    )
-
-    for team in team_objs:
-        logger.info(
-            f"Processing team {team.name} (slug: {team.slug}) for repository {repo.full_name}"
-        )
-
-        members: PaginatedList[NamedUser] | list[NamedUser] = (
-            _run_with_retry(
-                lambda: team.get_members(),
-                f"get members for team {team.name}",
-                github_client,
-            )
-            or []
-        )
-
-        team_members = []
-        for m in members:
-            user_info = UserInfo(login=m.login, name=m.name, email=m.email)
-            team_members.append(user_info)
-
-        team_info = TeamInfo(name=team.name, slug=team.slug, members=team_members)
-        teams_data.append(team_info)
-        logger.info(f"Team {team.name} has {len(team_members)} members")
-
-    logger.info(f"Fetched {len(teams_data)} teams for repository {repo.full_name}")
-    return teams_data
-
-
-def fetch_repository_team_slugs(
-    repo: Repository, github_client: Github, retry_count: int = 0
-) -> List[str]:
-    """Fetch team slugs with access to the repository."""
-    logger.info(f"Fetching team slugs for repository {repo.full_name}")
-    teams_data: List[str] = []
-
-    team_objs: PaginatedList[Team] | list[Team] = (
-        _run_with_retry(
-            lambda: repo.get_teams(),
-            f"get teams for repository {repo.full_name}",
-            github_client,
-        )
-        or []
-    )
-
-    for team in team_objs:
-        teams_data.append(team.slug)
-
-    logger.info(f"Fetched {len(teams_data)} team slugs for repository {repo.full_name}")
-    return teams_data
-
-
-def _get_collaborators_and_outside_collaborators(
-    github_client: Github,
-    repo: Repository,
-) -> Tuple[List[UserInfo], List[UserInfo]]:
-    """Fetch and categorize collaborators into regular and outside collaborators."""
-    collaborators: List[UserInfo] = []
-    outside_collaborators: List[UserInfo] = []
-    logger.info(f"Fetching collaborators for repository {repo.full_name}")
-
-    repo_collaborators: PaginatedList[NamedUser] | list[NamedUser] = (
-        _run_with_retry(
-            lambda: repo.get_collaborators(),
-            f"get collaborators for repository {repo.full_name}",
-            github_client,
-        )
-        or []
-    )
-
-    for collaborator in repo_collaborators:
-        is_outside = False
-
-        # Check if collaborator is outside the organization
-        if repo.organization:
-            org: Organization | None = _run_with_retry(
-                lambda: github_client.get_organization(repo.organization.login),
-                f"get organization {repo.organization.login}",
-                github_client,
-            )
-
-            if org is not None:
-                org_obj = org
-                membership = _run_with_retry(
-                    lambda: org_obj.has_in_members(collaborator),
-                    f"check membership for {collaborator.login} in org {org_obj.login}",
-                    github_client,
-                )
-                is_outside = membership is not None and not membership
-
-        info = UserInfo(
-            login=collaborator.login, name=collaborator.name, email=collaborator.email
-        )
-        if repo.organization and is_outside:
-            outside_collaborators.append(info)
-        else:
-            collaborators.append(info)
-
-    logger.info(
-        f"Categorized {len(collaborators)} regular and {len(outside_collaborators)} outside collaborators for {repo.full_name}"
-    )
-    return collaborators, outside_collaborators
-
-
-def form_collaborators_group_id(repository_id: int) -> str:
-    """Generate group ID for repository collaborators."""
-    if not repository_id:
-        logger.exception("Repository ID is required to generate collaborators group ID")
-        raise ValueError("Repository ID must be set to generate group ID.")
-    group_id = f"{repository_id}_collaborators"
-    return group_id
-
-
-def form_organization_group_id(organization_id: int) -> str:
-    """Generate group ID for organization using organization ID."""
-    if not organization_id:
-        logger.exception(
-            "Organization ID is required to generate organization group ID"
-        )
-        raise ValueError("Organization ID must be set to generate group ID.")
-    group_id = f"{organization_id}_organization"
-    return group_id
-
-
-def form_outside_collaborators_group_id(repository_id: int) -> str:
-    """Generate group ID for outside collaborators."""
-    if not repository_id:
-        logger.exception(
-            "Repository ID is required to generate outside collaborators group ID"
-        )
-        raise ValueError("Repository ID must be set to generate group ID.")
-    group_id = f"{repository_id}_outside_collaborators"
-    return group_id
-
-
-def get_repository_visibility(repo: Repository) -> GitHubVisibility:
-    """
-    Get the visibility of a repository.
-    Returns GitHubVisibility enum member.
-    """
-    if hasattr(repo, "visibility"):
-        visibility = repo.visibility
-        logger.info(
-            f"Repository {repo.full_name} visibility from attribute: {visibility}"
-        )
-        try:
-            return GitHubVisibility(visibility)
-        except ValueError:
-            logger.warning(
-                f"Unknown visibility '{visibility}' for repo {repo.full_name}, defaulting to private"
-            )
-            return GitHubVisibility.PRIVATE
-
-    logger.info(f"Repository {repo.full_name} is private")
-    return GitHubVisibility.PRIVATE
-
-
-def get_external_access_permission(
-    repo: Repository, github_client: Github, add_prefix: bool = False
-) -> ExternalAccess:
-    """
-    Get the external access permission for a repository.
-    Uses group-based permissions for efficiency and scalability.
-
-    add_prefix: When this method is called during the initial permission sync via the connector,
-                the group ID isn't prefixed with the source while inserting the document record.
-                So in that case, set add_prefix to True, allowing the method itself to handle
-                prefixing. However, when the same method is invoked from doc_sync, our system
-                already adds the prefix to the group ID while processing the ExternalAccess object.
-    """
-    # We maintain collaborators, and outside collaborators as two separate groups
-    # instead of adding individual user emails to ExternalAccess.external_user_emails for two reasons:
-    # 1. Changes in repo collaborators (additions/removals) would require updating all documents.
-    # 2. Repo permissions can change without updating the repo's updated_at timestamp,
-    #    forcing full permission syncs for all documents every time, which is inefficient.
-
-    repo_visibility = get_repository_visibility(repo)
-    logger.info(
-        f"Generating ExternalAccess for {repo.full_name}: visibility={repo_visibility.value}"
-    )
-
-    if repo_visibility == GitHubVisibility.PUBLIC:
-        logger.info(
-            f"Repository {repo.full_name} is public - allowing access to all users"
-        )
-        return ExternalAccess(
-            external_user_emails=set(),
-            external_user_group_ids=set(),
-            is_public=True,
-        )
-    elif repo_visibility == GitHubVisibility.PRIVATE:
-        logger.info(
-            f"Repository {repo.full_name} is private - setting up restricted access"
-        )
-
-        collaborators_group_id = form_collaborators_group_id(repo.id)
-        outside_collaborators_group_id = form_outside_collaborators_group_id(repo.id)
-        if add_prefix:
-            collaborators_group_id = build_ext_group_name_for_onyx(
-                source=DocumentSource.GITHUB,
-                ext_group_name=collaborators_group_id,
-            )
-            outside_collaborators_group_id = build_ext_group_name_for_onyx(
-                source=DocumentSource.GITHUB,
-                ext_group_name=outside_collaborators_group_id,
-            )
-        group_ids = {collaborators_group_id, outside_collaborators_group_id}
-
-        team_slugs = fetch_repository_team_slugs(repo, github_client)
-        if add_prefix:
-            team_slugs = [
-                build_ext_group_name_for_onyx(
-                    source=DocumentSource.GITHUB,
-                    ext_group_name=slug,
-                )
-                for slug in team_slugs
-            ]
-        group_ids.update(team_slugs)
-
-        logger.info(f"ExternalAccess groups for {repo.full_name}: {group_ids}")
-        return ExternalAccess(
-            external_user_emails=set(),
-            external_user_group_ids=group_ids,
-            is_public=False,
-        )
-    else:
-        # Internal repositories - accessible to organization members
-        logger.info(
-            f"Repository {repo.full_name} is internal - accessible to org members"
-        )
-        org_group_id = form_organization_group_id(repo.organization.id)
-        if add_prefix:
-            org_group_id = build_ext_group_name_for_onyx(
-                source=DocumentSource.GITHUB,
-                ext_group_name=org_group_id,
-            )
-        group_ids = {org_group_id}
-        logger.info(f"ExternalAccess groups for {repo.full_name}: {group_ids}")
-        return ExternalAccess(
-            external_user_emails=set(),
-            external_user_group_ids=group_ids,
-            is_public=False,
-        )
-
-
-def get_external_user_group(
-    repo: Repository, github_client: Github
-) -> list[ExternalUserGroup]:
-    """
-    Get the external user group for a repository.
-    Creates ExternalUserGroup objects with actual user emails for each permission group.
-    """
-    repo_visibility = get_repository_visibility(repo)
-    logger.info(
-        f"Generating ExternalUserGroups for {repo.full_name}: visibility={repo_visibility.value}"
-    )
-
-    if repo_visibility == GitHubVisibility.PRIVATE:
-        logger.info(f"Processing private repository {repo.full_name}")
-
-        collaborators, outside_collaborators = (
-            _get_collaborators_and_outside_collaborators(github_client, repo)
-        )
-        teams = _fetch_repository_teams_detailed(repo, github_client)
-        external_user_groups = []
-
-        user_emails = set()
-        for collab in collaborators:
-            if collab.email:
-                user_emails.add(collab.email)
-            else:
-                logger.error(f"Collaborator {collab.login} has no email")
-
-        if user_emails:
-            collaborators_group = ExternalUserGroup(
-                id=form_collaborators_group_id(repo.id),
-                user_emails=list(user_emails),
-            )
-            external_user_groups.append(collaborators_group)
-            logger.info(f"Created collaborators group with {len(user_emails)} emails")
-
-        # Create group for outside collaborators
-        user_emails = set()
-        for collab in outside_collaborators:
-            if collab.email:
-                user_emails.add(collab.email)
-            else:
-                logger.error(f"Outside collaborator {collab.login} has no email")
-
-        if user_emails:
-            outside_collaborators_group = ExternalUserGroup(
-                id=form_outside_collaborators_group_id(repo.id),
-                user_emails=list(user_emails),
-            )
-            external_user_groups.append(outside_collaborators_group)
-            logger.info(
-                f"Created outside collaborators group with {len(user_emails)} emails"
-            )
-
-        # Create groups for teams
-        for team in teams:
-            user_emails = set()
-            for member in team.members:
-                if member.email:
-                    user_emails.add(member.email)
-                else:
-                    logger.error(f"Team member {member.login} has no email")
-
-            if user_emails:
-                team_group = ExternalUserGroup(
-                    id=team.slug,
-                    user_emails=list(user_emails),
-                )
-                external_user_groups.append(team_group)
-                logger.info(
-                    f"Created team group {team.name} with {len(user_emails)} emails"
-                )
-
-        logger.info(
-            f"Created {len(external_user_groups)} ExternalUserGroups for private repository {repo.full_name}"
-        )
-        return external_user_groups
-
-    if repo_visibility == GitHubVisibility.INTERNAL:
-        logger.info(f"Processing internal repository {repo.full_name}")
-
-        org_group_id = form_organization_group_id(repo.organization.id)
-        org_members = _fetch_organization_members(
-            github_client, repo.organization.login
-        )
-
-        user_emails = set()
-        for member in org_members:
-            if member.email:
-                user_emails.add(member.email)
-            else:
-                logger.error(f"Org member {member.login} has no email")
-
-        org_group = ExternalUserGroup(
-            id=org_group_id,
-            user_emails=list(user_emails),
-        )
-        logger.info(
-            f"Created organization group with {len(user_emails)} emails for internal repository {repo.full_name}"
-        )
-        return [org_group]
-
-    logger.info(f"Repository {repo.full_name} is public - no user groups needed")
-    return []

backend/ee/onyx/external_permissions/gmail/doc_sync.py

@@ -3,7 +3,6 @@ from datetime import datetime
 from datetime import timezone
 
 from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsFunction
-from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsIdsFunction
 from onyx.access.models import DocExternalAccess
 from onyx.connectors.gmail.connector import GmailConnector
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
@@ -26,7 +25,7 @@ def _get_slim_doc_generator(
         else 0.0
     )
 
-    return gmail_connector.retrieve_all_slim_docs_perm_sync(
+    return gmail_connector.retrieve_all_slim_documents(
         start=start_time,
         end=current_time.timestamp(),
         callback=callback,
@@ -36,7 +35,6 @@ def _get_slim_doc_generator(
 def gmail_doc_sync(
     cc_pair: ConnectorCredentialPair,
     fetch_all_existing_docs_fn: FetchAllDocumentsFunction,
-    fetch_all_existing_docs_ids_fn: FetchAllDocumentsIdsFunction,
     callback: IndexingHeartbeatInterface | None,
 ) -> Generator[DocExternalAccess, None, None]:
     """

backend/ee/onyx/external_permissions/google_drive/doc_sync.py

@@ -8,7 +8,6 @@ from ee.onyx.external_permissions.google_drive.permission_retrieval import (
     get_permissions_by_ids,
 )
 from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsFunction
-from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsIdsFunction
 from onyx.access.models import DocExternalAccess
 from onyx.access.models import ExternalAccess
 from onyx.connectors.google_drive.connector import GoogleDriveConnector
@@ -34,35 +33,15 @@ def _get_slim_doc_generator(
         else 0.0
     )
 
-    return google_drive_connector.retrieve_all_slim_docs_perm_sync(
+    return google_drive_connector.retrieve_all_slim_documents(
         start=start_time,
         end=current_time.timestamp(),
         callback=callback,
     )
 
 
-def _merge_permissions_lists(
-    permission_lists: list[list[GoogleDrivePermission]],
-) -> list[GoogleDrivePermission]:
-    """
-    Merge a list of permission lists into a single list of permissions.
-    """
-    seen_permission_ids: set[str] = set()
-    merged_permissions: list[GoogleDrivePermission] = []
-    for permission_list in permission_lists:
-        for permission in permission_list:
-            if permission.id not in seen_permission_ids:
-                merged_permissions.append(permission)
-                seen_permission_ids.add(permission.id)
-
-    return merged_permissions
-
-
 def get_external_access_for_raw_gdrive_file(
-    file: GoogleDriveFileType,
-    company_domain: str,
-    retriever_drive_service: GoogleDriveService | None,
-    admin_drive_service: GoogleDriveService,
+    file: GoogleDriveFileType, company_domain: str, drive_service: GoogleDriveService
 ) -> ExternalAccess:
     """
     Get the external access for a raw Google Drive file.
@@ -83,28 +62,11 @@ def get_external_access_for_raw_gdrive_file(
             GoogleDrivePermission.from_drive_permission(p) for p in permissions
         ]
     elif permission_ids:
-
-        def _get_permissions(
-            drive_service: GoogleDriveService,
-        ) -> list[GoogleDrivePermission]:
-            return get_permissions_by_ids(
-                drive_service=drive_service,
-                doc_id=doc_id,
-                permission_ids=permission_ids,
-            )
-
-        permissions_list = _get_permissions(
-            retriever_drive_service or admin_drive_service
+        permissions_list = get_permissions_by_ids(
+            drive_service=drive_service,
+            doc_id=doc_id,
+            permission_ids=permission_ids,
         )
-        if len(permissions_list) != len(permission_ids) and retriever_drive_service:
-            logger.warning(
-                f"Failed to get all permissions for file {doc_id} with retriever service, "
-                "trying admin service"
-            )
-            backup_permissions_list = _get_permissions(admin_drive_service)
-            permissions_list = _merge_permissions_lists(
-                [permissions_list, backup_permissions_list]
-            )
 
     folder_ids_to_inherit_permissions_from: set[str] = set()
     user_emails: set[str] = set()
@@ -170,7 +132,6 @@ def get_external_access_for_raw_gdrive_file(
 def gdrive_doc_sync(
     cc_pair: ConnectorCredentialPair,
     fetch_all_existing_docs_fn: FetchAllDocumentsFunction,
-    fetch_all_existing_docs_ids_fn: FetchAllDocumentsIdsFunction,
     callback: IndexingHeartbeatInterface | None,
 ) -> Generator[DocExternalAccess, None, None]:
     """

backend/ee/onyx/external_permissions/google_drive/group_sync.py

@@ -1,5 +1,3 @@
-from collections.abc import Generator
-
 from googleapiclient.errors import HttpError  # type: ignore
 from pydantic import BaseModel
 
@@ -44,17 +42,11 @@ def _get_all_folders(
 
     TODO: tweak things so we can fetch deltas.
     """
-    MAX_FAILED_PERCENTAGE = 0.5
-
     all_folders: list[FolderInfo] = []
     seen_folder_ids: set[str] = set()
 
-    def _get_all_folders_for_user(
-        google_drive_connector: GoogleDriveConnector,
-        skip_folders_without_permissions: bool,
-        user_email: str,
-    ) -> None:
-        """Helper to get folders for a specific user + update shared seen_folder_ids"""
+    user_emails = google_drive_connector._get_all_user_emails()
+    for user_email in user_emails:
         drive_service = get_drive_service(
             google_drive_connector.creds,
             user_email,
@@ -104,61 +96,9 @@ def _get_all_folders(
                 )
             )
 
-    failed_count = 0
-    user_emails = google_drive_connector._get_all_user_emails()
-    for user_email in user_emails:
-        try:
-            _get_all_folders_for_user(
-                google_drive_connector, skip_folders_without_permissions, user_email
-            )
-        except Exception:
-            logger.exception(f"Error getting folders for user {user_email}")
-            failed_count += 1
-
-            if failed_count > MAX_FAILED_PERCENTAGE * len(user_emails):
-                raise RuntimeError("Too many failed folder fetches during group sync")
-
     return all_folders
 
 
-def _drive_folder_to_onyx_group(
-    folder: FolderInfo,
-    group_email_to_member_emails_map: dict[str, list[str]],
-) -> ExternalUserGroup:
-    """
-    Converts a folder into an Onyx group.
-    """
-    anyone_can_access = False
-    folder_member_emails: set[str] = set()
-
-    for permission in folder.permissions:
-        if permission.type == PermissionType.USER:
-            if permission.email_address is None:
-                logger.warning(
-                    f"User email is None for folder {folder.id} permission {permission}"
-                )
-                continue
-            folder_member_emails.add(permission.email_address)
-        elif permission.type == PermissionType.GROUP:
-            if permission.email_address not in group_email_to_member_emails_map:
-                logger.warning(
-                    f"Group email {permission.email_address} for folder {folder.id} "
-                    "not found in group_email_to_member_emails_map"
-                )
-                continue
-            folder_member_emails.update(
-                group_email_to_member_emails_map[permission.email_address]
-            )
-        elif permission.type == PermissionType.ANYONE:
-            anyone_can_access = True
-
-    return ExternalUserGroup(
-        id=folder.id,
-        user_emails=list(folder_member_emails),
-        gives_anyone_access=anyone_can_access,
-    )
-
-
 """Individual Shared Drive / My Drive Permission Sync"""
 
 
@@ -227,29 +167,7 @@ def _get_drive_members(
     return drive_id_to_members_map
 
 
-def _drive_member_map_to_onyx_groups(
-    drive_id_to_members_map: dict[str, tuple[set[str], set[str]]],
-    group_email_to_member_emails_map: dict[str, list[str]],
-) -> Generator[ExternalUserGroup, None, None]:
-    """The `user_emails` for the Shared Drive should be all individuals in the
-    Shared Drive + the union of all flattened group emails."""
-    for drive_id, (group_emails, user_emails) in drive_id_to_members_map.items():
-        drive_member_emails: set[str] = user_emails
-        for group_email in group_emails:
-            if group_email not in group_email_to_member_emails_map:
-                logger.warning(
-                    f"Group email {group_email} for drive {drive_id} not found in "
-                    "group_email_to_member_emails_map"
-                )
-                continue
-            drive_member_emails.update(group_email_to_member_emails_map[group_email])
-        yield ExternalUserGroup(
-            id=drive_id,
-            user_emails=list(drive_member_emails),
-        )
-
-
-def _get_all_google_groups(
+def _get_all_groups(
     admin_service: AdminService,
     google_domain: str,
 ) -> set[str]:
@@ -267,28 +185,6 @@ def _get_all_google_groups(
     return group_emails
 
 
-def _google_group_to_onyx_group(
-    admin_service: AdminService,
-    group_email: str,
-) -> ExternalUserGroup:
-    """
-    This maps google group emails to their member emails.
-    """
-    group_member_emails: set[str] = set()
-    for member in execute_paginated_retrieval(
-        admin_service.members().list,
-        list_key="members",
-        groupKey=group_email,
-        fields="members(email),nextPageToken",
-    ):
-        group_member_emails.add(member["email"])
-
-    return ExternalUserGroup(
-        id=group_email,
-        user_emails=list(group_member_emails),
-    )
-
-
 def _map_group_email_to_member_emails(
     admin_service: AdminService,
     group_emails: set[str],
@@ -386,7 +282,7 @@ def _build_onyx_groups(
 def gdrive_group_sync(
     tenant_id: str,
     cc_pair: ConnectorCredentialPair,
-) -> Generator[ExternalUserGroup, None, None]:
+) -> list[ExternalUserGroup]:
     # Initialize connector and build credential/service objects
     google_drive_connector = GoogleDriveConnector(
         **cc_pair.connector.connector_specific_config
@@ -400,27 +296,26 @@ def gdrive_group_sync(
     drive_id_to_members_map = _get_drive_members(google_drive_connector, admin_service)
 
     # Get all group emails
-    all_group_emails = _get_all_google_groups(
+    all_group_emails = _get_all_groups(
         admin_service, google_drive_connector.google_domain
     )
 
-    # Each google group is an Onyx group, yield those
-    group_email_to_member_emails_map: dict[str, list[str]] = {}
-    for group_email in all_group_emails:
-        onyx_group = _google_group_to_onyx_group(admin_service, group_email)
-        group_email_to_member_emails_map[group_email] = onyx_group.user_emails
-        yield onyx_group
-
-    # Each drive is a group, yield those
-    for onyx_group in _drive_member_map_to_onyx_groups(
-        drive_id_to_members_map, group_email_to_member_emails_map
-    ):
-        yield onyx_group
-
     # Get all folder permissions
     folder_info = _get_all_folders(
         google_drive_connector=google_drive_connector,
         skip_folders_without_permissions=True,
     )
-    for folder in folder_info:
-        yield _drive_folder_to_onyx_group(folder, group_email_to_member_emails_map)
+
+    # Map group emails to their members
+    group_email_to_member_emails_map = _map_group_email_to_member_emails(
+        admin_service, all_group_emails
+    )
+
+    # Convert the maps to onyx groups
+    onyx_groups = _build_onyx_groups(
+        drive_id_to_members_map=drive_id_to_members_map,
+        group_email_to_member_emails_map=group_email_to_member_emails_map,
+        folder_info=folder_info,
+    )
+
+    return onyx_groups

backend/ee/onyx/external_permissions/jira/doc_sync.py

@@ -1,36 +0,0 @@
-from collections.abc import Generator
-
-from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsFunction
-from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsIdsFunction
-from ee.onyx.external_permissions.utils import generic_doc_sync
-from onyx.access.models import DocExternalAccess
-from onyx.configs.constants import DocumentSource
-from onyx.connectors.jira.connector import JiraConnector
-from onyx.db.models import ConnectorCredentialPair
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-JIRA_DOC_SYNC_TAG = "jira_doc_sync"
-
-
-def jira_doc_sync(
-    cc_pair: ConnectorCredentialPair,
-    fetch_all_existing_docs_fn: FetchAllDocumentsFunction,
-    fetch_all_existing_docs_ids_fn: FetchAllDocumentsIdsFunction,
-    callback: IndexingHeartbeatInterface | None = None,
-) -> Generator[DocExternalAccess, None, None]:
-    jira_connector = JiraConnector(
-        **cc_pair.connector.connector_specific_config,
-    )
-    jira_connector.load_credentials(cc_pair.credential.credential_json)
-
-    yield from generic_doc_sync(
-        cc_pair=cc_pair,
-        fetch_all_existing_docs_ids_fn=fetch_all_existing_docs_ids_fn,
-        callback=callback,
-        doc_source=DocumentSource.JIRA,
-        slim_connector=jira_connector,
-        label=JIRA_DOC_SYNC_TAG,
-    )

backend/ee/onyx/external_permissions/jira/models.py

@@ -1,25 +0,0 @@
-from typing import Any
-
-from pydantic import BaseModel
-from pydantic import ConfigDict
-from pydantic.alias_generators import to_camel
-
-
-Holder = dict[str, Any]
-
-
-class Permission(BaseModel):
-    id: int
-    permission: str
-    holder: Holder | None
-
-
-class User(BaseModel):
-    account_id: str
-    email_address: str
-    display_name: str
-    active: bool
-
-    model_config = ConfigDict(
-        alias_generator=to_camel,
-    )

backend/ee/onyx/external_permissions/jira/page_access.py

@@ -1,209 +0,0 @@
-from collections import defaultdict
-
-from jira import JIRA
-from jira.resources import PermissionScheme
-from pydantic import ValidationError
-
-from ee.onyx.external_permissions.jira.models import Holder
-from ee.onyx.external_permissions.jira.models import Permission
-from ee.onyx.external_permissions.jira.models import User
-from onyx.access.models import ExternalAccess
-from onyx.utils.logger import setup_logger
-
-HolderMap = dict[str, list[Holder]]
-
-
-logger = setup_logger()
-
-
-def _build_holder_map(permissions: list[dict]) -> dict[str, list[Holder]]:
-    """
-    A "Holder" in JIRA is a person / entity who "holds" the corresponding permission.
-    It can have different types. They can be one of (but not limited to):
-        - user (an explicitly whitelisted user)
-        - projectRole (for project level "roles")
-        - reporter (the reporter of an issue)
-
-    A "Holder" usually has following structure:
-        - `{ "type": "user", "value": "$USER_ID", "user": { .. }, .. }`
-        - `{ "type": "projectRole", "value": "$PROJECT_ID", ..  }`
-
-    When we fetch the PermissionSchema from JIRA, we retrieve a list of "Holder"s.
-    The list of "Holder"s can have multiple "Holder"s of the same type in the list (e.g., you can have two `"type": "user"`s in
-    there, each corresponding to a different user).
-    This function constructs a map of "Holder" types to a list of the "Holder"s which contained that type.
-
-    Returns:
-        A dict from the "Holder" type to the actual "Holder" instance.
-
-    Example:
-        ```
-        {
-            "user": [
-                { "type": "user", "value": "10000", "user": { .. }, .. },
-                { "type": "user", "value": "10001", "user": { .. }, .. },
-            ],
-            "projectRole": [
-                { "type": "projectRole", "value": "10010", ..  },
-                { "type": "projectRole", "value": "10011", ..  },
-            ],
-            "applicationRole": [
-                { "type": "applicationRole" },
-            ],
-            ..
-        }
-        ```
-    """
-
-    holder_map: defaultdict[str, list[Holder]] = defaultdict(list)
-
-    for raw_perm in permissions:
-        if not hasattr(raw_perm, "raw"):
-            logger.warning(f"Expected a 'raw' field, but none was found: {raw_perm=}")
-            continue
-
-        permission = Permission(**raw_perm.raw)
-
-        # We only care about ability to browse through projects + issues (not other permissions such as read/write).
-        if permission.permission != "BROWSE_PROJECTS":
-            continue
-
-        # In order to associate this permission to some Atlassian entity, we need the "Holder".
-        # If this doesn't exist, then we cannot associate this permission to anyone; just skip.
-        if not permission.holder:
-            logger.warning(
-                f"Expected to find a permission holder, but none was found: {permission=}"
-            )
-            continue
-
-        type = permission.holder.get("type")
-        if not type:
-            logger.warning(
-                f"Expected to find the type of permission holder, but none was found: {permission=}"
-            )
-            continue
-
-        holder_map[type].append(permission.holder)
-
-    return holder_map
-
-
-def _get_user_emails(user_holders: list[Holder]) -> list[str]:
-    emails = []
-
-    for user_holder in user_holders:
-        if "user" not in user_holder:
-            continue
-        raw_user_dict = user_holder["user"]
-
-        try:
-            user_model = User.model_validate(raw_user_dict)
-        except ValidationError:
-            logger.error(
-                "Expected to be able to serialize the raw-user-dict into an instance of `User`, but validation failed;"
-                f"{raw_user_dict=}"
-            )
-            continue
-
-        emails.append(user_model.email_address)
-
-    return emails
-
-
-def _get_user_emails_from_project_roles(
-    jira_client: JIRA,
-    jira_project: str,
-    project_role_holders: list[Holder],
-) -> list[str]:
-    # NOTE (@raunakab) a `parallel_yield` may be helpful here...?
-    roles = [
-        jira_client.project_role(project=jira_project, id=project_role_holder["value"])
-        for project_role_holder in project_role_holders
-        if "value" in project_role_holder
-    ]
-
-    emails = []
-
-    for role in roles:
-        if not hasattr(role, "actors"):
-            continue
-
-        for actor in role.actors:
-            if not hasattr(actor, "actorUser") or not hasattr(
-                actor.actorUser, "accountId"
-            ):
-                continue
-
-            user = jira_client.user(id=actor.actorUser.accountId)
-            if not hasattr(user, "accountType") or user.accountType != "atlassian":
-                continue
-
-            if not hasattr(user, "emailAddress"):
-                msg = f"User's email address was not able to be retrieved;  {actor.actorUser.accountId=}"
-                if hasattr(user, "displayName"):
-                    msg += f" {actor.displayName=}"
-                logger.warn(msg)
-                continue
-
-            emails.append(user.emailAddress)
-
-    return emails
-
-
-def _build_external_access_from_holder_map(
-    jira_client: JIRA, jira_project: str, holder_map: HolderMap
-) -> ExternalAccess:
-    """
-    # Note:
-        If the `holder_map` contains an instance of "anyone", then this is a public JIRA project.
-        Otherwise, we fetch the "projectRole"s (i.e., the user-groups in JIRA speak), and the user emails.
-    """
-
-    if "anyone" in holder_map:
-        return ExternalAccess(
-            external_user_emails=set(), external_user_group_ids=set(), is_public=True
-        )
-
-    user_emails = (
-        _get_user_emails(user_holders=holder_map["user"])
-        if "user" in holder_map
-        else []
-    )
-    project_role_user_emails = (
-        _get_user_emails_from_project_roles(
-            jira_client=jira_client,
-            jira_project=jira_project,
-            project_role_holders=holder_map["projectRole"],
-        )
-        if "projectRole" in holder_map
-        else []
-    )
-
-    external_user_emails = set(user_emails + project_role_user_emails)
-
-    return ExternalAccess(
-        external_user_emails=external_user_emails,
-        external_user_group_ids=set(),
-        is_public=False,
-    )
-
-
-def get_project_permissions(
-    jira_client: JIRA,
-    jira_project: str,
-) -> ExternalAccess | None:
-    project_permissions: PermissionScheme = jira_client.project_permissionscheme(
-        project=jira_project
-    )
-
-    if not hasattr(project_permissions, "permissions"):
-        return None
-
-    if not isinstance(project_permissions.permissions, list):
-        return None
-
-    holder_map = _build_holder_map(permissions=project_permissions.permissions)
-
-    return _build_external_access_from_holder_map(
-        jira_client=jira_client, jira_project=jira_project, holder_map=holder_map
-    )